deepline 0.1.199 → 0.1.200

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -0,0 +1,882 @@
+import { Daytona } from '@daytonaio/sdk';
+import type {
+  PlayRunnerBackend,
+  PlayRunnerPreparedExecution,
+  PlayRunnerPrepareInput,
+} from '../types';
+import { buildPlayRunnerBundle } from '../bundle';
+import {
+  findPlayRunnerResult,
+  parsePlayRunnerEventLine,
+  parsePlayRunnerEvents,
+} from '../runner-events';
+import type {
+  PlayRunnerEvent,
+  PlayRunnerExecutionConfig,
+  PlayRunnerRuntimeTiming,
+  PlayRunnerResult,
+} from '@shared_libs/play-runtime/protocol';
+import {
+  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+import {
+  loadDaytonaRequiredConfig,
+  loadDaytonaRunnerPathsConfig,
+} from '@shared_libs/play-runtime/daytona-runtime-config';
+import {
+  DAYTONA_CANCELLED_ERROR,
+  createDaytonaSandboxCleanupManager,
+  createOneShotDaytonaSandboxLifecycle,
+  type DaytonaClient,
+  type DaytonaExecutionContext,
+  type DaytonaSandbox,
+  type OneShotDaytonaSandboxLifecycle,
+  validateDaytonaExecutionContext,
+} from './daytona-lifecycle';
+import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
+
+const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
+const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
+const DAYTONA_OUTPUT_TAIL_CHARS = 2_000;
+const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
+const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
+const DAYTONA_PROGRESS_SIDECAR_POLL_MS = 1_000;
+const DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS = 2;
+const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
+const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
+const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
+const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
+  /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
+const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
+  /\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete)\b/i;
+
+type DaytonaExecution = { exitCode: number | null; result: string };
+type DaytonaUploadAttemptTiming = {
+  attempt: number;
+  sandboxId: string;
+  acquireElapsedMs: number;
+  uploadElapsedMs: number;
+  attemptElapsedMs: number;
+  totalElapsedMs: number;
+};
+type DaytonaPreparedExecution = PlayRunnerPreparedExecution & {
+  kind: 'daytona';
+  bundlePromise: Promise<string>;
+  sandboxLifecycle: OneShotDaytonaSandboxLifecycle;
+};
+
+class DaytonaUploadDeadlineBreachError extends Error {
+  readonly attemptTimings: DaytonaUploadAttemptTiming[];
+  readonly deadlineMs: number;
+
+  constructor(input: {
+    deadlineMs: number;
+    attemptTimings: DaytonaUploadAttemptTiming[];
+  }) {
+    super(
+      `Daytona upload deadline breached after ${input.attemptTimings.length} attempts (deadlineMs=${input.deadlineMs}). ${formatDaytonaUploadAttemptTimings(input.attemptTimings)}`,
+    );
+    this.name = 'DaytonaUploadDeadlineBreachError';
+    this.deadlineMs = input.deadlineMs;
+    this.attemptTimings = input.attemptTimings;
+  }
+}
+
+class DaytonaCommandTransportError extends Error {
+  readonly sandboxId: string;
+  readonly commandError: unknown;
+  readonly recoveryError: unknown;
+
+  constructor(input: {
+    sandboxId: string;
+    commandError: unknown;
+    recoveryError: unknown;
+  }) {
+    super(
+      `Daytona command transport failed for sandbox ${input.sandboxId}: ${formatDaytonaError(input.commandError)}. Recovery could not read the command result: ${formatDaytonaError(input.recoveryError)}.`,
+      { cause: input.commandError },
+    );
+    this.name = 'DaytonaCommandTransportError';
+    this.sandboxId = input.sandboxId;
+    this.commandError = input.commandError;
+    this.recoveryError = input.recoveryError;
+  }
+}
+
+export const daytonaSdkClientFactory = {
+  create(input: ConstructorParameters<typeof Daytona>[0]): DaytonaClient {
+    return new Daytona(input);
+  },
+};
+
+function formatDaytonaExecutionError(error: unknown): string {
+  const message = formatDaytonaError(error);
+  return isConfiguredDaytonaRuntimeLimit(error)
+    ? STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR
+    : message;
+}
+
+function formatDaytonaError(error: unknown): string {
+  if (!(error instanceof Error)) {
+    return String(error);
+  }
+  const errorCode = (error as Error & { code?: unknown }).code;
+  const code = typeof errorCode === 'string' ? ` code=${errorCode}` : '';
+  const cause =
+    error.cause && error.cause !== error
+      ? ` cause=${formatDaytonaError(error.cause)}`
+      : '';
+  const details =
+    error.name && error.name !== 'Error'
+      ? [error.name, error.message].filter(Boolean).join(': ')
+      : error.message || error.name || 'Error';
+  return `${details || 'Error'}${code}${cause}`;
+}
+
+function isConfiguredDaytonaRuntimeLimit(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return new RegExp(
+    `(?:execution|command).{0,80}(?:timed out|timeout).{0,80}${DAYTONA_EXECUTE_TIMEOUT_SECONDS}|${DAYTONA_EXECUTE_TIMEOUT_SECONDS}.{0,80}(?:timed out|timeout)`,
+    'i',
+  ).test(message);
+}
+
+function getDaytonaOutputTail(output: unknown): string | null {
+  const text = String(output ?? '').trim();
+  if (!text) {
+    return null;
+  }
+  return text.slice(-DAYTONA_OUTPUT_TAIL_CHARS);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function formatDaytonaUploadAttemptTimings(
+  timings: readonly DaytonaUploadAttemptTiming[],
+): string {
+  return `attempts=[${timings
+    .map(
+      (timing) =>
+        `{attempt=${timing.attempt}, sandboxId=${timing.sandboxId}, acquireElapsedMs=${timing.acquireElapsedMs}, uploadElapsedMs=${timing.uploadElapsedMs}, attemptElapsedMs=${timing.attemptElapsedMs}, totalElapsedMs=${timing.totalElapsedMs}}`,
+    )
+    .join(', ')}]`;
+}
+
+function emitDaytonaStage(
+  _callbacks: Parameters<PlayRunnerBackend['execute']>[1],
+  context: DaytonaExecutionContext,
+  stage: string,
+  extra: Record<string, unknown> = {},
+): void {
+  const payload = {
+    workflowId: context.workflowId ?? null,
+    runId: context.runId ?? null,
+    playName: context.playName ?? null,
+    stage,
+    ...extra,
+  };
+  console.info('[play-runner.daytona.stage]', JSON.stringify(payload));
+}
+
+function dispatchDaytonaRunnerEvent(input: {
+  event: PlayRunnerEvent;
+  callbacks: Parameters<PlayRunnerBackend['execute']>[1];
+  context: DaytonaExecutionContext;
+  sandboxId: string;
+  startedAt: number;
+}): void {
+  const { event, callbacks, context, sandboxId, startedAt } = input;
+  if (event.type === 'log') {
+    if (
+      event.line.startsWith('[perf]') ||
+      event.line.includes('Executing tool batch')
+    ) {
+      emitDaytonaStage(callbacks, context, 'runner:log', {
+        sandboxId,
+        line: event.line,
+        elapsedMs: Date.now() - startedAt,
+      });
+    }
+    callbacks?.onLog?.(event);
+    return;
+  }
+  if (event.type === 'checkpoint') {
+    callbacks?.onCheckpoint?.(event.checkpoint);
+    return;
+  }
+  if (event.type === 'row_update') {
+    callbacks?.onRowUpdate?.(event.update);
+    return;
+  }
+  if (event.type === 'execution_event') {
+    callbacks?.onExecutionEvent?.(event.event);
+  }
+}
+
+function createDaytonaProgressSidecarPoller(input: {
+  sandbox: DaytonaSandbox;
+  progressEventPath: string;
+  callbacks: Parameters<PlayRunnerBackend['execute']>[1];
+  context: DaytonaExecutionContext;
+  startedAt: number;
+  processedEventKeys: Set<string>;
+}): { start(): void; stop(): Promise<void> } {
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  let stopped = false;
+  let inFlight: Promise<void> = Promise.resolve();
+  let offsetBytes = 0;
+  let pendingLine = '';
+  let warned = false;
+
+  const processText = (text: string, flushPartial = false) => {
+    const combined = pendingLine + text;
+    const lines = combined.split(/\r?\n/);
+    const completeLines = flushPartial ? lines : lines.slice(0, -1);
+    pendingLine = flushPartial ? '' : (lines.at(-1) ?? '');
+    for (const rawLine of completeLines) {
+      const line = rawLine.trim();
+      if (!line) {
+        continue;
+      }
+      const event = parsePlayRunnerEventLine(line);
+      if (!event) {
+        continue;
+      }
+      const eventKey = JSON.stringify(event);
+      input.processedEventKeys.add(eventKey);
+      dispatchDaytonaRunnerEvent({
+        event,
+        callbacks: input.callbacks,
+        context: input.context,
+        sandboxId: input.sandbox.id,
+        startedAt: input.startedAt,
+      });
+    }
+  };
+
+  const poll = async (flushPartial = false): Promise<void> => {
+    try {
+      const file = await input.sandbox.fs.downloadFile(
+        input.progressEventPath,
+        5,
+      );
+      const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
+      if (buffer.length < offsetBytes) {
+        offsetBytes = 0;
+        pendingLine = '';
+      }
+      if (buffer.length > offsetBytes) {
+        const chunk = buffer.subarray(offsetBytes).toString('utf-8');
+        offsetBytes = buffer.length;
+        processText(chunk, flushPartial);
+      } else if (flushPartial && pendingLine.trim()) {
+        processText('\n', true);
+      }
+    } catch (error) {
+      if (!warned && !/not found|no such file|ENOENT/i.test(String(error))) {
+        warned = true;
+        console.warn('[play-runner.daytona.progress_sidecar_poll_failed]', {
+          sandboxId: input.sandbox.id,
+          progressEventPath: input.progressEventPath,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+  };
+
+  const schedule = () => {
+    if (stopped) {
+      return;
+    }
+    timer = setTimeout(() => {
+      inFlight = inFlight
+        .catch(() => {})
+        .then(() => poll())
+        .finally(schedule);
+    }, DAYTONA_PROGRESS_SIDECAR_POLL_MS);
+    timer.unref?.();
+  };
+
+  return {
+    start() {
+      inFlight = inFlight.then(() => poll()).finally(schedule);
+    },
+    async stop() {
+      stopped = true;
+      if (timer) {
+        clearTimeout(timer);
+        timer = null;
+      }
+      await inFlight.catch(() => {});
+      await poll(true);
+    },
+  };
+}
+
+async function downloadDaytonaTextFile(
+  sandbox: DaytonaSandbox,
+  remotePath: string,
+): Promise<string> {
+  const file = await sandbox.fs.downloadFile(remotePath, 15);
+  return file.toString('utf-8');
+}
+
+async function recoverDaytonaCommandExecution(input: {
+  sandbox: DaytonaSandbox;
+  outputPath: string;
+  exitCodePath: string;
+  timeoutMs?: number;
+}): Promise<{ execution: DaytonaExecution | null; lastError: unknown }> {
+  const deadline =
+    Date.now() + (input.timeoutMs ?? DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS);
+  let lastError: unknown = null;
+
+  while (Date.now() < deadline) {
+    try {
+      const exitCodeText = await downloadDaytonaTextFile(
+        input.sandbox,
+        input.exitCodePath,
+      );
+      const output = await downloadDaytonaTextFile(
+        input.sandbox,
+        input.outputPath,
+      );
+      const exitCode = Number.parseInt(exitCodeText.trim(), 10);
+      return {
+        execution: {
+          exitCode: Number.isFinite(exitCode) ? exitCode : null,
+          result: output,
+        },
+        lastError: null,
+      };
+    } catch (error) {
+      lastError = error;
+      await sleep(DAYTONA_COMMAND_RECOVERY_POLL_MS);
+    }
+  }
+
+  return { execution: null, lastError };
+}
+
+async function resolveDaytonaCommandOutput(input: {
+  sandbox: DaytonaSandbox;
+  outputPath: string;
+  exitCodePath: string;
+  executionResult: unknown;
+}): Promise<string> {
+  const recovered = await recoverDaytonaCommandExecution({
+    sandbox: input.sandbox,
+    outputPath: input.outputPath,
+    exitCodePath: input.exitCodePath,
+  });
+  if (recovered.execution) {
+    return recovered.execution.result;
+  }
+  return String(input.executionResult ?? '');
+}
+
+function createDaytonaCancelledResult(
+  config: PlayRunnerExecutionConfig,
+): PlayRunnerResult {
+  return {
+    status: 'failed',
+    error: DAYTONA_CANCELLED_ERROR,
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: config.checkpoint ?? null,
+    tableNamespace: null,
+  };
+}
+
+function createDaytonaFailedResult(input: {
+  config: PlayRunnerExecutionConfig;
+  error: string;
+  runtimeTiming?: PlayRunnerRuntimeTiming;
+}): PlayRunnerResult {
+  return {
+    status: 'failed',
+    error: input.error,
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: input.config.checkpoint ?? null,
+    tableNamespace: null,
+    ...(input.runtimeTiming ? { runtimeTiming: input.runtimeTiming } : {}),
+  };
+}
+
+function isRetryableRuntimePostgresConnectFailure(
+  result: PlayRunnerResult,
+): result is Extract<PlayRunnerResult, { status: 'failed' }> {
+  return (
+    result.status === 'failed' &&
+    RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(result.error)
+  );
+}
+
+function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
+  if (error instanceof DaytonaCommandTransportError) {
+    return false;
+  }
+  const message = error instanceof Error ? error.message : String(error);
+  if (!message || /cancelled|aborted/i.test(message)) {
+    return false;
+  }
+  return DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(message);
+}
+
+function prepareDaytonaExecution(
+  input: PlayRunnerPrepareInput,
+  callbacks: Parameters<PlayRunnerBackend['execute']>[1],
+): DaytonaPreparedExecution {
+  const { clientOptions: daytonaOptions } = loadDaytonaRequiredConfig();
+  const daytona = daytonaSdkClientFactory.create(daytonaOptions);
+  const bundlePromise = buildPlayRunnerBundle();
+  const sandboxLifecycle = createOneShotDaytonaSandboxLifecycle({
+    daytona,
+    context: input.context,
+    emitStage: (stage, extra) =>
+      emitDaytonaStage(callbacks, input.context, stage, extra),
+  });
+  return {
+    kind: 'daytona',
+    bundlePromise,
+    sandboxLifecycle,
+    dispose: sandboxLifecycle.dispose,
+  };
+}
+
+function isDaytonaPreparedExecution(
+  prepared: PlayRunnerPreparedExecution | undefined,
+): prepared is DaytonaPreparedExecution {
+  return prepared?.kind === 'daytona';
+}
+
+export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
+  async prepare(input, callbacks) {
+    return prepareDaytonaExecution(input, callbacks);
+  },
+
+  async execute(config, callbacks, prepared) {
+    try {
+      validateDaytonaExecutionContext(config.context);
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error instanceof Error ? error.message : String(error),
+        logs: [],
+        stats: {},
+        steps: [],
+        checkpoint: config.checkpoint ?? null,
+        tableNamespace: null,
+      };
+    }
+    let preparedExecution: DaytonaPreparedExecution;
+    try {
+      preparedExecution = isDaytonaPreparedExecution(prepared)
+        ? prepared
+        : prepareDaytonaExecution({ context: config.context }, callbacks);
+    } catch (error) {
+      return {
+        status: 'failed',
+        error: error instanceof Error ? error.message : String(error),
+        logs: [],
+        stats: {},
+        steps: [],
+        checkpoint: config.checkpoint ?? null,
+        tableNamespace: null,
+      };
+    }
+
+    if (callbacks?.cancellationSignal?.aborted) {
+      await preparedExecution.dispose?.();
+      return createDaytonaCancelledResult(config);
+    }
+
+    const bundlePromise = preparedExecution.bundlePromise;
+    const sandboxLifecycle = preparedExecution.sandboxLifecycle;
+    const startedAt = sandboxLifecycle.startedAt;
+    const runtimeTiming: PlayRunnerRuntimeTiming = { backend: 'daytona' };
+    const sandboxCleanup = createDaytonaSandboxCleanupManager();
+    const withCleanup = (result: PlayRunnerResult): PlayRunnerResult => {
+      return sandboxCleanup.withCleanup(
+        {
+          ...result,
+          runtimeTiming: result.runtimeTiming ?? runtimeTiming,
+        },
+        {
+          cancellationCleanupStarted,
+        },
+      );
+    };
+    let cancellationRequested = false;
+    let cancellationCleanupStarted = false;
+    const uploadDeadlineBreaches: DaytonaUploadAttemptTiming[] = [];
+    let cancelExecution!: (error: Error) => void;
+    const cancellationPromise = new Promise<never>((_resolve, reject) => {
+      cancelExecution = reject;
+    });
+    const onCancel = () => {
+      cancellationRequested = true;
+      if (cancellationCleanupStarted) {
+        cancelExecution(new Error(DAYTONA_CANCELLED_ERROR));
+        return;
+      }
+      if (sandboxCleanup.cleanupActiveSandboxForCancellation()) {
+        cancellationCleanupStarted = true;
+      } else {
+        cancellationCleanupStarted = true;
+        void preparedExecution.dispose?.();
+      }
+      cancelExecution(new Error(DAYTONA_CANCELLED_ERROR));
+    };
+    callbacks?.cancellationSignal?.addEventListener('abort', onCancel, {
+      once: true,
+    });
+    const throwIfCancellationRequested = () => {
+      if (cancellationRequested || callbacks?.cancellationSignal?.aborted) {
+        onCancel();
+        throw new Error(DAYTONA_CANCELLED_ERROR);
+      }
+    };
+    let acquiredSandboxPromise = sandboxLifecycle.acquiredSandboxPromise;
+    try {
+      for (
+        let executionAttempt = 1;
+        executionAttempt <=
+        Math.max(
+          DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS,
+          DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS,
+        );
+        executionAttempt += 1
+      ) {
+        let sandboxForAttempt: DaytonaSandbox | null = null;
+        const attemptStartedAt = Date.now();
+        try {
+          const acquiredSandbox = await Promise.race([
+            acquiredSandboxPromise,
+            cancellationPromise,
+          ]);
+          sandboxCleanup.activate(acquiredSandbox);
+          runtimeTiming.daytonaCreateMs =
+            acquiredSandbox.billingStartedAt - startedAt;
+          const sandbox = acquiredSandbox.sandbox;
+          sandboxForAttempt = sandbox;
+          await callbacks?.onRuntimeResourceAcquired?.({
+            kind: 'daytona_sandbox',
+            sandboxId: sandbox.id,
+            billingStartedAt: acquiredSandbox.billingStartedAt,
+            cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
+            memoryGiB:
+              typeof sandbox.memory === 'number' ? sandbox.memory : null,
+            diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
+          });
+          throwIfCancellationRequested();
+
+          const { workdir: workDir } = loadDaytonaRunnerPathsConfig();
+          emitDaytonaStage(callbacks, config.context, 'workdir:resolved', {
+            sandboxId: sandbox.id,
+            workDir,
+          });
+
+          const uploadStartedAt = Date.now();
+          let uploadDeadlineTimer: ReturnType<typeof setTimeout> | null = null;
+          const uploadDeadlinePromise = new Promise<never>(
+            (_resolve, reject) => {
+              uploadDeadlineTimer = setTimeout(() => {
+                const now = Date.now();
+                const timing: DaytonaUploadAttemptTiming = {
+                  attempt: executionAttempt,
+                  sandboxId: sandbox.id,
+                  acquireElapsedMs: uploadStartedAt - attemptStartedAt,
+                  uploadElapsedMs: now - uploadStartedAt,
+                  attemptElapsedMs: now - attemptStartedAt,
+                  totalElapsedMs: now - startedAt,
+                };
+                uploadDeadlineBreaches.push(timing);
+                reject(
+                  new DaytonaUploadDeadlineBreachError({
+                    deadlineMs: DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS,
+                    attemptTimings: uploadDeadlineBreaches,
+                  }),
+                );
+              }, DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS);
+            },
+          );
+          let stagedPayload: Awaited<
+            ReturnType<typeof stageDaytonaRunnerPayload>
+          >;
+          try {
+            stagedPayload = await Promise.race([
+              stageDaytonaRunnerPayload({
+                sandbox,
+                bundlePromise,
+                config,
+                workDir,
+                startedAt,
+                emitStage: (stage, extra) =>
+                  emitDaytonaStage(callbacks, config.context, stage, extra),
+              }),
+              uploadDeadlinePromise,
+              cancellationPromise,
+            ]);
+          } catch (error) {
+            if (error instanceof DaytonaUploadDeadlineBreachError) {
+              const latestTiming =
+                error.attemptTimings[error.attemptTimings.length - 1];
+              emitDaytonaStage(callbacks, config.context, 'upload:timeout', {
+                sandboxId: sandbox.id,
+                attempt: executionAttempt,
+                deadlineMs: DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS,
+                uploadElapsedMs: latestTiming?.uploadElapsedMs ?? null,
+                attemptElapsedMs: latestTiming?.attemptElapsedMs ?? null,
+                elapsedMs: Date.now() - startedAt,
+              });
+              if (executionAttempt < DAYTONA_UPLOAD_MAX_ATTEMPTS) {
+                sandboxCleanup.stashActiveSandboxForRetry();
+                emitDaytonaStage(callbacks, config.context, 'recreate:start', {
+                  previousSandboxId: sandbox.id,
+                  attempt: executionAttempt + 1,
+                  reason: 'upload_deadline',
+                  elapsedMs: Date.now() - startedAt,
+                });
+                acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
+                continue;
+              }
+            }
+            throw error;
+          } finally {
+            if (uploadDeadlineTimer) {
+              clearTimeout(uploadDeadlineTimer);
+            }
+          }
+          runtimeTiming.daytonaUploadMs = Date.now() - uploadStartedAt;
+          throwIfCancellationRequested();
+
+          emitDaytonaStage(callbacks, config.context, 'execute:start', {
+            sandboxId: sandbox.id,
+            timeoutSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
+            mode: 'command',
+          });
+          const executionStartedAt = Date.now();
+          const streamedEventKeys = new Set<string>();
+          const progressSidecar = createDaytonaProgressSidecarPoller({
+            sandbox,
+            progressEventPath: stagedPayload.progressEventPath,
+            callbacks,
+            context: config.context,
+            startedAt,
+            processedEventKeys: streamedEventKeys,
+          });
+          let execution: DaytonaExecution;
+          try {
+            progressSidecar.start();
+            execution = await Promise.race([
+              sandbox.process.executeCommand(
+                stagedPayload.command,
+                stagedPayload.workDir,
+                undefined,
+                DAYTONA_EXECUTE_TIMEOUT_SECONDS,
+              ),
+              cancellationPromise,
+            ]);
+          } catch (error) {
+            await progressSidecar.stop();
+            if (
+              cancellationRequested ||
+              callbacks?.cancellationSignal?.aborted
+            ) {
+              onCancel();
+              throw new Error(DAYTONA_CANCELLED_ERROR);
+            }
+            const recovered = await recoverDaytonaCommandExecution({
+              sandbox,
+              outputPath: stagedPayload.outputPath,
+              exitCodePath: stagedPayload.exitCodePath,
+            });
+            emitDaytonaStage(
+              callbacks,
+              config.context,
+              'execute:transport_error',
+              {
+                sandboxId: sandbox.id,
+                operation: 'sandbox.process.executeCommand',
+                error: formatDaytonaError(error),
+                recovery: recovered.execution ? 'recovered' : 'unavailable',
+                recoveryError: recovered.execution
+                  ? null
+                  : formatDaytonaError(recovered.lastError),
+                elapsedMs: Date.now() - startedAt,
+              },
+            );
+            if (!recovered.execution) {
+              throw new DaytonaCommandTransportError({
+                sandboxId: sandbox.id,
+                commandError: error,
+                recoveryError: recovered.lastError,
+              });
+            }
+            emitDaytonaStage(callbacks, config.context, 'execute:recovered', {
+              sandboxId: sandbox.id,
+              error: formatDaytonaError(error),
+              exitCode: recovered.execution.exitCode,
+              outputBytes: recovered.execution.result.length,
+              elapsedMs: Date.now() - startedAt,
+            });
+            execution = recovered.execution;
+          } finally {
+            await progressSidecar.stop();
+          }
+          const runnerOutput = await resolveDaytonaCommandOutput({
+            sandbox,
+            outputPath: stagedPayload.outputPath,
+            exitCodePath: stagedPayload.exitCodePath,
+            executionResult: execution.result,
+          });
+          emitDaytonaStage(callbacks, config.context, 'execute:done', {
+            sandboxId: sandbox.id,
+            exitCode: execution.exitCode,
+            outputBytes: runnerOutput.length,
+            executeElapsedMs: Date.now() - executionStartedAt,
+            elapsedMs: Date.now() - startedAt,
+          });
+          runtimeTiming.daytonaExecuteMs = Date.now() - executionStartedAt;
+          const events = parsePlayRunnerEvents(runnerOutput);
+          for (const event of events) {
+            const eventKey = JSON.stringify(event);
+            if (streamedEventKeys.has(eventKey)) {
+              continue;
+            }
+            dispatchDaytonaRunnerEvent({
+              event,
+              callbacks,
+              context: config.context,
+              sandboxId: sandbox.id,
+              startedAt,
+            });
+          }
+
+          const result = findPlayRunnerResult(events);
+          if (result) {
+            if (
+              executionAttempt < DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS &&
+              isRetryableRuntimePostgresConnectFailure(result)
+            ) {
+              emitDaytonaStage(callbacks, config.context, 'execute:retry', {
+                sandboxId: sandbox.id,
+                attempt: executionAttempt + 1,
+                reason: 'runtime_postgres_connect',
+                error: result.error,
+                elapsedMs: Date.now() - startedAt,
+              });
+              sandboxCleanup.stashActiveSandboxForRetry();
+              acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
+              continue;
+            }
+            if (result.status === 'failed') {
+              emitDaytonaStage(
+                callbacks,
+                config.context,
+                'runner:failed_result',
+                {
+                  sandboxId: sandbox.id,
+                  exitCode: execution.exitCode,
+                  outputBytes: runnerOutput.length,
+                  error: result.error,
+                  outputTail: getDaytonaOutputTail(runnerOutput),
+                  elapsedMs: Date.now() - startedAt,
+                },
+              );
+            }
+            return withCleanup(result);
+          }
+
+          // executeCommand only returns the transport wrapper's captured-file
+          // path. The downloaded runner output contains the actual exception.
+          const outputTail = getDaytonaOutputTail(runnerOutput);
+          if (outputTail) {
+            emitDaytonaStage(
+              callbacks,
+              config.context,
+              'runner:missing_result',
+              {
+                sandboxId: sandbox.id,
+                exitCode: execution.exitCode,
+                outputTail,
+                elapsedMs: Date.now() - startedAt,
+              },
+            );
+          }
+
+          return withCleanup(
+            createDaytonaFailedResult({
+              config,
+              error:
+                typeof execution.exitCode === 'number' &&
+                execution.exitCode !== 0
+                  ? `Daytona play runner exited with code ${execution.exitCode}.${
+                      outputTail ? ` Output tail: ${outputTail}` : ''
+                    }`
+                  : 'Daytona play runner produced no result.',
+              runtimeTiming,
+            }),
+          );
+        } catch (error) {
+          if (cancellationRequested || callbacks?.cancellationSignal?.aborted) {
+            onCancel();
+            throw new Error(DAYTONA_CANCELLED_ERROR);
+          }
+          if (
+            executionAttempt < DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS &&
+            isRetryableDaytonaInfrastructureFailure(error)
+          ) {
+            emitDaytonaStage(callbacks, config.context, 'execute:retry', {
+              sandboxId: sandboxForAttempt?.id ?? null,
+              attempt: executionAttempt + 1,
+              reason: 'daytona_infrastructure',
+              error: error instanceof Error ? error.message : String(error),
+              elapsedMs: Date.now() - startedAt,
+            });
+            if (sandboxForAttempt) {
+              sandboxCleanup.stashActiveSandboxForRetry();
+            }
+            acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
+            continue;
+          }
+          throw error;
+        }
+      }
+
+      return withCleanup(
+        createDaytonaFailedResult({
+          config,
+          error: 'Daytona play runner exhausted retry attempts.',
+          runtimeTiming,
+        }),
+      );
+    } catch (error) {
+      emitDaytonaStage(callbacks, config.context, 'execute:error', {
+        sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
+        error: formatDaytonaError(error),
+        operation:
+          error instanceof DaytonaCommandTransportError
+            ? 'sandbox.process.executeCommand'
+            : 'runner_backend',
+        elapsedMs: Date.now() - startedAt,
+      });
+      return withCleanup(
+        createDaytonaFailedResult({
+          config,
+          error: formatDaytonaExecutionError(error),
+          runtimeTiming,
+        }),
+      );
+    } finally {
+      callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
+    }
+  },
+};