deepline 0.2.0 → 0.2.2

package/dist/bundling-sources/sdk/src/client.ts

@@ -187,7 +187,7 @@ function resolvePlayRunRuntimeSelection(
     const runtime = normalizePlayRuntimeSelection(request.runtime);
     if (!runtime) {
       throw new DeeplineError(
-        'runtime must be exactly { environment: "preview", namespace } with namespace matching ^[a-z][a-z0-9-]{0,30}$.',
+        'runtime must be { environment: "preview", namespace, backend?: "daytona" | "modal" } with namespace matching ^[a-z][a-z0-9-]{0,30}$.',
         undefined,
         'INVALID_RUNTIME_SELECTION',
       );
@@ -229,7 +229,18 @@ function resolvePlayRunRuntimeSelection(
       'INVALID_RUNTIME_NAMESPACE',
     );
   }
-  return { environment, namespace };
+  const configuredBackend = process.env.DEEPLINE_PLAY_RUNNER_BACKEND?.trim();
+  if (!configuredBackend) {
+    return { environment, namespace };
+  }
+  if (configuredBackend !== 'daytona' && configuredBackend !== 'modal') {
+    throw new DeeplineError(
+      `DEEPLINE_PLAY_RUNNER_BACKEND must be daytona or modal for preview runtime selection. Received "${configuredBackend}".`,
+      undefined,
+      'INVALID_RUNTIME_BACKEND',
+    );
+  }
+  return { environment, namespace, backend: configuredBackend };
 }
 
 function runtimeSelectionHeaders(
@@ -733,7 +744,10 @@ export type MonitorUpdateChangeSummary = {
   };
   upstream: {
     resource_replaced: boolean;
-    strategy: 'unchanged' | 'create_then_delete_previous';
+    strategy:
+      | 'unchanged'
+      | 'create_then_delete_previous'
+      | 'deferred_until_reactivation';
   };
 };
 export type MonitorUpdateResult = {

package/dist/bundling-sources/sdk/src/release.ts

@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
   // 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
   // exposed storage-dependent synchronous access. This deliberate minor
   // release keeps lazy paging semantics independent of row residency.
-  version: '0.2.0',
+  version: '0.2.2',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/observability/scheduled-job-errors.ts

@@ -0,0 +1,95 @@
+import { redactTelemetryText } from './redaction';
+
+const MAX_ERROR_SUMMARY_LENGTH = 900;
+
+const SECRET_ASSIGNMENT =
+  /\b((?:[A-Za-z][A-Za-z0-9_-]*[_-])?(?:token|secret|password|api[_ -]?key|access[_ -]?key))\b(\s*[:=]\s*)([^\s,;]+)/gi;
+const AUTHORIZATION_VALUE =
+  /\bauthorization\b(\s*[:=]\s*)(?:Bearer\s+)?[^\s,;|]+/gi;
+const BEARER_TOKEN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi;
+const SLACK_WEBHOOK =
+  /https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/_-]+/gi;
+const URL_SECRET = /([?&](?:token|secret|key|signature|sig|code)=)[^&\s]+/gi;
+const BARE_CREDENTIAL_PATTERNS = [
+  /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/gi,
+  /\bgithub_pat_[A-Za-z0-9_]{20,}\b/gi,
+  /\b(?:xaat|xain)-[A-Za-z0-9_-]{16,}\b/gi,
+  /\bsk-ant-(?:api|oat)[A-Za-z0-9_-]{8,}\b/gi,
+  /\bxox(?:a|b|p|r|s)-[A-Za-z0-9-]{10,}\b/gi,
+] as const;
+
+export type ScheduledJobFailure = {
+  jobId: string;
+  jobName: string;
+  scheduler: string;
+  error: unknown;
+  runUrl?: string | null;
+  stage?: string | null;
+  attempt?: number | null;
+  occurredAt?: string;
+  test?: boolean;
+};
+
+export function sanitizeScheduledJobError(
+  error: unknown,
+  maxLength = MAX_ERROR_SUMMARY_LENGTH,
+): string {
+  const raw =
+    error instanceof Error
+      ? `${error.name}: ${error.message}`
+      : typeof error === 'string'
+        ? error
+        : safeJson(error);
+  let compact = redactTelemetryText(raw)
+    .replace(/\u001b\[[0-9;]*m/g, '')
+    .replace(/\r/g, '')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join(' | ')
+    .replace(
+      AUTHORIZATION_VALUE,
+      (_match, separator) => `Authorization${separator}[REDACTED]`,
+    )
+    .replace(BEARER_TOKEN, 'Bearer [REDACTED]')
+    .replace(SLACK_WEBHOOK, '[REDACTED_SLACK_WEBHOOK]')
+    .replace(SECRET_ASSIGNMENT, (_match, label, separator) => {
+      return `${label}${separator}[REDACTED]`;
+    })
+    .replace(URL_SECRET, '$1[REDACTED]')
+    .replace(/\s+/g, ' ')
+    .trim();
+  for (const pattern of BARE_CREDENTIAL_PATTERNS) {
+    compact = compact.replace(pattern, '[REDACTED_CREDENTIAL]');
+  }
+  if (!compact) return 'No error detail was captured.';
+  return compact.length <= maxLength
+    ? compact
+    : `${compact.slice(0, Math.max(0, maxLength - 1))}…`;
+}
+
+export function formatScheduledJobFailure(input: ScheduledJobFailure): string {
+  const title = input.test
+    ? '[TEST] Scheduled job failure alert'
+    : 'Scheduled job failed';
+  return [
+    title,
+    `Job: ${input.jobName} (${input.jobId})`,
+    `Scheduler: ${input.scheduler}`,
+    input.stage ? `Stage: ${input.stage}` : null,
+    input.attempt ? `Attempt: ${input.attempt}` : null,
+    `Error: ${sanitizeScheduledJobError(input.error)}`,
+    `Occurred: ${input.occurredAt ?? new Date().toISOString()}`,
+    input.runUrl ? `Diagnostics: ${input.runUrl}` : null,
+  ]
+    .filter((line): line is string => Boolean(line))
+    .join('\n');
+}
+
+function safeJson(value: unknown): string {
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/backend.ts

@@ -1,6 +1,7 @@
 export const PLAY_RUNTIME_BACKENDS = {
   localProcess: 'local_process',
   daytona: 'daytona',
+  modal: 'modal',
 } as const;
 
 export type PlayRuntimeBackendId =
@@ -39,6 +40,11 @@ export const PLAY_BACKEND_DESCRIPTORS: Record<
     artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
     label: 'Daytona sandbox',
   },
+  [PLAY_RUNTIME_BACKENDS.modal]: {
+    id: PLAY_RUNTIME_BACKENDS.modal,
+    artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
+    label: 'Modal sandbox',
+  },
 };
 
 export function describePlayBackend(
@@ -65,6 +71,10 @@ export function normalizePlayRuntimeBackend(
     return PLAY_RUNTIME_BACKENDS.daytona;
   }
 
+  if (normalized === PLAY_RUNTIME_BACKENDS.modal) {
+    return PLAY_RUNTIME_BACKENDS.modal;
+  }
+
   if (normalized === PLAY_RUNTIME_BACKENDS.localProcess) {
     return PLAY_RUNTIME_BACKENDS.localProcess;
   }
@@ -118,3 +128,12 @@ export function isPlayRuntimeBackendId(
     Object.values(PLAY_RUNTIME_BACKENDS).includes(value as PlayRuntimeBackendId)
   );
 }
+
+export function isManagedSandboxRuntimeBackend(
+  value: string | null | undefined,
+): boolean {
+  return (
+    value === PLAY_RUNTIME_BACKENDS.daytona ||
+    value === PLAY_RUNTIME_BACKENDS.modal
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts

@@ -0,0 +1,104 @@
+import type { ModalClient } from 'modal';
+import { isIsolatedRuntimeSchedulerSchema } from './runtime-scheduler-topology';
+import {
+  STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
+  validatePlaySandboxRuntimeLimits,
+  type PlaySandboxRuntimeLimits,
+} from './sandbox-runtime-limits';
+
+export const MODAL_RUNNER_APP_NAME = 'deepline-play-runner';
+export const MODAL_RUNNER_IMAGE = 'node:20-bookworm-slim';
+export const MODAL_RUNNER_WORKDIR = '/root/deepline';
+
+// Modal bills one physical core as two vCPUs. This matches the standard
+// Deepline one-vCPU / one-GiB sandbox.
+export const MODAL_SANDBOX_CPU_CORES = 0.5;
+export const MODAL_SANDBOX_MEMORY_MIB = 1024;
+
+export type ModalRequiredConfig = {
+  client: ModalClient;
+  appName: string;
+  image: string;
+  workdir: string;
+  outboundCidrAllowlist: string[] | null;
+  limits: PlaySandboxRuntimeLimits;
+};
+
+export type ModalClientConfig = Omit<
+  ModalRequiredConfig,
+  'outboundCidrAllowlist' | 'limits'
+>;
+
+function requiredEnv(
+  env: NodeJS.ProcessEnv,
+  name: 'MODAL_TOKEN_ID' | 'MODAL_TOKEN_SECRET',
+): string {
+  const value = env[name]?.trim();
+  if (!value) throw new Error(`Missing required Modal configuration: ${name}.`);
+  return value;
+}
+
+function cidrAllowlist(raw: string | null | undefined): string[] | null {
+  const values = (raw ?? '')
+    .split(',')
+    .map((value) => value.trim())
+    .filter(Boolean)
+    .filter((value) => !value.includes(':'));
+  return values.length > 0 ? values : null;
+}
+
+export function resolveModalSandboxNetworkPolicy(input: {
+  runtimeSchedulerSchema: string | null | undefined;
+  env?: NodeJS.ProcessEnv;
+}): { outboundCidrAllowlist: string[] | null } {
+  const env = input.env ?? process.env;
+  const outboundCidrAllowlist = cidrAllowlist(
+    env.DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST,
+  );
+  if (
+    !outboundCidrAllowlist &&
+    env.NODE_ENV === 'production' &&
+    !isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
+  ) {
+    throw new Error(
+      'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST is required for the Modal capacity fallback in production. Refusing to run customer code with unrestricted outbound network access.',
+    );
+  }
+  return { outboundCidrAllowlist };
+}
+
+export async function loadModalRequiredConfig(input: {
+  env?: NodeJS.ProcessEnv;
+  runtimeSchedulerSchema?: string | null;
+  limits?: PlaySandboxRuntimeLimits | null;
+}): Promise<ModalRequiredConfig> {
+  const env = input.env ?? process.env;
+  const clientConfig = await loadModalClientConfig({ env });
+  const { outboundCidrAllowlist } = resolveModalSandboxNetworkPolicy({
+    env,
+    runtimeSchedulerSchema: input.runtimeSchedulerSchema,
+  });
+  return {
+    ...clientConfig,
+    outboundCidrAllowlist,
+    limits: validatePlaySandboxRuntimeLimits(
+      input.limits ?? { ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS },
+    ),
+  };
+}
+
+export async function loadModalClientConfig(
+  input: { env?: NodeJS.ProcessEnv } = {},
+): Promise<ModalClientConfig> {
+  const env = input.env ?? process.env;
+  const tokenId = requiredEnv(env, 'MODAL_TOKEN_ID');
+  const tokenSecret = requiredEnv(env, 'MODAL_TOKEN_SECRET');
+  const { ModalClient } = await import('modal');
+  return {
+    client: new ModalClient({ tokenId, tokenSecret }),
+    appName:
+      env.DEEPLINE_MODAL_RUNNER_APP_NAME?.trim() || MODAL_RUNNER_APP_NAME,
+    image: env.DEEPLINE_MODAL_RUNNER_IMAGE?.trim() || MODAL_RUNNER_IMAGE,
+    workdir: env.DEEPLINE_MODAL_RUNNER_WORKDIR?.trim() || MODAL_RUNNER_WORKDIR,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -231,10 +231,13 @@ export type PlayRunnerEvent =
     };
 
 export type PlayRunnerRuntimeTiming = {
-  backend: 'daytona';
+  backend: 'daytona' | 'modal';
   daytonaCreateMs?: number;
   daytonaUploadMs?: number;
   daytonaExecuteMs?: number;
+  modalCreateMs?: number;
+  modalUploadMs?: number;
+  modalExecuteMs?: number;
 };
 
 /**

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -9,9 +9,13 @@ import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
   validatePlaySandboxRuntimeLimits,
 } from '@shared_libs/play-runtime/sandbox-runtime-limits';
+import type { PlayRunnerRuntimeLifecycleEvent } from '../types';
 
 const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
 const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
+const DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS = [
+  0, 250, 750, 1_500, 3_000,
+] as const;
 // Explicit runner deadline + scheduler GC own the normal lifecycle. Daytona's
 // inactivity stop is a wider crash backstop measured from sandbox creation, so
 // setup time cannot consume the terminal-flush grace.
@@ -38,6 +42,10 @@ export type DaytonaStageEmitter = (
   extra?: Record<string, unknown>,
 ) => void;
 
+type DaytonaCreateCallObserver = (
+  event: PlayRunnerRuntimeLifecycleEvent,
+) => Promise<void>;
+
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
   daytonaOrganizationId: string;
@@ -54,12 +62,63 @@ export type OneShotDaytonaSandboxLifecycle = {
   dispose: () => Promise<void>;
 };
 
+export type DaytonaSandboxAcquisitionUnavailableReason =
+  | 'daytona_total_cpu_limit_exceeded'
+  | 'daytona_sandbox_start_timeout';
+
+/**
+ * A typed, pre-customer-code acquisition rejection. Only this error may move
+ * managed placement to another provider.
+ */
+export class DaytonaSandboxAcquisitionUnavailableError extends Error {
+  readonly reason: DaytonaSandboxAcquisitionUnavailableReason;
+
+  constructor(
+    reason: DaytonaSandboxAcquisitionUnavailableReason,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'DaytonaSandboxAcquisitionUnavailableError';
+    this.reason = reason;
+  }
+}
+
+export function resolveDaytonaSandboxAcquisitionUnavailableReason(
+  errors: readonly string[],
+): DaytonaSandboxAcquisitionUnavailableReason | null {
+  if (
+    errors.length > 0 &&
+    errors.every((error) =>
+      /Total CPU limit exceeded\.\s*Maximum allowed:\s*\d+/i.test(error),
+    )
+  ) {
+    return 'daytona_total_cpu_limit_exceeded';
+  }
+  return null;
+}
+
+function isDaytonaSandboxStartTimeout(error: string): boolean {
+  return /Failed to create and start sandbox within 10 seconds\. Operation timed out\./.test(
+    error,
+  );
+}
+
 type DaytonaCreateResult = {
   sandbox: DaytonaSandbox;
   attempt: number;
   attemptElapsedMs: number;
 };
 
+function daytonaCreateErrorClass(
+  error: unknown,
+): 'timeout' | 'capacity' | 'rate_limit' | 'other' {
+  const message = error instanceof Error ? error.message : String(error);
+  if (/timed out|timeout/i.test(message)) return 'timeout';
+  if (/total cpu limit|capacity/i.test(message)) return 'capacity';
+  if (/too many requests|rate limit|429/i.test(message)) return 'rate_limit';
+  return 'other';
+}
+
 async function rejectAcquiredSandbox(
   sandbox: DaytonaSandbox,
   reason: string,
@@ -177,6 +236,7 @@ async function createOneShotDaytonaSandbox(input: {
   daytona: DaytonaClient;
   orgId: string;
   context: DaytonaExecutionContext;
+  sandboxName: string;
 }): Promise<DaytonaSandbox> {
   const limits = validatePlaySandboxRuntimeLimits(
     input.context.sandboxRuntimeLimits ?? {
@@ -205,8 +265,13 @@ async function createOneShotDaytonaSandbox(input: {
   });
 
   const commonParams = {
+    name: input.sandboxName,
     labels,
+    // This is also the provider-owned backstop for an HTTP create timeout
+    // whose named sandbox never becomes lookup-addressable to this process:
+    // Daytona deletes an ephemeral sandbox immediately when it stops.
     ephemeral: true,
+    autoDeleteInterval: 0,
     autoStopInterval: Math.ceil(
       (limits.timeoutSeconds + PLAY_RUNNER_TIMEOUT_SECONDS - 30 * 60) / 60,
     ),
@@ -224,16 +289,47 @@ async function createOneShotDaytonaSandbox(input: {
   });
 }
 
+async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
+  daytona: DaytonaClient;
+  sandboxName: string;
+}): Promise<boolean> {
+  if (!input.daytona.get) return false;
+  for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
+    if (delayMs > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+    let sandbox: DaytonaSandbox;
+    try {
+      sandbox = await input.daytona.get(input.sandboxName);
+    } catch {
+      continue;
+    }
+    try {
+      await sandbox.delete(30);
+    } catch (error) {
+      throw new Error(
+        `Daytona timed-out sandbox ${sandbox.id} was reconciled by name but could not be deleted. Modal fallback suppressed.`,
+        { cause: error },
+      );
+    }
+    return true;
+  }
+  return false;
+}
+
 async function createRetriedOneShotDaytonaSandbox(input: {
   daytona: DaytonaClient;
   orgId: string;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
+  nextProviderAttempt: () => number;
   startedAt: number;
 }): Promise<DaytonaCreateResult> {
   const errors: string[] = [];
   for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
-    const attempt = index + 1;
+    const attempt = input.nextProviderAttempt();
+    const sandboxName = `dl-${crypto.randomUUID()}`;
     if (delayMs > 0) {
       input.emitStage('create:retry', {
         attempt: index,
@@ -244,18 +340,26 @@ async function createRetriedOneShotDaytonaSandbox(input: {
       await new Promise((resolve) => setTimeout(resolve, delayMs));
     }
     const attemptStartedAt = Date.now();
+    await input.observeCreateCall?.({
+      type: 'daytona_create_call_started',
+      occurredAtMs: attemptStartedAt,
+      providerAttempt: attempt,
+    });
+    let sandbox: DaytonaSandbox;
     try {
-      const sandbox = await createOneShotDaytonaSandbox({
+      sandbox = await createOneShotDaytonaSandbox({
         daytona: input.daytona,
         orgId: input.orgId,
         context: input.context,
+        sandboxName,
       });
-      return {
-        sandbox,
-        attempt,
-        attemptElapsedMs: Date.now() - attemptStartedAt,
-      };
     } catch (error) {
+      await input.observeCreateCall?.({
+        type: 'daytona_create_call_failed',
+        occurredAtMs: Date.now(),
+        providerAttempt: attempt,
+        errorClass: daytonaCreateErrorClass(error),
+      });
       const message = error instanceof Error ? error.message : String(error);
       errors.push(message);
       input.emitStage('create:attempt_failed', {
@@ -270,11 +374,65 @@ async function createRetriedOneShotDaytonaSandbox(input: {
         attempt,
         error: message,
       });
+      if (isDaytonaSandboxStartTimeout(message)) {
+        const deleted = await reconcileAndDeleteTimedOutDaytonaSandbox({
+          daytona: input.daytona,
+          sandboxName,
+        });
+        if (deleted) {
+          throw new DaytonaSandboxAcquisitionUnavailableError(
+            'daytona_sandbox_start_timeout',
+            `${message} Timed-out Daytona sandbox was reconciled and deleted before fallback.`,
+          );
+        }
+        throw new Error(
+          `${message} Daytona did not return a cleanup-addressable sandbox identity; Modal fallback suppressed. The named ephemeral sandbox retains Daytona's auto-delete-on-stop backstop.`,
+          { cause: error },
+        );
+      }
+      continue;
+    }
+    // The outcome must be attributed immediately after Daytona acknowledges
+    // creation, before resource-policy validation can reject it. Otherwise a
+    // created-but-rejected sandbox is indistinguishable from an unknown
+    // create result. Do not turn a failed *post-create* journal write into a
+    // new provider create: the sandbox is already real. The missing durable
+    // outcome is deliberately loud in worker logs and makes the capture gate
+    // fail as incomplete telemetry, while the normal resource ledger still
+    // records the known sandbox for cleanup below.
+    const acquiredAt = Date.now();
+    try {
+      await input.observeCreateCall?.({
+        type: 'daytona_create_call_succeeded',
+        occurredAtMs: acquiredAt,
+        providerAttempt: attempt,
+        sandboxId: sandbox.id,
+      });
+    } catch {
+      console.error('[play-runner.daytona.create_lifecycle_event_unrecorded]', {
+        workflowId: input.context.workflowId ?? null,
+        runId: input.context.runId ?? null,
+        attempt,
+        sandboxId: sandbox.id,
+        eventType: 'daytona_create_call_succeeded',
+      });
     }
+    return {
+      sandbox,
+      attempt,
+      attemptElapsedMs: acquiredAt - attemptStartedAt,
+    };
   }
-  throw new Error(
-    `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`,
-  );
+  const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`;
+  const fallbackReason =
+    resolveDaytonaSandboxAcquisitionUnavailableReason(errors);
+  if (fallbackReason) {
+    throw new DaytonaSandboxAcquisitionUnavailableError(
+      fallbackReason,
+      message,
+    );
+  }
+  throw new Error(message);
 }
 
 async function acquireOneShotDaytonaSandbox(input: {
@@ -282,6 +440,8 @@ async function acquireOneShotDaytonaSandbox(input: {
   orgId: string;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
+  nextProviderAttempt: () => number;
   startedAt: number;
 }): Promise<AcquiredDaytonaSandbox> {
   const limits = validatePlaySandboxRuntimeLimits(
@@ -361,11 +521,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   daytona: DaytonaClient;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
+  observeCreateCall?: DaytonaCreateCallObserver;
   startedAt?: number;
 }): OneShotDaytonaSandboxLifecycle {
   const orgId = validateDaytonaExecutionContext(input.context);
   const startedAt = input.startedAt ?? Date.now();
   let disposed = false;
+  let providerAttempt = 0;
   const acquiredSandboxes = new Map<string, AcquiredDaytonaSandbox>();
   let latestAcquiredSandboxPromise: Promise<AcquiredDaytonaSandbox>;
   const createFreshSandbox = () => {
@@ -374,6 +536,11 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       orgId,
       context: input.context,
       emitStage: input.emitStage,
+      observeCreateCall: input.observeCreateCall,
+      nextProviderAttempt: () => {
+        providerAttempt += 1;
+        return providerAttempt;
+      },
       startedAt,
     }).then((acquired) => {
       acquiredSandboxes.set(acquired.sandbox.id, acquired);