deepline 0.3.17 → 0.3.18

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -24,11 +24,92 @@ import type { PlaySandboxRuntimeLimits } from '@shared_libs/play-runtime/sandbox
 import { validateDaytonaExecutionContext } from './daytona-lifecycle';
 import { stageRunnerPayload } from './daytona-payload-transport';
 import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
+import {
+  isPlayRunRecoveryError,
+  recoveryForOpenModalCircuit,
+  recoveryForModalSandboxCreateFailure,
+} from '@shared_libs/play-runtime/play-run-recovery-policy';
+import { shouldInjectModalSandboxCreateResourceExhausted } from '@shared_libs/play-runtime/test-runtime-seams';
+import { RUNTIME_RELIABILITY_POLICY } from '@shared_libs/play-runtime/runtime-reliability-policy';
 
-const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
+const MODAL_RUNNER_READY_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.sandbox.runnerReadyTimeoutMs;
+const MODAL_SANDBOX_CREATE_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.sandbox.modalSandboxCreateTimeoutMs;
 const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
+const MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
 
+/**
+ * A create RPC that outlives its client deadline is fundamentally ambiguous:
+ * Modal may still create the sandbox after the caller stops waiting.  It is
+ * therefore neither a safe fresh retry nor a capacity rejection.  The caller
+ * must terminalize/reconcile it and retain any local capacity reservation
+ * until a later provider-confirmed cleanup can release it.
+ */
+export class ModalSandboxCreateOutcomeUnknownError extends Error {
+  constructor(readonly timeoutMs: number) {
+    super(
+      `RUNTIME_MODAL_SANDBOX_CREATE_OUTCOME_UNKNOWN: Modal SandboxCreate did not settle within ${timeoutMs}ms. ` +
+        'No fresh sandbox retry is safe because the provider may still create the requested sandbox.',
+    );
+    this.name = 'ModalSandboxCreateOutcomeUnknownError';
+  }
+}
+
+export async function awaitModalSandboxCreate<T>(input: {
+  create: Promise<T>;
+  timeoutMs?: number;
+  onLateSuccess?: (value: T) => Promise<void> | void;
+  onLateFailure?: (error: unknown) => Promise<void> | void;
+}): Promise<T> {
+  const timeoutMs = input.timeoutMs ?? MODAL_SANDBOX_CREATE_TIMEOUT_MS;
+  let settled = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  return new Promise<T>((resolve, reject) => {
+    timer = setTimeout(() => {
+      settled = true;
+      reject(new ModalSandboxCreateOutcomeUnknownError(timeoutMs));
+    }, timeoutMs);
+    void input.create.then(
+      (value) => {
+        if (!settled) {
+          settled = true;
+          if (timer) clearTimeout(timer);
+          resolve(value);
+          return;
+        }
+        void Promise.resolve(input.onLateSuccess?.(value)).catch((error) => {
+          console.error('[play-runner.modal] late_create_cleanup_failed', {
+            error: error instanceof Error ? error.message : String(error),
+          });
+        });
+      },
+      (error) => {
+        if (!settled) {
+          settled = true;
+          if (timer) clearTimeout(timer);
+          reject(error);
+          return;
+        }
+        void Promise.resolve(input.onLateFailure?.(error)).catch(
+          (callbackError) => {
+            console.error(
+              '[play-runner.modal] late_create_failure_cleanup_failed',
+              {
+                error:
+                  callbackError instanceof Error
+                    ? callbackError.message
+                    : String(callbackError),
+              },
+            );
+          },
+        );
+      },
+    );
+  });
+}
+
 export function modalSandboxLifetimeMs(
   limits: PlaySandboxRuntimeLimits,
 ): number {
@@ -137,6 +218,37 @@ async function releaseModalSandboxCapacityAfterConfirmedTermination(input: {
   throw lastError;
 }
 
+/**
+ * A create that settles after its caller timed out is already an external
+ * fact. Persist it before attempting any further provider operation: the
+ * durable cleanup controller then owns deletion, capacity release, and the
+ * late billing adjustment. A short retry absorbs a transient scheduler-store
+ * outage without turning this into a fresh Play attempt.
+ */
+async function persistLateModalSandboxResource(input: {
+  resource: Parameters<
+    NonNullable<PlayRunnerCallbacks['onLateRuntimeResourceAcquired']>
+  >[0];
+  record: NonNullable<PlayRunnerCallbacks['onLateRuntimeResourceAcquired']>;
+}): Promise<void> {
+  let lastError: unknown;
+  for (const delayMs of MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS) {
+    if (delayMs > 0) {
+      await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
+    }
+    try {
+      await input.record(input.resource);
+      return;
+    } catch (error) {
+      lastError = error;
+    }
+  }
+  throw new AggregateError(
+    [lastError],
+    'RUNTIME_MODAL_LATE_CREATE_PERSIST_FAILED: Modal created a sandbox after the caller deadline, but its durable cleanup obligation could not be recorded.',
+  );
+}
+
 export const modalPlayRunnerBackend: PlayRunnerBackend = {
   async execute(config, callbacks) {
     let runtimeResourceRegistrationError: unknown;
@@ -160,6 +272,11 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
       const startedAt = Date.now();
       const runtimeTiming: PlayRunnerRuntimeTiming = { backend: 'modal' };
       emitModalStage(config.context, 'create:start');
+      const providerCircuitClaim =
+        await callbacks?.shouldAttemptSandboxProvider?.('modal');
+      if (providerCircuitClaim && !providerCircuitClaim.allowed) {
+        throw recoveryForOpenModalCircuit();
+      }
       const modalConfig = await loadModalRequiredConfig({
         runtimeSchedulerSchema: config.context.runtimeSchedulerSchema,
         limits: config.context.sandboxRuntimeLimits,
@@ -170,33 +287,157 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
       const image = modalConfig.client.images.fromRegistry(modalConfig.image);
       const capacityReservation =
         await callbacks?.reserveSandboxCapacity?.('modal');
-      const sandbox = await modalConfig.client.sandboxes
-        .create(app, image, {
-          cpu: MODAL_SANDBOX_CPU_CORES,
-          cpuLimit: MODAL_SANDBOX_CPU_CORES,
-          memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
-          memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
-          timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
-          idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
-          workdir: modalConfig.workdir,
-          tags: {
-            source: 'deepline-play-runner',
-            orgId: config.context.orgId ?? 'unknown',
-            workflowId: config.context.workflowId ?? 'unknown',
-            runId: config.context.runId ?? 'unknown',
-          },
-          ...(modalConfig.outboundCidrAllowlist
-            ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
+      const createTags = {
+        source: 'deepline-play-runner',
+        orgId: config.context.orgId ?? 'unknown',
+        workflowId: config.context.workflowId ?? 'unknown',
+        runId: config.context.runId ?? 'unknown',
+        attempt: String(config.context.runAttempt ?? 0),
+      };
+      let sandbox: ModalSandbox;
+      try {
+        // This is deliberately adjacent to the real provider boundary. The
+        // reservation proves the scheduler observes the same no-resource
+        // cleanup path as a genuine Modal rejection; a later persisted
+        // attempt calls the real Modal SDK normally.
+        if (
+          shouldInjectModalSandboxCreateResourceExhausted({
+            runtimeTestFaultHeader: config.context.runtimeTestFaultHeader,
+            runAttempt: config.context.runAttempt,
+          })
+        ) {
+          throw Object.assign(
+            new Error(
+              'SandboxCreate RESOURCE_EXHAUSTED: injected preview runtime test fault',
+            ),
+            { code: 8, details: 'RESOURCE_EXHAUSTED' },
+          );
+        }
+        const modalCreateRequestedAt = Date.now();
+        // A client timeout does not cancel Modal's create RPC. Persist the
+        // provider-scoped lookup key before beginning it, so a worker loss or
+        // failed late callback leaves an actionable reconciliation obligation
+        // rather than an invisible sandbox/capacity leak.
+        if (!callbacks?.onAmbiguousModalCreateIntent) {
+          throw new Error(
+            'RUNTIME_MODAL_CREATE_RECONCILIATION_UNAVAILABLE: Modal create requires a durable ambiguity recorder.',
+          );
+        }
+        await callbacks.onAmbiguousModalCreateIntent({
+          modalAppId: app.appId,
+          runtimeEnvironment:
+            process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+              ? 'preview'
+              : 'production',
+          tags: createTags,
+          ...(capacityReservation
+            ? { sandboxCapacityLeaseId: capacityReservation.leaseId }
             : {}),
-        })
-        .catch(async (error) => {
-          if (capacityReservation) {
-            await callbacks?.releaseSandboxCapacity?.(
-              capacityReservation.leaseId,
+        });
+        sandbox = await awaitModalSandboxCreate({
+          create: modalConfig.client.sandboxes.create(app, image, {
+            cpu: MODAL_SANDBOX_CPU_CORES,
+            cpuLimit: MODAL_SANDBOX_CPU_CORES,
+            memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
+            memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
+            timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
+            idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
+            workdir: modalConfig.workdir,
+            tags: createTags,
+            ...(modalConfig.outboundCidrAllowlist
+              ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
+              : {}),
+          }),
+          onLateSuccess: async (lateSandbox) => {
+            // Do not directly delete a late sandbox. First store it as a
+            // durable cleanup obligation, then let the normal cleanup worker
+            // perform provider-confirmed deletion, capacity release, and
+            // billing settlement. That remains recoverable if this process
+            // exits between any of those operations.
+            const record = callbacks?.onLateRuntimeResourceAcquired;
+            if (!record) {
+              throw new Error(
+                'RUNTIME_MODAL_LATE_CREATE_PERSIST_UNAVAILABLE: Modal created a sandbox after the caller deadline, but no durable resource recorder is configured.',
+              );
+            }
+            await persistLateModalSandboxResource({
+              record,
+              resource: {
+                kind: 'modal_sandbox',
+                sandboxId: lateSandbox.sandboxId,
+                runtimeEnvironment:
+                  process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+                    ? 'preview'
+                    : 'production',
+                modalAppId: app.appId,
+                billingStartedAt: modalCreateRequestedAt,
+                lateAcquired: true,
+                maxBillingDurationSeconds:
+                  modalConfig.limits.timeoutSeconds +
+                  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+                cpu: MODAL_SANDBOX_CPU_CORES,
+                memoryGiB: MODAL_SANDBOX_MEMORY_MIB / 1024,
+                diskGiB: 0,
+                ...(capacityReservation
+                  ? { sandboxCapacityLeaseId: capacityReservation.leaseId }
+                  : {}),
+              },
+            });
+          },
+          onLateFailure: async (lateError) => {
+            console.warn('[play-runner.modal] late_create_failed', {
+              error:
+                lateError instanceof Error
+                  ? lateError.message
+                  : String(lateError),
+            });
+            // The provider has now rejected the timed-out create, proving no
+            // sandbox exists. Release the provisional lease instead of
+            // holding organization capacity until the crash-backstop TTL.
+            if (capacityReservation && callbacks?.releaseSandboxCapacity) {
+              await releaseModalSandboxCapacityAfterConfirmedTermination({
+                leaseId: capacityReservation.leaseId,
+                release: callbacks.releaseSandboxCapacity,
+              });
+            }
+            // The late rejection is provider-confirmed absence.  Do not leave
+            // the pre-create selector pending merely because it crossed the
+            // caller deadline; otherwise the durable reconciler scans a fact
+            // we already know is safe.
+            await callbacks?.onAmbiguousModalCreateResolved?.();
+          },
+        });
+      } catch (error) {
+        if (
+          capacityReservation &&
+          !(error instanceof ModalSandboxCreateOutcomeUnknownError)
+        ) {
+          await callbacks?.releaseSandboxCapacity?.(
+            capacityReservation.leaseId,
+          );
+        }
+        // A settled rejection is known-safe; a timeout deliberately keeps the
+        // pre-create reconciliation obligation pending for recovery.
+        if (!(error instanceof ModalSandboxCreateOutcomeUnknownError)) {
+          try {
+            await callbacks?.onAmbiguousModalCreateResolved?.();
+          } catch (reconciliationError) {
+            console.warn(
+              '[play-runner.modal] create_reconciliation_resolve_failed',
+              {
+                error:
+                  reconciliationError instanceof Error
+                    ? reconciliationError.message
+                    : String(reconciliationError),
+              },
             );
           }
-          throw error;
-        });
+        }
+        // A create rejection proves no sandbox and no customer code exist.
+        // Preserve that fact for the scheduler rather than converting it
+        // into a terminal runner result below.
+        throw recoveryForModalSandboxCreateFailure(error) ?? error;
+      }
       const billingStartedAt = Date.now();
       runtimeTiming.modalCreateMs = billingStartedAt - startedAt;
       emitModalStage(config.context, 'create:done', {
@@ -237,6 +478,46 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
               ? { sandboxCapacityLeaseId: capacityReservation.leaseId }
               : {}),
           });
+          // Do not clear the pre-create obligation until the sandbox itself
+          // is durable. A process failure between Modal's success response and
+          // resource recording must remain reconciliable rather than looking
+          // like a known-safe rejection.
+          try {
+            await callbacks?.onAmbiguousModalCreateResolved?.();
+          } catch (reconciliationError) {
+            // The resource record is the authority now. Leaving this intent
+            // pending is conservative and observable; do not turn a settled
+            // resource into a fresh execution path just because the advisory
+            // intent cleanup was temporarily unavailable.
+            console.warn(
+              '[play-runner.modal] create_reconciliation_resolve_failed',
+              {
+                sandboxId: sandbox.sandboxId,
+                error:
+                  reconciliationError instanceof Error
+                    ? reconciliationError.message
+                    : String(reconciliationError),
+              },
+            );
+          }
+          // Circuit state is an operational hint, not part of the durable
+          // resource transaction. A successful resource record proves this
+          // provider path is healthy even if the best-effort circuit close
+          // cannot reach the scheduler control plane right now.
+          try {
+            await callbacks?.markSandboxProviderHealthy?.(
+              'modal',
+              providerCircuitClaim?.probeToken ?? null,
+            );
+          } catch (circuitError) {
+            console.warn('[play-runner.modal] recovery_circuit_close_failed', {
+              sandboxId: sandbox.sandboxId,
+              error:
+                circuitError instanceof Error
+                  ? circuitError.message
+                  : String(circuitError),
+            });
+          }
         } catch (error) {
           runtimeResourceRegistrationError = error;
           throw error;
@@ -278,6 +559,10 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
           cancellationPromise,
         ]);
         runtimeTiming.modalExecuteMs = Date.now() - executeStartedAt;
+        // A readiness observation after the command is accepted is ambiguous:
+        // the detached runner may already have received its durable park and
+        // started customer code while this worker cannot read the scheduler.
+        // Never translate that ambiguity into a fresh sandbox retry.
         await confirmDetachedModalRunnerReady({
           readiness: readRunnerReadiness,
           baselineHeartbeatAt,
@@ -372,7 +657,8 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
       if (
         error === runtimeResourceRegistrationError ||
         error instanceof RuntimeResourceFenceLostError ||
-        isRuntimeSandboxCapacityLimitError(error)
+        isRuntimeSandboxCapacityLimitError(error) ||
+        isPlayRunRecoveryError(error)
       ) {
         throw error;
       }
@@ -431,6 +717,38 @@ export async function deleteModalSandboxById(input: {
   }
 }
 
+/**
+ * Read the provider inventory using the exact app + tag selector persisted
+ * before a Modal create call.  This is deliberately a tiny provider adapter:
+ * the scheduler owns claim/fencing, resource recording, cleanup, and policy.
+ */
+export async function findModalSandboxIdsByTags(input: {
+  modalAppId: string;
+  tags: Record<string, string>;
+  environment: 'preview' | 'production';
+}): Promise<string[]> {
+  const modalAppId = input.modalAppId.trim();
+  if (!modalAppId) throw new Error('Modal reconciliation requires modalAppId.');
+  const tags = Object.fromEntries(
+    Object.entries(input.tags).filter(
+      ([key, value]) => key.trim() && value.trim(),
+    ),
+  );
+  if (Object.keys(tags).length === 0) {
+    throw new Error('Modal reconciliation requires non-empty create tags.');
+  }
+  const { client } = await loadModalClientConfig();
+  const sandboxIds: string[] = [];
+  for await (const sandbox of client.sandboxes.list({
+    appId: modalAppId,
+    tags,
+    environment: input.environment,
+  })) {
+    if (sandbox.sandboxId?.trim()) sandboxIds.push(sandbox.sandboxId.trim());
+  }
+  return [...new Set(sandboxIds)];
+}
+
 export async function readDetachedModalRuntimeCompletion(input: {
   sandboxId: string;
   runtimeCompletedPath: string;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -21,6 +21,13 @@ export type PlayRunnerRuntimeResource = {
   modalAppId?: string;
   billingStartedAt: number;
   billingEndedAt?: number | null;
+  /**
+   * The provider create settled only after the owning scheduler attempt had
+   * already timed out.  This resource is billed through its own durable
+   * adjustment session; it must never be silently folded into an already
+   * finalized run session.
+   */
+  lateAcquired?: boolean;
   /** Customer liability ceiling for this physical sandbox. */
   maxBillingDurationSeconds?: number | null;
   terminalReason?: RuntimeResourceTerminalReason | null;
@@ -32,6 +39,19 @@ export type PlayRunnerRuntimeResource = {
   sandboxCapacityLeaseId?: string;
 };
 
+/**
+ * Durable identity for a Modal create that can outlive the caller's timeout.
+ * These fields are only provider routing and correlation data; payloads,
+ * commands, and credentials never cross this boundary.
+ */
+export type PlayRunnerAmbiguousModalCreateIntent = {
+  modalAppId: string;
+  runtimeEnvironment: 'preview' | 'production';
+  tags: Record<string, string>;
+  /** The provisional soft-capacity reservation, if admission is enabled. */
+  sandboxCapacityLeaseId?: string;
+};
+
 export type RuntimeResourceTerminalReason =
   | 'completed'
   | 'suspended'
@@ -91,12 +111,42 @@ export interface PlayRunnerCallbacks {
   onRuntimeResourceAcquired?: (
     resource: PlayRunnerRuntimeResource,
   ) => void | Promise<void>;
+  /**
+   * A provider create timed out locally, then later produced a sandbox which
+   * could not be deleted. Persist it as a cleanup obligation even though the
+   * original attempt may already be terminal; it must never be an invisible
+   * provider resource.
+   */
+  onLateRuntimeResourceAcquired?: (
+    resource: PlayRunnerRuntimeResource,
+  ) => void | Promise<void>;
+  /**
+   * Record a reconciliation obligation before Modal create begins. It is
+   * resolved only after a known rejection or a durably recorded sandbox.
+   */
+  onAmbiguousModalCreateIntent?: (
+    intent: PlayRunnerAmbiguousModalCreateIntent,
+  ) => void | Promise<void>;
+  onAmbiguousModalCreateResolved?: () => void | Promise<void>;
   /** Reserve one physical-sandbox slot immediately before a provider create. */
   reserveSandboxCapacity?: (
     provider: 'daytona' | 'modal',
   ) => Promise<{ leaseId: string }>;
   /** Release an unbound reservation when provider create never produced a resource. */
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  /**
+   * Scheduler-owned dependency circuit admission. A false result means no
+   * provider call is safe to make yet; the backend must return/throw a typed
+   * pre-code recovery outcome instead.
+   */
+  shouldAttemptSandboxProvider?: (
+    provider: 'daytona' | 'modal',
+  ) => Promise<{ allowed: boolean; probeToken: string | null }>;
+  /** A durably recorded resource may close only the probe that owns it. */
+  markSandboxProviderHealthy?: (
+    provider: 'daytona' | 'modal',
+    probeToken: string | null,
+  ) => Promise<void>;
   /**
    * Durable scheduler-owned evidence for a provider create call. Backends
    * await this callback at the create boundary; it is not best-effort logging.

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -105,8 +105,8 @@ import {
 import {
   parseRuntimeTestFaultCounts,
   PLAY_RUNTIME_TEST_FAULT_HEADER,
-  type RuntimeTestFaultName,
 } from './test-runtime-seams';
+import type { RuntimeTestFaultName } from './runtime-incident-drills';
 import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from './durable-receipt-execution';
 import { vercelProtectionBypassHeader } from './vercel-protection';
 import {