deepline 0.3.247 → 0.3.248

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -1,4 +1,10 @@
-import { Daytona } from '@daytonaio/sdk';
+import { createDaytonaLaunchClient } from './daytona-launch-transport';
+import type { DaytonaClientOptions } from '../../daytona-runtime-config';
+import {
+  createDaytonaControlClient,
+  DAYTONA_CONTROL_TIMEOUT_MS,
+  DaytonaControlTimeoutError,
+} from './daytona-control-transport';
 import { randomUUID } from 'node:crypto';
 import type {
   PlayRunnerBackend,
@@ -51,8 +57,6 @@ import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
 import { daytonaSandboxStartAdmission } from './daytona-start-admission';
 import { RUNNER_MEMORY_CGROUP_ROOT } from './daytona-runtime-watchdog';
 
-const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS =
-  RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryTimeoutMs;
 const DAYTONA_COMMAND_RECOVERY_POLL_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryPollMs;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
@@ -275,6 +279,7 @@ type DaytonaPreparedExecution = PlayRunnerPreparedExecution & {
   kind: 'daytona';
   bundlePromise: Promise<string>;
   sandboxLifecycle: OneShotDaytonaSandboxLifecycle;
+  daytona: DaytonaClient;
 };
 
 class DaytonaUploadDeadlineBreachError extends Error {
@@ -295,13 +300,11 @@ class DaytonaUploadDeadlineBreachError extends Error {
 }
 
 export const daytonaSdkClientFactory = {
-  create(input: ConstructorParameters<typeof Daytona>[0]): DaytonaClient {
-    return new Daytona(input);
-  },
-  /** Full client (get/delete by id) for best-effort GC of orphaned sandboxes. */
-  createFull(input: ConstructorParameters<typeof Daytona>[0]): Daytona {
-    return new Daytona(input);
+  create(input: DaytonaClientOptions, signal?: AbortSignal): DaytonaClient {
+    return createDaytonaLaunchClient(input, signal);
   },
+  /** Request-scoped native deadlines for detached recovery and cleanup. */
+  createControl: createDaytonaControlClient,
 };
 
 function stringProperty(value: unknown, key: string): string | null {
@@ -667,11 +670,11 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
     resourceSnapshot: null,
     collectionFailure,
   });
+  let control: ReturnType<typeof createDaytonaControlClient> | undefined;
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
-    const sandbox = (await daytonaSdkClientFactory
-      .createFull(clientOptions)
-      .get(input.sandboxId)) as DaytonaSandbox;
+    control = daytonaSdkClientFactory.createControl(clientOptions);
+    const sandbox = (await control.get(input.sandboxId)) as DaytonaSandbox;
     const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
     const observedOrganizationId = sandbox.organizationId?.trim() || null;
     if (
@@ -781,6 +784,8 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
     };
   } catch (error) {
     return unavailable(describeDaytonaLookupFailure(error));
+  } finally {
+    control?.dispose();
   }
 }
 
@@ -829,10 +834,11 @@ export async function inspectDetachedDaytonaRunner(input: {
     detail: string | null;
   };
 }> {
+  let control: ReturnType<typeof createDaytonaControlClient> | undefined;
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
-    const daytona = daytonaSdkClientFactory.createFull(clientOptions);
-    const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
+    control = daytonaSdkClientFactory.createControl(clientOptions);
+    const sandbox = (await control.get(input.sandboxId)) as DaytonaSandbox;
     const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
     const observedOrganizationId = sandbox.organizationId?.trim() || null;
     if (
@@ -868,11 +874,15 @@ export async function inspectDetachedDaytonaRunner(input: {
         error: error instanceof Error ? error.message : String(error),
       });
     }
+    // The full captured stdout file can scale with customer output. Metadata
+    // time must not consume its existing independent file-recovery budget.
+    const recoveryFiles = control.legacyOutputRecovery(sandbox);
+    control.dispose();
     const recovered = await recoverDaytonaCommandExecution({
-      sandbox,
+      sandbox: recoveryFiles,
       outputPath: input.outputPath,
       exitCodePath: input.exitCodePath,
-      timeoutMs: 10_000,
+      deadlineAtMs: Date.now() + 10_000,
     });
     if (!recovered.execution) {
       return {
@@ -916,6 +926,8 @@ export async function inspectDetachedDaytonaRunner(input: {
         detail: failure.detail,
       },
     };
+  } finally {
+    control?.dispose();
   }
 }
 
@@ -925,11 +937,11 @@ export async function readDetachedDaytonaRuntimeCompletion(input: {
   runtimeCompletedPath: string;
   expectedOrganizationId?: string | null;
 }): Promise<number | null> {
+  let control: ReturnType<typeof createDaytonaControlClient> | undefined;
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
-    const sandbox = (await daytonaSdkClientFactory
-      .createFull(clientOptions)
-      .get(input.sandboxId)) as DaytonaSandbox;
+    control = daytonaSdkClientFactory.createControl(clientOptions);
+    const sandbox = (await control.get(input.sandboxId)) as DaytonaSandbox;
     const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
     const observedOrganizationId = sandbox.organizationId?.trim() || null;
     if (
@@ -965,6 +977,8 @@ export async function readDetachedDaytonaRuntimeCompletion(input: {
       },
     );
     return null;
+  } finally {
+    control?.dispose();
   }
 }
 
@@ -996,10 +1010,14 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
       detail: 'Sandbox ID is required.',
     };
   }
+  let control: ReturnType<typeof createDaytonaControlClient> | undefined;
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
-    const daytona = daytonaSdkClientFactory.createFull(clientOptions);
-    const sandbox = await daytona.get(sandboxId);
+    control = daytonaSdkClientFactory.createControl(
+      clientOptions,
+      (input.timeoutSeconds ?? 30) * 1_000,
+    );
+    const sandbox = await control.get(sandboxId);
     const observedOrganizationId = sandbox.organizationId?.trim() || null;
     if (
       expectedOrganizationId &&
@@ -1012,7 +1030,7 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
         detail: 'Sandbox belongs to a different Daytona organization.',
       };
     }
-    await daytona.delete(sandbox, input.timeoutSeconds ?? 30);
+    await control.delete(sandbox);
     return {
       kind: 'deleted',
       organizationId: observedOrganizationId,
@@ -1077,6 +1095,8 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
         (failure.httpStatus ? `http_${failure.httpStatus}` : 'delete_failed'),
       detail: failure.detail,
     };
+  } finally {
+    control?.dispose();
   }
 }
 
@@ -1140,7 +1160,7 @@ function emitDaytonaStage(
 }
 
 async function downloadDaytonaTextFile(
-  sandbox: DaytonaSandbox,
+  sandbox: Pick<DaytonaSandbox, 'fs'>,
   remotePath: string,
 ): Promise<string> {
   const file = await sandbox.fs.downloadFile(remotePath, 15);
@@ -1148,13 +1168,12 @@ async function downloadDaytonaTextFile(
 }
 
 async function recoverDaytonaCommandExecution(input: {
-  sandbox: DaytonaSandbox;
+  sandbox: Pick<DaytonaSandbox, 'fs'>;
   outputPath: string;
   exitCodePath: string;
-  timeoutMs?: number;
+  deadlineAtMs: number;
 }): Promise<{ execution: DaytonaExecution | null; lastError: unknown }> {
-  const deadline =
-    Date.now() + (input.timeoutMs ?? DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS);
+  const deadline = input.deadlineAtMs;
   let lastError: unknown = null;
 
   while (Date.now() < deadline) {
@@ -1177,7 +1196,9 @@ async function recoverDaytonaCommandExecution(input: {
       };
     } catch (error) {
       lastError = error;
-      await sleep(DAYTONA_COMMAND_RECOVERY_POLL_MS);
+      const remainingMs = deadline - Date.now();
+      if (remainingMs <= 0) break;
+      await sleep(Math.min(DAYTONA_COMMAND_RECOVERY_POLL_MS, remainingMs));
     }
   }
 
@@ -1273,6 +1294,7 @@ export function retryableRuntimeInitializationFailureReason(
 
 function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
   if (error instanceof DaytonaRunnerInitializationError) return false;
+  if (error instanceof DaytonaControlTimeoutError) return true;
   const message = error instanceof Error ? error.message : String(error);
   if (!message || /cancelled|aborted/i.test(message)) {
     return false;
@@ -1304,7 +1326,10 @@ function prepareDaytonaExecution(
   callbacks: Parameters<PlayRunnerBackend['execute']>[1],
 ): DaytonaPreparedExecution {
   const { clientOptions: daytonaOptions } = loadDaytonaRequiredConfig();
-  const daytona = daytonaSdkClientFactory.create(daytonaOptions);
+  const daytona = daytonaSdkClientFactory.create(
+    daytonaOptions,
+    callbacks?.cancellationSignal,
+  );
   const bundlePromise = buildPlayRunnerBundle();
   const sandboxLifecycle = createOneShotDaytonaSandboxLifecycle({
     daytona,
@@ -1318,9 +1343,11 @@ function prepareDaytonaExecution(
       : undefined,
     releaseSandboxCapacity: callbacks?.releaseSandboxCapacity,
     startAdmission: daytonaSandboxStartAdmission,
+    cancellationSignal: callbacks?.cancellationSignal,
   });
   return {
     kind: 'daytona',
+    daytona,
     bundlePromise,
     sandboxLifecycle,
     dispose: sandboxLifecycle.dispose,
@@ -1566,6 +1593,17 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             sandbox,
             sessionId,
             readiness: readRunnerReadiness,
+            runControl: async (operation) => {
+              const scope = preparedExecution.daytona.scope(
+                sandbox,
+                DAYTONA_CONTROL_TIMEOUT_MS,
+              );
+              try {
+                return await operation(scope.sandbox);
+              } finally {
+                scope.dispose();
+              }
+            },
             cancellation: cancellationPromise,
             onCommandAccepted: async () => {
               await callbacks?.onRunnerCommandAccepted?.({
@@ -1610,10 +1648,14 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           let stagedPayload: Awaited<
             ReturnType<typeof stageDaytonaRunnerPayload>
           >;
+          const uploadScope = preparedExecution.daytona.scope(
+            sandbox,
+            DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS,
+          );
           try {
             stagedPayload = await Promise.race([
               stageDaytonaRunnerPayload({
-                sandbox,
+                sandbox: uploadScope.sandbox,
                 bundlePromise,
                 config,
                 workDir,
@@ -1639,7 +1681,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               if (executionAttempt < DAYTONA_UPLOAD_MAX_ATTEMPTS) {
                 acquiredSandbox.billingEndedAt = Date.now();
                 await reportRetiringRuntimeResource(acquiredSandbox);
-                sandboxCleanup.stashActiveSandboxForRetry();
+                await sandboxCleanup.stashActiveSandboxForRetry();
                 emitDaytonaStage(callbacks, config.context, 'recreate:start', {
                   previousSandboxId: sandbox.id,
                   attempt: executionAttempt + 1,
@@ -1652,6 +1694,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             }
             throw error;
           } finally {
+            uploadScope.dispose();
             if (uploadDeadlineTimer) {
               clearTimeout(uploadDeadlineTimer);
             }
@@ -1768,7 +1811,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             if (sandboxForAttempt) {
               activeAcquiredResource!.billingEndedAt = Date.now();
               await reportRetiringRuntimeResource(activeAcquiredResource!);
-              sandboxCleanup.stashActiveSandboxForRetry();
+              await sandboxCleanup.stashActiveSandboxForRetry();
             }
             acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
             continue;
@@ -1796,7 +1839,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
         const sandbox = sandboxCleanup.currentSandbox();
         if (sandbox) {
           try {
-            await sandbox.delete(30);
+            await preparedExecution.daytona.delete(sandbox, 30);
             console.info(
               '[play-runner.daytona.resource_report_failure_cleanup_done]',
               { sandboxId: sandbox.id },

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -2885,21 +2885,33 @@ async function executeAssignedAbsurdAttempt(input: {
           ),
         ),
     });
+    const finalizeSuspensionProgress = () =>
+      runWithAbsurdExecutionLeaseHeartbeat({
+        ctx,
+        runId,
+        operation: () =>
+          input.progressReporter.finalize(input.runnerResult, {
+            status: 'running',
+            required: suspension.kind !== 'detached_runner',
+            replayAllTerminalLogs: input.fromDetachedRunner,
+            terminalLogProducerAttempt: input.terminalLogProducerAttempt,
+          }),
+      });
+    // A real suspension closes an invocation's counter segment before its
+    // wait or handoff. Detached terminal recovery can replay the retained
+    // result and its idempotent final segment after a crash. A detached_runner
+    // placeholder still owns a live runner; preserve its park-first recovery.
+    if (suspension.kind !== 'detached_runner') {
+      await finalizeSuspensionProgress();
+    }
     const outcome = await createPlayRunAttemptSuspensionJob({
       store: attemptStore,
       engine,
       lifecycle: {
         waiting: async (lifecycle) => {
-          await runWithAbsurdExecutionLeaseHeartbeat({
-            ctx,
-            runId,
-            operation: () =>
-              input.progressReporter.finalize(input.runnerResult, {
-                status: 'running',
-                replayAllTerminalLogs: input.fromDetachedRunner,
-                terminalLogProducerAttempt: input.terminalLogProducerAttempt,
-              }),
-          });
+          if (suspension.kind === 'detached_runner') {
+            await finalizeSuspensionProgress();
+          }
           await projectLifecycleOutbox({
             ctx,
             deps,
@@ -6996,30 +7008,27 @@ async function reconcileReservedDaytonaCancellationOnce(input: {
     ),
   );
   if (!operation) return;
-  const [{ Daytona }, { loadDaytonaRequiredConfig }] = await Promise.all([
-    import('@daytonaio/sdk'),
-    import('../daytona-runtime-config'),
-  ]);
-  const daytona = new Daytona(
+  const [{ createDaytonaControlClient }, { loadDaytonaRequiredConfig }] =
+    await Promise.all([
+      import('../runner-backends/backends/daytona-control-transport'),
+      import('../daytona-runtime-config'),
+    ]);
+  const control = createDaytonaControlClient(
     loadDaytonaRequiredConfig({ environment: input.environment }).clientOptions,
+    ABSURD_SANDBOX_GC_TIMEOUT_MS,
   );
   const sandboxName = `dl-${operation.providerOperationToken}`;
-  let sandbox: Awaited<ReturnType<typeof daytona.get>>;
+  let sandbox: Awaited<ReturnType<typeof control.get>>;
   try {
-    sandbox = await deleteRuntimeSandboxBounded({
-      deleteRuntimeSandbox: (name) => daytona.get(name),
-      sandboxId: sandboxName,
-      timeoutMs: ABSURD_SANDBOX_GC_TIMEOUT_MS,
-    });
+    sandbox = await control.get(sandboxName);
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     if (!/\b(?:sandbox )?not found\b|\b404\b/i.test(message)) throw error;
     if (
       Date.now() - operation.createdAtMs <
       RUNTIME_RELIABILITY_POLICY.recovery.horizonMs
-    ) {
+    )
       return;
-    }
     await input.runWithClient((client) =>
       abandonRuntimeSandboxOperationReservation(
         client,
@@ -7028,6 +7037,8 @@ async function reconcileReservedDaytonaCancellationOnce(input: {
       ),
     );
     return;
+  } finally {
+    control.dispose();
   }
   if (sandbox.labels?.operationKey !== operation.providerOperationToken) {
     throw new Error(
@@ -7422,6 +7433,49 @@ export type RuntimeWorkerHost = {
   close(): Promise<void>;
 };
 
+/** Own shutdown ordering across the primary queue, listener and maintenance.
+ * Stop accepting claims synchronously; accepted work and every controller
+ * must settle before shutdown succeeds or reports its collected failures. */
+export function createAbsurdRuntimeWorkerHostLifecycle(input: {
+  worker: RuntimeWorkerHost;
+  closeWake: () => void;
+  stopControllers: () => void;
+  controllers: readonly (Promise<unknown> | null)[];
+}): RuntimeWorkerHost {
+  let closing: Promise<void> | null = null;
+  return {
+    close: () => {
+      if (closing) return closing;
+      const start = (operation: () => unknown): Promise<unknown> => {
+        try {
+          return Promise.resolve(operation());
+        } catch (error) {
+          return Promise.reject(error);
+        }
+      };
+      // Invoke all stop operations now, before awaiting any controller. Even
+      // a synchronous listener failure must not leave the queue claiming.
+      const stops = [
+        start(input.stopControllers),
+        start(() => input.worker.close()),
+        start(input.closeWake),
+      ];
+      closing = Promise.allSettled([...stops, ...input.controllers]).then(
+        (results) => {
+          const errors = results.flatMap((result) =>
+            result.status === 'rejected' ? [result.reason] : [],
+          );
+          if (errors.length === 1) throw errors[0];
+          if (errors.length > 1) {
+            throw new AggregateError(errors, 'Absurd worker shutdown failed.');
+          }
+        },
+      );
+      return closing;
+    },
+  };
+}
+
 function resolveAbsurdDrainExit(explicit?: boolean | null): boolean {
   if (explicit != null) return explicit;
   return process.env[ABSURD_DRAIN_EXIT_ENV]?.trim() === '1';
@@ -7777,23 +7831,24 @@ export async function startRuntimeWorkerHost(
     workerName: workerBase,
   });
 
-  return {
-    close: async () => {
-      // Accepted handlers and maintenance can take minutes to settle. Stop
-      // owning a direct LISTEN session before waiting for that drain, so a
-      // replacement can elect its listener without exhausting primary slots.
-      onWake?.close();
+  return createAbsurdRuntimeWorkerHostLifecycle({
+    worker,
+    // Release LISTEN before joining accepted work, so a replacement can elect
+    // its listener without exhausting primary scheduler sessions.
+    closeWake: () => onWake?.close(),
+    stopControllers: () => {
       drainStopped = true;
       billingStopped = true;
       cleanupStopped = true;
       adoptionStopped = true;
-      await drainWatch;
-      await adoptionSupervisor;
-      await billingController;
-      await cleanupController;
-      await worker.close();
     },
-  };
+    controllers: [
+      drainWatch,
+      adoptionSupervisor,
+      billingController,
+      cleanupController,
+    ],
+  });
 }
 
 // Re-export for the entrypoint script's telemetry/observability wiring.

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts

@@ -69,6 +69,22 @@ export function normalizePostgresConnectionStringForPg(url: string): string {
   }
 }
 
+/** PlanetScale's connection login includes a branch suffix; SQL sees the role before it. */
+export function planetScalePooledSqlRole(connectionString: string): string {
+  const url = new URL(connectionString);
+  const username = decodeURIComponent(url.username);
+  const match = /^([A-Za-z_][A-Za-z0-9_]*)\.[A-Za-z0-9_]+$/.exec(username);
+  if (
+    !['postgres:', 'postgresql:'].includes(url.protocol) ||
+    !url.hostname.endsWith('.psdb.cloud') ||
+    url.port !== '6432' ||
+    !match
+  ) {
+    throw new Error('Expected a pooled PlanetScale Postgres role URL.');
+  }
+  return match[1];
+}
+
 /**
  * Canonical env var for the workflow runtime control-plane Postgres
  * (scheduler queue, leases, receipts, outbox). Hard-cutover runtime wiring