deepline 0.1.200 → 0.1.202

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -60,16 +60,18 @@ function nodeMaterializePayloadCommand(input: {
 function remoteRuntimeContextForDaytona(
   context: DaytonaExecutionContext,
 ): DaytonaRemoteRuntimeContext {
-  if ((context.preloadedDbSessions?.length ?? 0) > 0) {
-    return {
-      ...context,
-      dbSessionStrategy: 'preloaded',
-      preloadedDbSessions: context.preloadedDbSessions,
-    };
+  const gatewayBaseUrl = context.receiptGatewayBaseUrl?.trim();
+  if (!gatewayBaseUrl) {
+    throw new Error(
+      'Daytona execution requires DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL. Refusing to put app credentials or database sessions in the customer-code sandbox.',
+    );
   }
   return {
     ...context,
-    dbSessionStrategy: 'sandbox_public_key',
+    baseUrl: gatewayBaseUrl,
+    receiptGatewayBaseUrl: gatewayBaseUrl,
+    vercelProtectionBypassToken: null,
+    dbSessionStrategy: 'gateway_only',
     preloadedDbSessions: undefined,
   };
 }
@@ -215,7 +217,7 @@ export async function stageDaytonaRunnerPayload(input: {
     configPath,
     artifactCodePath,
   })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -46,6 +46,71 @@ const DAYTONA_RUNTIME_DB_CONNECT_MAX_ATTEMPTS = 2;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
+/**
+ * Client-side wall-clock grace for `sandbox.process.executeCommand`. The
+ * `timeout` argument we pass is a SERVER-side directive; the Daytona SDK's
+ * underlying HTTP call has no client deadline, so a wedged socket (observed:
+ * `AggregateError [ETIMEDOUT]`) leaves the await hanging forever and holds the
+ * worker slot. We race the call against this deadline = server timeout + grace,
+ * so a genuine server-side runtime-limit hit still returns first, but a wedged
+ * client transport fails the attempt loudly and lets the engine retry ladder
+ * take over instead of livelocking the slot.
+ */
+const DAYTONA_EXECUTE_CLIENT_GRACE_MS = 60_000;
+const DAYTONA_EXECUTE_CLIENT_DEADLINE_MS =
+  DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000 + DAYTONA_EXECUTE_CLIENT_GRACE_MS;
+/**
+ * Bound for individual sandbox/runtime API calls that have no natural timeout
+ * (e.g. recording the acquired runtime resource). A wedged one of these must
+ * fail the attempt, not hold the slot.
+ */
+const DAYTONA_API_CALL_DEADLINE_MS = 60_000;
+
+export class DaytonaOperationDeadlineError extends Error {
+  readonly operation: string;
+  readonly deadlineMs: number;
+
+  constructor(input: { operation: string; deadlineMs: number }) {
+    super(
+      `Daytona operation "${input.operation}" exceeded client deadline of ${input.deadlineMs}ms and was abandoned to free the worker slot.`,
+    );
+    this.name = 'DaytonaOperationDeadlineError';
+    this.operation = input.operation;
+    this.deadlineMs = input.deadlineMs;
+  }
+}
+
+/**
+ * Race `promise` against a wall-clock deadline. On breach, reject loudly with a
+ * `DaytonaOperationDeadlineError` (the caller's retry/failure classification
+ * decides what happens next). The timer is always cleared so a fast-resolving
+ * promise never leaves a dangling handle.
+ */
+export async function withDaytonaOperationDeadline<T>(input: {
+  operation: string;
+  deadlineMs: number;
+  promise: Promise<T>;
+}): Promise<T> {
+  let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
+  const deadlinePromise = new Promise<never>((_resolve, reject) => {
+    deadlineTimer = setTimeout(() => {
+      reject(
+        new DaytonaOperationDeadlineError({
+          operation: input.operation,
+          deadlineMs: input.deadlineMs,
+        }),
+      );
+    }, input.deadlineMs);
+    deadlineTimer.unref?.();
+  });
+  try {
+    return await Promise.race([input.promise, deadlinePromise]);
+  } finally {
+    if (deadlineTimer) {
+      clearTimeout(deadlineTimer);
+    }
+  }
+}
 const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
   /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
 const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
@@ -223,10 +288,11 @@ function createDaytonaProgressSidecarPoller(input: {
   context: DaytonaExecutionContext;
   startedAt: number;
   processedEventKeys: Set<string>;
-}): { start(): void; stop(): Promise<void> } {
+}): { start(): void; stop(flush?: boolean): Promise<void> } {
   let timer: ReturnType<typeof setTimeout> | null = null;
   let stopped = false;
   let inFlight: Promise<void> = Promise.resolve();
+  let segment = 0;
   let offsetBytes = 0;
   let pendingLine = '';
   let warned = false;
@@ -259,20 +325,39 @@ function createDaytonaProgressSidecarPoller(input: {
 
   const poll = async (flushPartial = false): Promise<void> => {
     try {
-      const file = await input.sandbox.fs.downloadFile(
+      const manifest = await input.sandbox.fs.downloadFile(
         input.progressEventPath,
         5,
       );
-      const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
-      if (buffer.length < offsetBytes) {
+      const parsedLatestSegment = Number.parseInt(
+        Buffer.isBuffer(manifest)
+          ? manifest.toString('utf-8').trim()
+          : String(manifest).trim(),
+        10,
+      );
+      const latestSegment = Number.isFinite(parsedLatestSegment)
+        ? Math.max(segment, parsedLatestSegment)
+        : segment;
+      while (segment <= latestSegment) {
+        const segmentPath = `${input.progressEventPath}.${String(
+          segment,
+        ).padStart(6, '0')}`;
+        const file = await input.sandbox.fs.downloadFile(segmentPath, 5);
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
+        if (buffer.length < offsetBytes) {
+          offsetBytes = 0;
+          pendingLine = '';
+        }
+        if (buffer.length > offsetBytes) {
+          const chunk = buffer.subarray(offsetBytes).toString('utf-8');
+          offsetBytes = buffer.length;
+          processText(chunk, flushPartial && segment === latestSegment);
+        }
+        if (segment >= latestSegment) break;
+        segment += 1;
         offsetBytes = 0;
-        pendingLine = '';
       }
-      if (buffer.length > offsetBytes) {
-        const chunk = buffer.subarray(offsetBytes).toString('utf-8');
-        offsetBytes = buffer.length;
-        processText(chunk, flushPartial);
-      } else if (flushPartial && pendingLine.trim()) {
+      if (flushPartial && pendingLine.trim()) {
         processText('\n', true);
       }
     } catch (error) {
@@ -304,14 +389,14 @@ function createDaytonaProgressSidecarPoller(input: {
     start() {
       inFlight = inFlight.then(() => poll()).finally(schedule);
     },
-    async stop() {
+    async stop(flush = true) {
       stopped = true;
       if (timer) {
         clearTimeout(timer);
         timer = null;
       }
       await inFlight.catch(() => {});
-      await poll(true);
+      if (flush) await poll(true);
     },
   };
 }
@@ -422,6 +507,12 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
   if (error instanceof DaytonaCommandTransportError) {
     return false;
   }
+  // A client-side deadline breach is a wedged transport (hung socket / SDK call
+  // with no client timeout). Retry once on a fresh sandbox before spending an
+  // engine attempt, same as the other transport-level infrastructure failures.
+  if (error instanceof DaytonaOperationDeadlineError) {
+    return true;
+  }
   const message = error instanceof Error ? error.message : String(error);
   if (!message || /cancelled|aborted/i.test(message)) {
     return false;
@@ -566,15 +657,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             acquiredSandbox.billingStartedAt - startedAt;
           const sandbox = acquiredSandbox.sandbox;
           sandboxForAttempt = sandbox;
-          await callbacks?.onRuntimeResourceAcquired?.({
-            kind: 'daytona_sandbox',
-            sandboxId: sandbox.id,
-            billingStartedAt: acquiredSandbox.billingStartedAt,
-            cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
-            memoryGiB:
-              typeof sandbox.memory === 'number' ? sandbox.memory : null,
-            diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
-          });
+          const resourceAcquiredNotice = callbacks?.onRuntimeResourceAcquired?.(
+            {
+              kind: 'daytona_sandbox',
+              sandboxId: sandbox.id,
+              billingStartedAt: acquiredSandbox.billingStartedAt,
+              cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
+              memoryGiB:
+                typeof sandbox.memory === 'number' ? sandbox.memory : null,
+              diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
+            },
+          );
+          if (resourceAcquiredNotice) {
+            await withDaytonaOperationDeadline({
+              operation: 'onRuntimeResourceAcquired',
+              deadlineMs: DAYTONA_API_CALL_DEADLINE_MS,
+              promise: resourceAcquiredNotice,
+            });
+          }
           throwIfCancellationRequested();
 
           const { workdir: workDir } = loadDaytonaRunnerPathsConfig();
@@ -676,16 +776,19 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           try {
             progressSidecar.start();
             execution = await Promise.race([
-              sandbox.process.executeCommand(
-                stagedPayload.command,
-                stagedPayload.workDir,
-                undefined,
-                DAYTONA_EXECUTE_TIMEOUT_SECONDS,
-              ),
+              withDaytonaOperationDeadline({
+                operation: 'sandbox.process.executeCommand',
+                deadlineMs: DAYTONA_EXECUTE_CLIENT_DEADLINE_MS,
+                promise: sandbox.process.executeCommand(
+                  stagedPayload.command,
+                  stagedPayload.workDir,
+                  undefined,
+                  DAYTONA_EXECUTE_TIMEOUT_SECONDS,
+                ),
+              }),
               cancellationPromise,
             ]);
           } catch (error) {
-            await progressSidecar.stop();
             if (
               cancellationRequested ||
               callbacks?.cancellationSignal?.aborted
@@ -729,7 +832,10 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             });
             execution = recovered.execution;
           } finally {
-            await progressSidecar.stop();
+            await progressSidecar.stop(
+              !cancellationRequested &&
+                callbacks?.cancellationSignal?.aborted !== true,
+            );
           }
           const runnerOutput = await resolveDaytonaCommandOutput({
             sandbox,
@@ -818,9 +924,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               error:
                 typeof execution.exitCode === 'number' &&
                 execution.exitCode !== 0
-                  ? `Daytona play runner exited with code ${execution.exitCode}.${
-                      outputTail ? ` Output tail: ${outputTail}` : ''
-                    }`
+                  ? `Daytona play runner exited with code ${execution.exitCode}${
+                      execution.exitCode === 137
+                        ? ' (sandbox OOM or forced SIGKILL)'
+                        : ''
+                    }.${outputTail ? ` Output tail: ${outputTail}` : ''}`
                   : 'Daytona play runner produced no result.',
               runtimeTiming,
             }),