deepline 0.1.238 → 0.1.240

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -1,6 +1,8 @@
 import type { DaytonaSandbox } from './daytona-lifecycle';
 
 export type DetachedDaytonaStartupDiagnostic = {
+  readyMarkerObserved: boolean;
+  startAcknowledgementAttempted: boolean;
   commandFound: boolean;
   exitCode: number | null;
   sessionFound: boolean;
@@ -51,6 +53,8 @@ export async function inspectDetachedDaytonaStartup(input: {
 }): Promise<DetachedDaytonaStartupDiagnostic> {
   const errors: string[] = [];
   const diagnostic: DetachedDaytonaStartupDiagnostic = {
+    readyMarkerObserved: false,
+    startAcknowledgementAttempted: false,
     commandFound: false,
     exitCode: null,
     sessionFound: false,
@@ -112,11 +116,17 @@ export type DetachedDaytonaCommandStart = {
   cmdId: string;
 };
 
-const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
+// The marker now proves artifact materialization plus an accepted gateway
+// heartbeat. Keep the handoff bounded, while allowing a cold Fly gateway and
+// one timed-out heartbeat request to recover before the worker rejects it.
+export const DAYTONA_RUNNER_READY_TIMEOUT_MS = 60_000;
 const DAYTONA_RUNNER_READY_POLL_MS = 250;
 
 export class DaytonaRunnerInitializationError extends Error {
-  constructor(message: string) {
+  constructor(
+    message: string,
+    readonly startupDiagnostic: DetachedDaytonaStartupDiagnostic,
+  ) {
     super(message);
     this.name = 'DaytonaRunnerInitializationError';
   }
@@ -150,15 +160,17 @@ export async function startDetachedDaytonaCommand(input: {
 
 /**
  * A detached command id only proves that Daytona accepted the request. Before
- * the worker parks, require a marker written by the initialized runner process.
- * This closes the unobservable handoff where a sandbox or command disappears
- * after executeSessionCommand but before the first gateway heartbeat.
+ * the worker parks, require a marker written only after the initialized runner
+ * materializes its artifact and the receipt gateway accepts its first
+ * heartbeat. This closes the unobservable handoff where a sandbox or command
+ * disappears after executeSessionCommand but before durable runner liveness.
  */
 export async function confirmDetachedDaytonaRunnerReady(input: {
   sandbox: DaytonaSandbox;
   sessionId: string;
   cmdId: string;
   readyPath: string;
+  startAckPath: string;
   exitCodePath: string;
   timeoutMs?: number;
   pollMs?: number;
@@ -176,6 +188,8 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
   const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
   const deadline = now() + timeoutMs;
   let markerDiagnosis = 'ready_marker_unavailable';
+  let readyMarkerObserved = false;
+  let startAcknowledgementAttempted = false;
 
   while (now() < deadline) {
     try {
@@ -183,8 +197,21 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
       const parsed = JSON.parse(marker.toString('utf-8')) as {
         ready?: unknown;
       };
-      if (parsed.ready === true) return;
-      markerDiagnosis = 'ready_marker_invalid';
+      if (parsed.ready === true) {
+        readyMarkerObserved = true;
+        startAcknowledgementAttempted = true;
+        try {
+          await input.sandbox.fs.uploadFile(
+            Buffer.from('{"start":true}', 'utf-8'),
+            input.startAckPath,
+          );
+          return;
+        } catch {
+          markerDiagnosis = 'start_ack_unavailable';
+        }
+      } else {
+        markerDiagnosis = 'ready_marker_invalid';
+      }
     } catch {
       markerDiagnosis = 'ready_marker_unavailable';
     }
@@ -197,9 +224,12 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
     cmdId: input.cmdId,
     exitCodePath: input.exitCodePath,
   });
+  diagnostic.readyMarkerObserved = readyMarkerObserved;
+  diagnostic.startAcknowledgementAttempted = startAcknowledgementAttempted;
   const oomIndicated =
     diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
   throw new DaytonaRunnerInitializationError(
-    `RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
+    `RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not materialize and establish receipt-gateway liveness within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The worker will not park this runner; the failed runner was stopped and was not retried in place.`,
+    diagnostic,
   );
 }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -474,6 +474,27 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
   return DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(message);
 }
 
+function isRetryableUnstartedDaytonaRunner(
+  error: unknown,
+): error is DaytonaRunnerInitializationError {
+  if (!(error instanceof DaytonaRunnerInitializationError)) return false;
+  const diagnostic = error.startupDiagnostic;
+  // The ready fence runs before any customer play code. If Daytona accepted
+  // the command but the worker never observed its ready marker, the runner
+  // cannot pass the reverse start-ack fence and customer code cannot begin.
+  // Terminate that sandbox and retry once on a fresh one even when its Node
+  // process is still alive. Do not retry after a start acknowledgement was
+  // attempted (its write outcome may be ambiguous), or after a runner exit.
+  return (
+    diagnostic.commandFound &&
+    diagnostic.sessionFound &&
+    diagnostic.exitCode === null &&
+    diagnostic.exitCodeFile === null &&
+    !diagnostic.readyMarkerObserved &&
+    !diagnostic.startAcknowledgementAttempted
+  );
+}
+
 function prepareDaytonaExecution(
   input: PlayRunnerPrepareInput,
   callbacks: Parameters<PlayRunnerBackend['execute']>[1],
@@ -785,11 +806,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           });
           // Push execution (B2-final, park-and-wake): start the runner command
           // DETACHED via a Daytona session and return a `detached_runner`
-          // suspension after the runner writes its one-time ready marker. The
-          // marker is the handoff proof: a Daytona cmdId alone does not prove
-          // the sandbox process initialized. After that bounded startup check,
-          // the worker parks (releasing its claim and slot) and wakes only on
-          // the runner's pushed terminal or the ceiling timeout.
+          // suspension only after the runner writes its one-time ready marker.
+          // The marker proves artifact materialization and an accepted gateway
+          // heartbeat: a Daytona cmdId alone does not prove durable liveness.
+          // After that bounded startup check, the worker parks (releasing its
+          // claim and slot) and wakes on terminal push or the ceiling timeout.
           const sessionId = `deepline-play-${randomUUID()}`;
           let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
           try {
@@ -819,6 +840,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               sessionId: start.sessionId,
               cmdId: start.cmdId,
               readyPath: stagedPayload.readyPath,
+              startAckPath: stagedPayload.startAckPath,
               exitCodePath: stagedPayload.exitCodePath,
             }),
             cancellationPromise,
@@ -868,6 +890,25 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             onCancel();
             throw new Error(DAYTONA_CANCELLED_ERROR);
           }
+          if (
+            executionAttempt < DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS &&
+            isRetryableUnstartedDaytonaRunner(error)
+          ) {
+            emitDaytonaStage(callbacks, config.context, 'execute:retry', {
+              sandboxId: sandboxForAttempt?.id ?? null,
+              attempt: executionAttempt + 1,
+              reason: 'runner_startup_not_live',
+              error: error.message,
+              elapsedMs: Date.now() - startedAt,
+            });
+            if (sandboxForAttempt) {
+              activeAcquiredResource!.billingEndedAt = Date.now();
+              await reportRetiringRuntimeResource(activeAcquiredResource!);
+              sandboxCleanup.stashActiveSandboxForRetry();
+            }
+            acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
+            continue;
+          }
           if (
             executionAttempt < DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS &&
             isRetryableDaytonaInfrastructureFailure(error)

package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts

@@ -2,7 +2,11 @@ export const SECRET_REDACTION_PLACEHOLDER = '[REDACTED_SECRET]';
 
 const BEARER_TOKEN_RE = /\bBearer\s+[A-Za-z0-9._~+/=-]{12,}\b/g;
 const BASIC_CREDENTIAL_RE = /\bBasic\s+[A-Za-z0-9+/]{4,}={0,2}\b/g;
-const JWT_RE = /\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{1,}\.[A-Za-z0-9_-]{8,}\b/g;
+// Bound each segment so V8 cannot recursively backtrack across a multi-megabyte
+// opaque receipt value that contains no dots. Real executor/provider JWT
+// segments are far smaller; 16 KiB per segment still leaves ample headroom.
+const JWT_RE =
+  /\b[A-Za-z0-9_-]{8,16384}\.[A-Za-z0-9_-]{1,16384}\.[A-Za-z0-9_-]{8,16384}\b/g;
 const PRIVATE_KEY_RE =
   /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
 const SENSITIVE_FIELD_NAME =