deepline 0.3.16 → 0.3.18

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -76,6 +76,7 @@ import {
 } from './run-execution-scope';
 import { DEFAULT_RUNTIME_EXECUTION_CAPABILITIES } from './execution-capabilities';
 import { vercelProtectionBypassHeader } from './vercel-protection';
+import { RUNTIME_RELIABILITY_POLICY } from './runtime-reliability-policy';
 export {
   RuntimeSheetRowsBlockedError,
   type RuntimeSheetBlockedRowDetail,
@@ -418,11 +419,15 @@ const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // a bounded, repairable transport failure that row-isolation settles per row
 // instead of hanging the whole run.
 const DEFAULT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000 + 30_000;
-const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
+const FETCH_TRANSPORT_MAX_ATTEMPTS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchMaxAttempts;
 const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
-const CTX_FETCH_HEADERS_TIMEOUT_MS = 30_000;
-const CTX_FETCH_BODY_TIMEOUT_MS = 30_000;
-const CTX_FETCH_TOTAL_TIMEOUT_MS = 120_000;
+const CTX_FETCH_HEADERS_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchHeadersTimeoutMs;
+const CTX_FETCH_BODY_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchBodyTimeoutMs;
+const CTX_FETCH_TOTAL_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchTotalTimeoutMs;
 // cloudflared returns this branded HTML when its connection to the local app
 // resets. It is a transport failure before the app can reach a provider, not a
 // provider 502; require both the branded page and our development tunnel host.
@@ -430,6 +435,12 @@ const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
   /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
 const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
 const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
+// A newly claimed receipt should normally use its scheduled heartbeat. When a
+// test/SEV policy deliberately gives it only a moment of lease life, however,
+// scheduler setup and the first provider-dispatch turn can consume that entire
+// window before the timer fires. Renew once before fan-out in that narrow case.
+// Production's normal multi-minute lease remains on the zero-extra-query path.
+const IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS = 100;
 // Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
 // (governor) already bounds nesting; this bounds fan-WIDTH from one active row
 // resolver. The dedupe set is row-local, never retained for the full run.
@@ -5369,6 +5380,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     entries: Array<{
       request: ToolCallRequest;
       result: unknown | null;
+      status?: string;
       metadata?: ToolResultMetadataInput | null;
       jobId?: string;
       meta?: Record<string, unknown>;
@@ -5380,7 +5392,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         ...entry,
         wrapped: await this.wrapToolExecutionResult({
           toolId,
-          status: entry.result == null ? 'no_result' : 'completed',
+          status:
+            entry.status ?? (entry.result == null ? 'no_result' : 'completed'),
           jobId: entry.jobId,
           result: entry.result,
           metadata: entry.metadata,
@@ -10352,6 +10365,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
         // explicit backend capability; retain the immediate ownership check
         // for those receipts.
         if (claimsEstablishedExecutionFence) {
+          if (
+            ownershipConfirmedUntilMs - Date.now() <=
+            IMMEDIATE_PENDING_RECEIPT_HEARTBEAT_THRESHOLD_MS
+          ) {
+            const outcome = await heartbeatOwnedRequests();
+            if (outcome === 'active') supervisor.start();
+            return;
+          }
           supervisor.start();
           return;
         }
@@ -11282,12 +11303,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       }
                       continue;
                     }
+                    const batchExecution = entry.result;
                     try {
                       const splitResults =
-                        entry.result != null
+                        batchExecution != null
                           ? entry.request.splitResults(
                               legacyResultForBatchSplitter(
-                                entry.result.execution,
+                                batchExecution.execution,
                               ),
                             )
                           : entry.request.memberRequests.map(() => null);
@@ -11298,11 +11320,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                             (request, index) => ({
                               request,
                               result: splitResults[index] ?? null,
+                              status: batchExecution?.execution.status,
                               toolResponse:
-                                entry.result == null
+                                batchExecution == null
                                   ? undefined
                                   : publicToolResponseForBatchedItem(
-                                      entry.result.execution,
+                                      batchExecution.execution,
                                       splitResults[index] ?? null,
                                       this.currentToolResponseContract ===
                                         RAW_V2_TOOL_RESPONSE_CONTRACT,
@@ -11323,7 +11346,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                         resolveLiveFollowers(request, resolvedResults[index]);
                       }
                     } finally {
-                      entry.result?.releaseToolSlot();
+                      batchExecution?.releaseToolSlot();
                     }
                   }
 
@@ -11340,6 +11363,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               const completionBuffer: Array<{
                 request: ToolCallRequest;
                 result: unknown | null;
+                status?: string;
                 metadata?: ToolResultMetadataInput | null;
                 jobId?: string;
                 meta?: Record<string, unknown>;
@@ -11359,6 +11383,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                       entries.map((entry) => ({
                         request: entry.request,
                         result: entry.result,
+                        status: entry.status,
                         metadata: entry.metadata,
                         jobId: entry.jobId,
                         meta: entry.meta,
@@ -11392,6 +11417,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   completionBuffer.push({
                     request,
                     result: execution.result ?? null,
+                    status: execution.status,
                     metadata: execution.metadata ?? null,
                     jobId: execution.jobId,
                     meta: execution.meta,
@@ -12176,26 +12202,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
-              if (error instanceof ToolExecuteInvalidJsonError) {
-                throw createToolHttpError(
-                  toolErrorSchemaVersion,
-                  `Tool ${toolId} returned an invalid JSON response body.`,
-                  null,
-                  response?.status ?? 0,
-                  'repairable',
-                );
-              }
               const transportError = abortController?.signal.aborted
                 ? abortController.signal.reason instanceof Error
                   ? abortController.signal.reason
                   : new Error(
                       `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
                     )
-                : error instanceof ToolExecuteResponseBodyTransportError
+                : error instanceof ToolExecuteResponseBodyTransportError ||
+                    error instanceof ToolExecuteInvalidJsonError
                   ? error.cause
                   : error;
               if (
-                error instanceof ToolExecuteResponseBodyTransportError &&
+                (error instanceof ToolExecuteResponseBodyTransportError ||
+                  error instanceof ToolExecuteInvalidJsonError) &&
                 response?.status === 402
               ) {
                 const diagnostic = describeTransportError(transportError);
@@ -12214,7 +12233,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               const ambiguousDispatchedFailure =
                 fetchDispatched &&
                 (response === null ||
-                  error instanceof ToolExecuteResponseBodyTransportError);
+                  error instanceof ToolExecuteResponseBodyTransportError ||
+                  error instanceof ToolExecuteInvalidJsonError);
               const hasDurableInvocationIdentity = Boolean(
                 durableCallReceiptKey &&
                 providerIdempotencyKey &&
@@ -12270,8 +12290,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
                   })}`,
                 );
                 const failureBoundary =
-                  error instanceof ToolExecuteResponseBodyTransportError
-                    ? 'response body transport failed after response headers'
+                  error instanceof ToolExecuteResponseBodyTransportError ||
+                  error instanceof ToolExecuteInvalidJsonError
+                    ? 'response body could not be verified after response headers'
                     : 'request transport failed after dispatch before response headers';
                 throw createToolHttpError(
                   toolErrorSchemaVersion,

package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts

@@ -0,0 +1,254 @@
+import { RUNTIME_RELIABILITY_POLICY } from './runtime-reliability-policy';
+
+/**
+ * Typed recovery facts that cross the provider-backend seam.
+ *
+ * A runner backend is allowed to say only that customer code has not begun
+ * and name the provider-side condition it observed.  The scheduler owns the
+ * durable attempt/defer counters, deadline, and circuit state; keeping those
+ * controls out of provider adapters prevents every adapter from inventing a
+ * subtly different retry loop.
+ */
+
+export const PLAY_RUN_RECOVERY_ERROR_NAME = 'PlayRunRecoveryError';
+
+/**
+ * Hard platform defaults for a dependency-wide pre-code outage. They are
+ * deliberately finite; scheduler persistence records every defer so an
+ * operator can inspect or redrive exhausted work instead of inheriting an
+ * invisible infinite loop. A later control-plane policy may narrow these
+ * values, never exceed these ceilings.
+ */
+export const PLAY_RUN_RECOVERY_DEFER_POLICY = {
+  maxDefers: RUNTIME_RELIABILITY_POLICY.recovery.maxDefers,
+  horizonMs: RUNTIME_RELIABILITY_POLICY.recovery.horizonMs,
+  deferBaseSeconds: RUNTIME_RELIABILITY_POLICY.recovery.deferBaseSeconds,
+  deferMaxSeconds: RUNTIME_RELIABILITY_POLICY.recovery.deferMaxSeconds,
+} as const;
+
+export function recoveryDeferDelaySeconds(deferCount: number): number {
+  const index = Math.max(0, Math.floor(deferCount) - 1);
+  return Math.min(
+    PLAY_RUN_RECOVERY_DEFER_POLICY.deferMaxSeconds,
+    PLAY_RUN_RECOVERY_DEFER_POLICY.deferBaseSeconds * 2 ** index,
+  );
+}
+
+export type PlayRunRecoveryAction = 'engine_retry' | 'defer';
+
+export type PlayRunRecoveryReason =
+  | 'modal_capacity_exhausted_before_execution'
+  | 'modal_circuit_open_before_execution'
+  | 'modal_sandbox_unavailable_before_execution'
+  | 'modal_runner_start_failed_before_execution'
+  | 'runtime_initialization_transport_before_execution'
+  | 'runtime_traffic_hold_before_execution'
+  | 'runtime_traffic_throttled_before_execution';
+
+export type PlayRunRecoveryDecision = Readonly<{
+  /** A fresh sandbox is safe only before customer code can have started. */
+  stage: 'pre_code';
+  /**
+   * `engine_retry` consumes the scheduler engine's bounded attempt ladder.
+   * `defer` is for dependency-wide pressure and MUST consume the scheduler's
+   * durable defer budget/deadline instead of looping in an adapter.
+   */
+  action: PlayRunRecoveryAction;
+  reason: PlayRunRecoveryReason;
+  /** Non-secret scope used by the scheduler's dependency circuit breaker. */
+  dependency: 'modal' | 'runtime';
+}>;
+
+/**
+ * The scheduler must interpret this error.  It is intentionally not a
+ * user-facing failure and must not be converted into a PlayRunnerResult:
+ * doing so would terminalize work which has not begun executing customer code.
+ */
+export class PlayRunRecoveryError extends Error {
+  readonly decision: PlayRunRecoveryDecision;
+  readonly cause?: unknown;
+
+  constructor(input: {
+    decision: PlayRunRecoveryDecision;
+    message: string;
+    cause?: unknown;
+  }) {
+    super(input.message);
+    this.name = PLAY_RUN_RECOVERY_ERROR_NAME;
+    this.decision = input.decision;
+    if (input.cause !== undefined) {
+      this.cause = input.cause;
+    }
+  }
+}
+
+export function isPlayRunRecoveryError(
+  error: unknown,
+): error is PlayRunRecoveryError {
+  return (
+    error instanceof Error &&
+    error.name === PLAY_RUN_RECOVERY_ERROR_NAME &&
+    'decision' in error
+  );
+}
+
+function errorCode(error: unknown): string | number | null {
+  if (!error || typeof error !== 'object') return null;
+  const code = (error as { code?: unknown }).code;
+  return typeof code === 'string' || typeof code === 'number' ? code : null;
+}
+
+function errorStatus(error: unknown): number | null {
+  if (!error || typeof error !== 'object') return null;
+  const status = (error as { status?: unknown }).status;
+  if (typeof status === 'number' && Number.isFinite(status)) return status;
+  const responseStatus = (error as { response?: { status?: unknown } }).response
+    ?.status;
+  return typeof responseStatus === 'number' && Number.isFinite(responseStatus)
+    ? responseStatus
+    : null;
+}
+
+function messageOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * Classify Modal's sandbox-create response at the provider edge, while it is
+ * still known that no sandbox/runner/customer code exists.  Do not use this
+ * classifier after `sandboxes.create` succeeds: a later failure may be
+ * ambiguous and must not receive a blind fresh replay.
+ */
+export function recoveryForModalSandboxCreateFailure(
+  error: unknown,
+): PlayRunRecoveryError | null {
+  const code = errorCode(error);
+  const status = errorStatus(error);
+  const message = messageOf(error);
+  const resourceExhausted =
+    code === 8 ||
+    (typeof code === 'string' && /RESOURCE_EXHAUSTED/i.test(code)) ||
+    /\bRESOURCE_EXHAUSTED\b/i.test(message);
+  if (resourceExhausted) {
+    return new PlayRunRecoveryError({
+      decision: {
+        stage: 'pre_code',
+        action: 'defer',
+        reason: 'modal_capacity_exhausted_before_execution',
+        dependency: 'modal',
+      },
+      message:
+        'Modal sandbox capacity is unavailable before customer execution began.',
+      cause: error,
+    });
+  }
+
+  const unavailable =
+    code === 14 ||
+    code === 4 ||
+    (typeof code === 'string' &&
+      /(?:UNAVAILABLE|DEADLINE_EXCEEDED)/i.test(code)) ||
+    status === 429 ||
+    (status !== null && status >= 500 && status <= 599) ||
+    /\b(?:UNAVAILABLE|DEADLINE_EXCEEDED)\b/i.test(message);
+  if (!unavailable) return null;
+  return new PlayRunRecoveryError({
+    decision: {
+      stage: 'pre_code',
+      action: 'engine_retry',
+      reason: 'modal_sandbox_unavailable_before_execution',
+      dependency: 'modal',
+    },
+    message:
+      'Modal sandbox creation was temporarily unavailable before customer execution began.',
+    cause: error,
+  });
+}
+
+/**
+ * The scheduler circuit blocked provider acquisition before the backend made a
+ * Modal API call. This is equivalent to a proven pre-code capacity rejection,
+ * but avoids turning a shared-account outage into one create request per run.
+ */
+export function recoveryForOpenModalCircuit(): PlayRunRecoveryError {
+  return new PlayRunRecoveryError({
+    decision: {
+      stage: 'pre_code',
+      action: 'defer',
+      reason: 'modal_circuit_open_before_execution',
+      dependency: 'modal',
+    },
+    message:
+      'Modal sandbox acquisition is deferred while the shared dependency circuit is open.',
+  });
+}
+
+/**
+ * An operator traffic policy is evaluated before a first managed sandbox is
+ * created. It becomes a bounded durable recovery defer, never a generic task
+ * retry, so a forgotten SEV mitigation cannot spin accepted webhooks forever.
+ */
+export function recoveryForRuntimeTrafficPolicy(input: {
+  mode: 'hold' | 'throttle';
+}): PlayRunRecoveryError {
+  const reason =
+    input.mode === 'hold'
+      ? 'runtime_traffic_hold_before_execution'
+      : 'runtime_traffic_throttled_before_execution';
+  return new PlayRunRecoveryError({
+    decision: {
+      stage: 'pre_code',
+      action: 'defer',
+      reason,
+      dependency: 'runtime',
+    },
+    message:
+      input.mode === 'hold'
+        ? 'New sandbox starts are temporarily held by the runtime incident policy.'
+        : 'New sandbox starts are temporarily paced by the runtime incident policy.',
+  });
+}
+
+/**
+ * This is constructed only by the detached-runner readiness check.  That
+ * check has not parked the runner or observed scheduler liveness, so it is a
+ * pre-code failure for recovery purposes.  The scheduler's bounded engine
+ * ladder, not this module, chooses how many fresh starts are allowed.
+ */
+export function recoveryForModalRunnerStartupFailure(
+  error: unknown,
+): PlayRunRecoveryError {
+  return new PlayRunRecoveryError({
+    decision: {
+      stage: 'pre_code',
+      action: 'engine_retry',
+      reason: 'modal_runner_start_failed_before_execution',
+      dependency: 'modal',
+    },
+    message:
+      'Modal sandbox did not become ready before customer execution began.',
+    cause: error,
+  });
+}
+
+/**
+ * A runner transport classifier has already proved that the result has no
+ * customer rows or steps. Keep its engine retry typed so an ordinary failed
+ * result cannot accidentally bypass the engine's bounded retry ladder.
+ */
+export function recoveryForRuntimeInitializationTransportFailure(
+  reason:
+    | 'runtime_postgres_connect'
+    | 'runtime_api_transport'
+    | 'runtime_receipt_gateway_transport',
+): PlayRunRecoveryError {
+  return new PlayRunRecoveryError({
+    decision: {
+      stage: 'pre_code',
+      action: 'engine_retry',
+      reason: 'runtime_initialization_transport_before_execution',
+      dependency: 'runtime',
+    },
+    message: `Runtime initialization transport failed before customer execution began (${reason}).`,
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts

@@ -1,4 +1,5 @@
 import type { AnyBatchOperationStrategy } from './batching-types';
+import { bettercontactBatchStrategies } from './bettercontact-batching';
 import { DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES } from './default-batch-strategies';
 import { fullenrichBatchStrategies } from './fullenrich-batching';
 import { opensosdataBatchStrategies } from './opensosdata-batching';
@@ -8,6 +9,7 @@ export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
   AnyBatchOperationStrategy
 > = {
   ...DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES,
+  ...bettercontactBatchStrategies,
   ...fullenrichBatchStrategies,
   ...opensosdataBatchStrategies,
 };

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -32,7 +32,7 @@ export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
 export const RUNTIME_SANDBOX_LOST_MESSAGE =
   'The execution sandbox became unreachable before returning a terminal result. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.';
 export const RUNTIME_SANDBOX_OOM_MESSAGE =
-  'The execution sandbox ran out of memory. Completed work is durably recorded. Retry the same command safely; Deepline reuses completed steps and does not repeat their provider calls. If it happens again, reduce the batch size or row payload.';
+  'The execution sandbox ran out of memory. Completed work is durably recorded. Deepline does not automatically retry this run because the same resource profile is unlikely to succeed. Reduce the batch size or row payload before starting a new run.';
 export const RUNTIME_SANDBOX_KILLED_MESSAGE =
   'The execution sandbox was killed before it could return a result, and the kill reason is unavailable. Completed work is durably recorded. Retry the same command safely; Deepline reuses completed steps and does not repeat their provider calls.';
 export const RUNTIME_LIMIT_EXCEEDED_MESSAGE =
@@ -385,7 +385,11 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       code: 'RUNTIME_SANDBOX_OOM',
       phase: 'infrastructure',
       message: RUNTIME_SANDBOX_OOM_MESSAGE,
-      retryable: true,
+      // A confirmed runtime OOM is poison work for the current resource
+      // profile. Retrying it automatically spends capacity without changing
+      // the cause; completed receipts remain available to a deliberate run
+      // after the input or limits are changed.
+      retryable: false,
       cause,
     };
   }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -14,6 +14,10 @@ import {
   PLAY_RUNNER_STARTUP_GRACE_SECONDS,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '@shared_libs/play-runtime/runtime-constants';
+import {
+  RUNTIME_RELIABILITY_ENV,
+  RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV,
+} from '@shared_libs/play-runtime/runtime-reliability-policy';
 import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
   validatePlaySandboxRuntimeLimits,
@@ -80,6 +84,26 @@ function shellQuote(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`;
 }
 
+/**
+ * Sandboxes intentionally receive no general worker environment. The bounded,
+ * non-secret reliability controls are the exception: copy only their declared
+ * names so an incident override affects the runner's gateway/egress deadlines
+ * too, without ever widening this into an environment pass-through.
+ */
+function runtimeReliabilityEnvironmentPrefix(): string {
+  const names = [
+    RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV,
+    ...Object.values(RUNTIME_RELIABILITY_ENV),
+  ];
+  const entries = names.flatMap((name) => {
+    const value = process.env[name];
+    return value === undefined || value.trim() === ''
+      ? []
+      : [`${name}=${shellQuote(value)}`];
+  });
+  return entries.length > 0 ? `${entries.join(' ')} ` : '';
+}
+
 function gzipUtf8(value: string): Buffer {
   return gzipSync(Buffer.from(value, 'utf-8'));
 }
@@ -326,7 +350,7 @@ async function main() {
           code: 'RUNTIME_SANDBOX_OOM',
           phase: 'infrastructure',
           message: ${JSON.stringify(RUNTIME_SANDBOX_OOM_MESSAGE)},
-          retryable: true,
+          retryable: false,
           cause: synthesizedError,
         }] }
       : {}),
@@ -579,6 +603,7 @@ export async function stageRunnerPayload(input: {
     process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
       ? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
       : '';
+  const runtimeReliabilityEnv = runtimeReliabilityEnvironmentPrefix();
   const watchedRunnerCommand = [
     'node',
     '-e',
@@ -600,7 +625,7 @@ export async function stageRunnerPayload(input: {
     artifactCodePath,
     artifactSourceMapPath,
     crashPusherPath,
-  })} && DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH=${shellQuote(runtimeStartedPath)} DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH=${shellQuote(runtimeCompletedPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH=${shellQuote(startupDiagnosticPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}${watchedRunnerCommand}`;
+  })} && DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH=${shellQuote(runtimeStartedPath)} DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH=${shellQuote(runtimeCompletedPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STARTUP_DIAGNOSTIC_PATH=${shellQuote(startupDiagnosticPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runtimeReliabilityEnv}${runnerTraceEnv}${watchedRunnerCommand}`;
   // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
   // pushes the parsed (or synthesized) terminal to the gateway so the parked
   // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
@@ -609,7 +634,7 @@ export async function stageRunnerPayload(input: {
   // Its own retry diagnostics are kept in a separate log: appending them to
   // `outputPath` after a near-gateway-sized result could hide that result from
   // a later bounded-tail recovery read.
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)} ${shellQuote(startupDiagnosticPath)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)} ${shellQuote(startupDiagnosticPath)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; ${runtimeReliabilityEnv}node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -1,4 +1,5 @@
 import type { DaytonaSandbox } from './daytona-lifecycle';
+import { RUNTIME_RELIABILITY_POLICY } from '@shared_libs/play-runtime/runtime-reliability-policy';
 
 export type DetachedDaytonaStartupDiagnostic = {
   schedulerReadinessObserved: boolean;
@@ -229,7 +230,8 @@ export type DetachedDaytonaRunnerSupervisor = {
 
 // The durable heartbeat proves runner-process liveness plus gateway reachability.
 // Keep the handoff bounded while allowing a cold gateway to recover.
-export const DAYTONA_RUNNER_READY_TIMEOUT_MS = 30_000;
+export const DAYTONA_RUNNER_READY_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.sandbox.runnerReadyTimeoutMs;
 const DAYTONA_RUNNER_READY_POLL_MS = 50;
 
 export type DetachedRunnerReadinessPort = (

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -41,9 +41,12 @@ import {
   DaytonaRunnerInitializationError,
   prepareDetachedDaytonaRunner,
 } from './daytona-session-execution';
+import { RUNTIME_RELIABILITY_POLICY } from '@shared_libs/play-runtime/runtime-reliability-policy';
 
-const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
-const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
+const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryTimeoutMs;
+const DAYTONA_COMMAND_RECOVERY_POLL_MS =
+  RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryPollMs;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;