deepline 0.3.461 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -114,6 +114,16 @@ export const RUNNER_MEMORY_CGROUP_ROOT = '/sys/fs/cgroup';
  */
 export const RUNNER_MEMORY_CGROUP_UNLIMITED_BYTES = 2 ** 50;
 
+/** How often the watchdog checks that its own timers still fire on time. */
+export const RUNNER_SANDBOX_STALL_TICK_MS = 1_000;
+
+/**
+ * A timer this late is worth reporting. It sits far above ordinary scheduling
+ * jitter on a one-core sandbox and well below the shortest observed sandbox
+ * silence, so a report is evidence of a stall and quiet sandboxes send nothing.
+ */
+export const RUNNER_SANDBOX_STALL_REPORT_MS = 5_000;
+
 /** `reason` written into the termination diagnostic on a memory abort. */
 export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
   RUNTIME_SANDBOX_MEMORY_ABORT_CODE;
@@ -194,12 +204,107 @@ let heartbeatCadenceMs = 0;
 let heartbeatController = null;
 let heartbeatInFlight = false;
 let heartbeatStopped = false;
+let heartbeatSilenced = false;
 let heartbeatConfigRead = false;
 let heartbeatConfig = null;
 let heartbeatLastSuccessAtMs = 0;
 let heartbeatAttempt = 0;
 let stopRequested = false;
 
+// --- sandbox stall evidence -------------------------------------------------
+// When a sandbox goes silent, only something inside it can say why, and it can
+// only say so afterwards. Comparing two clocks separates the causes: a paused
+// VM stops the monotonic clock while wall time jumps, a frozen container jumps
+// both, and a network cut jumps neither while heartbeats fail. Pressure deltas
+// show a stall in disk, memory or CPU.
+let stallWallAtMs = Date.now();
+let stallMonotonicAtMs = performance.now();
+let stallMaxWallGapMs = 0;
+let stallMaxMonotonicGapMs = 0;
+let heartbeatFailureCount = 0;
+let heartbeatLastErrorCode = null;
+let pressureBaseline = readSandboxPressure();
+
+// Only this sandbox's own counters. /proc/pressure covers the whole host on a
+// shared kernel, so another workload's contention would read as a stall here.
+function readPressureTotalUs(resource) {
+  try {
+    const match = /^some .*total=(\\d+)$/m.exec(fs.readFileSync(cgroupRoot + '/' + resource + '.pressure', 'utf8'));
+    if (match) return Number(match[1]);
+  } catch {}
+  return null;
+}
+
+function readCpuThrottledUs() {
+  try {
+    const match = /^throttled_usec (\\d+)$/m.exec(fs.readFileSync(cgroupRoot + '/cpu.stat', 'utf8'));
+    if (match) return Number(match[1]);
+  } catch {}
+  return null;
+}
+
+function readSandboxPressure() {
+  return {
+    cpuPressureUs: readPressureTotalUs('cpu'),
+    ioPressureUs: readPressureTotalUs('io'),
+    memoryPressureUs: readPressureTotalUs('memory'),
+    cpuThrottledUs: readCpuThrottledUs(),
+  };
+}
+
+function pressureDelta(current, baseline) {
+  return current === null || baseline === null ? null : Math.max(0, current - baseline);
+}
+
+function observeStallTick() {
+  const wallMs = Date.now();
+  const monotonicMs = performance.now();
+  stallMaxWallGapMs = Math.max(stallMaxWallGapMs, wallMs - stallWallAtMs - ${RUNNER_SANDBOX_STALL_TICK_MS});
+  stallMaxMonotonicGapMs = Math.max(stallMaxMonotonicGapMs, monotonicMs - stallMonotonicAtMs - ${RUNNER_SANDBOX_STALL_TICK_MS});
+  stallWallAtMs = wallMs;
+  stallMonotonicAtMs = monotonicMs;
+}
+
+// Gaps clear when taken, so a stall that begins while this heartbeat is in
+// flight waits for the next one instead of being cleared by its response.
+function takeSandboxHealth() {
+  observeStallTick();
+  const pressure = readSandboxPressure();
+  if (
+    stallMaxWallGapMs < ${RUNNER_SANDBOX_STALL_REPORT_MS} &&
+    stallMaxMonotonicGapMs < ${RUNNER_SANDBOX_STALL_REPORT_MS} &&
+    heartbeatFailureCount === 0
+  ) {
+    pressureBaseline = pressure;
+    return null;
+  }
+  const report = {
+    maxTimerGapWallMs: Math.round(stallMaxWallGapMs),
+    maxTimerGapMonotonicMs: Math.round(stallMaxMonotonicGapMs),
+    heartbeatFailures: heartbeatFailureCount,
+    lastHeartbeatErrorCode: heartbeatLastErrorCode,
+    heartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs),
+    cpuPressureUs: pressureDelta(pressure.cpuPressureUs, pressureBaseline.cpuPressureUs),
+    ioPressureUs: pressureDelta(pressure.ioPressureUs, pressureBaseline.ioPressureUs),
+    memoryPressureUs: pressureDelta(pressure.memoryPressureUs, pressureBaseline.memoryPressureUs),
+    cpuThrottledUs: pressureDelta(pressure.cpuThrottledUs, pressureBaseline.cpuThrottledUs),
+  };
+  stallMaxWallGapMs = 0;
+  stallMaxMonotonicGapMs = 0;
+  pressureBaseline = pressure;
+  return report;
+}
+
+function restoreSandboxHealth(report, baseline) {
+  if (!report) return;
+  stallMaxWallGapMs = Math.max(stallMaxWallGapMs, report.maxTimerGapWallMs);
+  stallMaxMonotonicGapMs = Math.max(stallMaxMonotonicGapMs, report.maxTimerGapMonotonicMs);
+  pressureBaseline = baseline;
+}
+
+const stallWatch = setInterval(observeStallTick, ${RUNNER_SANDBOX_STALL_TICK_MS});
+stallWatch.unref?.();
+
 function readRunnerHeartbeatConfig() {
   if (heartbeatConfigRead) return heartbeatConfig;
   heartbeatConfigRead = true;
@@ -211,6 +316,7 @@ function readRunnerHeartbeatConfig() {
     const executorToken = context && context.executorToken;
     const runId = push && push.runId;
     const leaseSeconds = Number(push && push.leaseSeconds);
+    const silenceMs = Number(push && push.heartbeatSilenceMs);
     if (
       !gatewayBaseUrl ||
       typeof executorToken !== 'string' ||
@@ -227,6 +333,7 @@ function readRunnerHeartbeatConfig() {
       executorToken,
       runId,
       leaseMs: Math.max(1_000, Math.floor(leaseSeconds * 1_000)),
+      silenceMs: Number.isSafeInteger(silenceMs) && silenceMs > 0 ? silenceMs : 0,
     };
   } catch {}
   return heartbeatConfig;
@@ -249,8 +356,12 @@ function stopRunnerHeartbeat() {
 }
 
 function runnerHeartbeatErrorCode(error) {
-  const code = error && typeof error.code === 'string' ? error.code : '';
-  if (/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+|http_[0-9]{3}|redirect_[0-9]{3}|protocol_error)$/.test(code)) return code;
+  // fetch wraps every transport failure in a TypeError and keeps the network
+  // error on its cause. That code is what separates DNS from connect from reset.
+  const code = [error, error && error.cause]
+    .map((candidate) => (candidate && typeof candidate.code === 'string' ? candidate.code : ''))
+    .find((candidate) => /^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+|http_[0-9]{3}|redirect_[0-9]{3}|protocol_error)$/.test(candidate));
+  if (code) return code;
   const name = error && typeof error.name === 'string' ? error.name : '';
   if (name === 'AbortError' || name === 'TimeoutError') return name;
   return 'unknown';
@@ -273,13 +384,34 @@ function startRunnerHeartbeatSupervisor() {
       '\\n',
   );
   heartbeatCadenceMs = cadenceMs;
+  if (config.silenceMs > 0) {
+    // Test drill: the runner keeps working while the scheduler hears nothing.
+    // Only the worker sets this, and never on a Production launch.
+    heartbeatSilenced = true;
+    process.stderr.write(
+      '[play-runner.heartbeat] watchdog_owner_silence_started ' +
+        JSON.stringify({ runId: config.runId, silenceMs: config.silenceMs }) +
+        '\\n',
+    );
+    heartbeatRetryTimer = setTimeout(() => {
+      heartbeatRetryTimer = null;
+      heartbeatSilenced = false;
+      process.stderr.write(
+        '[play-runner.heartbeat] watchdog_owner_silence_finished ' +
+          JSON.stringify({ runId: config.runId }) +
+          '\\n',
+      );
+      void runnerHeartbeatTick();
+    }, config.silenceMs);
+    heartbeatRetryTimer.unref?.();
+  }
   void runnerHeartbeatTick();
   heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
   heartbeatTimer.unref?.();
 }
 
 async function runnerHeartbeatTick() {
-  if (heartbeatStopped || heartbeatInFlight || settled || !child || child.exitCode !== null) return;
+  if (heartbeatStopped || heartbeatSilenced || heartbeatInFlight || settled || !child || child.exitCode !== null) return;
   const config = readRunnerHeartbeatConfig();
   if (!config) return;
   heartbeatInFlight = true;
@@ -288,6 +420,8 @@ async function runnerHeartbeatTick() {
   const controller = new AbortController();
   heartbeatController = controller;
   const timeout = setTimeout(() => controller.abort(), ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS});
+  const reportPressureBaseline = pressureBaseline;
+  const sandboxHealth = takeSandboxHealth();
   try {
     const response = await fetch(
       config.gatewayBaseUrl + '/api/v2/plays/runtime',
@@ -298,7 +432,11 @@ async function runnerHeartbeatTick() {
           authorization: 'Bearer ' + config.executorToken,
           ${JSON.stringify(PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER)}: require('node:crypto').randomUUID(),
         },
-        body: JSON.stringify({ action: 'runner_heartbeat', runId: config.runId }),
+        body: JSON.stringify({
+          action: 'runner_heartbeat',
+          runId: config.runId,
+          ...(sandboxHealth ? { sandboxHealth } : {}),
+        }),
         redirect: 'manual',
         signal: controller.signal,
       },
@@ -326,12 +464,17 @@ async function runnerHeartbeatTick() {
       throw Object.assign(new Error('gateway_protocol_error'), { code: 'protocol_error' });
     }
     heartbeatLastSuccessAtMs = Date.now();
+    heartbeatFailureCount = 0;
+    heartbeatLastErrorCode = null;
   } catch (error) {
     if (heartbeatStopped || settled) return;
     const errorCode = controller.signal.aborted
       ? 'TimeoutError'
       : runnerHeartbeatErrorCode(error);
     const heartbeatAgeMs = Math.max(0, Date.now() - heartbeatLastSuccessAtMs);
+    heartbeatFailureCount += 1;
+    heartbeatLastErrorCode = errorCode;
+    restoreSandboxHealth(sandboxHealth, reportPressureBaseline);
     process.stderr.write(
       '[play-runner.heartbeat] watchdog_owner_retrying ' +
         JSON.stringify({ runId: config.runId, attempt: heartbeatAttempt, durationMs: Math.max(0, Date.now() - startedAt), heartbeatAgeMs, errorCode }) +
@@ -605,6 +748,7 @@ function finish(code) {
   settled = true;
   clearInterval(runtimeBoundaryPoll);
   clearInterval(memoryWatch);
+  clearInterval(stallWatch);
   stopRunnerHeartbeat();
   clearTimeout(startupTimer);
   if (runtimeTimer) clearTimeout(runtimeTimer);

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -360,6 +360,11 @@ export function prepareDetachedDaytonaRunner(input: {
   readiness: DetachedRunnerReadinessPort;
   cancellation?: Promise<never>;
   runControl<T>(operation: (sandbox: DaytonaSandbox) => Promise<T>): Promise<T>;
+  /**
+   * Invoked just before the command is sent. From then on a runner may exist
+   * in the sandbox, even if the request fails without an answer.
+   */
+  onCommandSending?: () => void;
   /** Invoked immediately after Daytona accepts the detached command. */
   onCommandAccepted?: (
     state: Extract<DetachedDaytonaRunnerState, { phase: 'command_accepted' }>,
@@ -400,6 +405,7 @@ export function prepareDetachedDaytonaRunner(input: {
       if (session.ok === false) throw session.error;
       const baseline = await Promise.race([readinessBaseline, cancellation]);
       if (baseline.ok === false) throw baseline.error;
+      input.onCommandSending?.();
       const response = await Promise.race([
         input.runControl((sandbox) =>
           sandbox.process.executeSessionCommand(input.sessionId, {