deepline 0.1.208 → 0.1.210

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -2,6 +2,10 @@ import { randomUUID } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
 import { gzipSync } from 'node:zlib';
 import type { PlayRunnerExecutionConfig } from '@shared_libs/play-runtime/protocol';
+import {
+  PLAY_RUNTIME_CONTRACT,
+  PLAY_RUNTIME_CONTRACT_HEADER,
+} from '@shared_libs/play-runtime/runtime-contract';
 import { compactPlayArtifactForRuntimeTransport } from '@shared_libs/plays/artifact-transport';
 import type {
   DaytonaExecutionContext,
@@ -16,6 +20,8 @@ type DaytonaPayloadEnvelope = {
   runnerCode: string;
   artifactBundledCode: string;
   config: PlayRunnerExecutionConfig;
+  /** Crash-containment epilogue script (see buildDaytonaCrashTerminalPusherSource). */
+  crashPusherCode: string;
 };
 
 export type StagedDaytonaPayload = {
@@ -43,9 +49,10 @@ function nodeMaterializePayloadCommand(input: {
   runnerPath: string;
   configPath: string;
   artifactCodePath: string;
+  crashPusherPath: string;
 }): string {
   const script =
-    "const fs=require('node:fs');const zlib=require('node:zlib');const p=JSON.parse(zlib.gunzipSync(fs.readFileSync(process.argv[1])));if(p.schemaVersion!==1)throw new Error('Unsupported Daytona payload envelope');fs.writeFileSync(process.argv[2],p.runnerCode);fs.writeFileSync(process.argv[3],JSON.stringify(p.config));fs.writeFileSync(process.argv[4],p.artifactBundledCode);";
+    "const fs=require('node:fs');const zlib=require('node:zlib');const p=JSON.parse(zlib.gunzipSync(fs.readFileSync(process.argv[1])));if(p.schemaVersion!==1)throw new Error('Unsupported Daytona payload envelope');fs.writeFileSync(process.argv[2],p.runnerCode);fs.writeFileSync(process.argv[3],JSON.stringify(p.config));fs.writeFileSync(process.argv[4],p.artifactBundledCode);fs.writeFileSync(process.argv[5],p.crashPusherCode);";
   return [
     'node',
     '-e',
@@ -54,9 +61,133 @@ function nodeMaterializePayloadCommand(input: {
     shellQuote(input.runnerPath),
     shellQuote(input.configPath),
     shellQuote(input.artifactCodePath),
+    shellQuote(input.crashPusherPath),
   ].join(' ');
 }
 
+/**
+ * Crash-containment epilogue (push execution). The parked worker no longer
+ * holds anything that returns when the runner process dies, so a runner that
+ * exits WITHOUT pushing its terminal (play code calling `process.exit`, OOM
+ * SIGKILL of the node process, a crash before the in-process push) would only
+ * be detected by the ceiling timeout. The staged shell command therefore runs
+ * this tiny script AFTER the runner exits, unconditionally:
+ *
+ *  1. parse the captured output for the structured `result` event (the runner
+ *     may have printed it even when its own gateway push failed), and
+ *  2. push it — or a synthesized crash failure naming the exit code — to the
+ *     gateway's `runner_terminal` action.
+ *
+ * The gateway records terminals FIRST-WRITE-WINS and absurd wake events are
+ * first-emit-wins, so a duplicate push after a successful in-runner push is a
+ * no-op; this needs no coordination with the runner.
+ *
+ * Exported (with the builder) for the unit test that executes the script
+ * against a real local HTTP server.
+ */
+export function buildDaytonaCrashTerminalPusherSource(): string {
+  return `
+const fs = require('node:fs');
+const configPath = process.argv[2];
+const exitCodeRaw = Number.parseInt(process.argv[3] ?? '', 10);
+const outputPath = process.argv[4];
+const exitCode = Number.isFinite(exitCodeRaw) ? exitCodeRaw : null;
+const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+
+function readResultFromOutput() {
+  try {
+    const stat = fs.statSync(outputPath);
+    const start = Math.max(0, stat.size - MAX_OUTPUT_BYTES);
+    const fd = fs.openSync(outputPath, 'r');
+    const buffer = Buffer.alloc(stat.size - start);
+    fs.readSync(fd, buffer, 0, buffer.length, start);
+    fs.closeSync(fd);
+    const lines = buffer.toString('utf-8').split(/\\r?\\n/);
+    for (let index = lines.length - 1; index >= 0; index -= 1) {
+      const line = lines[index].trim();
+      if (!line.startsWith('{')) continue;
+      try {
+        const event = JSON.parse(line);
+        if (
+          event &&
+          event.type === 'result' &&
+          event.result &&
+          ['completed', 'failed', 'suspended'].includes(event.result.status)
+        ) {
+          return event.result;
+        }
+      } catch {}
+    }
+  } catch {}
+  return null;
+}
+
+async function main() {
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+  const context = (config && config.context) || {};
+  const push = context.runnerPushExecution;
+  const gateway = String(context.receiptGatewayBaseUrl || '').replace(/\\/$/, '');
+  const token = context.executorToken;
+  if (!push || !push.runId || !gateway || !token) {
+    console.log('[crash-terminal] no push config; skipping');
+    return;
+  }
+  const result = readResultFromOutput() || {
+    status: 'failed',
+    error:
+      'Daytona play runner exited with code ' +
+      (exitCode === null ? 'unknown' : exitCode) +
+      (exitCode === 137 ? ' (sandbox OOM or forced SIGKILL)' : '') +
+      ' without pushing a terminal (crash containment epilogue).',
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: null,
+    tableNamespace: null,
+  };
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), 10000);
+      const response = await fetch(gateway + '/api/v2/plays/runtime', {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: 'Bearer ' + token,
+          '${PLAY_RUNTIME_CONTRACT_HEADER}': '${String(PLAY_RUNTIME_CONTRACT)}',
+        },
+        body: JSON.stringify({
+          action: 'runner_terminal',
+          runId: push.runId,
+          result,
+        }),
+        signal: controller.signal,
+        redirect: 'manual',
+      });
+      clearTimeout(timer);
+      if (response.ok) {
+        console.log('[crash-terminal] pushed status=' + result.status);
+        return;
+      }
+      console.log('[crash-terminal] rejected http=' + response.status);
+      // Deterministic scope/fence rejections cannot succeed on retry.
+      if (response.status >= 400 && response.status < 500) return;
+    } catch (error) {
+      console.log(
+        '[crash-terminal] transport ' + (error && error.message ? error.message : String(error)),
+      );
+    }
+    await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
+  }
+  console.log('[crash-terminal] exhausted retries');
+}
+
+main().catch((error) => {
+  console.log('[crash-terminal] fatal ' + (error && error.message ? error.message : String(error)));
+});
+`;
+}
+
 function remoteRuntimeContextForDaytona(
   context: DaytonaExecutionContext,
 ): DaytonaRemoteRuntimeContext {
@@ -104,6 +235,7 @@ export async function stageDaytonaRunnerPayload(input: {
   const envelopePath = `${workDir}/deepline-play-payload-${randomUUID()}.json.gz`;
   const configPath = `${workDir}/deepline-play-config-${randomUUID()}.json`;
   const artifactCodePath = `${workDir}/deepline-play-artifact-${randomUUID()}.cjs`;
+  const crashPusherPath = `${workDir}/deepline-play-crash-terminal-${randomUUID()}.cjs`;
 
   const remoteMaterializedFiles: Record<string, string> = {};
   const hasInlineCsv = Boolean(input.config.csvSourceContentBase64);
@@ -198,6 +330,7 @@ export async function stageDaytonaRunnerPayload(input: {
         runnerCode: bundle,
         artifactBundledCode: compactedArtifact.bundledCode,
         config: remoteConfig,
+        crashPusherCode: buildDaytonaCrashTerminalPusherSource(),
       }),
     }),
   );
@@ -216,8 +349,14 @@ export async function stageDaytonaRunnerPayload(input: {
     runnerPath,
     configPath,
     artifactCodePath,
+    crashPusherPath,
   })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
+  // pushes the parsed (or synthesized) terminal to the gateway so the parked
+  // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
+  // SIGKILL, or a crash before the in-process push. Idempotent against a
+  // successful in-runner push (terminals are first-write-wins).
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -0,0 +1,45 @@
+import type { DaytonaSandbox } from './daytona-lifecycle';
+
+/**
+ * Detached Daytona command start (push-execution B2-final).
+ *
+ * The runner command is started DETACHED via a Daytona session
+ * (`createSession` + `executeSessionCommand({ runAsync: true })`, which returns
+ * a cmdId immediately) and the worker PARKS on a `detached_runner` suspension —
+ * nothing holds a socket or polls for completion. The parked attempt wakes on
+ * the runner's pushed terminal (receipt-gateway wake event) or on the ceiling
+ * timeout, where `inspectDetachedDaytonaRunner` (daytona.ts) reads the session
+ * command's exit code and salvages the captured output file as the fallback
+ * verification.
+ */
+
+export type DetachedDaytonaCommandStart = {
+  sessionId: string;
+  cmdId: string;
+};
+
+/**
+ * Start `command` detached in a fresh session. Returns the session + command id
+ * immediately (no await on completion).
+ */
+export async function startDetachedDaytonaCommand(input: {
+  sandbox: DaytonaSandbox;
+  sessionId: string;
+  command: string;
+}): Promise<DetachedDaytonaCommandStart> {
+  await input.sandbox.process.createSession(input.sessionId);
+  const response = await input.sandbox.process.executeSessionCommand(
+    input.sessionId,
+    {
+      command: input.command,
+      runAsync: true,
+    },
+  );
+  const cmdId = response?.cmdId;
+  if (!cmdId) {
+    throw new Error(
+      `Daytona executeSessionCommand did not return a cmdId for session ${input.sessionId}.`,
+    );
+  }
+  return { sessionId: input.sessionId, cmdId };
+}