deepline 0.1.207 → 0.1.209

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +2 -12
  2. package/dist/bundling-sources/sdk/src/client.ts +10 -0
  3. package/dist/bundling-sources/sdk/src/http.ts +10 -1
  4. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +16 -31
  6. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +18 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +312 -207
  8. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +15 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +0 -5
  10. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +1 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +429 -102
  12. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +27 -5
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +31 -12
  15. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +103 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +105 -10
  17. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +21 -1
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +32 -86
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +141 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +45 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +180 -447
  22. package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +33 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  25. package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +112 -0
  26. package/dist/cli/index.js +38 -5
  27. package/dist/cli/index.mjs +38 -5
  28. package/dist/index.d.mts +10 -0
  29. package/dist/index.d.ts +10 -0
  30. package/dist/index.js +32 -5
  31. package/dist/index.mjs +32 -5
  32. package/package.json +1 -1
  33. package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +0 -70
@@ -1,4 +1,5 @@
1
1
  import { Daytona } from '@daytonaio/sdk';
2
+ import { randomUUID } from 'node:crypto';
2
3
  import type {
3
4
  PlayRunnerBackend,
4
5
  PlayRunnerPreparedExecution,
@@ -6,13 +7,8 @@ import type {
6
7
  } from '../types';
7
8
  import { RuntimeResourceFenceLostError } from '../types';
8
9
  import { buildPlayRunnerBundle } from '../bundle';
9
- import {
10
- findPlayRunnerResult,
11
- parsePlayRunnerEventLine,
12
- parsePlayRunnerEvents,
13
- } from '../runner-events';
10
+ import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
14
11
  import type {
15
- PlayRunnerEvent,
16
12
  PlayRunnerExecutionConfig,
17
13
  PlayRunnerRuntimeTiming,
18
14
  PlayRunnerResult,
@@ -36,82 +32,17 @@ import {
36
32
  validateDaytonaExecutionContext,
37
33
  } from './daytona-lifecycle';
38
34
  import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
35
+ import { startDetachedDaytonaCommand } from './daytona-session-execution';
39
36
 
40
37
  const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
41
38
  const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
42
- const DAYTONA_OUTPUT_TAIL_CHARS = 2_000;
43
39
  const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
44
40
  const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
45
- const DAYTONA_PROGRESS_SIDECAR_POLL_MS = 1_000;
46
41
  const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
47
42
  const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
48
43
  const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
49
44
  const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
50
- /**
51
- * Client-side wall-clock grace for `sandbox.process.executeCommand`. The
52
- * `timeout` argument we pass is a SERVER-side directive; the Daytona SDK's
53
- * underlying HTTP call has no client deadline, so a wedged socket (observed:
54
- * `AggregateError [ETIMEDOUT]`) leaves the await hanging forever and holds the
55
- * worker slot. We race the call against this deadline = server timeout + grace,
56
- * so a genuine server-side runtime-limit hit still returns first, but a wedged
57
- * client transport fails the attempt loudly and lets the engine retry ladder
58
- * take over instead of livelocking the slot.
59
- */
60
- const DAYTONA_EXECUTE_CLIENT_GRACE_MS = 60_000;
61
- const DAYTONA_EXECUTE_CLIENT_DEADLINE_MS =
62
- DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000 + DAYTONA_EXECUTE_CLIENT_GRACE_MS;
63
- /**
64
- * Bound for individual sandbox/runtime API calls that have no natural timeout
65
- * (e.g. recording the acquired runtime resource). A wedged one of these must
66
- * fail the attempt, not hold the slot.
67
- */
68
- const DAYTONA_API_CALL_DEADLINE_MS = 60_000;
69
-
70
- export class DaytonaOperationDeadlineError extends Error {
71
- readonly operation: string;
72
- readonly deadlineMs: number;
73
-
74
- constructor(input: { operation: string; deadlineMs: number }) {
75
- super(
76
- `Daytona operation "${input.operation}" exceeded client deadline of ${input.deadlineMs}ms and was abandoned to free the worker slot.`,
77
- );
78
- this.name = 'DaytonaOperationDeadlineError';
79
- this.operation = input.operation;
80
- this.deadlineMs = input.deadlineMs;
81
- }
82
- }
83
45
 
84
- /**
85
- * Race `promise` against a wall-clock deadline. On breach, reject loudly with a
86
- * `DaytonaOperationDeadlineError` (the caller's retry/failure classification
87
- * decides what happens next). The timer is always cleared so a fast-resolving
88
- * promise never leaves a dangling handle.
89
- */
90
- export async function withDaytonaOperationDeadline<T>(input: {
91
- operation: string;
92
- deadlineMs: number;
93
- promise: Promise<T>;
94
- }): Promise<T> {
95
- let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
96
- const deadlinePromise = new Promise<never>((_resolve, reject) => {
97
- deadlineTimer = setTimeout(() => {
98
- reject(
99
- new DaytonaOperationDeadlineError({
100
- operation: input.operation,
101
- deadlineMs: input.deadlineMs,
102
- }),
103
- );
104
- }, input.deadlineMs);
105
- deadlineTimer.unref?.();
106
- });
107
- try {
108
- return await Promise.race([input.promise, deadlinePromise]);
109
- } finally {
110
- if (deadlineTimer) {
111
- clearTimeout(deadlineTimer);
112
- }
113
- }
114
- }
115
46
  const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
116
47
  /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
117
48
  const RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN =
@@ -153,33 +84,110 @@ class DaytonaUploadDeadlineBreachError extends Error {
153
84
  }
154
85
  }
155
86
 
156
- class DaytonaCommandTransportError extends Error {
157
- readonly sandboxId: string;
158
- readonly commandError: unknown;
159
- readonly recoveryError: unknown;
160
-
161
- constructor(input: {
162
- sandboxId: string;
163
- commandError: unknown;
164
- recoveryError: unknown;
165
- }) {
166
- super(
167
- `Daytona command transport failed for sandbox ${input.sandboxId}: ${formatDaytonaError(input.commandError)}. Recovery could not read the command result: ${formatDaytonaError(input.recoveryError)}.`,
168
- { cause: input.commandError },
169
- );
170
- this.name = 'DaytonaCommandTransportError';
171
- this.sandboxId = input.sandboxId;
172
- this.commandError = input.commandError;
173
- this.recoveryError = input.recoveryError;
174
- }
175
- }
176
-
177
87
  export const daytonaSdkClientFactory = {
178
88
  create(input: ConstructorParameters<typeof Daytona>[0]): DaytonaClient {
179
89
  return new Daytona(input);
180
90
  },
91
+ /** Full client (get/delete by id) for best-effort GC of orphaned sandboxes. */
92
+ createFull(input: ConstructorParameters<typeof Daytona>[0]): Daytona {
93
+ return new Daytona(input);
94
+ },
181
95
  };
182
96
 
97
+ /**
98
+ * Best-effort deletion of an orphaned Daytona sandbox by id (push-execution B4).
99
+ *
100
+ * When the scheduler's expired-claim sweeper reclaims a run whose worker was
101
+ * babysitting a sandbox, nothing tears that sandbox down — `ephemeral +
102
+ * autoStop` is the only backstop, which leaks paid compute until it fires. The
103
+ * worker records each acquired sandbox id under the fenced lease, so on the
104
+ * reclaim / terminal-failure path it can look the sandbox up by id and delete
105
+ * it. Failures are swallowed: the run already ended, and the autoStop backstop
106
+ * still applies.
107
+ */
108
+ /**
109
+ * Timeout-wake verification for a detached push-execution runner (B2-final).
110
+ *
111
+ * When the parked attempt's ceiling timeout fires without a pushed terminal,
112
+ * the wake leg calls this to distinguish "runner finished but its terminal
113
+ * push was lost" from "runner died silently": read the session command's exit
114
+ * code, then salvage the structured `result` event from the captured output
115
+ * file. Everything is best-effort — a deleted/expired sandbox yields
116
+ * `{ exitCode: null, result: null }` and the attempt fails through the normal
117
+ * engine retry/fencing path.
118
+ */
119
+ export async function inspectDetachedDaytonaRunner(input: {
120
+ sandboxId: string;
121
+ sessionId: string;
122
+ cmdId: string;
123
+ outputPath: string;
124
+ exitCodePath: string;
125
+ }): Promise<{ exitCode: number | null; result: PlayRunnerResult | null }> {
126
+ try {
127
+ const { clientOptions } = loadDaytonaRequiredConfig();
128
+ const daytona = daytonaSdkClientFactory.createFull(clientOptions);
129
+ const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
130
+ let exitCode: number | null = null;
131
+ try {
132
+ const command = await sandbox.process.getSessionCommand(
133
+ input.sessionId,
134
+ input.cmdId,
135
+ );
136
+ exitCode = typeof command?.exitCode === 'number' ? command.exitCode : null;
137
+ } catch (error) {
138
+ console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
139
+ sandboxId: input.sandboxId,
140
+ cmdId: input.cmdId,
141
+ error: error instanceof Error ? error.message : String(error),
142
+ });
143
+ }
144
+ const recovered = await recoverDaytonaCommandExecution({
145
+ sandbox,
146
+ outputPath: input.outputPath,
147
+ exitCodePath: input.exitCodePath,
148
+ timeoutMs: 10_000,
149
+ });
150
+ if (!recovered.execution) {
151
+ return { exitCode, result: null };
152
+ }
153
+ const result =
154
+ findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
155
+ null;
156
+ return {
157
+ exitCode: exitCode ?? recovered.execution.exitCode,
158
+ result,
159
+ };
160
+ } catch (error) {
161
+ console.warn('[play-runner.daytona.detached_inspect_failed]', {
162
+ sandboxId: input.sandboxId,
163
+ cmdId: input.cmdId,
164
+ error: error instanceof Error ? error.message : String(error),
165
+ });
166
+ return { exitCode: null, result: null };
167
+ }
168
+ }
169
+
170
+ export async function deleteDaytonaSandboxById(input: {
171
+ sandboxId: string;
172
+ timeoutSeconds?: number;
173
+ }): Promise<boolean> {
174
+ const sandboxId = input.sandboxId?.trim();
175
+ if (!sandboxId) return false;
176
+ try {
177
+ const { clientOptions } = loadDaytonaRequiredConfig();
178
+ const daytona = daytonaSdkClientFactory.createFull(clientOptions);
179
+ const sandbox = await daytona.get(sandboxId);
180
+ await daytona.delete(sandbox, input.timeoutSeconds ?? 30);
181
+ return true;
182
+ } catch (error) {
183
+ console.warn('[play-runner.daytona.reclaim_sandbox_delete_failed]', {
184
+ sandboxId,
185
+ error: error instanceof Error ? error.message : String(error),
186
+ });
187
+ return false;
188
+ }
189
+ }
190
+
183
191
  function formatDaytonaExecutionError(error: unknown): string {
184
192
  const message = formatDaytonaError(error);
185
193
  return isConfiguredDaytonaRuntimeLimit(error)
@@ -212,14 +220,6 @@ function isConfiguredDaytonaRuntimeLimit(error: unknown): boolean {
212
220
  ).test(message);
213
221
  }
214
222
 
215
- function getDaytonaOutputTail(output: unknown): string | null {
216
- const text = String(output ?? '').trim();
217
- if (!text) {
218
- return null;
219
- }
220
- return text.slice(-DAYTONA_OUTPUT_TAIL_CHARS);
221
- }
222
-
223
223
  function sleep(ms: number): Promise<void> {
224
224
  return new Promise((resolve) => setTimeout(resolve, ms));
225
225
  }
@@ -251,161 +251,6 @@ function emitDaytonaStage(
251
251
  console.info('[play-runner.daytona.stage]', JSON.stringify(payload));
252
252
  }
253
253
 
254
- function dispatchDaytonaRunnerEvent(input: {
255
- event: PlayRunnerEvent;
256
- callbacks: Parameters<PlayRunnerBackend['execute']>[1];
257
- context: DaytonaExecutionContext;
258
- sandboxId: string;
259
- startedAt: number;
260
- }): void {
261
- const { event, callbacks, context, sandboxId, startedAt } = input;
262
- if (event.type === 'log') {
263
- if (
264
- event.line.startsWith('[perf]') ||
265
- event.line.includes('Executing tool batch')
266
- ) {
267
- emitDaytonaStage(callbacks, context, 'runner:log', {
268
- sandboxId,
269
- line: event.line,
270
- elapsedMs: Date.now() - startedAt,
271
- });
272
- }
273
- callbacks?.onLog?.(event);
274
- return;
275
- }
276
- if (event.type === 'checkpoint') {
277
- callbacks?.onCheckpoint?.(event.checkpoint);
278
- return;
279
- }
280
- if (event.type === 'row_update') {
281
- callbacks?.onRowUpdate?.(event.update);
282
- return;
283
- }
284
- if (event.type === 'execution_event') {
285
- callbacks?.onExecutionEvent?.(event.event);
286
- }
287
- }
288
-
289
- function createDaytonaProgressSidecarPoller(input: {
290
- sandbox: DaytonaSandbox;
291
- progressEventPath: string;
292
- callbacks: Parameters<PlayRunnerBackend['execute']>[1];
293
- context: DaytonaExecutionContext;
294
- startedAt: number;
295
- processedEventKeys: Set<string>;
296
- }): { start(): void; stop(flush?: boolean): Promise<void> } {
297
- let timer: ReturnType<typeof setTimeout> | null = null;
298
- let stopped = false;
299
- let inFlight: Promise<void> = Promise.resolve();
300
- let segment = 0;
301
- let offsetBytes = 0;
302
- let pendingLine = '';
303
- let warned = false;
304
-
305
- const processText = (text: string, flushPartial = false) => {
306
- const combined = pendingLine + text;
307
- const lines = combined.split(/\r?\n/);
308
- const completeLines = flushPartial ? lines : lines.slice(0, -1);
309
- pendingLine = flushPartial ? '' : (lines.at(-1) ?? '');
310
- for (const rawLine of completeLines) {
311
- const line = rawLine.trim();
312
- if (!line) {
313
- continue;
314
- }
315
- const event = parsePlayRunnerEventLine(line);
316
- if (!event) {
317
- continue;
318
- }
319
- const eventKey = JSON.stringify(event);
320
- input.processedEventKeys.add(eventKey);
321
- dispatchDaytonaRunnerEvent({
322
- event,
323
- callbacks: input.callbacks,
324
- context: input.context,
325
- sandboxId: input.sandbox.id,
326
- startedAt: input.startedAt,
327
- });
328
- }
329
- };
330
-
331
- const poll = async (flushPartial = false): Promise<void> => {
332
- try {
333
- const manifest = await input.sandbox.fs.downloadFile(
334
- input.progressEventPath,
335
- 5,
336
- );
337
- const parsedLatestSegment = Number.parseInt(
338
- Buffer.isBuffer(manifest)
339
- ? manifest.toString('utf-8').trim()
340
- : String(manifest).trim(),
341
- 10,
342
- );
343
- const latestSegment = Number.isFinite(parsedLatestSegment)
344
- ? Math.max(segment, parsedLatestSegment)
345
- : segment;
346
- while (segment <= latestSegment) {
347
- const segmentPath = `${input.progressEventPath}.${String(
348
- segment,
349
- ).padStart(6, '0')}`;
350
- const file = await input.sandbox.fs.downloadFile(segmentPath, 5);
351
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
352
- if (buffer.length < offsetBytes) {
353
- offsetBytes = 0;
354
- pendingLine = '';
355
- }
356
- if (buffer.length > offsetBytes) {
357
- const chunk = buffer.subarray(offsetBytes).toString('utf-8');
358
- offsetBytes = buffer.length;
359
- processText(chunk, flushPartial && segment === latestSegment);
360
- }
361
- if (segment >= latestSegment) break;
362
- segment += 1;
363
- offsetBytes = 0;
364
- }
365
- if (flushPartial && pendingLine.trim()) {
366
- processText('\n', true);
367
- }
368
- } catch (error) {
369
- if (!warned && !/not found|no such file|ENOENT/i.test(String(error))) {
370
- warned = true;
371
- console.warn('[play-runner.daytona.progress_sidecar_poll_failed]', {
372
- sandboxId: input.sandbox.id,
373
- progressEventPath: input.progressEventPath,
374
- error: error instanceof Error ? error.message : String(error),
375
- });
376
- }
377
- }
378
- };
379
-
380
- const schedule = () => {
381
- if (stopped) {
382
- return;
383
- }
384
- timer = setTimeout(() => {
385
- inFlight = inFlight
386
- .catch(() => {})
387
- .then(() => poll())
388
- .finally(schedule);
389
- }, DAYTONA_PROGRESS_SIDECAR_POLL_MS);
390
- timer.unref?.();
391
- };
392
-
393
- return {
394
- start() {
395
- inFlight = inFlight.then(() => poll()).finally(schedule);
396
- },
397
- async stop(flush = true) {
398
- stopped = true;
399
- if (timer) {
400
- clearTimeout(timer);
401
- timer = null;
402
- }
403
- await inFlight.catch(() => {});
404
- if (flush) await poll(true);
405
- },
406
- };
407
- }
408
-
409
254
  async function downloadDaytonaTextFile(
410
255
  sandbox: DaytonaSandbox,
411
256
  remotePath: string,
@@ -451,23 +296,6 @@ async function recoverDaytonaCommandExecution(input: {
451
296
  return { execution: null, lastError };
452
297
  }
453
298
 
454
- async function resolveDaytonaCommandOutput(input: {
455
- sandbox: DaytonaSandbox;
456
- outputPath: string;
457
- exitCodePath: string;
458
- executionResult: unknown;
459
- }): Promise<string> {
460
- const recovered = await recoverDaytonaCommandExecution({
461
- sandbox: input.sandbox,
462
- outputPath: input.outputPath,
463
- exitCodePath: input.exitCodePath,
464
- });
465
- if (recovered.execution) {
466
- return recovered.execution.result;
467
- }
468
- return String(input.executionResult ?? '');
469
- }
470
-
471
299
  function createDaytonaCancelledResult(
472
300
  config: PlayRunnerExecutionConfig,
473
301
  ): PlayRunnerResult {
@@ -499,7 +327,14 @@ function createDaytonaFailedResult(input: {
499
327
  };
500
328
  }
501
329
 
502
- function retryableRuntimeInitializationFailureReason(
330
+ /**
331
+ * Classify a failed runner result as a retryable runtime-initialization /
332
+ * transport failure. Exported for the absurd worker's detached wake leg
333
+ * (B2-final): the backend no longer sees the runner result (it arrives via the
334
+ * gateway terminal push), so the fresh-sandbox retry decision moved to the
335
+ * worker, which throws a retryable error into the engine retry ladder.
336
+ */
337
+ export function retryableRuntimeInitializationFailureReason(
503
338
  result: PlayRunnerResult,
504
339
  ):
505
340
  | 'runtime_postgres_connect'
@@ -525,15 +360,6 @@ function retryableRuntimeInitializationFailureReason(
525
360
  }
526
361
 
527
362
  function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
528
- if (error instanceof DaytonaCommandTransportError) {
529
- return false;
530
- }
531
- // A client-side deadline breach is a wedged transport (hung socket / SDK call
532
- // with no client timeout). Retry once on a fresh sandbox before spending an
533
- // engine attempt, same as the other transport-level infrastructure failures.
534
- if (error instanceof DaytonaOperationDeadlineError) {
535
- return true;
536
- }
537
363
  const message = error instanceof Error ? error.message : String(error);
538
364
  if (!message || /cancelled|aborted/i.test(message)) {
539
365
  return false;
@@ -587,6 +413,23 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
587
413
  tableNamespace: null,
588
414
  };
589
415
  }
416
+ // Push execution is THE Daytona shape (B2-final): the worker parks after
417
+ // the detached start and finalizes from the runner's pushed terminal.
418
+ // There is deliberately no worker-babysat fallback — a launch without the
419
+ // push config is a wiring bug, not a mode.
420
+ const push = config.context.runnerPushExecution;
421
+ if (!push) {
422
+ return {
423
+ status: 'failed',
424
+ error:
425
+ 'Daytona runner backend requires push-execution config (context.runnerPushExecution). Worker-babysat Daytona execution was removed; submit through the absurd scheduler, which stamps it.',
426
+ logs: [],
427
+ stats: {},
428
+ steps: [],
429
+ checkpoint: config.checkpoint ?? null,
430
+ tableNamespace: null,
431
+ };
432
+ }
590
433
  let preparedExecution: DaytonaPreparedExecution;
591
434
  try {
592
435
  preparedExecution = isDaytonaPreparedExecution(prepared)
@@ -831,31 +674,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
831
674
  emitDaytonaStage(callbacks, config.context, 'execute:start', {
832
675
  sandboxId: sandbox.id,
833
676
  timeoutSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
834
- mode: 'command',
835
- });
836
- const executionStartedAt = Date.now();
837
- const streamedEventKeys = new Set<string>();
838
- const progressSidecar = createDaytonaProgressSidecarPoller({
839
- sandbox,
840
- progressEventPath: stagedPayload.progressEventPath,
841
- callbacks,
842
- context: config.context,
843
- startedAt,
844
- processedEventKeys: streamedEventKeys,
677
+ mode: 'session_detached',
845
678
  });
846
- let execution: DaytonaExecution;
679
+ // Push execution (B2-final, park-and-wake): start the runner command
680
+ // DETACHED via a Daytona session and return a `detached_runner`
681
+ // suspension immediately — the worker parks on it (releasing its
682
+ // claim, freeing the slot) and wakes on the runner's pushed terminal
683
+ // or the ceiling timeout. Nothing here awaits completion: no held
684
+ // socket, no poll loop, no per-second sandbox file reads. The sandbox
685
+ // MUST outlive this return (the runner is executing inside it); the
686
+ // wake leg owns finalize + sandbox GC.
687
+ const sessionId = `deepline-play-${randomUUID()}`;
688
+ let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
847
689
  try {
848
- progressSidecar.start();
849
- execution = await Promise.race([
850
- withDaytonaOperationDeadline({
851
- operation: 'sandbox.process.executeCommand',
852
- deadlineMs: DAYTONA_EXECUTE_CLIENT_DEADLINE_MS,
853
- promise: sandbox.process.executeCommand(
854
- stagedPayload.command,
855
- stagedPayload.workDir,
856
- undefined,
857
- DAYTONA_EXECUTE_TIMEOUT_SECONDS,
858
- ),
690
+ start = await Promise.race([
691
+ startDetachedDaytonaCommand({
692
+ sandbox,
693
+ sessionId,
694
+ command: stagedPayload.command,
859
695
  }),
860
696
  cancellationPromise,
861
697
  ]);
@@ -867,148 +703,48 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
867
703
  onCancel();
868
704
  throw new Error(DAYTONA_CANCELLED_ERROR);
869
705
  }
870
- const recovered = await recoverDaytonaCommandExecution({
871
- sandbox,
872
- outputPath: stagedPayload.outputPath,
873
- exitCodePath: stagedPayload.exitCodePath,
874
- });
875
- emitDaytonaStage(
876
- callbacks,
877
- config.context,
878
- 'execute:transport_error',
879
- {
880
- sandboxId: sandbox.id,
881
- operation: 'sandbox.process.executeCommand',
882
- error: formatDaytonaError(error),
883
- recovery: recovered.execution ? 'recovered' : 'unavailable',
884
- recoveryError: recovered.execution
885
- ? null
886
- : formatDaytonaError(recovered.lastError),
887
- elapsedMs: Date.now() - startedAt,
888
- },
889
- );
890
- if (!recovered.execution) {
891
- throw new DaytonaCommandTransportError({
892
- sandboxId: sandbox.id,
893
- commandError: error,
894
- recoveryError: recovered.lastError,
895
- });
896
- }
897
- emitDaytonaStage(callbacks, config.context, 'execute:recovered', {
898
- sandboxId: sandbox.id,
899
- error: formatDaytonaError(error),
900
- exitCode: recovered.execution.exitCode,
901
- outputBytes: recovered.execution.result.length,
902
- elapsedMs: Date.now() - startedAt,
903
- });
904
- execution = recovered.execution;
905
- } finally {
906
- await progressSidecar.stop(
907
- !cancellationRequested &&
908
- callbacks?.cancellationSignal?.aborted !== true,
909
- );
706
+ // Start never launched customer code; let the infrastructure retry
707
+ // classification below decide fresh-sandbox retry vs loud failure.
708
+ throw error;
910
709
  }
911
- const runnerOutput = await resolveDaytonaCommandOutput({
912
- sandbox,
913
- outputPath: stagedPayload.outputPath,
914
- exitCodePath: stagedPayload.exitCodePath,
915
- executionResult: execution.result,
916
- });
917
- emitDaytonaStage(callbacks, config.context, 'execute:done', {
710
+ const runnerAttempt = Math.max(
711
+ 0,
712
+ Math.floor(config.context.runAttempt ?? 0),
713
+ );
714
+ emitDaytonaStage(callbacks, config.context, 'execute:detached', {
918
715
  sandboxId: sandbox.id,
919
- exitCode: execution.exitCode,
920
- outputBytes: runnerOutput.length,
921
- executeElapsedMs: Date.now() - executionStartedAt,
716
+ sessionId: start.sessionId,
717
+ cmdId: start.cmdId,
718
+ runnerAttempt,
719
+ ceilingSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
922
720
  elapsedMs: Date.now() - startedAt,
923
721
  });
924
- runtimeTiming.daytonaExecuteMs = Date.now() - executionStartedAt;
925
- const events = parsePlayRunnerEvents(runnerOutput);
926
- for (const event of events) {
927
- const eventKey = JSON.stringify(event);
928
- if (streamedEventKeys.has(eventKey)) {
929
- continue;
930
- }
931
- dispatchDaytonaRunnerEvent({
932
- event,
933
- callbacks,
934
- context: config.context,
722
+ return {
723
+ status: 'suspended',
724
+ suspension: {
725
+ kind: 'detached_runner',
726
+ boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
727
+ runnerAttempt,
935
728
  sandboxId: sandbox.id,
936
- startedAt,
937
- });
938
- }
939
-
940
- const result = findPlayRunnerResult(events);
941
- if (result) {
942
- const initializationRetryReason =
943
- retryableRuntimeInitializationFailureReason(result);
944
- if (
945
- result.status === 'failed' &&
946
- executionAttempt < DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS &&
947
- initializationRetryReason
948
- ) {
949
- emitDaytonaStage(callbacks, config.context, 'execute:retry', {
950
- sandboxId: sandbox.id,
951
- attempt: executionAttempt + 1,
952
- reason: initializationRetryReason,
953
- error: result.error,
954
- elapsedMs: Date.now() - startedAt,
955
- });
956
- acquiredSandbox.billingEndedAt = Date.now();
957
- await reportRetiringRuntimeResource(acquiredSandbox);
958
- sandboxCleanup.stashActiveSandboxForRetry();
959
- acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
960
- continue;
961
- }
962
- if (result.status === 'failed') {
963
- emitDaytonaStage(
964
- callbacks,
965
- config.context,
966
- 'runner:failed_result',
967
- {
968
- sandboxId: sandbox.id,
969
- exitCode: execution.exitCode,
970
- outputBytes: runnerOutput.length,
971
- error: result.error,
972
- outputTail: getDaytonaOutputTail(runnerOutput),
973
- elapsedMs: Date.now() - startedAt,
974
- },
975
- );
976
- }
977
- return withCleanup(result);
978
- }
979
-
980
- // executeCommand only returns the transport wrapper's captured-file
981
- // path. The downloaded runner output contains the actual exception.
982
- const outputTail = getDaytonaOutputTail(runnerOutput);
983
- if (outputTail) {
984
- emitDaytonaStage(
985
- callbacks,
986
- config.context,
987
- 'runner:missing_result',
988
- {
989
- sandboxId: sandbox.id,
990
- exitCode: execution.exitCode,
991
- outputTail,
992
- elapsedMs: Date.now() - startedAt,
993
- },
994
- );
995
- }
996
-
997
- return withCleanup(
998
- createDaytonaFailedResult({
999
- config,
1000
- error:
1001
- typeof execution.exitCode === 'number' &&
1002
- execution.exitCode !== 0
1003
- ? `Daytona play runner exited with code ${execution.exitCode}${
1004
- execution.exitCode === 137
1005
- ? ' (sandbox OOM or forced SIGKILL)'
1006
- : ''
1007
- }.${outputTail ? ` Output tail: ${outputTail}` : ''}`
1008
- : 'Daytona play runner produced no result.',
1009
- runtimeTiming,
1010
- }),
1011
- );
729
+ sessionId: start.sessionId,
730
+ cmdId: start.cmdId,
731
+ outputPath: stagedPayload.outputPath,
732
+ exitCodePath: stagedPayload.exitCodePath,
733
+ startedAtMs: Date.now(),
734
+ ceilingMs: DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000,
735
+ },
736
+ logs: [],
737
+ stats: {},
738
+ steps: [],
739
+ checkpoint: config.checkpoint ?? {
740
+ completedBatches: {},
741
+ completedToolBatches: {},
742
+ resolvedWaterfalls: {},
743
+ resolvedBoundaries: {},
744
+ },
745
+ tableNamespace: null,
746
+ runtimeTiming,
747
+ };
1012
748
  } catch (error) {
1013
749
  if (cancellationRequested || callbacks?.cancellationSignal?.aborted) {
1014
750
  onCancel();
@@ -1053,10 +789,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1053
789
  emitDaytonaStage(callbacks, config.context, 'execute:error', {
1054
790
  sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
1055
791
  error: formatDaytonaError(error),
1056
- operation:
1057
- error instanceof DaytonaCommandTransportError
1058
- ? 'sandbox.process.executeCommand'
1059
- : 'runner_backend',
792
+ operation: 'runner_backend',
1060
793
  elapsedMs: Date.now() - startedAt,
1061
794
  });
1062
795
  return withCleanup(