deepline 0.1.214 → 0.1.215

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +107 -0
  2. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +85 -10
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-runtime-proxy.ts +34 -0
  4. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +50 -36
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +12 -0
  6. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +4 -0
  7. package/dist/bundling-sources/sdk/src/client.ts +1 -2
  8. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  9. package/dist/bundling-sources/sdk/src/types.ts +2 -3
  10. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -3
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +6 -6
  12. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +14 -3
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +19 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts +22 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +32 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +13 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +66 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +18 -55
  20. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +57 -25
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +17 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -2
  23. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +11 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
  25. package/dist/cli/index.js +35 -12
  26. package/dist/cli/index.mjs +35 -12
  27. package/dist/index.d.mts +2 -3
  28. package/dist/index.d.ts +2 -3
  29. package/dist/index.js +7 -6
  30. package/dist/index.mjs +7 -6
  31. package/package.json +1 -1
@@ -38,7 +38,7 @@ import {
38
38
  PLAY_RUNTIME_CONTRACT,
39
39
  PLAY_RUNTIME_CONTRACT_HEADER,
40
40
  } from '@shared_libs/play-runtime/runtime-contract';
41
- import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
41
+ import { PLAY_RUNTIME_API_CURRENT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
42
42
  import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
43
43
  import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
44
44
 
@@ -569,6 +569,19 @@ function summarizeAppRuntimeErrorBody(body: string): string {
569
569
  return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
570
570
  }
571
571
 
572
+ function appRuntimeErrorCode(response: Response, body: string): string | null {
573
+ const header = response.headers.get('x-deepline-error-code')?.trim();
574
+ if (header) return header;
575
+ try {
576
+ const parsed = JSON.parse(body) as { code?: unknown };
577
+ return typeof parsed?.code === 'string' && parsed.code.trim()
578
+ ? parsed.code.trim()
579
+ : null;
580
+ } catch {
581
+ return null;
582
+ }
583
+ }
584
+
572
585
  export function isTransientAppRuntimeFailureBody(body: string): boolean {
573
586
  return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
574
587
  body,
@@ -775,7 +788,7 @@ export class AppRuntimeApiTransportError extends Error {
775
788
 
776
789
  function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
777
790
  const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
778
- return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
791
+ return `${baseUrl}${PLAY_RUNTIME_API_CURRENT_PATH}`;
779
792
  }
780
793
 
781
794
  async function postAppRuntimeApi<TResponse>(
@@ -892,8 +905,13 @@ async function postAppRuntimeApi<TResponse>(
892
905
  );
893
906
  continue;
894
907
  }
908
+ const code = appRuntimeErrorCode(response, responseText);
909
+ const requestId = response.headers.get('x-deepline-request-id')?.trim();
895
910
  throw new Error(
896
- `App runtime API ${body.action} failed with status ${response.status}: ${summarizeAppRuntimeErrorBody(responseText)}`,
911
+ `App runtime API ${body.action} failed with status ${response.status}` +
912
+ `${code ? ` code=${code}` : ''}` +
913
+ `${requestId ? ` request_id=${requestId}` : ''}: ` +
914
+ summarizeAppRuntimeErrorBody(responseText),
897
915
  );
898
916
  }
899
917
 
@@ -48,8 +48,8 @@ import {
48
48
  SYNTHETIC_RUN_HEADER,
49
49
  } from './coordinator-headers';
50
50
  import {
51
- PLAY_RUNTIME_API_COMPAT_PATH,
52
- PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
51
+ PLAY_RUNTIME_API_CURRENT_PATH,
52
+ PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH,
53
53
  } from './runtime-api-paths';
54
54
  import {
55
55
  createRootRunExecutionScope,
@@ -1702,7 +1702,7 @@ export class PlayContextImpl {
1702
1702
  | null
1703
1703
  > {
1704
1704
  const response = await fetch(
1705
- `${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_COMPAT_PATH}`,
1705
+ `${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_CURRENT_PATH}`,
1706
1706
  {
1707
1707
  method: 'POST',
1708
1708
  headers: await this.runtimeApiHeaders(),
@@ -1817,7 +1817,7 @@ export class PlayContextImpl {
1817
1817
  this.#options.runId
1818
1818
  ) {
1819
1819
  const response = await fetch(
1820
- `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`,
1820
+ `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_CURRENT_PATH}`,
1821
1821
  {
1822
1822
  method: 'POST',
1823
1823
  headers: {
@@ -6707,7 +6707,7 @@ export class PlayContextImpl {
6707
6707
  : inlineChildGovernor!.snapshot();
6708
6708
  const childPlaySlot = launchThroughScheduler
6709
6709
  ? await this.governor.acquireChildSubmitSlot()
6710
- : null;
6710
+ : await this.governor.acquireInlineChildSlot();
6711
6711
  const rowStore = rowContext.getStore();
6712
6712
  const producer = {
6713
6713
  kind: 'play' as const,
@@ -6964,7 +6964,7 @@ export class PlayContextImpl {
6964
6964
  return this.#options.executorToken;
6965
6965
  }
6966
6966
  const response = await fetch(
6967
- `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH}`,
6967
+ `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH}`,
6968
6968
  {
6969
6969
  method: 'POST',
6970
6970
  headers: {
@@ -429,7 +429,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
429
429
  receipt: RuntimeStepReceipt,
430
430
  source: DurableReceiptRecoverySource = 'cache',
431
431
  ): Promise<T> => {
432
- input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
432
+ input.log(
433
+ source === 'owner'
434
+ ? `ctx.${input.operation}(${input.id}): executed and converged through immutable publication`
435
+ : `ctx.${input.operation}(${input.id}): reused completed work`,
436
+ );
433
437
  if (receipt.output === undefined) {
434
438
  return receipt.output as T;
435
439
  }
@@ -489,8 +493,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
489
493
  const afterLock = await input.store.get(input.receiptKey);
490
494
  if (
491
495
  input.force !== true &&
492
- (afterLock?.status === 'completed' ||
493
- afterLock?.status === 'skipped')
496
+ (afterLock?.status === 'completed' || afterLock?.status === 'skipped')
494
497
  ) {
495
498
  await input.store.releaseExecutionLock({
496
499
  receiptKey: input.receiptKey,
@@ -566,6 +569,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
566
569
  completed?.status === 'completed' ||
567
570
  completed?.status === 'skipped'
568
571
  ) {
572
+ // Completion is a compact acknowledgement. This execution already
573
+ // owns the just-produced value; only a later receipt read needs the
574
+ // persisted payload for replay.
575
+ if (completed.output === undefined) return result;
569
576
  return await recoverCompletedReceipt(completed, 'owner');
570
577
  }
571
578
  const winner = await input.store.get(input.receiptKey);
@@ -828,6 +835,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
828
835
  (completed.status === 'completed' || completed.status === 'skipped')
829
836
  ) {
830
837
  logPhase('total', lifecycleStartedAt);
838
+ // Completion endpoints acknowledge ownership and status only. The caller
839
+ // already owns the live value; replay reads are the only path that needs
840
+ // the receipt payload returned over the transport.
841
+ if (completed.output === undefined) return result;
831
842
  return await recoverCompletedReceipt(completed, 'owner');
832
843
  }
833
844
  if (input.store.canPersistCompletion) {
@@ -106,6 +106,13 @@ export interface PlayExecutionGovernor {
106
106
  acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
107
107
  /** Block until a child-play submit slot is free. Released after submit, not terminal. */
108
108
  acquireChildSubmitSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
109
+ /**
110
+ * Block until an in-process child-composition slot is free. Unlike a
111
+ * scheduled child's submit slot, this lease is held through child terminal
112
+ * execution so a large row fan-out cannot materialize thousands of child
113
+ * contexts and receipt closures at once.
114
+ */
115
+ acquireInlineChildSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
109
116
  /**
110
117
  * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
111
118
  * permit are free, then charge the tool-call budget and return a lease. Order:
@@ -397,6 +404,7 @@ export function createPlayExecutionGovernor(
397
404
 
398
405
  acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
399
406
  acquireChildSubmitSlot: (opts) => childPlaySlots.acquire(opts?.signal),
407
+ acquireInlineChildSlot: (opts) => childPlaySlots.acquire(opts?.signal),
400
408
 
401
409
  async acquireToolSlot(toolId, opts) {
402
410
  // 1. global tool-concurrency slot.
@@ -612,6 +620,17 @@ function createInlineChildGovernor(
612
620
  policy: root.policy,
613
621
  acquireRowSlot: (opts) => root.acquireRowSlot(opts),
614
622
  acquireChildSubmitSlot: (opts) => root.acquireChildSubmitSlot(opts),
623
+ async acquireInlineChildSlot(opts) {
624
+ if (opts?.signal?.aborted) {
625
+ throw opts.signal.reason instanceof Error
626
+ ? opts.signal.reason
627
+ : new Error('Slot acquire aborted.');
628
+ }
629
+ // This governor already runs inside a root inline-child lease. Descendant
630
+ // composition shares that admitted subtree slot; reacquiring the root
631
+ // semaphore here can deadlock when every admitted parent awaits a child.
632
+ return { release: () => {} };
633
+ },
615
634
  acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
616
635
  suggestedParallelism: (toolId, fallback) =>
617
636
  root.suggestedParallelism(toolId, fallback),
@@ -6,6 +6,7 @@ import {
6
6
  PLAY_RUNTIME_PROVIDERS,
7
7
  PLAY_RUNTIME_PROVIDER_IDS,
8
8
  defaultPlayRuntimeProvider,
9
+ resolveEnabledPlayRuntimeProvider,
9
10
  resolvePlayRuntimeProvider,
10
11
  type PlayRuntimeProviderId,
11
12
  } from './providers';
@@ -50,3 +51,24 @@ export function resolveExecutionProfile(
50
51
  throw error;
51
52
  }
52
53
  }
54
+
55
+ export function resolveEnabledExecutionProfile(
56
+ override?: string | null,
57
+ ): PlayExecutionProfile {
58
+ try {
59
+ return resolveEnabledPlayRuntimeProvider(override);
60
+ } catch (error) {
61
+ if (
62
+ override?.trim() &&
63
+ error instanceof Error &&
64
+ /Unsupported play runtime provider/.test(error.message)
65
+ ) {
66
+ throw new Error(
67
+ `Unknown execution profile "${override.trim()}". Expected one of: ${Object.keys(
68
+ PLAY_EXECUTION_PROFILES,
69
+ ).join(', ')}.`,
70
+ );
71
+ }
72
+ throw error;
73
+ }
74
+ }
@@ -27,6 +27,22 @@ export type PlayRuntimeProviderDescriptor = {
27
27
  label: string;
28
28
  };
29
29
 
30
+ export const PLAY_RUNTIME_PROVIDER_DISABLED_CODE =
31
+ 'PLAY_RUNTIME_PROVIDER_DISABLED' as const;
32
+
33
+ export class PlayRuntimeProviderDisabledError extends Error {
34
+ readonly code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
35
+ readonly profile: PlayRuntimeProviderId;
36
+
37
+ constructor(profile: PlayRuntimeProviderId) {
38
+ super(
39
+ `Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`,
40
+ );
41
+ this.name = 'PlayRuntimeProviderDisabledError';
42
+ this.profile = profile;
43
+ }
44
+ }
45
+
30
46
  export const PLAY_RUNTIME_PROVIDERS: Record<
31
47
  PlayRuntimeProviderId,
32
48
  PlayRuntimeProviderDescriptor
@@ -57,6 +73,14 @@ export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
57
73
  return PLAY_RUNTIME_PROVIDERS.absurd;
58
74
  }
59
75
 
76
+ export function assertPlayRuntimeProviderEnabled(
77
+ provider: PlayRuntimeProviderDescriptor,
78
+ ): void {
79
+ if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
80
+ throw new PlayRuntimeProviderDisabledError(provider.id);
81
+ }
82
+ }
83
+
60
84
  export function resolvePlayRuntimeProvider(
61
85
  override?: string | null,
62
86
  ): PlayRuntimeProviderDescriptor {
@@ -73,3 +97,11 @@ export function resolvePlayRuntimeProvider(
73
97
  }
74
98
  return defaultPlayRuntimeProvider();
75
99
  }
100
+
101
+ export function resolveEnabledPlayRuntimeProvider(
102
+ override?: string | null,
103
+ ): PlayRuntimeProviderDescriptor {
104
+ const provider = resolvePlayRuntimeProvider(override);
105
+ assertPlayRuntimeProviderEnabled(provider);
106
+ return provider;
107
+ }
@@ -2,6 +2,8 @@ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
2
2
  /Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
3
3
  const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
4
4
  /Too many subrequests by single Worker invocation/i;
5
+ const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
6
+ /The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
5
7
  const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
6
8
  /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
7
9
 
@@ -9,6 +11,8 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
9
11
  'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
10
12
  export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
11
13
  'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
14
+ export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE =
15
+ 'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.';
12
16
 
13
17
  export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
14
18
  'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
@@ -171,6 +175,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
171
175
  cause,
172
176
  };
173
177
  }
178
+ if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(cause)) {
179
+ return {
180
+ code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED',
181
+ phase: 'runtime',
182
+ message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE,
183
+ retryable: true,
184
+ cause,
185
+ };
186
+ }
174
187
  const playDepthBudgetMatch = cause.match(
175
188
  /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
176
189
  );
@@ -27,6 +27,7 @@ type DaytonaPayloadEnvelope = {
27
27
  export type StagedDaytonaPayload = {
28
28
  workDir: string;
29
29
  command: string;
30
+ readyPath: string;
30
31
  outputPath: string;
31
32
  exitCodePath: string;
32
33
  progressEventPath: string;
@@ -343,6 +344,7 @@ export async function stageDaytonaRunnerPayload(input: {
343
344
 
344
345
  const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
345
346
  const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
347
+ const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
346
348
  const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
347
349
  const runnerTraceEnv =
348
350
  process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
@@ -354,17 +356,18 @@ export async function stageDaytonaRunnerPayload(input: {
354
356
  configPath,
355
357
  artifactCodePath,
356
358
  crashPusherPath,
357
- })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
359
+ })} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
358
360
  // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
359
361
  // pushes the parsed (or synthesized) terminal to the gateway so the parked
360
362
  // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
361
363
  // SIGKILL, or a crash before the in-process push. Idempotent against a
362
364
  // successful in-runner push (terminals are first-write-wins).
363
- const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
365
+ const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
364
366
 
365
367
  return {
366
368
  workDir: input.workDir,
367
369
  command,
370
+ readyPath,
368
371
  outputPath,
369
372
  exitCodePath,
370
373
  progressEventPath,
@@ -112,6 +112,16 @@ export type DetachedDaytonaCommandStart = {
112
112
  cmdId: string;
113
113
  };
114
114
 
115
+ const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
116
+ const DAYTONA_RUNNER_READY_POLL_MS = 250;
117
+
118
+ export class DaytonaRunnerInitializationError extends Error {
119
+ constructor(message: string) {
120
+ super(message);
121
+ this.name = 'DaytonaRunnerInitializationError';
122
+ }
123
+ }
124
+
115
125
  /**
116
126
  * Start `command` detached in a fresh session. Returns the session + command id
117
127
  * immediately (no await on completion).
@@ -137,3 +147,59 @@ export async function startDetachedDaytonaCommand(input: {
137
147
  }
138
148
  return { sessionId: input.sessionId, cmdId };
139
149
  }
150
+
151
+ /**
152
+ * A detached command id only proves that Daytona accepted the request. Before
153
+ * the worker parks, require a marker written by the initialized runner process.
154
+ * This closes the unobservable handoff where a sandbox or command disappears
155
+ * after executeSessionCommand but before the first gateway heartbeat.
156
+ */
157
+ export async function confirmDetachedDaytonaRunnerReady(input: {
158
+ sandbox: DaytonaSandbox;
159
+ sessionId: string;
160
+ cmdId: string;
161
+ readyPath: string;
162
+ exitCodePath: string;
163
+ timeoutMs?: number;
164
+ pollMs?: number;
165
+ now?: () => number;
166
+ sleep?: (ms: number) => Promise<void>;
167
+ }): Promise<void> {
168
+ const now = input.now ?? Date.now;
169
+ const sleep =
170
+ input.sleep ??
171
+ ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
172
+ const timeoutMs = Math.max(
173
+ 1,
174
+ input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
175
+ );
176
+ const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
177
+ const deadline = now() + timeoutMs;
178
+ let markerDiagnosis = 'ready_marker_unavailable';
179
+
180
+ while (now() < deadline) {
181
+ try {
182
+ const marker = await input.sandbox.fs.downloadFile(input.readyPath, 2);
183
+ const parsed = JSON.parse(marker.toString('utf-8')) as {
184
+ ready?: unknown;
185
+ };
186
+ if (parsed.ready === true) return;
187
+ markerDiagnosis = 'ready_marker_invalid';
188
+ } catch {
189
+ markerDiagnosis = 'ready_marker_unavailable';
190
+ }
191
+ await sleep(Math.min(pollMs, Math.max(1, deadline - now())));
192
+ }
193
+
194
+ const diagnostic = await inspectDetachedDaytonaStartup({
195
+ sandbox: input.sandbox,
196
+ sessionId: input.sessionId,
197
+ cmdId: input.cmdId,
198
+ exitCodePath: input.exitCodePath,
199
+ });
200
+ const oomIndicated =
201
+ diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
202
+ throw new DaytonaRunnerInitializationError(
203
+ `RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
204
+ );
205
+ }
@@ -33,7 +33,8 @@ import {
33
33
  } from './daytona-lifecycle';
34
34
  import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
35
35
  import {
36
- inspectDetachedDaytonaStartup,
36
+ confirmDetachedDaytonaRunnerReady,
37
+ DaytonaRunnerInitializationError,
37
38
  startDetachedDaytonaCommand,
38
39
  } from './daytona-session-execution';
39
40
 
@@ -45,8 +46,6 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
45
46
  const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
46
47
  const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
47
48
  const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
48
- const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
49
- const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
50
49
 
51
50
  const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
52
51
  /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
@@ -397,6 +396,7 @@ export function retryableRuntimeInitializationFailureReason(
397
396
  }
398
397
 
399
398
  function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
399
+ if (error instanceof DaytonaRunnerInitializationError) return false;
400
400
  const message = error instanceof Error ? error.message : String(error);
401
401
  if (!message || /cancelled|aborted/i.test(message)) {
402
402
  return false;
@@ -715,12 +715,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
715
715
  });
716
716
  // Push execution (B2-final, park-and-wake): start the runner command
717
717
  // DETACHED via a Daytona session and return a `detached_runner`
718
- // suspension immediately the worker parks on it (releasing its
719
- // claim, freeing the slot) and wakes on the runner's pushed terminal
720
- // or the ceiling timeout. Nothing here awaits completion: no held
721
- // socket, no poll loop, no per-second sandbox file reads. The sandbox
722
- // MUST outlive this return (the runner is executing inside it); the
723
- // wake leg owns finalize + sandbox GC.
718
+ // suspension after the runner writes its one-time ready marker. The
719
+ // marker is the handoff proof: a Daytona cmdId alone does not prove
720
+ // the sandbox process initialized. After that bounded startup check,
721
+ // the worker parks (releasing its claim and slot) and wakes only on
722
+ // the runner's pushed terminal or the ceiling timeout.
724
723
  const sessionId = `deepline-play-${randomUUID()}`;
725
724
  let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
726
725
  try {
@@ -744,6 +743,16 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
744
743
  // classification below decide fresh-sandbox retry vs loud failure.
745
744
  throw error;
746
745
  }
746
+ await Promise.race([
747
+ confirmDetachedDaytonaRunnerReady({
748
+ sandbox,
749
+ sessionId: start.sessionId,
750
+ cmdId: start.cmdId,
751
+ readyPath: stagedPayload.readyPath,
752
+ exitCodePath: stagedPayload.exitCodePath,
753
+ }),
754
+ cancellationPromise,
755
+ ]);
747
756
  const runnerAttempt = Math.max(
748
757
  0,
749
758
  Math.floor(config.context.runAttempt ?? 0),
@@ -758,52 +767,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
758
767
  exitCodePath: stagedPayload.exitCodePath,
759
768
  elapsedMs: Date.now() - startedAt,
760
769
  });
761
- // Daytona acknowledges an async command with only a cmdId. It does
762
- // not expose a queued/running state, and the backend parks before the
763
- // runner contacts the runtime sheet. During startup investigations,
764
- // retain a bounded snapshot while the sandbox is still alive; the
765
- // normal cancellation path otherwise deletes the only evidence.
766
- // This is opt-in because it adds Daytona control-plane calls to every
767
- // run. The diagnostic excludes command lines and environment values.
768
- if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
769
- const diagnosticTimer = setTimeout(() => {
770
- void inspectDetachedDaytonaStartup({
771
- sandbox,
772
- sessionId: start.sessionId,
773
- cmdId: start.cmdId,
774
- exitCodePath: stagedPayload.exitCodePath,
775
- }).then(
776
- (diagnostic) =>
777
- emitDaytonaStage(
778
- callbacks,
779
- config.context,
780
- 'execute:startup_diagnostic',
781
- {
782
- sandboxId: sandbox.id,
783
- sessionId: start.sessionId,
784
- cmdId: start.cmdId,
785
- diagnostic,
786
- elapsedMs: Date.now() - startedAt,
787
- },
788
- ),
789
- (error: unknown) =>
790
- emitDaytonaStage(
791
- callbacks,
792
- config.context,
793
- 'execute:startup_diagnostic_failed',
794
- {
795
- sandboxId: sandbox.id,
796
- sessionId: start.sessionId,
797
- cmdId: start.cmdId,
798
- errorType:
799
- error instanceof Error ? error.name : 'unknown',
800
- elapsedMs: Date.now() - startedAt,
801
- },
802
- ),
803
- );
804
- }, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
805
- diagnosticTimer.unref?.();
806
- }
807
770
  return {
808
771
  status: 'suspended',
809
772
  suspension: {