deepline 0.2.55 → 0.2.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +14 -0
  2. package/dist/bundling-sources/sdk/src/http.ts +19 -1
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/stream-reconnect.ts +6 -0
  5. package/dist/bundling-sources/sdk/src/types.ts +7 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +146 -12
  7. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +6 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1631 -748
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +20 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +81 -52
  11. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +146 -6
  12. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +56 -22
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +14 -11
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +21 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +81 -21
  16. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +7 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +21 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +27 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +49 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +96 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +341 -67
  23. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +10 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +8 -1
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +17 -2
  26. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +4 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +17 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +16 -2
  29. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +25 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +2 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +80 -1
  33. package/dist/bundling-sources/shared_libs/plays/docflow.ts +113 -14
  34. package/dist/bundling-sources/shared_libs/plays/play-exports.ts +53 -4
  35. package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +9 -0
  36. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +85 -48
  37. package/dist/cli/index.js +429 -54
  38. package/dist/cli/index.mjs +409 -28
  39. package/dist/{compiler-manifest-Bl8kmLx9.d.mts → compiler-manifest-TgaC4DeD.d.mts} +13 -0
  40. package/dist/{compiler-manifest-Bl8kmLx9.d.ts → compiler-manifest-TgaC4DeD.d.ts} +13 -0
  41. package/dist/index.d.mts +21 -3
  42. package/dist/index.d.ts +21 -3
  43. package/dist/index.js +29 -2
  44. package/dist/index.mjs +29 -2
  45. package/dist/install-integrity.json +2 -2
  46. package/dist/plays/bundle-play-file.d.mts +2 -2
  47. package/dist/plays/bundle-play-file.d.ts +2 -2
  48. package/dist/plays/bundle-play-file.mjs +78 -18
  49. package/package.json +1 -1
@@ -79,37 +79,88 @@ export interface ResolvedExecutionPolicy {
79
79
  readonly pacing: ExecutionPacingPolicy;
80
80
  }
81
81
 
82
+ export const DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS = 64;
83
+ export const MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS = 256;
84
+ export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
85
+ // Dispatch groups are only a scheduler envelope. The external-call semaphore
86
+ // remains the resource bound, so do not impose a lower hidden ceiling here.
87
+ // This lets an explicit run-level call limit describe the actual concurrency
88
+ // available to long-residence providers while still bounding live calls.
89
+ export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS =
90
+ MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS;
91
+ // Launches persisted before this field existed used the original fixed
92
+ // eight-slot policy. Preserve that value on their retries and resumes.
93
+ export const LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS = 8;
94
+
95
+ export function resolveMaxConcurrentExternalCalls(
96
+ requested?: number | null,
97
+ ): number {
98
+ if (requested === undefined || requested === null) {
99
+ return DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS;
100
+ }
101
+ if (
102
+ !Number.isSafeInteger(requested) ||
103
+ requested < 1 ||
104
+ requested > MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS
105
+ ) {
106
+ throw new Error(
107
+ `maxConcurrentExternalCalls must be a whole number from 1 through ${MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS}.`,
108
+ );
109
+ }
110
+ return requested;
111
+ }
112
+
113
+ export function resolveMaxConcurrentRows(
114
+ requested?: number | null,
115
+ ): number | null {
116
+ if (requested === undefined || requested === null) return null;
117
+ if (
118
+ !Number.isSafeInteger(requested) ||
119
+ requested < 1 ||
120
+ requested > MAX_CONFIGURABLE_CONCURRENT_ROWS
121
+ ) {
122
+ throw new Error(
123
+ `maxConcurrentRows must be a whole number from 1 through ${MAX_CONFIGURABLE_CONCURRENT_ROWS}.`,
124
+ );
125
+ }
126
+ return requested;
127
+ }
128
+
82
129
  /**
83
130
  * The shared default policy. Both substrates use this verbatim unless an entry
84
131
  * in {@link ADAPTER_POLICY_OVERRIDES} forces a documented difference.
85
132
  */
86
133
  export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
87
134
  concurrency: {
88
- // Logical row target. A 1,000-row window can fill five 200-item provider
89
- // batches or keep a 100-RPS unbatched provider fed; the runtime's shared
90
- // resident-byte admission calculator clamps fat rows below this target.
91
- // Pending rows remain indexes, not promises or pre-claimed receipts.
92
- rowDefault: 1_000,
93
- // Finite global ceiling across concurrent maps in one run. This intentionally
94
- // matches the default: throughput grows through provider pacing/batching,
95
- // while row payload size can only reduce admission through the byte budget.
135
+ // Bound the complete live row promise graph, not only outbound calls.
136
+ // Cached provider payloads can expand far beyond their encoded receipt size
137
+ // while Node parses and delivers result lists. Twenty rows leave enough
138
+ // headroom in a standard 1 GiB sandbox while still filling the default
139
+ // external-call scheduling group. Pending rows remain compact indexes.
140
+ // Pure-compute maps
141
+ // use a separate fast path and do not pay this bound.
142
+ rowDefault: 20,
143
+ // Internal callers can request a wider window, but the default remains the
144
+ // safe standard-sandbox resident set. The input-byte estimator may reduce
145
+ // either value further for large source rows.
96
146
  rowMax: 1_000,
97
- // Global logical-call backstop. A logical call can spend most of its life
98
- // waiting on a provider/fixture response without holding a network socket.
99
- toolCalls: 256,
147
+ // Global logical-call backstop. The default is intentionally close to the
148
+ // standard row cohort so slow small responses do not serialize most rows.
149
+ // Runs may request a value through the validated 1..256 launch contract;
150
+ // provider pacing and the runner-owned hard ceiling still apply.
151
+ toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
100
152
  // Physical runtime-to-app fetch/body admission. Keep short socket bursts
101
153
  // below the sandbox/Vercel transport cliff while allowing many more logical
102
154
  // calls to remain resident. This is deliberately independent of provider
103
155
  // rate limits: it protects Deepline's own transport edge.
104
156
  integrationRequests: 64,
105
- // Scheduling groups contain bounded-dispatch workers and receipt buffers.
106
- // Keep their promise topology finite even when pipelined replacement rows
107
- // become ready one at a time behind a slower provider pacer.
108
- toolDispatchGroups: 32,
109
- // Preserve limited overlap so a fast row can enter its next column while a
110
- // prior group's receipt persistence is still settling. Provider permits
111
- // enforce declared maxConcurrency across these overlapping groups.
112
- toolDispatchGroupsPerLane: 4,
157
+ // Scheduling cohorts overlap provider-response tails, but each cohort may
158
+ // itself contain many logical calls. Keep this materially above the legacy
159
+ // eight-group ceiling without creating one independently flushed cohort
160
+ // per configured tool slot. The latter fragments batchable providers and
161
+ // adds receipt/gateway round trips under wide (128+) runs.
162
+ toolDispatchGroups: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
163
+ toolDispatchGroupsPerLane: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
113
164
  },
114
165
  budgets: {
115
166
  // Runaway guards, not workload limits. A 5,000-row map calling several tools
@@ -181,13 +232,22 @@ export function resolveExecutionPolicy(
181
232
  export function resolveRowConcurrency(
182
233
  policy: ResolvedExecutionPolicy,
183
234
  requested?: number,
235
+ runMaximum?: number | null,
184
236
  ): number {
237
+ const normalizedRunMaximum = resolveMaxConcurrentRows(runMaximum);
185
238
  if (
186
239
  typeof requested === 'number' &&
187
240
  Number.isFinite(requested) &&
188
241
  requested > 0
189
242
  ) {
190
- return Math.min(Math.floor(requested), policy.concurrency.rowMax);
243
+ return Math.min(
244
+ Math.floor(requested),
245
+ normalizedRunMaximum ?? policy.concurrency.rowMax,
246
+ policy.concurrency.rowMax,
247
+ );
191
248
  }
192
- return policy.concurrency.rowDefault;
249
+ return Math.min(
250
+ normalizedRunMaximum ?? policy.concurrency.rowDefault,
251
+ policy.concurrency.rowMax,
252
+ );
193
253
  }
@@ -47,6 +47,13 @@ export const RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
47
47
  * 10 MiB receipt fits while compatible fat receipts split before transport.
48
48
  */
49
49
  export const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
50
+ /**
51
+ * Aim below the hard gateway ceiling when coalescing receipts. One legal large
52
+ * receipt may exceed this target and is still sent alone; the target only
53
+ * prevents several multi-megabyte results from monopolizing the serialized
54
+ * gateway writer behind one request.
55
+ */
56
+ export const RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES = 4 * 1024 * 1024;
50
57
  /**
51
58
  * The terminal `result` line is the crash-recovery record in Daytona stdout.
52
59
  * Once that line is written, retries may still emit runner-owned diagnostics.
@@ -133,6 +133,9 @@ export interface PlayRunnerContextConfig {
133
133
  fixtureBehavior?: FixtureBehavior | null;
134
134
  /** Preview/dev test seam that applies provider pacing to fixture responses. */
135
135
  enforceFixtureProviderPacing?: boolean;
136
+ /** Validated per-run ceiling for provider-tool executions and ctx.fetch. */
137
+ maxConcurrentExternalCalls?: number | null;
138
+ maxConcurrentRows?: number | null;
136
139
  /** Immutable tool-error payload schema copied from the run contract. */
137
140
  toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
138
141
  orgId?: string;
@@ -109,6 +109,11 @@ function isDaytonaSandboxStartTimeout(error: string): boolean {
109
109
  );
110
110
  }
111
111
 
112
+ function isDaytonaSandboxStateChangeConflict(error: unknown): boolean {
113
+ const message = error instanceof Error ? error.message : String(error);
114
+ return /Sandbox state change in progress/i.test(message);
115
+ }
116
+
112
117
  type DaytonaCreateResult = {
113
118
  sandbox: DaytonaSandbox;
114
119
  attempt: number;
@@ -300,6 +305,7 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
300
305
  sandboxName: string;
301
306
  }): Promise<boolean> {
302
307
  if (!input.daytona.get) return false;
308
+ let lastDeleteConflict: unknown = null;
303
309
  for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
304
310
  if (delayMs > 0) {
305
311
  await new Promise((resolve) => setTimeout(resolve, delayMs));
@@ -313,6 +319,15 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
313
319
  try {
314
320
  await sandbox.delete(30);
315
321
  } catch (error) {
322
+ // Daytona can make the named sandbox lookup-addressable before its
323
+ // timed-out create transition has settled. That exact conflict is not a
324
+ // cleanup verdict: re-read the same provider identity after the bounded
325
+ // reconcile delay instead of failing the run or launching a duplicate
326
+ // sandbox while the first one is still becoming ready.
327
+ if (isDaytonaSandboxStateChangeConflict(error)) {
328
+ lastDeleteConflict = error;
329
+ continue;
330
+ }
316
331
  throw new Error(
317
332
  `Daytona timed-out sandbox ${sandbox.id} was reconciled by name but could not be deleted. Modal fallback suppressed.`,
318
333
  { cause: error },
@@ -320,6 +335,12 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
320
335
  }
321
336
  return true;
322
337
  }
338
+ if (lastDeleteConflict) {
339
+ throw new Error(
340
+ `Daytona timed-out sandbox ${input.sandboxName} remained in a state transition across the bounded cleanup window. Modal fallback suppressed.`,
341
+ { cause: lastDeleteConflict },
342
+ );
343
+ }
323
344
  return false;
324
345
  }
325
346
 
@@ -66,6 +66,7 @@ export type StagedDaytonaPayload = {
66
66
  outputPath: string;
67
67
  exitCodePath: string;
68
68
  runtimeCompletedPath: string;
69
+ terminationDiagnosticPath: string;
69
70
  progressEventPath: string;
70
71
  };
71
72
 
@@ -142,6 +143,7 @@ const outputPath = process.argv[4];
142
143
  const runtimeLimitMarkerPath = process.argv[5] || '';
143
144
  const oomKillBaselinePath = process.argv[6] || '';
144
145
  const memoryEventsPath = process.argv[7] || '/sys/fs/cgroup/memory.events';
146
+ const terminationDiagnosticPath = process.argv[8] || '';
145
147
  const exitCode = Number.isFinite(exitCodeRaw) ? exitCodeRaw : null;
146
148
  const MAX_OUTPUT_BYTES = ${DAYTONA_CRASH_PUSHER_OUTPUT_TAIL_MAX_BYTES};
147
149
  const MAX_POST_TERMINAL_DIAGNOSTIC_BYTES = ${RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES};
@@ -221,6 +223,23 @@ function cgroupOomKillObserved() {
221
223
  return false;
222
224
  }
223
225
 
226
+ function watchdogObservedChildSigkill() {
227
+ try {
228
+ if (!terminationDiagnosticPath) return false;
229
+ const diagnostic = JSON.parse(
230
+ fs.readFileSync(terminationDiagnosticPath, 'utf8'),
231
+ );
232
+ return Boolean(
233
+ diagnostic &&
234
+ diagnostic.schemaVersion === 1 &&
235
+ diagnostic.reason === 'child_exit' &&
236
+ diagnostic.childExitCode === null &&
237
+ diagnostic.childSignal === 'SIGKILL',
238
+ );
239
+ } catch {}
240
+ return false;
241
+ }
242
+
224
243
  function terminalResultForTransport(result) {
225
244
  if (
226
245
  result &&
@@ -273,7 +292,10 @@ async function main() {
273
292
  // 137 is SIGKILL, not proof of an OOM. A watchdog, platform eviction, or
274
293
  // operator action can produce the same status. Only label this an OOM when
275
294
  // the captured runtime output carries an explicit V8 allocation signature.
276
- const sandboxKilled = exitCode === 137;
295
+ // The shell sees the watchdog's exit code, not necessarily the nested
296
+ // runner child's signal. Read the watchdog's strict marker so a child
297
+ // SIGKILL does not collapse into the wrapper's generic exit code 1.
298
+ const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
277
299
  const sandboxOom =
278
300
  sandboxKilled && (outputHasExplicitOomSignature() || cgroupOomKillObserved());
279
301
  const synthesizedError = runtimeLimitExceeded
@@ -549,6 +571,7 @@ export async function stageRunnerPayload(input: {
549
571
  const runtimeStartedPath = `${workDir}/deepline-play-runtime-started-${randomUUID()}.json`;
550
572
  const runtimeCompletedPath = `${workDir}/deepline-play-runtime-completed-${randomUUID()}.json`;
551
573
  const runtimeLimitMarkerPath = `${workDir}/deepline-play-runtime-limit-${randomUUID()}.txt`;
574
+ const terminationDiagnosticPath = `${workDir}/deepline-play-termination-${randomUUID()}.json`;
552
575
  const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
553
576
  const runnerTraceEnv =
554
577
  process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
@@ -566,6 +589,7 @@ export async function stageRunnerPayload(input: {
566
589
  String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
567
590
  String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
568
591
  shellQuote(runtimeLimitMarkerPath),
592
+ shellQuote(terminationDiagnosticPath),
569
593
  ].join(' ');
570
594
  const runnerCommand = `${nodeMaterializePayloadCommand({
571
595
  envelopePath,
@@ -583,7 +607,7 @@ export async function stageRunnerPayload(input: {
583
607
  // Its own retry diagnostics are kept in a separate log: appending them to
584
608
  // `outputPath` after a near-gateway-sized result could hide that result from
585
609
  // a later bounded-tail recovery read.
586
- const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
610
+ const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
587
611
 
588
612
  return {
589
613
  workDir: input.workDir,
@@ -591,6 +615,7 @@ export async function stageRunnerPayload(input: {
591
615
  outputPath,
592
616
  exitCodePath,
593
617
  runtimeCompletedPath,
618
+ terminationDiagnosticPath,
594
619
  progressEventPath,
595
620
  };
596
621
  }
@@ -18,11 +18,16 @@ const limitMs = Math.max(1, Number.parseInt(process.argv[5] || '', 10) || 1);
18
18
  const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) || 1);
19
19
  const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
20
20
  const runtimeLimitMarkerPath = process.argv[8] || '';
21
+ const terminationDiagnosticPath = process.argv[9] || '';
21
22
  const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
22
- const child = spawn(process.execPath, [runnerPath, configPath], {
23
+ const child = spawn(
24
+ process.execPath,
25
+ ['--max-old-space-size=512', runnerPath, configPath],
26
+ {
23
27
  env: process.env,
24
28
  stdio: 'inherit',
25
- });
29
+ },
30
+ );
26
31
  let timedOut = false;
27
32
  let startupTimedOut = false;
28
33
  let terminalFlushTimedOut = false;
@@ -31,7 +36,25 @@ let forceKillTimer = null;
31
36
  let runtimeTimer = null;
32
37
  let terminalGraceTimer = null;
33
38
 
34
- function stopChild() {
39
+ function writeTerminationDiagnostic(reason, details = {}) {
40
+ if (!terminationDiagnosticPath) return;
41
+ try {
42
+ fs.writeFileSync(
43
+ terminationDiagnosticPath,
44
+ JSON.stringify({
45
+ schemaVersion: 1,
46
+ reason,
47
+ runtimeStarted: fs.existsSync(runtimeStartedPath),
48
+ runtimeCompleted: fs.existsSync(runtimeCompletedPath),
49
+ ...details,
50
+ }),
51
+ { flag: 'wx' },
52
+ );
53
+ } catch {}
54
+ }
55
+
56
+ function stopChild(reason, details) {
57
+ writeTerminationDiagnostic(reason, details);
35
58
  child.kill('SIGTERM');
36
59
  forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000);
37
60
  }
@@ -60,7 +83,7 @@ function armRuntimeDeadline() {
60
83
  runtimeLimitSeconds +
61
84
  ' second runtime limit and was stopped by the external watchdog.\\n',
62
85
  );
63
- stopChild();
86
+ stopChild('runtime_limit');
64
87
  }, limitMs);
65
88
  }
66
89
 
@@ -75,7 +98,7 @@ function armTerminalGrace() {
75
98
  terminalGraceTimer = setTimeout(() => {
76
99
  terminalFlushTimedOut = true;
77
100
  process.stderr.write('RUNTIME_TERMINAL_FLUSH_LIMIT_EXCEEDED: terminal settlement exceeded its grace period.\\n');
78
- stopChild();
101
+ stopChild('terminal_flush_limit');
79
102
  }, terminalGraceMs);
80
103
  }
81
104
 
@@ -92,7 +115,7 @@ const startupTimer = setTimeout(() => {
92
115
  startupTimedOut = true;
93
116
  clearInterval(runtimeBoundaryPoll);
94
117
  process.stderr.write('RUNTIME_STARTUP_LIMIT_EXCEEDED: runner never reached the customer-code boundary.\\n');
95
- stopChild();
118
+ stopChild('startup_limit');
96
119
  }, startupLimitMs);
97
120
  if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
98
121
  if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
@@ -116,9 +139,25 @@ function finish(code) {
116
139
  : 1;
117
140
  }
118
141
 
119
- child.once('error', () => finish(1));
120
- child.once('exit', (code) => finish(code));
121
- process.once('SIGTERM', stopChild);
122
- process.once('SIGINT', stopChild);
142
+ child.once('error', (error) => {
143
+ writeTerminationDiagnostic('child_spawn_error', {
144
+ errorName: error && typeof error.name === 'string' ? error.name : 'Error',
145
+ errorCode: error && typeof error.code === 'string' ? error.code : null,
146
+ });
147
+ finish(1);
148
+ });
149
+ child.once('exit', (code, signal) => {
150
+ writeTerminationDiagnostic('child_exit', {
151
+ childExitCode: typeof code === 'number' ? code : null,
152
+ childSignal: typeof signal === 'string' ? signal : null,
153
+ });
154
+ finish(code);
155
+ });
156
+ process.once('SIGTERM', () =>
157
+ stopChild('watchdog_signal', { watchdogSignal: 'SIGTERM' }),
158
+ );
159
+ process.once('SIGINT', () =>
160
+ stopChild('watchdog_signal', { watchdogSignal: 'SIGINT' }),
161
+ );
123
162
  `;
124
163
  }
@@ -74,8 +74,93 @@ export type DaytonaCrashDiagnostic = {
74
74
  tailTruncated: boolean;
75
75
  signals: string[];
76
76
  unclassifiedLineCount: number;
77
+ termination: DaytonaTerminationDiagnostic | null;
77
78
  collectionFailure: DaytonaLookupFailure | null;
78
79
  };
80
+ export type DaytonaTerminationDiagnostic = {
81
+ schemaVersion: 1;
82
+ reason:
83
+ | 'runtime_limit'
84
+ | 'terminal_flush_limit'
85
+ | 'startup_limit'
86
+ | 'watchdog_signal'
87
+ | 'child_spawn_error'
88
+ | 'child_exit';
89
+ runtimeStarted: boolean;
90
+ runtimeCompleted: boolean;
91
+ childExitCode?: number | null;
92
+ childSignal?: string | null;
93
+ watchdogSignal?: 'SIGTERM' | 'SIGINT';
94
+ errorName?: string;
95
+ errorCode?: string | null;
96
+ };
97
+
98
+ function parseDaytonaTerminationDiagnostic(
99
+ value: unknown,
100
+ ): DaytonaTerminationDiagnostic | null {
101
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
102
+ const candidate = value as Record<string, unknown>;
103
+ const reasons = new Set<DaytonaTerminationDiagnostic['reason']>([
104
+ 'runtime_limit',
105
+ 'terminal_flush_limit',
106
+ 'startup_limit',
107
+ 'watchdog_signal',
108
+ 'child_spawn_error',
109
+ 'child_exit',
110
+ ]);
111
+ if (
112
+ candidate.schemaVersion !== 1 ||
113
+ typeof candidate.reason !== 'string' ||
114
+ !reasons.has(candidate.reason as DaytonaTerminationDiagnostic['reason']) ||
115
+ typeof candidate.runtimeStarted !== 'boolean' ||
116
+ typeof candidate.runtimeCompleted !== 'boolean'
117
+ ) {
118
+ return null;
119
+ }
120
+ const diagnostic: DaytonaTerminationDiagnostic = {
121
+ schemaVersion: 1,
122
+ reason: candidate.reason as DaytonaTerminationDiagnostic['reason'],
123
+ runtimeStarted: candidate.runtimeStarted,
124
+ runtimeCompleted: candidate.runtimeCompleted,
125
+ };
126
+ if (candidate.childExitCode === null) {
127
+ diagnostic.childExitCode = null;
128
+ } else if (
129
+ typeof candidate.childExitCode === 'number' &&
130
+ Number.isSafeInteger(candidate.childExitCode)
131
+ ) {
132
+ diagnostic.childExitCode = candidate.childExitCode;
133
+ }
134
+ if (candidate.childSignal === null) {
135
+ diagnostic.childSignal = null;
136
+ } else if (
137
+ typeof candidate.childSignal === 'string' &&
138
+ /^SIG[A-Z0-9]+$/.test(candidate.childSignal)
139
+ ) {
140
+ diagnostic.childSignal = candidate.childSignal;
141
+ }
142
+ if (
143
+ candidate.watchdogSignal === 'SIGTERM' ||
144
+ candidate.watchdogSignal === 'SIGINT'
145
+ ) {
146
+ diagnostic.watchdogSignal = candidate.watchdogSignal;
147
+ }
148
+ if (
149
+ typeof candidate.errorName === 'string' &&
150
+ /^[A-Za-z][A-Za-z0-9]{0,39}$/.test(candidate.errorName)
151
+ ) {
152
+ diagnostic.errorName = candidate.errorName;
153
+ }
154
+ if (
155
+ candidate.errorCode === null ||
156
+ (typeof candidate.errorCode === 'string' &&
157
+ /^[A-Z0-9_]{1,40}$/.test(candidate.errorCode))
158
+ ) {
159
+ diagnostic.errorCode =
160
+ typeof candidate.errorCode === 'string' ? candidate.errorCode : null;
161
+ }
162
+ return diagnostic;
163
+ }
79
164
  type DaytonaUploadAttemptTiming = {
80
165
  attempt: number;
81
166
  sandboxId: string;
@@ -198,7 +283,8 @@ function daytonaCrashDiagnosticSignature(line: string): string | null {
198
283
  if (/\bunhandled(?:promiserejection)?\b/i.test(line))
199
284
  signals.add('exception:unhandled_rejection');
200
285
  if (/\bFATAL ERROR\b/i.test(line)) signals.add('runtime:fatal_error');
201
- if (/heap out of memory/i.test(line)) signals.add('runtime:heap_out_of_memory');
286
+ if (/heap out of memory/i.test(line))
287
+ signals.add('runtime:heap_out_of_memory');
202
288
 
203
289
  for (const code of line.matchAll(/\b(ENOMEM|EPIPE|ECONNRESET|ETIMEDOUT)\b/gi))
204
290
  signals.add(`transport:${code[1]!.toUpperCase()}`);
@@ -268,6 +354,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
268
354
  cmdId: string;
269
355
  outputPath: string;
270
356
  exitCodePath: string;
357
+ terminationDiagnosticPath?: string;
271
358
  expectedOrganizationId?: string | null;
272
359
  }): Promise<DaytonaCrashDiagnostic> {
273
360
  const unavailable = (
@@ -282,6 +369,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
282
369
  tailTruncated: false,
283
370
  signals: [],
284
371
  unclassifiedLineCount: 0,
372
+ termination: null,
285
373
  collectionFailure,
286
374
  });
287
375
  try {
@@ -307,10 +395,10 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
307
395
  };
308
396
  }
309
397
 
310
- const script = `const fs=require('node:fs');const output=process.argv[1];const exit=process.argv[2];const max=Number(process.argv[3]);const stat=fs.statSync(output);const start=Math.max(0,stat.size-max);const fd=fs.openSync(output,'r');const tail=Buffer.alloc(stat.size-start);fs.readSync(fd,tail,0,tail.length,start);fs.closeSync(fd);let exitCode=null;try{const value=Number.parseInt(fs.readFileSync(exit,'utf8').trim(),10);if(Number.isFinite(value))exitCode=value}catch{}process.stdout.write(JSON.stringify({outputBytes:stat.size,tail:tail.toString('base64'),exitCode}));`;
398
+ const script = `const fs=require('node:fs');const output=process.argv[1];const exit=process.argv[2];const max=Number(process.argv[3]);const terminationPath=process.argv[4]||'';const stat=fs.statSync(output);const start=Math.max(0,stat.size-max);const fd=fs.openSync(output,'r');const tail=Buffer.alloc(stat.size-start);fs.readSync(fd,tail,0,tail.length,start);fs.closeSync(fd);let exitCode=null;try{const value=Number.parseInt(fs.readFileSync(exit,'utf8').trim(),10);if(Number.isFinite(value))exitCode=value}catch{}let termination=null;try{const value=JSON.parse(fs.readFileSync(terminationPath,'utf8'));if(value&&typeof value==='object')termination=value}catch{}process.stdout.write(JSON.stringify({outputBytes:stat.size,tail:tail.toString('base64'),exitCode,termination}));`;
311
399
  const [snapshotResult, sessionResult] = await Promise.allSettled([
312
400
  sandbox.process.executeCommand(
313
- `node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES}`,
401
+ `node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES} ${shellQuoteDaytonaDiagnostic(input.terminationDiagnosticPath ?? '')}`,
314
402
  undefined,
315
403
  {},
316
404
  8,
@@ -324,6 +412,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
324
412
  outputBytes?: unknown;
325
413
  tail?: unknown;
326
414
  exitCode?: unknown;
415
+ termination?: unknown;
327
416
  };
328
417
  const tailBuffer =
329
418
  typeof snapshot.tail === 'string'
@@ -342,6 +431,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
342
431
  typeof sessionResult.value?.exitCode === 'number'
343
432
  ? sessionResult.value.exitCode
344
433
  : null;
434
+ const termination = parseDaytonaTerminationDiagnostic(snapshot.termination);
345
435
  return {
346
436
  schemaVersion: 1,
347
437
  collectionStatus: 'collected',
@@ -358,6 +448,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
358
448
  outputBytes > DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES,
359
449
  signals,
360
450
  unclassifiedLineCount,
451
+ termination,
361
452
  collectionFailure:
362
453
  sessionResult.status === 'rejected'
363
454
  ? describeDaytonaLookupFailure(sessionResult.reason)
@@ -1251,6 +1342,8 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1251
1342
  outputPath: stagedPayload.outputPath,
1252
1343
  exitCodePath: stagedPayload.exitCodePath,
1253
1344
  runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
1345
+ terminationDiagnosticPath:
1346
+ stagedPayload.terminationDiagnosticPath,
1254
1347
  startedAtMs: Date.now(),
1255
1348
  heartbeatTimeoutMs: push.leaseSeconds * 1_000,
1256
1349
  runtimeLimitSeconds: executionTimeoutSeconds,