deepline 0.2.55 → 0.2.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +14 -0
- package/dist/bundling-sources/sdk/src/http.ts +19 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/stream-reconnect.ts +6 -0
- package/dist/bundling-sources/sdk/src/types.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +146 -12
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1631 -748
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +81 -52
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +146 -6
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +56 -22
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +14 -11
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +21 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +81 -21
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +27 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +49 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +96 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +341 -67
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +17 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +17 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +16 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +25 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +80 -1
- package/dist/bundling-sources/shared_libs/plays/docflow.ts +113 -14
- package/dist/bundling-sources/shared_libs/plays/play-exports.ts +53 -4
- package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +9 -0
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +85 -48
- package/dist/cli/index.js +429 -54
- package/dist/cli/index.mjs +409 -28
- package/dist/{compiler-manifest-Bl8kmLx9.d.mts → compiler-manifest-TgaC4DeD.d.mts} +13 -0
- package/dist/{compiler-manifest-Bl8kmLx9.d.ts → compiler-manifest-TgaC4DeD.d.ts} +13 -0
- package/dist/index.d.mts +21 -3
- package/dist/index.d.ts +21 -3
- package/dist/index.js +29 -2
- package/dist/index.mjs +29 -2
- package/dist/install-integrity.json +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +78 -18
- package/package.json +1 -1
|
@@ -79,37 +79,88 @@ export interface ResolvedExecutionPolicy {
|
|
|
79
79
|
readonly pacing: ExecutionPacingPolicy;
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
export const DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS = 64;
|
|
83
|
+
export const MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS = 256;
|
|
84
|
+
export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
|
|
85
|
+
// Dispatch groups are only a scheduler envelope. The external-call semaphore
|
|
86
|
+
// remains the resource bound, so do not impose a lower hidden ceiling here.
|
|
87
|
+
// This lets an explicit run-level call limit describe the actual concurrency
|
|
88
|
+
// available to long-residence providers while still bounding live calls.
|
|
89
|
+
export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS =
|
|
90
|
+
MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS;
|
|
91
|
+
// Launches persisted before this field existed used the original fixed
|
|
92
|
+
// eight-slot policy. Preserve that value on their retries and resumes.
|
|
93
|
+
export const LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS = 8;
|
|
94
|
+
|
|
95
|
+
export function resolveMaxConcurrentExternalCalls(
|
|
96
|
+
requested?: number | null,
|
|
97
|
+
): number {
|
|
98
|
+
if (requested === undefined || requested === null) {
|
|
99
|
+
return DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS;
|
|
100
|
+
}
|
|
101
|
+
if (
|
|
102
|
+
!Number.isSafeInteger(requested) ||
|
|
103
|
+
requested < 1 ||
|
|
104
|
+
requested > MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS
|
|
105
|
+
) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`maxConcurrentExternalCalls must be a whole number from 1 through ${MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS}.`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return requested;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function resolveMaxConcurrentRows(
|
|
114
|
+
requested?: number | null,
|
|
115
|
+
): number | null {
|
|
116
|
+
if (requested === undefined || requested === null) return null;
|
|
117
|
+
if (
|
|
118
|
+
!Number.isSafeInteger(requested) ||
|
|
119
|
+
requested < 1 ||
|
|
120
|
+
requested > MAX_CONFIGURABLE_CONCURRENT_ROWS
|
|
121
|
+
) {
|
|
122
|
+
throw new Error(
|
|
123
|
+
`maxConcurrentRows must be a whole number from 1 through ${MAX_CONFIGURABLE_CONCURRENT_ROWS}.`,
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
return requested;
|
|
127
|
+
}
|
|
128
|
+
|
|
82
129
|
/**
|
|
83
130
|
* The shared default policy. Both substrates use this verbatim unless an entry
|
|
84
131
|
* in {@link ADAPTER_POLICY_OVERRIDES} forces a documented difference.
|
|
85
132
|
*/
|
|
86
133
|
export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
87
134
|
concurrency: {
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
|
|
135
|
+
// Bound the complete live row promise graph, not only outbound calls.
|
|
136
|
+
// Cached provider payloads can expand far beyond their encoded receipt size
|
|
137
|
+
// while Node parses and delivers result lists. Twenty rows leave enough
|
|
138
|
+
// headroom in a standard 1 GiB sandbox while still filling the default
|
|
139
|
+
// external-call scheduling group. Pending rows remain compact indexes.
|
|
140
|
+
// Pure-compute maps
|
|
141
|
+
// use a separate fast path and do not pay this bound.
|
|
142
|
+
rowDefault: 20,
|
|
143
|
+
// Internal callers can request a wider window, but the default remains the
|
|
144
|
+
// safe standard-sandbox resident set. The input-byte estimator may reduce
|
|
145
|
+
// either value further for large source rows.
|
|
96
146
|
rowMax: 1_000,
|
|
97
|
-
// Global logical-call backstop.
|
|
98
|
-
//
|
|
99
|
-
|
|
147
|
+
// Global logical-call backstop. The default is intentionally close to the
|
|
148
|
+
// standard row cohort so slow small responses do not serialize most rows.
|
|
149
|
+
// Runs may request a value through the validated 1..256 launch contract;
|
|
150
|
+
// provider pacing and the runner-owned hard ceiling still apply.
|
|
151
|
+
toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
|
|
100
152
|
// Physical runtime-to-app fetch/body admission. Keep short socket bursts
|
|
101
153
|
// below the sandbox/Vercel transport cliff while allowing many more logical
|
|
102
154
|
// calls to remain resident. This is deliberately independent of provider
|
|
103
155
|
// rate limits: it protects Deepline's own transport edge.
|
|
104
156
|
integrationRequests: 64,
|
|
105
|
-
// Scheduling
|
|
106
|
-
//
|
|
107
|
-
//
|
|
108
|
-
|
|
109
|
-
//
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
toolDispatchGroupsPerLane: 4,
|
|
157
|
+
// Scheduling cohorts overlap provider-response tails, but each cohort may
|
|
158
|
+
// itself contain many logical calls. Keep this materially above the legacy
|
|
159
|
+
// eight-group ceiling without creating one independently flushed cohort
|
|
160
|
+
// per configured tool slot. The latter fragments batchable providers and
|
|
161
|
+
// adds receipt/gateway round trips under wide (128+) runs.
|
|
162
|
+
toolDispatchGroups: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
|
|
163
|
+
toolDispatchGroupsPerLane: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
|
|
113
164
|
},
|
|
114
165
|
budgets: {
|
|
115
166
|
// Runaway guards, not workload limits. A 5,000-row map calling several tools
|
|
@@ -181,13 +232,22 @@ export function resolveExecutionPolicy(
|
|
|
181
232
|
export function resolveRowConcurrency(
|
|
182
233
|
policy: ResolvedExecutionPolicy,
|
|
183
234
|
requested?: number,
|
|
235
|
+
runMaximum?: number | null,
|
|
184
236
|
): number {
|
|
237
|
+
const normalizedRunMaximum = resolveMaxConcurrentRows(runMaximum);
|
|
185
238
|
if (
|
|
186
239
|
typeof requested === 'number' &&
|
|
187
240
|
Number.isFinite(requested) &&
|
|
188
241
|
requested > 0
|
|
189
242
|
) {
|
|
190
|
-
return Math.min(
|
|
243
|
+
return Math.min(
|
|
244
|
+
Math.floor(requested),
|
|
245
|
+
normalizedRunMaximum ?? policy.concurrency.rowMax,
|
|
246
|
+
policy.concurrency.rowMax,
|
|
247
|
+
);
|
|
191
248
|
}
|
|
192
|
-
return
|
|
249
|
+
return Math.min(
|
|
250
|
+
normalizedRunMaximum ?? policy.concurrency.rowDefault,
|
|
251
|
+
policy.concurrency.rowMax,
|
|
252
|
+
);
|
|
193
253
|
}
|
|
@@ -47,6 +47,13 @@ export const RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
|
47
47
|
* 10 MiB receipt fits while compatible fat receipts split before transport.
|
|
48
48
|
*/
|
|
49
49
|
export const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
|
|
50
|
+
/**
|
|
51
|
+
* Aim below the hard gateway ceiling when coalescing receipts. One legal large
|
|
52
|
+
* receipt may exceed this target and is still sent alone; the target only
|
|
53
|
+
* prevents several multi-megabyte results from monopolizing the serialized
|
|
54
|
+
* gateway writer behind one request.
|
|
55
|
+
*/
|
|
56
|
+
export const RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES = 4 * 1024 * 1024;
|
|
50
57
|
/**
|
|
51
58
|
* The terminal `result` line is the crash-recovery record in Daytona stdout.
|
|
52
59
|
* Once that line is written, retries may still emit runner-owned diagnostics.
|
|
@@ -133,6 +133,9 @@ export interface PlayRunnerContextConfig {
|
|
|
133
133
|
fixtureBehavior?: FixtureBehavior | null;
|
|
134
134
|
/** Preview/dev test seam that applies provider pacing to fixture responses. */
|
|
135
135
|
enforceFixtureProviderPacing?: boolean;
|
|
136
|
+
/** Validated per-run ceiling for provider-tool executions and ctx.fetch. */
|
|
137
|
+
maxConcurrentExternalCalls?: number | null;
|
|
138
|
+
maxConcurrentRows?: number | null;
|
|
136
139
|
/** Immutable tool-error payload schema copied from the run contract. */
|
|
137
140
|
toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
138
141
|
orgId?: string;
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -109,6 +109,11 @@ function isDaytonaSandboxStartTimeout(error: string): boolean {
|
|
|
109
109
|
);
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
function isDaytonaSandboxStateChangeConflict(error: unknown): boolean {
|
|
113
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
114
|
+
return /Sandbox state change in progress/i.test(message);
|
|
115
|
+
}
|
|
116
|
+
|
|
112
117
|
type DaytonaCreateResult = {
|
|
113
118
|
sandbox: DaytonaSandbox;
|
|
114
119
|
attempt: number;
|
|
@@ -300,6 +305,7 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
|
|
|
300
305
|
sandboxName: string;
|
|
301
306
|
}): Promise<boolean> {
|
|
302
307
|
if (!input.daytona.get) return false;
|
|
308
|
+
let lastDeleteConflict: unknown = null;
|
|
303
309
|
for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
|
|
304
310
|
if (delayMs > 0) {
|
|
305
311
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
@@ -313,6 +319,15 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
|
|
|
313
319
|
try {
|
|
314
320
|
await sandbox.delete(30);
|
|
315
321
|
} catch (error) {
|
|
322
|
+
// Daytona can make the named sandbox lookup-addressable before its
|
|
323
|
+
// timed-out create transition has settled. That exact conflict is not a
|
|
324
|
+
// cleanup verdict: re-read the same provider identity after the bounded
|
|
325
|
+
// reconcile delay instead of failing the run or launching a duplicate
|
|
326
|
+
// sandbox while the first one is still becoming ready.
|
|
327
|
+
if (isDaytonaSandboxStateChangeConflict(error)) {
|
|
328
|
+
lastDeleteConflict = error;
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
316
331
|
throw new Error(
|
|
317
332
|
`Daytona timed-out sandbox ${sandbox.id} was reconciled by name but could not be deleted. Modal fallback suppressed.`,
|
|
318
333
|
{ cause: error },
|
|
@@ -320,6 +335,12 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
|
|
|
320
335
|
}
|
|
321
336
|
return true;
|
|
322
337
|
}
|
|
338
|
+
if (lastDeleteConflict) {
|
|
339
|
+
throw new Error(
|
|
340
|
+
`Daytona timed-out sandbox ${input.sandboxName} remained in a state transition across the bounded cleanup window. Modal fallback suppressed.`,
|
|
341
|
+
{ cause: lastDeleteConflict },
|
|
342
|
+
);
|
|
343
|
+
}
|
|
323
344
|
return false;
|
|
324
345
|
}
|
|
325
346
|
|
|
@@ -66,6 +66,7 @@ export type StagedDaytonaPayload = {
|
|
|
66
66
|
outputPath: string;
|
|
67
67
|
exitCodePath: string;
|
|
68
68
|
runtimeCompletedPath: string;
|
|
69
|
+
terminationDiagnosticPath: string;
|
|
69
70
|
progressEventPath: string;
|
|
70
71
|
};
|
|
71
72
|
|
|
@@ -142,6 +143,7 @@ const outputPath = process.argv[4];
|
|
|
142
143
|
const runtimeLimitMarkerPath = process.argv[5] || '';
|
|
143
144
|
const oomKillBaselinePath = process.argv[6] || '';
|
|
144
145
|
const memoryEventsPath = process.argv[7] || '/sys/fs/cgroup/memory.events';
|
|
146
|
+
const terminationDiagnosticPath = process.argv[8] || '';
|
|
145
147
|
const exitCode = Number.isFinite(exitCodeRaw) ? exitCodeRaw : null;
|
|
146
148
|
const MAX_OUTPUT_BYTES = ${DAYTONA_CRASH_PUSHER_OUTPUT_TAIL_MAX_BYTES};
|
|
147
149
|
const MAX_POST_TERMINAL_DIAGNOSTIC_BYTES = ${RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES};
|
|
@@ -221,6 +223,23 @@ function cgroupOomKillObserved() {
|
|
|
221
223
|
return false;
|
|
222
224
|
}
|
|
223
225
|
|
|
226
|
+
function watchdogObservedChildSigkill() {
|
|
227
|
+
try {
|
|
228
|
+
if (!terminationDiagnosticPath) return false;
|
|
229
|
+
const diagnostic = JSON.parse(
|
|
230
|
+
fs.readFileSync(terminationDiagnosticPath, 'utf8'),
|
|
231
|
+
);
|
|
232
|
+
return Boolean(
|
|
233
|
+
diagnostic &&
|
|
234
|
+
diagnostic.schemaVersion === 1 &&
|
|
235
|
+
diagnostic.reason === 'child_exit' &&
|
|
236
|
+
diagnostic.childExitCode === null &&
|
|
237
|
+
diagnostic.childSignal === 'SIGKILL',
|
|
238
|
+
);
|
|
239
|
+
} catch {}
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
|
|
224
243
|
function terminalResultForTransport(result) {
|
|
225
244
|
if (
|
|
226
245
|
result &&
|
|
@@ -273,7 +292,10 @@ async function main() {
|
|
|
273
292
|
// 137 is SIGKILL, not proof of an OOM. A watchdog, platform eviction, or
|
|
274
293
|
// operator action can produce the same status. Only label this an OOM when
|
|
275
294
|
// the captured runtime output carries an explicit V8 allocation signature.
|
|
276
|
-
|
|
295
|
+
// The shell sees the watchdog's exit code, not necessarily the nested
|
|
296
|
+
// runner child's signal. Read the watchdog's strict marker so a child
|
|
297
|
+
// SIGKILL does not collapse into the wrapper's generic exit code 1.
|
|
298
|
+
const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
|
|
277
299
|
const sandboxOom =
|
|
278
300
|
sandboxKilled && (outputHasExplicitOomSignature() || cgroupOomKillObserved());
|
|
279
301
|
const synthesizedError = runtimeLimitExceeded
|
|
@@ -549,6 +571,7 @@ export async function stageRunnerPayload(input: {
|
|
|
549
571
|
const runtimeStartedPath = `${workDir}/deepline-play-runtime-started-${randomUUID()}.json`;
|
|
550
572
|
const runtimeCompletedPath = `${workDir}/deepline-play-runtime-completed-${randomUUID()}.json`;
|
|
551
573
|
const runtimeLimitMarkerPath = `${workDir}/deepline-play-runtime-limit-${randomUUID()}.txt`;
|
|
574
|
+
const terminationDiagnosticPath = `${workDir}/deepline-play-termination-${randomUUID()}.json`;
|
|
552
575
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
553
576
|
const runnerTraceEnv =
|
|
554
577
|
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
@@ -566,6 +589,7 @@ export async function stageRunnerPayload(input: {
|
|
|
566
589
|
String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
|
|
567
590
|
String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
|
|
568
591
|
shellQuote(runtimeLimitMarkerPath),
|
|
592
|
+
shellQuote(terminationDiagnosticPath),
|
|
569
593
|
].join(' ');
|
|
570
594
|
const runnerCommand = `${nodeMaterializePayloadCommand({
|
|
571
595
|
envelopePath,
|
|
@@ -583,7 +607,7 @@ export async function stageRunnerPayload(input: {
|
|
|
583
607
|
// Its own retry diagnostics are kept in a separate log: appending them to
|
|
584
608
|
// `outputPath` after a near-gateway-sized result could hide that result from
|
|
585
609
|
// a later bounded-tail recovery read.
|
|
586
|
-
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
610
|
+
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
587
611
|
|
|
588
612
|
return {
|
|
589
613
|
workDir: input.workDir,
|
|
@@ -591,6 +615,7 @@ export async function stageRunnerPayload(input: {
|
|
|
591
615
|
outputPath,
|
|
592
616
|
exitCodePath,
|
|
593
617
|
runtimeCompletedPath,
|
|
618
|
+
terminationDiagnosticPath,
|
|
594
619
|
progressEventPath,
|
|
595
620
|
};
|
|
596
621
|
}
|
|
@@ -18,11 +18,16 @@ const limitMs = Math.max(1, Number.parseInt(process.argv[5] || '', 10) || 1);
|
|
|
18
18
|
const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) || 1);
|
|
19
19
|
const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
|
|
20
20
|
const runtimeLimitMarkerPath = process.argv[8] || '';
|
|
21
|
+
const terminationDiagnosticPath = process.argv[9] || '';
|
|
21
22
|
const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
|
|
22
|
-
const child = spawn(
|
|
23
|
+
const child = spawn(
|
|
24
|
+
process.execPath,
|
|
25
|
+
['--max-old-space-size=512', runnerPath, configPath],
|
|
26
|
+
{
|
|
23
27
|
env: process.env,
|
|
24
28
|
stdio: 'inherit',
|
|
25
|
-
}
|
|
29
|
+
},
|
|
30
|
+
);
|
|
26
31
|
let timedOut = false;
|
|
27
32
|
let startupTimedOut = false;
|
|
28
33
|
let terminalFlushTimedOut = false;
|
|
@@ -31,7 +36,25 @@ let forceKillTimer = null;
|
|
|
31
36
|
let runtimeTimer = null;
|
|
32
37
|
let terminalGraceTimer = null;
|
|
33
38
|
|
|
34
|
-
function
|
|
39
|
+
function writeTerminationDiagnostic(reason, details = {}) {
|
|
40
|
+
if (!terminationDiagnosticPath) return;
|
|
41
|
+
try {
|
|
42
|
+
fs.writeFileSync(
|
|
43
|
+
terminationDiagnosticPath,
|
|
44
|
+
JSON.stringify({
|
|
45
|
+
schemaVersion: 1,
|
|
46
|
+
reason,
|
|
47
|
+
runtimeStarted: fs.existsSync(runtimeStartedPath),
|
|
48
|
+
runtimeCompleted: fs.existsSync(runtimeCompletedPath),
|
|
49
|
+
...details,
|
|
50
|
+
}),
|
|
51
|
+
{ flag: 'wx' },
|
|
52
|
+
);
|
|
53
|
+
} catch {}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function stopChild(reason, details) {
|
|
57
|
+
writeTerminationDiagnostic(reason, details);
|
|
35
58
|
child.kill('SIGTERM');
|
|
36
59
|
forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000);
|
|
37
60
|
}
|
|
@@ -60,7 +83,7 @@ function armRuntimeDeadline() {
|
|
|
60
83
|
runtimeLimitSeconds +
|
|
61
84
|
' second runtime limit and was stopped by the external watchdog.\\n',
|
|
62
85
|
);
|
|
63
|
-
stopChild();
|
|
86
|
+
stopChild('runtime_limit');
|
|
64
87
|
}, limitMs);
|
|
65
88
|
}
|
|
66
89
|
|
|
@@ -75,7 +98,7 @@ function armTerminalGrace() {
|
|
|
75
98
|
terminalGraceTimer = setTimeout(() => {
|
|
76
99
|
terminalFlushTimedOut = true;
|
|
77
100
|
process.stderr.write('RUNTIME_TERMINAL_FLUSH_LIMIT_EXCEEDED: terminal settlement exceeded its grace period.\\n');
|
|
78
|
-
stopChild();
|
|
101
|
+
stopChild('terminal_flush_limit');
|
|
79
102
|
}, terminalGraceMs);
|
|
80
103
|
}
|
|
81
104
|
|
|
@@ -92,7 +115,7 @@ const startupTimer = setTimeout(() => {
|
|
|
92
115
|
startupTimedOut = true;
|
|
93
116
|
clearInterval(runtimeBoundaryPoll);
|
|
94
117
|
process.stderr.write('RUNTIME_STARTUP_LIMIT_EXCEEDED: runner never reached the customer-code boundary.\\n');
|
|
95
|
-
stopChild();
|
|
118
|
+
stopChild('startup_limit');
|
|
96
119
|
}, startupLimitMs);
|
|
97
120
|
if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
|
|
98
121
|
if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
|
|
@@ -116,9 +139,25 @@ function finish(code) {
|
|
|
116
139
|
: 1;
|
|
117
140
|
}
|
|
118
141
|
|
|
119
|
-
child.once('error', () =>
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
142
|
+
child.once('error', (error) => {
|
|
143
|
+
writeTerminationDiagnostic('child_spawn_error', {
|
|
144
|
+
errorName: error && typeof error.name === 'string' ? error.name : 'Error',
|
|
145
|
+
errorCode: error && typeof error.code === 'string' ? error.code : null,
|
|
146
|
+
});
|
|
147
|
+
finish(1);
|
|
148
|
+
});
|
|
149
|
+
child.once('exit', (code, signal) => {
|
|
150
|
+
writeTerminationDiagnostic('child_exit', {
|
|
151
|
+
childExitCode: typeof code === 'number' ? code : null,
|
|
152
|
+
childSignal: typeof signal === 'string' ? signal : null,
|
|
153
|
+
});
|
|
154
|
+
finish(code);
|
|
155
|
+
});
|
|
156
|
+
process.once('SIGTERM', () =>
|
|
157
|
+
stopChild('watchdog_signal', { watchdogSignal: 'SIGTERM' }),
|
|
158
|
+
);
|
|
159
|
+
process.once('SIGINT', () =>
|
|
160
|
+
stopChild('watchdog_signal', { watchdogSignal: 'SIGINT' }),
|
|
161
|
+
);
|
|
123
162
|
`;
|
|
124
163
|
}
|
|
@@ -74,8 +74,93 @@ export type DaytonaCrashDiagnostic = {
|
|
|
74
74
|
tailTruncated: boolean;
|
|
75
75
|
signals: string[];
|
|
76
76
|
unclassifiedLineCount: number;
|
|
77
|
+
termination: DaytonaTerminationDiagnostic | null;
|
|
77
78
|
collectionFailure: DaytonaLookupFailure | null;
|
|
78
79
|
};
|
|
80
|
+
export type DaytonaTerminationDiagnostic = {
|
|
81
|
+
schemaVersion: 1;
|
|
82
|
+
reason:
|
|
83
|
+
| 'runtime_limit'
|
|
84
|
+
| 'terminal_flush_limit'
|
|
85
|
+
| 'startup_limit'
|
|
86
|
+
| 'watchdog_signal'
|
|
87
|
+
| 'child_spawn_error'
|
|
88
|
+
| 'child_exit';
|
|
89
|
+
runtimeStarted: boolean;
|
|
90
|
+
runtimeCompleted: boolean;
|
|
91
|
+
childExitCode?: number | null;
|
|
92
|
+
childSignal?: string | null;
|
|
93
|
+
watchdogSignal?: 'SIGTERM' | 'SIGINT';
|
|
94
|
+
errorName?: string;
|
|
95
|
+
errorCode?: string | null;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
function parseDaytonaTerminationDiagnostic(
|
|
99
|
+
value: unknown,
|
|
100
|
+
): DaytonaTerminationDiagnostic | null {
|
|
101
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
|
102
|
+
const candidate = value as Record<string, unknown>;
|
|
103
|
+
const reasons = new Set<DaytonaTerminationDiagnostic['reason']>([
|
|
104
|
+
'runtime_limit',
|
|
105
|
+
'terminal_flush_limit',
|
|
106
|
+
'startup_limit',
|
|
107
|
+
'watchdog_signal',
|
|
108
|
+
'child_spawn_error',
|
|
109
|
+
'child_exit',
|
|
110
|
+
]);
|
|
111
|
+
if (
|
|
112
|
+
candidate.schemaVersion !== 1 ||
|
|
113
|
+
typeof candidate.reason !== 'string' ||
|
|
114
|
+
!reasons.has(candidate.reason as DaytonaTerminationDiagnostic['reason']) ||
|
|
115
|
+
typeof candidate.runtimeStarted !== 'boolean' ||
|
|
116
|
+
typeof candidate.runtimeCompleted !== 'boolean'
|
|
117
|
+
) {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
const diagnostic: DaytonaTerminationDiagnostic = {
|
|
121
|
+
schemaVersion: 1,
|
|
122
|
+
reason: candidate.reason as DaytonaTerminationDiagnostic['reason'],
|
|
123
|
+
runtimeStarted: candidate.runtimeStarted,
|
|
124
|
+
runtimeCompleted: candidate.runtimeCompleted,
|
|
125
|
+
};
|
|
126
|
+
if (candidate.childExitCode === null) {
|
|
127
|
+
diagnostic.childExitCode = null;
|
|
128
|
+
} else if (
|
|
129
|
+
typeof candidate.childExitCode === 'number' &&
|
|
130
|
+
Number.isSafeInteger(candidate.childExitCode)
|
|
131
|
+
) {
|
|
132
|
+
diagnostic.childExitCode = candidate.childExitCode;
|
|
133
|
+
}
|
|
134
|
+
if (candidate.childSignal === null) {
|
|
135
|
+
diagnostic.childSignal = null;
|
|
136
|
+
} else if (
|
|
137
|
+
typeof candidate.childSignal === 'string' &&
|
|
138
|
+
/^SIG[A-Z0-9]+$/.test(candidate.childSignal)
|
|
139
|
+
) {
|
|
140
|
+
diagnostic.childSignal = candidate.childSignal;
|
|
141
|
+
}
|
|
142
|
+
if (
|
|
143
|
+
candidate.watchdogSignal === 'SIGTERM' ||
|
|
144
|
+
candidate.watchdogSignal === 'SIGINT'
|
|
145
|
+
) {
|
|
146
|
+
diagnostic.watchdogSignal = candidate.watchdogSignal;
|
|
147
|
+
}
|
|
148
|
+
if (
|
|
149
|
+
typeof candidate.errorName === 'string' &&
|
|
150
|
+
/^[A-Za-z][A-Za-z0-9]{0,39}$/.test(candidate.errorName)
|
|
151
|
+
) {
|
|
152
|
+
diagnostic.errorName = candidate.errorName;
|
|
153
|
+
}
|
|
154
|
+
if (
|
|
155
|
+
candidate.errorCode === null ||
|
|
156
|
+
(typeof candidate.errorCode === 'string' &&
|
|
157
|
+
/^[A-Z0-9_]{1,40}$/.test(candidate.errorCode))
|
|
158
|
+
) {
|
|
159
|
+
diagnostic.errorCode =
|
|
160
|
+
typeof candidate.errorCode === 'string' ? candidate.errorCode : null;
|
|
161
|
+
}
|
|
162
|
+
return diagnostic;
|
|
163
|
+
}
|
|
79
164
|
type DaytonaUploadAttemptTiming = {
|
|
80
165
|
attempt: number;
|
|
81
166
|
sandboxId: string;
|
|
@@ -198,7 +283,8 @@ function daytonaCrashDiagnosticSignature(line: string): string | null {
|
|
|
198
283
|
if (/\bunhandled(?:promiserejection)?\b/i.test(line))
|
|
199
284
|
signals.add('exception:unhandled_rejection');
|
|
200
285
|
if (/\bFATAL ERROR\b/i.test(line)) signals.add('runtime:fatal_error');
|
|
201
|
-
if (/heap out of memory/i.test(line))
|
|
286
|
+
if (/heap out of memory/i.test(line))
|
|
287
|
+
signals.add('runtime:heap_out_of_memory');
|
|
202
288
|
|
|
203
289
|
for (const code of line.matchAll(/\b(ENOMEM|EPIPE|ECONNRESET|ETIMEDOUT)\b/gi))
|
|
204
290
|
signals.add(`transport:${code[1]!.toUpperCase()}`);
|
|
@@ -268,6 +354,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
268
354
|
cmdId: string;
|
|
269
355
|
outputPath: string;
|
|
270
356
|
exitCodePath: string;
|
|
357
|
+
terminationDiagnosticPath?: string;
|
|
271
358
|
expectedOrganizationId?: string | null;
|
|
272
359
|
}): Promise<DaytonaCrashDiagnostic> {
|
|
273
360
|
const unavailable = (
|
|
@@ -282,6 +369,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
282
369
|
tailTruncated: false,
|
|
283
370
|
signals: [],
|
|
284
371
|
unclassifiedLineCount: 0,
|
|
372
|
+
termination: null,
|
|
285
373
|
collectionFailure,
|
|
286
374
|
});
|
|
287
375
|
try {
|
|
@@ -307,10 +395,10 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
307
395
|
};
|
|
308
396
|
}
|
|
309
397
|
|
|
310
|
-
const script = `const fs=require('node:fs');const output=process.argv[1];const exit=process.argv[2];const max=Number(process.argv[3]);const stat=fs.statSync(output);const start=Math.max(0,stat.size-max);const fd=fs.openSync(output,'r');const tail=Buffer.alloc(stat.size-start);fs.readSync(fd,tail,0,tail.length,start);fs.closeSync(fd);let exitCode=null;try{const value=Number.parseInt(fs.readFileSync(exit,'utf8').trim(),10);if(Number.isFinite(value))exitCode=value}catch{}process.stdout.write(JSON.stringify({outputBytes:stat.size,tail:tail.toString('base64'),exitCode}));`;
|
|
398
|
+
const script = `const fs=require('node:fs');const output=process.argv[1];const exit=process.argv[2];const max=Number(process.argv[3]);const terminationPath=process.argv[4]||'';const stat=fs.statSync(output);const start=Math.max(0,stat.size-max);const fd=fs.openSync(output,'r');const tail=Buffer.alloc(stat.size-start);fs.readSync(fd,tail,0,tail.length,start);fs.closeSync(fd);let exitCode=null;try{const value=Number.parseInt(fs.readFileSync(exit,'utf8').trim(),10);if(Number.isFinite(value))exitCode=value}catch{}let termination=null;try{const value=JSON.parse(fs.readFileSync(terminationPath,'utf8'));if(value&&typeof value==='object')termination=value}catch{}process.stdout.write(JSON.stringify({outputBytes:stat.size,tail:tail.toString('base64'),exitCode,termination}));`;
|
|
311
399
|
const [snapshotResult, sessionResult] = await Promise.allSettled([
|
|
312
400
|
sandbox.process.executeCommand(
|
|
313
|
-
`node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES}`,
|
|
401
|
+
`node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES} ${shellQuoteDaytonaDiagnostic(input.terminationDiagnosticPath ?? '')}`,
|
|
314
402
|
undefined,
|
|
315
403
|
{},
|
|
316
404
|
8,
|
|
@@ -324,6 +412,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
324
412
|
outputBytes?: unknown;
|
|
325
413
|
tail?: unknown;
|
|
326
414
|
exitCode?: unknown;
|
|
415
|
+
termination?: unknown;
|
|
327
416
|
};
|
|
328
417
|
const tailBuffer =
|
|
329
418
|
typeof snapshot.tail === 'string'
|
|
@@ -342,6 +431,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
342
431
|
typeof sessionResult.value?.exitCode === 'number'
|
|
343
432
|
? sessionResult.value.exitCode
|
|
344
433
|
: null;
|
|
434
|
+
const termination = parseDaytonaTerminationDiagnostic(snapshot.termination);
|
|
345
435
|
return {
|
|
346
436
|
schemaVersion: 1,
|
|
347
437
|
collectionStatus: 'collected',
|
|
@@ -358,6 +448,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
358
448
|
outputBytes > DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES,
|
|
359
449
|
signals,
|
|
360
450
|
unclassifiedLineCount,
|
|
451
|
+
termination,
|
|
361
452
|
collectionFailure:
|
|
362
453
|
sessionResult.status === 'rejected'
|
|
363
454
|
? describeDaytonaLookupFailure(sessionResult.reason)
|
|
@@ -1251,6 +1342,8 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1251
1342
|
outputPath: stagedPayload.outputPath,
|
|
1252
1343
|
exitCodePath: stagedPayload.exitCodePath,
|
|
1253
1344
|
runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
|
|
1345
|
+
terminationDiagnosticPath:
|
|
1346
|
+
stagedPayload.terminationDiagnosticPath,
|
|
1254
1347
|
startedAtMs: Date.now(),
|
|
1255
1348
|
heartbeatTimeoutMs: push.leaseSeconds * 1_000,
|
|
1256
1349
|
runtimeLimitSeconds: executionTimeoutSeconds,
|