deepline 0.1.260 → 0.1.261
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +2 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +147 -40
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +57 -71
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +8 -0
- package/dist/cli/index.js +468 -149
- package/dist/cli/index.mjs +479 -160
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
|
@@ -123,7 +123,7 @@ export const SDK_RELEASE = {
|
|
|
123
123
|
// Deepline-native radars. Older clients must update before discovering,
|
|
124
124
|
// checking, or deploying an unlaunched monitor integration.
|
|
125
125
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
126
|
-
version: '0.1.
|
|
126
|
+
version: '0.1.261',
|
|
127
127
|
apiContract: '2026-07-native-monitor-launch-hard-cutover',
|
|
128
128
|
supportPolicy: {
|
|
129
129
|
minimumSupported: '0.1.53',
|
|
@@ -8,6 +8,12 @@ export type GatewayAuthSession<TAuth, TLaunch> = {
|
|
|
8
8
|
source: 'hit' | 'miss' | 'coalesced';
|
|
9
9
|
};
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Launch rows are immutable for an executor token's lifetime. The cache still
|
|
13
|
+
* caps each entry at the token expiry in getOrEstablish.
|
|
14
|
+
*/
|
|
15
|
+
export const GATEWAY_IMMUTABLE_SESSION_TTL_MS = 45_000;
|
|
16
|
+
|
|
11
17
|
type StoredSession<TAuth, TLaunch> = {
|
|
12
18
|
auth: TAuth;
|
|
13
19
|
launch: TLaunch;
|
|
@@ -55,8 +55,6 @@ export const DAYTONA_CRASH_PUSHER_OUTPUT_TAIL_MAX_BYTES =
|
|
|
55
55
|
export type StagedDaytonaPayload = {
|
|
56
56
|
workDir: string;
|
|
57
57
|
command: string;
|
|
58
|
-
readyPath: string;
|
|
59
|
-
startAckPath: string;
|
|
60
58
|
outputPath: string;
|
|
61
59
|
exitCodePath: string;
|
|
62
60
|
progressEventPath: string;
|
|
@@ -456,8 +454,6 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
456
454
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
457
455
|
const crashPusherLogPath = `${workDir}/deepline-play-crash-terminal-${randomUUID()}.log`;
|
|
458
456
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
459
|
-
const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
|
|
460
|
-
const startAckPath = `${workDir}/deepline-play-start-ack-${randomUUID()}.json`;
|
|
461
457
|
const runtimeStartedPath = `${workDir}/deepline-play-runtime-started-${randomUUID()}.json`;
|
|
462
458
|
const runtimeCompletedPath = `${workDir}/deepline-play-runtime-completed-${randomUUID()}.json`;
|
|
463
459
|
const runtimeLimitMarkerPath = `${workDir}/deepline-play-runtime-limit-${randomUUID()}.txt`;
|
|
@@ -486,7 +482,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
486
482
|
artifactCodePath,
|
|
487
483
|
artifactSourceMapPath,
|
|
488
484
|
crashPusherPath,
|
|
489
|
-
})} &&
|
|
485
|
+
})} && DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH=${shellQuote(runtimeStartedPath)} DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH=${shellQuote(runtimeCompletedPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}${watchedRunnerCommand}`;
|
|
490
486
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
491
487
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
492
488
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
@@ -495,13 +491,11 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
495
491
|
// Its own retry diagnostics are kept in a separate log: appending them to
|
|
496
492
|
// `outputPath` after a near-gateway-sized result could hide that result from
|
|
497
493
|
// a later bounded-tail recovery read.
|
|
498
|
-
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(
|
|
494
|
+
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
499
495
|
|
|
500
496
|
return {
|
|
501
497
|
workDir: input.workDir,
|
|
502
498
|
command,
|
|
503
|
-
readyPath,
|
|
504
|
-
startAckPath,
|
|
505
499
|
outputPath,
|
|
506
500
|
exitCodePath,
|
|
507
501
|
progressEventPath,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { DaytonaSandbox } from './daytona-lifecycle';
|
|
2
2
|
|
|
3
3
|
export type DetachedDaytonaStartupDiagnostic = {
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
schedulerReadinessObserved: boolean;
|
|
5
|
+
schedulerReadinessResponses: number;
|
|
6
6
|
commandFound: boolean;
|
|
7
7
|
exitCode: number | null;
|
|
8
8
|
sessionFound: boolean;
|
|
@@ -53,8 +53,8 @@ export async function inspectDetachedDaytonaStartup(input: {
|
|
|
53
53
|
}): Promise<DetachedDaytonaStartupDiagnostic> {
|
|
54
54
|
const errors: string[] = [];
|
|
55
55
|
const diagnostic: DetachedDaytonaStartupDiagnostic = {
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
schedulerReadinessObserved: false,
|
|
57
|
+
schedulerReadinessResponses: 0,
|
|
58
58
|
commandFound: false,
|
|
59
59
|
exitCode: null,
|
|
60
60
|
sessionFound: false,
|
|
@@ -116,11 +116,113 @@ export type DetachedDaytonaCommandStart = {
|
|
|
116
116
|
cmdId: string;
|
|
117
117
|
};
|
|
118
118
|
|
|
119
|
-
// The
|
|
120
|
-
//
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
119
|
+
// The durable heartbeat proves runner-process liveness plus gateway reachability.
|
|
120
|
+
// Keep the handoff bounded while allowing a cold gateway to recover.
|
|
121
|
+
export const DAYTONA_RUNNER_READY_TIMEOUT_MS = 30_000;
|
|
122
|
+
const DAYTONA_RUNNER_READY_POLL_MS = 50;
|
|
123
|
+
|
|
124
|
+
export type DetachedRunnerReadinessPort = (
|
|
125
|
+
baselineHeartbeatAt?: string | null,
|
|
126
|
+
) => Promise<RunnerReadinessResponse | null>;
|
|
127
|
+
|
|
128
|
+
type RunnerReadinessResponse = {
|
|
129
|
+
ready: boolean;
|
|
130
|
+
heartbeatAt: string | null;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
async function requestRunnerReadiness(input: {
|
|
134
|
+
readiness: DetachedRunnerReadinessPort;
|
|
135
|
+
baselineHeartbeatAt?: string | null;
|
|
136
|
+
timeoutMs: number;
|
|
137
|
+
}): Promise<{
|
|
138
|
+
response: RunnerReadinessResponse | null;
|
|
139
|
+
diagnosis: string;
|
|
140
|
+
timedOut: boolean;
|
|
141
|
+
}> {
|
|
142
|
+
const timedOut = Symbol('readiness_timeout');
|
|
143
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
144
|
+
try {
|
|
145
|
+
// Bound the only in-flight scheduler-store read by the remaining startup
|
|
146
|
+
// budget. If it times out, callers stop polling instead of layering a new
|
|
147
|
+
// query over the abandoned one and amplifying a degraded connection pool.
|
|
148
|
+
const response = await Promise.race([
|
|
149
|
+
input.readiness(input.baselineHeartbeatAt),
|
|
150
|
+
new Promise<typeof timedOut>((resolve) => {
|
|
151
|
+
timeout = setTimeout(
|
|
152
|
+
() => resolve(timedOut),
|
|
153
|
+
Math.max(1, input.timeoutMs),
|
|
154
|
+
);
|
|
155
|
+
}),
|
|
156
|
+
]);
|
|
157
|
+
if (response === timedOut) {
|
|
158
|
+
return {
|
|
159
|
+
response: null,
|
|
160
|
+
diagnosis: 'readiness_timeout',
|
|
161
|
+
timedOut: true,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
if (!response)
|
|
165
|
+
return {
|
|
166
|
+
response: null,
|
|
167
|
+
diagnosis: 'readiness_unavailable',
|
|
168
|
+
timedOut: false,
|
|
169
|
+
};
|
|
170
|
+
return {
|
|
171
|
+
response,
|
|
172
|
+
diagnosis: 'ok',
|
|
173
|
+
timedOut: false,
|
|
174
|
+
};
|
|
175
|
+
} catch {
|
|
176
|
+
return {
|
|
177
|
+
response: null,
|
|
178
|
+
diagnosis: 'readiness_unavailable',
|
|
179
|
+
timedOut: false,
|
|
180
|
+
};
|
|
181
|
+
} finally {
|
|
182
|
+
if (timeout) clearTimeout(timeout);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Capture the attempt heartbeat using the scheduler's Postgres clock before
|
|
188
|
+
* launching customer code. The post-launch readiness poll waits for this exact
|
|
189
|
+
* database value to advance, avoiding both a gateway protocol dependency and
|
|
190
|
+
* any comparison with the worker host clock.
|
|
191
|
+
*/
|
|
192
|
+
export async function captureDetachedDaytonaRunnerReadinessBaseline(input: {
|
|
193
|
+
readiness: DetachedRunnerReadinessPort;
|
|
194
|
+
timeoutMs?: number;
|
|
195
|
+
pollMs?: number;
|
|
196
|
+
now?: () => number;
|
|
197
|
+
sleep?: (ms: number) => Promise<void>;
|
|
198
|
+
}): Promise<string | null> {
|
|
199
|
+
const now = input.now ?? Date.now;
|
|
200
|
+
const sleep =
|
|
201
|
+
input.sleep ??
|
|
202
|
+
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
203
|
+
const timeoutMs = Math.max(
|
|
204
|
+
1,
|
|
205
|
+
input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
|
|
206
|
+
);
|
|
207
|
+
const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
|
|
208
|
+
const deadline = now() + timeoutMs;
|
|
209
|
+
let diagnosis = 'no_readiness_response';
|
|
210
|
+
let retryDelayMs = pollMs;
|
|
211
|
+
while (now() < deadline) {
|
|
212
|
+
const read = await requestRunnerReadiness({
|
|
213
|
+
readiness: input.readiness,
|
|
214
|
+
timeoutMs: Math.max(1, deadline - now()),
|
|
215
|
+
});
|
|
216
|
+
diagnosis = read.diagnosis;
|
|
217
|
+
if (read.response) return read.response.heartbeatAt;
|
|
218
|
+
if (read.timedOut) break;
|
|
219
|
+
await sleep(Math.min(retryDelayMs, Math.max(1, deadline - now())));
|
|
220
|
+
retryDelayMs = Math.min(1_000, retryDelayMs * 2);
|
|
221
|
+
}
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Daytona readiness baseline could not reach the scheduler store before command launch within ${timeoutMs}ms (${diagnosis}). Customer play code was not started.`,
|
|
224
|
+
);
|
|
225
|
+
}
|
|
124
226
|
|
|
125
227
|
export class DaytonaRunnerInitializationError extends Error {
|
|
126
228
|
constructor(
|
|
@@ -141,7 +243,22 @@ export async function startDetachedDaytonaCommand(input: {
|
|
|
141
243
|
sessionId: string;
|
|
142
244
|
command: string;
|
|
143
245
|
}): Promise<DetachedDaytonaCommandStart> {
|
|
246
|
+
await createDetachedDaytonaSession(input);
|
|
247
|
+
return executeDetachedDaytonaSessionCommand(input);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export async function createDetachedDaytonaSession(input: {
|
|
251
|
+
sandbox: DaytonaSandbox;
|
|
252
|
+
sessionId: string;
|
|
253
|
+
}): Promise<void> {
|
|
144
254
|
await input.sandbox.process.createSession(input.sessionId);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
export async function executeDetachedDaytonaSessionCommand(input: {
|
|
258
|
+
sandbox: DaytonaSandbox;
|
|
259
|
+
sessionId: string;
|
|
260
|
+
command: string;
|
|
261
|
+
}): Promise<DetachedDaytonaCommandStart> {
|
|
145
262
|
const response = await input.sandbox.process.executeSessionCommand(
|
|
146
263
|
input.sessionId,
|
|
147
264
|
{
|
|
@@ -169,9 +286,9 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
|
|
|
169
286
|
sandbox: DaytonaSandbox;
|
|
170
287
|
sessionId: string;
|
|
171
288
|
cmdId: string;
|
|
172
|
-
readyPath: string;
|
|
173
|
-
startAckPath: string;
|
|
174
289
|
exitCodePath: string;
|
|
290
|
+
readiness: DetachedRunnerReadinessPort;
|
|
291
|
+
baselineHeartbeatAt: string | null;
|
|
175
292
|
timeoutMs?: number;
|
|
176
293
|
pollMs?: number;
|
|
177
294
|
now?: () => number;
|
|
@@ -187,35 +304,25 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
|
|
|
187
304
|
);
|
|
188
305
|
const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
|
|
189
306
|
const deadline = now() + timeoutMs;
|
|
190
|
-
let
|
|
191
|
-
let
|
|
192
|
-
let
|
|
307
|
+
let schedulerDiagnosis = 'no_readiness_response';
|
|
308
|
+
let schedulerReadinessResponses = 0;
|
|
309
|
+
let retryDelayMs = pollMs;
|
|
193
310
|
|
|
194
311
|
while (now() < deadline) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
Buffer.from('{"start":true}', 'utf-8'),
|
|
206
|
-
input.startAckPath,
|
|
207
|
-
);
|
|
208
|
-
return;
|
|
209
|
-
} catch {
|
|
210
|
-
markerDiagnosis = 'start_ack_unavailable';
|
|
211
|
-
}
|
|
212
|
-
} else {
|
|
213
|
-
markerDiagnosis = 'ready_marker_invalid';
|
|
214
|
-
}
|
|
215
|
-
} catch {
|
|
216
|
-
markerDiagnosis = 'ready_marker_unavailable';
|
|
312
|
+
const read = await requestRunnerReadiness({
|
|
313
|
+
readiness: input.readiness,
|
|
314
|
+
baselineHeartbeatAt: input.baselineHeartbeatAt,
|
|
315
|
+
timeoutMs: Math.max(1, deadline - now()),
|
|
316
|
+
});
|
|
317
|
+
schedulerDiagnosis = read.diagnosis;
|
|
318
|
+
if (read.response) {
|
|
319
|
+
schedulerReadinessResponses += 1;
|
|
320
|
+
if (read.response.ready) return;
|
|
321
|
+
schedulerDiagnosis = 'runner_heartbeat_not_observed';
|
|
217
322
|
}
|
|
218
|
-
|
|
323
|
+
if (read.timedOut) break;
|
|
324
|
+
await sleep(Math.min(retryDelayMs, Math.max(1, deadline - now())));
|
|
325
|
+
retryDelayMs = Math.min(1_000, retryDelayMs * 2);
|
|
219
326
|
}
|
|
220
327
|
|
|
221
328
|
const diagnostic = await inspectDetachedDaytonaStartup({
|
|
@@ -224,12 +331,12 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
|
|
|
224
331
|
cmdId: input.cmdId,
|
|
225
332
|
exitCodePath: input.exitCodePath,
|
|
226
333
|
});
|
|
227
|
-
diagnostic.
|
|
228
|
-
diagnostic.
|
|
334
|
+
diagnostic.schedulerReadinessObserved = false;
|
|
335
|
+
diagnostic.schedulerReadinessResponses = schedulerReadinessResponses;
|
|
229
336
|
const oomIndicated =
|
|
230
337
|
diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
|
|
231
338
|
throw new DaytonaRunnerInitializationError(
|
|
232
|
-
`RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the
|
|
339
|
+
`RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the scheduler did not observe play-runner liveness within ${timeoutMs}ms. Scheduler diagnosis: ${schedulerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The worker will not park this runner; the failed runner was stopped and was not retried in place.`,
|
|
233
340
|
diagnostic,
|
|
234
341
|
);
|
|
235
342
|
}
|
|
@@ -34,9 +34,11 @@ import {
|
|
|
34
34
|
} from './daytona-lifecycle';
|
|
35
35
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
36
36
|
import {
|
|
37
|
+
captureDetachedDaytonaRunnerReadinessBaseline,
|
|
37
38
|
confirmDetachedDaytonaRunnerReady,
|
|
39
|
+
createDetachedDaytonaSession,
|
|
38
40
|
DaytonaRunnerInitializationError,
|
|
39
|
-
|
|
41
|
+
executeDetachedDaytonaSessionCommand,
|
|
40
42
|
} from './daytona-session-execution';
|
|
41
43
|
|
|
42
44
|
const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
@@ -45,7 +47,6 @@ const DAYTONA_DETACHED_CEILING_SECONDS =
|
|
|
45
47
|
const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
|
|
46
48
|
const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
|
|
47
49
|
const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
|
|
48
|
-
const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
|
|
49
50
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
50
51
|
const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
|
|
51
52
|
const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
|
|
@@ -56,15 +57,8 @@ const RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN =
|
|
|
56
57
|
/AppRuntimeApiTransportError: App runtime API transport exhausted action=(?:get|claim|mark|complete|fail|heartbeat|release|skip)_runtime_step_receipts?/i;
|
|
57
58
|
const RUNTIME_API_TRANSPORT_RETRY_PATTERN =
|
|
58
59
|
/\bRuntime API request to .+ failed before receiving a response:\s*(?:fetch failed|connection (?:terminated|timed out|closed|reset)|ECONNRESET|ETIMEDOUT|ECONNREFUSED)\b/i;
|
|
59
|
-
// The ready fence precedes customer play execution. A failed detached start is
|
|
60
|
-
// safe to retry only when the runner never wrote its ready marker or attempted
|
|
61
|
-
// the start acknowledgement, and Daytona has no exit evidence. This covers a
|
|
62
|
-
// transient Daytona control-plane blind spot where the accepted command cannot
|
|
63
|
-
// be inspected, while keeping post-start and crash failures terminal.
|
|
64
|
-
const RUNTIME_SANDBOX_UNSTARTED_RETRY_PATTERN =
|
|
65
|
-
/DaytonaRunnerInitializationError:\s*RUNTIME_SANDBOX_START_FAILED[\s\S]*Marker diagnosis:\s*ready_marker_unavailable[\s\S]*"readyMarkerObserved"\s*:\s*false[\s\S]*"startAcknowledgementAttempted"\s*:\s*false[\s\S]*"exitCode"\s*:\s*null[\s\S]*"exitCodeFile"\s*:\s*null/i;
|
|
66
60
|
const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
|
|
67
|
-
/\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete)\b/i;
|
|
61
|
+
/\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
|
|
68
62
|
|
|
69
63
|
type DaytonaExecution = { exitCode: number | null; result: string };
|
|
70
64
|
type DaytonaLookupFailure = {
|
|
@@ -468,7 +462,6 @@ export function retryableRuntimeInitializationFailureReason(
|
|
|
468
462
|
| 'runtime_postgres_connect'
|
|
469
463
|
| 'runtime_api_transport'
|
|
470
464
|
| 'runtime_receipt_gateway_transport'
|
|
471
|
-
| 'runtime_sandbox_unstarted'
|
|
472
465
|
| null {
|
|
473
466
|
if (result.status !== 'failed') return null;
|
|
474
467
|
if (RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(result.error)) {
|
|
@@ -478,13 +471,6 @@ export function retryableRuntimeInitializationFailureReason(
|
|
|
478
471
|
return 'runtime_receipt_gateway_transport';
|
|
479
472
|
}
|
|
480
473
|
const rowsProcessed = Number(result.stats?.rowsProcessed ?? 0);
|
|
481
|
-
if (
|
|
482
|
-
rowsProcessed === 0 &&
|
|
483
|
-
result.steps.length === 0 &&
|
|
484
|
-
RUNTIME_SANDBOX_UNSTARTED_RETRY_PATTERN.test(result.error)
|
|
485
|
-
) {
|
|
486
|
-
return 'runtime_sandbox_unstarted';
|
|
487
|
-
}
|
|
488
474
|
if (
|
|
489
475
|
rowsProcessed === 0 &&
|
|
490
476
|
result.steps.length === 0 &&
|
|
@@ -504,27 +490,6 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
|
|
|
504
490
|
return DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(message);
|
|
505
491
|
}
|
|
506
492
|
|
|
507
|
-
function isRetryableUnstartedDaytonaRunner(
|
|
508
|
-
error: unknown,
|
|
509
|
-
): error is DaytonaRunnerInitializationError {
|
|
510
|
-
if (!(error instanceof DaytonaRunnerInitializationError)) return false;
|
|
511
|
-
const diagnostic = error.startupDiagnostic;
|
|
512
|
-
// The ready fence runs before any customer play code. If Daytona accepted
|
|
513
|
-
// the command but the worker never observed its ready marker, the runner
|
|
514
|
-
// cannot pass the reverse start-ack fence and customer code cannot begin.
|
|
515
|
-
// Terminate that sandbox and retry once on a fresh one even when its Node
|
|
516
|
-
// process is still alive. Do not retry after a start acknowledgement was
|
|
517
|
-
// attempted (its write outcome may be ambiguous), or after a runner exit.
|
|
518
|
-
return (
|
|
519
|
-
diagnostic.commandFound &&
|
|
520
|
-
diagnostic.sessionFound &&
|
|
521
|
-
diagnostic.exitCode === null &&
|
|
522
|
-
diagnostic.exitCodeFile === null &&
|
|
523
|
-
!diagnostic.readyMarkerObserved &&
|
|
524
|
-
!diagnostic.startAcknowledgementAttempted
|
|
525
|
-
);
|
|
526
|
-
}
|
|
527
|
-
|
|
528
493
|
function prepareDaytonaExecution(
|
|
529
494
|
input: PlayRunnerPrepareInput,
|
|
530
495
|
callbacks: Parameters<PlayRunnerBackend['execute']>[1],
|
|
@@ -588,6 +553,22 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
588
553
|
tableNamespace: null,
|
|
589
554
|
};
|
|
590
555
|
}
|
|
556
|
+
const readRunnerReadiness = callbacks?.readDetachedRunnerReadiness;
|
|
557
|
+
if (!readRunnerReadiness) {
|
|
558
|
+
if (isDaytonaPreparedExecution(prepared)) {
|
|
559
|
+
await prepared.dispose?.();
|
|
560
|
+
}
|
|
561
|
+
return {
|
|
562
|
+
status: 'failed',
|
|
563
|
+
error:
|
|
564
|
+
'Daytona runner backend requires the scheduler-owned detached-runner readiness port.',
|
|
565
|
+
logs: [],
|
|
566
|
+
stats: {},
|
|
567
|
+
steps: [],
|
|
568
|
+
checkpoint: config.checkpoint ?? null,
|
|
569
|
+
tableNamespace: null,
|
|
570
|
+
};
|
|
571
|
+
}
|
|
591
572
|
let preparedExecution: DaytonaPreparedExecution;
|
|
592
573
|
try {
|
|
593
574
|
preparedExecution = isDaytonaPreparedExecution(prepared)
|
|
@@ -720,11 +701,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
720
701
|
try {
|
|
721
702
|
for (
|
|
722
703
|
let executionAttempt = 1;
|
|
723
|
-
executionAttempt <=
|
|
724
|
-
Math.max(
|
|
725
|
-
DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS,
|
|
726
|
-
DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS,
|
|
727
|
-
);
|
|
704
|
+
executionAttempt <= DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS;
|
|
728
705
|
executionAttempt += 1
|
|
729
706
|
) {
|
|
730
707
|
let sandboxForAttempt: DaytonaSandbox | null = null;
|
|
@@ -757,6 +734,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
757
734
|
workDir,
|
|
758
735
|
});
|
|
759
736
|
|
|
737
|
+
// Session allocation is independent of payload staging. Start it
|
|
738
|
+
// now so Daytona control-plane latency overlaps compression/upload.
|
|
739
|
+
const sessionId = `deepline-play-${randomUUID()}`;
|
|
740
|
+
const sessionCreation = createDetachedDaytonaSession({
|
|
741
|
+
sandbox,
|
|
742
|
+
sessionId,
|
|
743
|
+
}).then(
|
|
744
|
+
() => ({ ok: true as const }),
|
|
745
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
746
|
+
);
|
|
747
|
+
const readinessBaseline =
|
|
748
|
+
captureDetachedDaytonaRunnerReadinessBaseline({
|
|
749
|
+
readiness: readRunnerReadiness,
|
|
750
|
+
}).then(
|
|
751
|
+
(heartbeatAt) => ({ ok: true as const, heartbeatAt }),
|
|
752
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
753
|
+
);
|
|
754
|
+
|
|
760
755
|
const uploadStartedAt = Date.now();
|
|
761
756
|
let uploadDeadlineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
762
757
|
const uploadDeadlinePromise = new Promise<never>(
|
|
@@ -840,16 +835,26 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
840
835
|
});
|
|
841
836
|
// Push execution (B2-final, park-and-wake): start the runner command
|
|
842
837
|
// DETACHED via a Daytona session and return a `detached_runner`
|
|
843
|
-
// suspension only after the
|
|
844
|
-
//
|
|
845
|
-
// heartbeat: a Daytona cmdId alone does not prove durable liveness.
|
|
838
|
+
// suspension only after the scheduler observes the runner's durable
|
|
839
|
+
// gateway heartbeat. A Daytona cmdId alone does not prove liveness.
|
|
846
840
|
// After that bounded startup check, the worker parks (releasing its
|
|
847
841
|
// claim and slot) and wakes on terminal push or the ceiling timeout.
|
|
848
|
-
const
|
|
849
|
-
|
|
842
|
+
const session = await Promise.race([
|
|
843
|
+
sessionCreation,
|
|
844
|
+
cancellationPromise,
|
|
845
|
+
]);
|
|
846
|
+
if (session.ok === false) throw session.error;
|
|
847
|
+
const baseline = await Promise.race([
|
|
848
|
+
readinessBaseline,
|
|
849
|
+
cancellationPromise,
|
|
850
|
+
]);
|
|
851
|
+
if (baseline.ok === false) throw baseline.error;
|
|
852
|
+
let start: Awaited<
|
|
853
|
+
ReturnType<typeof executeDetachedDaytonaSessionCommand>
|
|
854
|
+
>;
|
|
850
855
|
try {
|
|
851
856
|
start = await Promise.race([
|
|
852
|
-
|
|
857
|
+
executeDetachedDaytonaSessionCommand({
|
|
853
858
|
sandbox,
|
|
854
859
|
sessionId,
|
|
855
860
|
command: stagedPayload.command,
|
|
@@ -873,9 +878,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
873
878
|
sandbox,
|
|
874
879
|
sessionId: start.sessionId,
|
|
875
880
|
cmdId: start.cmdId,
|
|
876
|
-
readyPath: stagedPayload.readyPath,
|
|
877
|
-
startAckPath: stagedPayload.startAckPath,
|
|
878
881
|
exitCodePath: stagedPayload.exitCodePath,
|
|
882
|
+
readiness: readRunnerReadiness,
|
|
883
|
+
baselineHeartbeatAt: baseline.heartbeatAt,
|
|
879
884
|
}),
|
|
880
885
|
cancellationPromise,
|
|
881
886
|
]);
|
|
@@ -925,25 +930,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
925
930
|
onCancel();
|
|
926
931
|
throw new Error(DAYTONA_CANCELLED_ERROR);
|
|
927
932
|
}
|
|
928
|
-
if (
|
|
929
|
-
executionAttempt < DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS &&
|
|
930
|
-
isRetryableUnstartedDaytonaRunner(error)
|
|
931
|
-
) {
|
|
932
|
-
emitDaytonaStage(callbacks, config.context, 'execute:retry', {
|
|
933
|
-
sandboxId: sandboxForAttempt?.id ?? null,
|
|
934
|
-
attempt: executionAttempt + 1,
|
|
935
|
-
reason: 'runner_startup_not_live',
|
|
936
|
-
error: error.message,
|
|
937
|
-
elapsedMs: Date.now() - startedAt,
|
|
938
|
-
});
|
|
939
|
-
if (sandboxForAttempt) {
|
|
940
|
-
activeAcquiredResource!.billingEndedAt = Date.now();
|
|
941
|
-
await reportRetiringRuntimeResource(activeAcquiredResource!);
|
|
942
|
-
sandboxCleanup.stashActiveSandboxForRetry();
|
|
943
|
-
}
|
|
944
|
-
acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
|
|
945
|
-
continue;
|
|
946
|
-
}
|
|
947
933
|
if (
|
|
948
934
|
executionAttempt < DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS &&
|
|
949
935
|
isRetryableDaytonaInfrastructureFailure(error)
|
|
@@ -46,6 +46,14 @@ export interface PlayRunnerCallbacks {
|
|
|
46
46
|
onRuntimeResourceAcquired?: (
|
|
47
47
|
resource: PlayRunnerRuntimeResource,
|
|
48
48
|
) => void | Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Scheduler-owned readiness read for a detached runner attempt. Daytona uses
|
|
51
|
+
* this direct control-plane port before parking; it is never serialized into
|
|
52
|
+
* the sandbox or routed through the runtime gateway.
|
|
53
|
+
*/
|
|
54
|
+
readDetachedRunnerReadiness?: (
|
|
55
|
+
baselineHeartbeatAt?: string | null,
|
|
56
|
+
) => Promise<{ ready: boolean; heartbeatAt: string | null } | null>;
|
|
49
57
|
cancellationSignal?: AbortSignal;
|
|
50
58
|
}
|
|
51
59
|
|