deepline 0.1.208 → 0.1.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +2 -12
- package/dist/bundling-sources/sdk/src/client.ts +10 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +16 -31
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +312 -207
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +0 -5
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +429 -102
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +27 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +31 -12
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +105 -10
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +32 -86
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +141 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +180 -447
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +112 -0
- package/dist/cli/index.js +32 -4
- package/dist/cli/index.mjs +32 -4
- package/dist/index.d.mts +10 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +26 -4
- package/dist/index.mjs +26 -4
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +0 -70
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Daytona } from '@daytonaio/sdk';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import type {
|
|
3
4
|
PlayRunnerBackend,
|
|
4
5
|
PlayRunnerPreparedExecution,
|
|
@@ -6,13 +7,8 @@ import type {
|
|
|
6
7
|
} from '../types';
|
|
7
8
|
import { RuntimeResourceFenceLostError } from '../types';
|
|
8
9
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
9
|
-
import {
|
|
10
|
-
findPlayRunnerResult,
|
|
11
|
-
parsePlayRunnerEventLine,
|
|
12
|
-
parsePlayRunnerEvents,
|
|
13
|
-
} from '../runner-events';
|
|
10
|
+
import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
|
|
14
11
|
import type {
|
|
15
|
-
PlayRunnerEvent,
|
|
16
12
|
PlayRunnerExecutionConfig,
|
|
17
13
|
PlayRunnerRuntimeTiming,
|
|
18
14
|
PlayRunnerResult,
|
|
@@ -36,82 +32,17 @@ import {
|
|
|
36
32
|
validateDaytonaExecutionContext,
|
|
37
33
|
} from './daytona-lifecycle';
|
|
38
34
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
35
|
+
import { startDetachedDaytonaCommand } from './daytona-session-execution';
|
|
39
36
|
|
|
40
37
|
const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
41
38
|
const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
|
|
42
|
-
const DAYTONA_OUTPUT_TAIL_CHARS = 2_000;
|
|
43
39
|
const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
|
|
44
40
|
const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
|
|
45
|
-
const DAYTONA_PROGRESS_SIDECAR_POLL_MS = 1_000;
|
|
46
41
|
const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
|
|
47
42
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
48
43
|
const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
|
|
49
44
|
const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
|
|
50
|
-
/**
|
|
51
|
-
* Client-side wall-clock grace for `sandbox.process.executeCommand`. The
|
|
52
|
-
* `timeout` argument we pass is a SERVER-side directive; the Daytona SDK's
|
|
53
|
-
* underlying HTTP call has no client deadline, so a wedged socket (observed:
|
|
54
|
-
* `AggregateError [ETIMEDOUT]`) leaves the await hanging forever and holds the
|
|
55
|
-
* worker slot. We race the call against this deadline = server timeout + grace,
|
|
56
|
-
* so a genuine server-side runtime-limit hit still returns first, but a wedged
|
|
57
|
-
* client transport fails the attempt loudly and lets the engine retry ladder
|
|
58
|
-
* take over instead of livelocking the slot.
|
|
59
|
-
*/
|
|
60
|
-
const DAYTONA_EXECUTE_CLIENT_GRACE_MS = 60_000;
|
|
61
|
-
const DAYTONA_EXECUTE_CLIENT_DEADLINE_MS =
|
|
62
|
-
DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000 + DAYTONA_EXECUTE_CLIENT_GRACE_MS;
|
|
63
|
-
/**
|
|
64
|
-
* Bound for individual sandbox/runtime API calls that have no natural timeout
|
|
65
|
-
* (e.g. recording the acquired runtime resource). A wedged one of these must
|
|
66
|
-
* fail the attempt, not hold the slot.
|
|
67
|
-
*/
|
|
68
|
-
const DAYTONA_API_CALL_DEADLINE_MS = 60_000;
|
|
69
|
-
|
|
70
|
-
export class DaytonaOperationDeadlineError extends Error {
|
|
71
|
-
readonly operation: string;
|
|
72
|
-
readonly deadlineMs: number;
|
|
73
|
-
|
|
74
|
-
constructor(input: { operation: string; deadlineMs: number }) {
|
|
75
|
-
super(
|
|
76
|
-
`Daytona operation "${input.operation}" exceeded client deadline of ${input.deadlineMs}ms and was abandoned to free the worker slot.`,
|
|
77
|
-
);
|
|
78
|
-
this.name = 'DaytonaOperationDeadlineError';
|
|
79
|
-
this.operation = input.operation;
|
|
80
|
-
this.deadlineMs = input.deadlineMs;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
45
|
|
|
84
|
-
/**
|
|
85
|
-
* Race `promise` against a wall-clock deadline. On breach, reject loudly with a
|
|
86
|
-
* `DaytonaOperationDeadlineError` (the caller's retry/failure classification
|
|
87
|
-
* decides what happens next). The timer is always cleared so a fast-resolving
|
|
88
|
-
* promise never leaves a dangling handle.
|
|
89
|
-
*/
|
|
90
|
-
export async function withDaytonaOperationDeadline<T>(input: {
|
|
91
|
-
operation: string;
|
|
92
|
-
deadlineMs: number;
|
|
93
|
-
promise: Promise<T>;
|
|
94
|
-
}): Promise<T> {
|
|
95
|
-
let deadlineTimer: ReturnType<typeof setTimeout> | null = null;
|
|
96
|
-
const deadlinePromise = new Promise<never>((_resolve, reject) => {
|
|
97
|
-
deadlineTimer = setTimeout(() => {
|
|
98
|
-
reject(
|
|
99
|
-
new DaytonaOperationDeadlineError({
|
|
100
|
-
operation: input.operation,
|
|
101
|
-
deadlineMs: input.deadlineMs,
|
|
102
|
-
}),
|
|
103
|
-
);
|
|
104
|
-
}, input.deadlineMs);
|
|
105
|
-
deadlineTimer.unref?.();
|
|
106
|
-
});
|
|
107
|
-
try {
|
|
108
|
-
return await Promise.race([input.promise, deadlinePromise]);
|
|
109
|
-
} finally {
|
|
110
|
-
if (deadlineTimer) {
|
|
111
|
-
clearTimeout(deadlineTimer);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
46
|
const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
|
|
116
47
|
/\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
|
|
117
48
|
const RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN =
|
|
@@ -153,33 +84,110 @@ class DaytonaUploadDeadlineBreachError extends Error {
|
|
|
153
84
|
}
|
|
154
85
|
}
|
|
155
86
|
|
|
156
|
-
class DaytonaCommandTransportError extends Error {
|
|
157
|
-
readonly sandboxId: string;
|
|
158
|
-
readonly commandError: unknown;
|
|
159
|
-
readonly recoveryError: unknown;
|
|
160
|
-
|
|
161
|
-
constructor(input: {
|
|
162
|
-
sandboxId: string;
|
|
163
|
-
commandError: unknown;
|
|
164
|
-
recoveryError: unknown;
|
|
165
|
-
}) {
|
|
166
|
-
super(
|
|
167
|
-
`Daytona command transport failed for sandbox ${input.sandboxId}: ${formatDaytonaError(input.commandError)}. Recovery could not read the command result: ${formatDaytonaError(input.recoveryError)}.`,
|
|
168
|
-
{ cause: input.commandError },
|
|
169
|
-
);
|
|
170
|
-
this.name = 'DaytonaCommandTransportError';
|
|
171
|
-
this.sandboxId = input.sandboxId;
|
|
172
|
-
this.commandError = input.commandError;
|
|
173
|
-
this.recoveryError = input.recoveryError;
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
|
|
177
87
|
export const daytonaSdkClientFactory = {
|
|
178
88
|
create(input: ConstructorParameters<typeof Daytona>[0]): DaytonaClient {
|
|
179
89
|
return new Daytona(input);
|
|
180
90
|
},
|
|
91
|
+
/** Full client (get/delete by id) for best-effort GC of orphaned sandboxes. */
|
|
92
|
+
createFull(input: ConstructorParameters<typeof Daytona>[0]): Daytona {
|
|
93
|
+
return new Daytona(input);
|
|
94
|
+
},
|
|
181
95
|
};
|
|
182
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Best-effort deletion of an orphaned Daytona sandbox by id (push-execution B4).
|
|
99
|
+
*
|
|
100
|
+
* When the scheduler's expired-claim sweeper reclaims a run whose worker was
|
|
101
|
+
* babysitting a sandbox, nothing tears that sandbox down — `ephemeral +
|
|
102
|
+
* autoStop` is the only backstop, which leaks paid compute until it fires. The
|
|
103
|
+
* worker records each acquired sandbox id under the fenced lease, so on the
|
|
104
|
+
* reclaim / terminal-failure path it can look the sandbox up by id and delete
|
|
105
|
+
* it. Failures are swallowed: the run already ended, and the autoStop backstop
|
|
106
|
+
* still applies.
|
|
107
|
+
*/
|
|
108
|
+
/**
|
|
109
|
+
* Timeout-wake verification for a detached push-execution runner (B2-final).
|
|
110
|
+
*
|
|
111
|
+
* When the parked attempt's ceiling timeout fires without a pushed terminal,
|
|
112
|
+
* the wake leg calls this to distinguish "runner finished but its terminal
|
|
113
|
+
* push was lost" from "runner died silently": read the session command's exit
|
|
114
|
+
* code, then salvage the structured `result` event from the captured output
|
|
115
|
+
* file. Everything is best-effort — a deleted/expired sandbox yields
|
|
116
|
+
* `{ exitCode: null, result: null }` and the attempt fails through the normal
|
|
117
|
+
* engine retry/fencing path.
|
|
118
|
+
*/
|
|
119
|
+
export async function inspectDetachedDaytonaRunner(input: {
|
|
120
|
+
sandboxId: string;
|
|
121
|
+
sessionId: string;
|
|
122
|
+
cmdId: string;
|
|
123
|
+
outputPath: string;
|
|
124
|
+
exitCodePath: string;
|
|
125
|
+
}): Promise<{ exitCode: number | null; result: PlayRunnerResult | null }> {
|
|
126
|
+
try {
|
|
127
|
+
const { clientOptions } = loadDaytonaRequiredConfig();
|
|
128
|
+
const daytona = daytonaSdkClientFactory.createFull(clientOptions);
|
|
129
|
+
const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
|
|
130
|
+
let exitCode: number | null = null;
|
|
131
|
+
try {
|
|
132
|
+
const command = await sandbox.process.getSessionCommand(
|
|
133
|
+
input.sessionId,
|
|
134
|
+
input.cmdId,
|
|
135
|
+
);
|
|
136
|
+
exitCode = typeof command?.exitCode === 'number' ? command.exitCode : null;
|
|
137
|
+
} catch (error) {
|
|
138
|
+
console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
|
|
139
|
+
sandboxId: input.sandboxId,
|
|
140
|
+
cmdId: input.cmdId,
|
|
141
|
+
error: error instanceof Error ? error.message : String(error),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
const recovered = await recoverDaytonaCommandExecution({
|
|
145
|
+
sandbox,
|
|
146
|
+
outputPath: input.outputPath,
|
|
147
|
+
exitCodePath: input.exitCodePath,
|
|
148
|
+
timeoutMs: 10_000,
|
|
149
|
+
});
|
|
150
|
+
if (!recovered.execution) {
|
|
151
|
+
return { exitCode, result: null };
|
|
152
|
+
}
|
|
153
|
+
const result =
|
|
154
|
+
findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
|
|
155
|
+
null;
|
|
156
|
+
return {
|
|
157
|
+
exitCode: exitCode ?? recovered.execution.exitCode,
|
|
158
|
+
result,
|
|
159
|
+
};
|
|
160
|
+
} catch (error) {
|
|
161
|
+
console.warn('[play-runner.daytona.detached_inspect_failed]', {
|
|
162
|
+
sandboxId: input.sandboxId,
|
|
163
|
+
cmdId: input.cmdId,
|
|
164
|
+
error: error instanceof Error ? error.message : String(error),
|
|
165
|
+
});
|
|
166
|
+
return { exitCode: null, result: null };
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export async function deleteDaytonaSandboxById(input: {
|
|
171
|
+
sandboxId: string;
|
|
172
|
+
timeoutSeconds?: number;
|
|
173
|
+
}): Promise<boolean> {
|
|
174
|
+
const sandboxId = input.sandboxId?.trim();
|
|
175
|
+
if (!sandboxId) return false;
|
|
176
|
+
try {
|
|
177
|
+
const { clientOptions } = loadDaytonaRequiredConfig();
|
|
178
|
+
const daytona = daytonaSdkClientFactory.createFull(clientOptions);
|
|
179
|
+
const sandbox = await daytona.get(sandboxId);
|
|
180
|
+
await daytona.delete(sandbox, input.timeoutSeconds ?? 30);
|
|
181
|
+
return true;
|
|
182
|
+
} catch (error) {
|
|
183
|
+
console.warn('[play-runner.daytona.reclaim_sandbox_delete_failed]', {
|
|
184
|
+
sandboxId,
|
|
185
|
+
error: error instanceof Error ? error.message : String(error),
|
|
186
|
+
});
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
183
191
|
function formatDaytonaExecutionError(error: unknown): string {
|
|
184
192
|
const message = formatDaytonaError(error);
|
|
185
193
|
return isConfiguredDaytonaRuntimeLimit(error)
|
|
@@ -212,14 +220,6 @@ function isConfiguredDaytonaRuntimeLimit(error: unknown): boolean {
|
|
|
212
220
|
).test(message);
|
|
213
221
|
}
|
|
214
222
|
|
|
215
|
-
function getDaytonaOutputTail(output: unknown): string | null {
|
|
216
|
-
const text = String(output ?? '').trim();
|
|
217
|
-
if (!text) {
|
|
218
|
-
return null;
|
|
219
|
-
}
|
|
220
|
-
return text.slice(-DAYTONA_OUTPUT_TAIL_CHARS);
|
|
221
|
-
}
|
|
222
|
-
|
|
223
223
|
function sleep(ms: number): Promise<void> {
|
|
224
224
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
225
225
|
}
|
|
@@ -251,161 +251,6 @@ function emitDaytonaStage(
|
|
|
251
251
|
console.info('[play-runner.daytona.stage]', JSON.stringify(payload));
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
-
function dispatchDaytonaRunnerEvent(input: {
|
|
255
|
-
event: PlayRunnerEvent;
|
|
256
|
-
callbacks: Parameters<PlayRunnerBackend['execute']>[1];
|
|
257
|
-
context: DaytonaExecutionContext;
|
|
258
|
-
sandboxId: string;
|
|
259
|
-
startedAt: number;
|
|
260
|
-
}): void {
|
|
261
|
-
const { event, callbacks, context, sandboxId, startedAt } = input;
|
|
262
|
-
if (event.type === 'log') {
|
|
263
|
-
if (
|
|
264
|
-
event.line.startsWith('[perf]') ||
|
|
265
|
-
event.line.includes('Executing tool batch')
|
|
266
|
-
) {
|
|
267
|
-
emitDaytonaStage(callbacks, context, 'runner:log', {
|
|
268
|
-
sandboxId,
|
|
269
|
-
line: event.line,
|
|
270
|
-
elapsedMs: Date.now() - startedAt,
|
|
271
|
-
});
|
|
272
|
-
}
|
|
273
|
-
callbacks?.onLog?.(event);
|
|
274
|
-
return;
|
|
275
|
-
}
|
|
276
|
-
if (event.type === 'checkpoint') {
|
|
277
|
-
callbacks?.onCheckpoint?.(event.checkpoint);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
if (event.type === 'row_update') {
|
|
281
|
-
callbacks?.onRowUpdate?.(event.update);
|
|
282
|
-
return;
|
|
283
|
-
}
|
|
284
|
-
if (event.type === 'execution_event') {
|
|
285
|
-
callbacks?.onExecutionEvent?.(event.event);
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
function createDaytonaProgressSidecarPoller(input: {
|
|
290
|
-
sandbox: DaytonaSandbox;
|
|
291
|
-
progressEventPath: string;
|
|
292
|
-
callbacks: Parameters<PlayRunnerBackend['execute']>[1];
|
|
293
|
-
context: DaytonaExecutionContext;
|
|
294
|
-
startedAt: number;
|
|
295
|
-
processedEventKeys: Set<string>;
|
|
296
|
-
}): { start(): void; stop(flush?: boolean): Promise<void> } {
|
|
297
|
-
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
298
|
-
let stopped = false;
|
|
299
|
-
let inFlight: Promise<void> = Promise.resolve();
|
|
300
|
-
let segment = 0;
|
|
301
|
-
let offsetBytes = 0;
|
|
302
|
-
let pendingLine = '';
|
|
303
|
-
let warned = false;
|
|
304
|
-
|
|
305
|
-
const processText = (text: string, flushPartial = false) => {
|
|
306
|
-
const combined = pendingLine + text;
|
|
307
|
-
const lines = combined.split(/\r?\n/);
|
|
308
|
-
const completeLines = flushPartial ? lines : lines.slice(0, -1);
|
|
309
|
-
pendingLine = flushPartial ? '' : (lines.at(-1) ?? '');
|
|
310
|
-
for (const rawLine of completeLines) {
|
|
311
|
-
const line = rawLine.trim();
|
|
312
|
-
if (!line) {
|
|
313
|
-
continue;
|
|
314
|
-
}
|
|
315
|
-
const event = parsePlayRunnerEventLine(line);
|
|
316
|
-
if (!event) {
|
|
317
|
-
continue;
|
|
318
|
-
}
|
|
319
|
-
const eventKey = JSON.stringify(event);
|
|
320
|
-
input.processedEventKeys.add(eventKey);
|
|
321
|
-
dispatchDaytonaRunnerEvent({
|
|
322
|
-
event,
|
|
323
|
-
callbacks: input.callbacks,
|
|
324
|
-
context: input.context,
|
|
325
|
-
sandboxId: input.sandbox.id,
|
|
326
|
-
startedAt: input.startedAt,
|
|
327
|
-
});
|
|
328
|
-
}
|
|
329
|
-
};
|
|
330
|
-
|
|
331
|
-
const poll = async (flushPartial = false): Promise<void> => {
|
|
332
|
-
try {
|
|
333
|
-
const manifest = await input.sandbox.fs.downloadFile(
|
|
334
|
-
input.progressEventPath,
|
|
335
|
-
5,
|
|
336
|
-
);
|
|
337
|
-
const parsedLatestSegment = Number.parseInt(
|
|
338
|
-
Buffer.isBuffer(manifest)
|
|
339
|
-
? manifest.toString('utf-8').trim()
|
|
340
|
-
: String(manifest).trim(),
|
|
341
|
-
10,
|
|
342
|
-
);
|
|
343
|
-
const latestSegment = Number.isFinite(parsedLatestSegment)
|
|
344
|
-
? Math.max(segment, parsedLatestSegment)
|
|
345
|
-
: segment;
|
|
346
|
-
while (segment <= latestSegment) {
|
|
347
|
-
const segmentPath = `${input.progressEventPath}.${String(
|
|
348
|
-
segment,
|
|
349
|
-
).padStart(6, '0')}`;
|
|
350
|
-
const file = await input.sandbox.fs.downloadFile(segmentPath, 5);
|
|
351
|
-
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
|
|
352
|
-
if (buffer.length < offsetBytes) {
|
|
353
|
-
offsetBytes = 0;
|
|
354
|
-
pendingLine = '';
|
|
355
|
-
}
|
|
356
|
-
if (buffer.length > offsetBytes) {
|
|
357
|
-
const chunk = buffer.subarray(offsetBytes).toString('utf-8');
|
|
358
|
-
offsetBytes = buffer.length;
|
|
359
|
-
processText(chunk, flushPartial && segment === latestSegment);
|
|
360
|
-
}
|
|
361
|
-
if (segment >= latestSegment) break;
|
|
362
|
-
segment += 1;
|
|
363
|
-
offsetBytes = 0;
|
|
364
|
-
}
|
|
365
|
-
if (flushPartial && pendingLine.trim()) {
|
|
366
|
-
processText('\n', true);
|
|
367
|
-
}
|
|
368
|
-
} catch (error) {
|
|
369
|
-
if (!warned && !/not found|no such file|ENOENT/i.test(String(error))) {
|
|
370
|
-
warned = true;
|
|
371
|
-
console.warn('[play-runner.daytona.progress_sidecar_poll_failed]', {
|
|
372
|
-
sandboxId: input.sandbox.id,
|
|
373
|
-
progressEventPath: input.progressEventPath,
|
|
374
|
-
error: error instanceof Error ? error.message : String(error),
|
|
375
|
-
});
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
};
|
|
379
|
-
|
|
380
|
-
const schedule = () => {
|
|
381
|
-
if (stopped) {
|
|
382
|
-
return;
|
|
383
|
-
}
|
|
384
|
-
timer = setTimeout(() => {
|
|
385
|
-
inFlight = inFlight
|
|
386
|
-
.catch(() => {})
|
|
387
|
-
.then(() => poll())
|
|
388
|
-
.finally(schedule);
|
|
389
|
-
}, DAYTONA_PROGRESS_SIDECAR_POLL_MS);
|
|
390
|
-
timer.unref?.();
|
|
391
|
-
};
|
|
392
|
-
|
|
393
|
-
return {
|
|
394
|
-
start() {
|
|
395
|
-
inFlight = inFlight.then(() => poll()).finally(schedule);
|
|
396
|
-
},
|
|
397
|
-
async stop(flush = true) {
|
|
398
|
-
stopped = true;
|
|
399
|
-
if (timer) {
|
|
400
|
-
clearTimeout(timer);
|
|
401
|
-
timer = null;
|
|
402
|
-
}
|
|
403
|
-
await inFlight.catch(() => {});
|
|
404
|
-
if (flush) await poll(true);
|
|
405
|
-
},
|
|
406
|
-
};
|
|
407
|
-
}
|
|
408
|
-
|
|
409
254
|
async function downloadDaytonaTextFile(
|
|
410
255
|
sandbox: DaytonaSandbox,
|
|
411
256
|
remotePath: string,
|
|
@@ -451,23 +296,6 @@ async function recoverDaytonaCommandExecution(input: {
|
|
|
451
296
|
return { execution: null, lastError };
|
|
452
297
|
}
|
|
453
298
|
|
|
454
|
-
async function resolveDaytonaCommandOutput(input: {
|
|
455
|
-
sandbox: DaytonaSandbox;
|
|
456
|
-
outputPath: string;
|
|
457
|
-
exitCodePath: string;
|
|
458
|
-
executionResult: unknown;
|
|
459
|
-
}): Promise<string> {
|
|
460
|
-
const recovered = await recoverDaytonaCommandExecution({
|
|
461
|
-
sandbox: input.sandbox,
|
|
462
|
-
outputPath: input.outputPath,
|
|
463
|
-
exitCodePath: input.exitCodePath,
|
|
464
|
-
});
|
|
465
|
-
if (recovered.execution) {
|
|
466
|
-
return recovered.execution.result;
|
|
467
|
-
}
|
|
468
|
-
return String(input.executionResult ?? '');
|
|
469
|
-
}
|
|
470
|
-
|
|
471
299
|
function createDaytonaCancelledResult(
|
|
472
300
|
config: PlayRunnerExecutionConfig,
|
|
473
301
|
): PlayRunnerResult {
|
|
@@ -499,7 +327,14 @@ function createDaytonaFailedResult(input: {
|
|
|
499
327
|
};
|
|
500
328
|
}
|
|
501
329
|
|
|
502
|
-
|
|
330
|
+
/**
|
|
331
|
+
* Classify a failed runner result as a retryable runtime-initialization /
|
|
332
|
+
* transport failure. Exported for the absurd worker's detached wake leg
|
|
333
|
+
* (B2-final): the backend no longer sees the runner result (it arrives via the
|
|
334
|
+
* gateway terminal push), so the fresh-sandbox retry decision moved to the
|
|
335
|
+
* worker, which throws a retryable error into the engine retry ladder.
|
|
336
|
+
*/
|
|
337
|
+
export function retryableRuntimeInitializationFailureReason(
|
|
503
338
|
result: PlayRunnerResult,
|
|
504
339
|
):
|
|
505
340
|
| 'runtime_postgres_connect'
|
|
@@ -525,15 +360,6 @@ function retryableRuntimeInitializationFailureReason(
|
|
|
525
360
|
}
|
|
526
361
|
|
|
527
362
|
function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
|
|
528
|
-
if (error instanceof DaytonaCommandTransportError) {
|
|
529
|
-
return false;
|
|
530
|
-
}
|
|
531
|
-
// A client-side deadline breach is a wedged transport (hung socket / SDK call
|
|
532
|
-
// with no client timeout). Retry once on a fresh sandbox before spending an
|
|
533
|
-
// engine attempt, same as the other transport-level infrastructure failures.
|
|
534
|
-
if (error instanceof DaytonaOperationDeadlineError) {
|
|
535
|
-
return true;
|
|
536
|
-
}
|
|
537
363
|
const message = error instanceof Error ? error.message : String(error);
|
|
538
364
|
if (!message || /cancelled|aborted/i.test(message)) {
|
|
539
365
|
return false;
|
|
@@ -587,6 +413,23 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
587
413
|
tableNamespace: null,
|
|
588
414
|
};
|
|
589
415
|
}
|
|
416
|
+
// Push execution is THE Daytona shape (B2-final): the worker parks after
|
|
417
|
+
// the detached start and finalizes from the runner's pushed terminal.
|
|
418
|
+
// There is deliberately no worker-babysat fallback — a launch without the
|
|
419
|
+
// push config is a wiring bug, not a mode.
|
|
420
|
+
const push = config.context.runnerPushExecution;
|
|
421
|
+
if (!push) {
|
|
422
|
+
return {
|
|
423
|
+
status: 'failed',
|
|
424
|
+
error:
|
|
425
|
+
'Daytona runner backend requires push-execution config (context.runnerPushExecution). Worker-babysat Daytona execution was removed; submit through the absurd scheduler, which stamps it.',
|
|
426
|
+
logs: [],
|
|
427
|
+
stats: {},
|
|
428
|
+
steps: [],
|
|
429
|
+
checkpoint: config.checkpoint ?? null,
|
|
430
|
+
tableNamespace: null,
|
|
431
|
+
};
|
|
432
|
+
}
|
|
590
433
|
let preparedExecution: DaytonaPreparedExecution;
|
|
591
434
|
try {
|
|
592
435
|
preparedExecution = isDaytonaPreparedExecution(prepared)
|
|
@@ -831,31 +674,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
831
674
|
emitDaytonaStage(callbacks, config.context, 'execute:start', {
|
|
832
675
|
sandboxId: sandbox.id,
|
|
833
676
|
timeoutSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
|
|
834
|
-
mode: '
|
|
835
|
-
});
|
|
836
|
-
const executionStartedAt = Date.now();
|
|
837
|
-
const streamedEventKeys = new Set<string>();
|
|
838
|
-
const progressSidecar = createDaytonaProgressSidecarPoller({
|
|
839
|
-
sandbox,
|
|
840
|
-
progressEventPath: stagedPayload.progressEventPath,
|
|
841
|
-
callbacks,
|
|
842
|
-
context: config.context,
|
|
843
|
-
startedAt,
|
|
844
|
-
processedEventKeys: streamedEventKeys,
|
|
677
|
+
mode: 'session_detached',
|
|
845
678
|
});
|
|
846
|
-
|
|
679
|
+
// Push execution (B2-final, park-and-wake): start the runner command
|
|
680
|
+
// DETACHED via a Daytona session and return a `detached_runner`
|
|
681
|
+
// suspension immediately — the worker parks on it (releasing its
|
|
682
|
+
// claim, freeing the slot) and wakes on the runner's pushed terminal
|
|
683
|
+
// or the ceiling timeout. Nothing here awaits completion: no held
|
|
684
|
+
// socket, no poll loop, no per-second sandbox file reads. The sandbox
|
|
685
|
+
// MUST outlive this return (the runner is executing inside it); the
|
|
686
|
+
// wake leg owns finalize + sandbox GC.
|
|
687
|
+
const sessionId = `deepline-play-${randomUUID()}`;
|
|
688
|
+
let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
|
|
847
689
|
try {
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
promise: sandbox.process.executeCommand(
|
|
854
|
-
stagedPayload.command,
|
|
855
|
-
stagedPayload.workDir,
|
|
856
|
-
undefined,
|
|
857
|
-
DAYTONA_EXECUTE_TIMEOUT_SECONDS,
|
|
858
|
-
),
|
|
690
|
+
start = await Promise.race([
|
|
691
|
+
startDetachedDaytonaCommand({
|
|
692
|
+
sandbox,
|
|
693
|
+
sessionId,
|
|
694
|
+
command: stagedPayload.command,
|
|
859
695
|
}),
|
|
860
696
|
cancellationPromise,
|
|
861
697
|
]);
|
|
@@ -867,148 +703,48 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
867
703
|
onCancel();
|
|
868
704
|
throw new Error(DAYTONA_CANCELLED_ERROR);
|
|
869
705
|
}
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
exitCodePath: stagedPayload.exitCodePath,
|
|
874
|
-
});
|
|
875
|
-
emitDaytonaStage(
|
|
876
|
-
callbacks,
|
|
877
|
-
config.context,
|
|
878
|
-
'execute:transport_error',
|
|
879
|
-
{
|
|
880
|
-
sandboxId: sandbox.id,
|
|
881
|
-
operation: 'sandbox.process.executeCommand',
|
|
882
|
-
error: formatDaytonaError(error),
|
|
883
|
-
recovery: recovered.execution ? 'recovered' : 'unavailable',
|
|
884
|
-
recoveryError: recovered.execution
|
|
885
|
-
? null
|
|
886
|
-
: formatDaytonaError(recovered.lastError),
|
|
887
|
-
elapsedMs: Date.now() - startedAt,
|
|
888
|
-
},
|
|
889
|
-
);
|
|
890
|
-
if (!recovered.execution) {
|
|
891
|
-
throw new DaytonaCommandTransportError({
|
|
892
|
-
sandboxId: sandbox.id,
|
|
893
|
-
commandError: error,
|
|
894
|
-
recoveryError: recovered.lastError,
|
|
895
|
-
});
|
|
896
|
-
}
|
|
897
|
-
emitDaytonaStage(callbacks, config.context, 'execute:recovered', {
|
|
898
|
-
sandboxId: sandbox.id,
|
|
899
|
-
error: formatDaytonaError(error),
|
|
900
|
-
exitCode: recovered.execution.exitCode,
|
|
901
|
-
outputBytes: recovered.execution.result.length,
|
|
902
|
-
elapsedMs: Date.now() - startedAt,
|
|
903
|
-
});
|
|
904
|
-
execution = recovered.execution;
|
|
905
|
-
} finally {
|
|
906
|
-
await progressSidecar.stop(
|
|
907
|
-
!cancellationRequested &&
|
|
908
|
-
callbacks?.cancellationSignal?.aborted !== true,
|
|
909
|
-
);
|
|
706
|
+
// Start never launched customer code; let the infrastructure retry
|
|
707
|
+
// classification below decide fresh-sandbox retry vs loud failure.
|
|
708
|
+
throw error;
|
|
910
709
|
}
|
|
911
|
-
const
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
});
|
|
917
|
-
emitDaytonaStage(callbacks, config.context, 'execute:done', {
|
|
710
|
+
const runnerAttempt = Math.max(
|
|
711
|
+
0,
|
|
712
|
+
Math.floor(config.context.runAttempt ?? 0),
|
|
713
|
+
);
|
|
714
|
+
emitDaytonaStage(callbacks, config.context, 'execute:detached', {
|
|
918
715
|
sandboxId: sandbox.id,
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
716
|
+
sessionId: start.sessionId,
|
|
717
|
+
cmdId: start.cmdId,
|
|
718
|
+
runnerAttempt,
|
|
719
|
+
ceilingSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
|
|
922
720
|
elapsedMs: Date.now() - startedAt,
|
|
923
721
|
});
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
}
|
|
931
|
-
dispatchDaytonaRunnerEvent({
|
|
932
|
-
event,
|
|
933
|
-
callbacks,
|
|
934
|
-
context: config.context,
|
|
722
|
+
return {
|
|
723
|
+
status: 'suspended',
|
|
724
|
+
suspension: {
|
|
725
|
+
kind: 'detached_runner',
|
|
726
|
+
boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
|
|
727
|
+
runnerAttempt,
|
|
935
728
|
sandboxId: sandbox.id,
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
});
|
|
956
|
-
acquiredSandbox.billingEndedAt = Date.now();
|
|
957
|
-
await reportRetiringRuntimeResource(acquiredSandbox);
|
|
958
|
-
sandboxCleanup.stashActiveSandboxForRetry();
|
|
959
|
-
acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
|
|
960
|
-
continue;
|
|
961
|
-
}
|
|
962
|
-
if (result.status === 'failed') {
|
|
963
|
-
emitDaytonaStage(
|
|
964
|
-
callbacks,
|
|
965
|
-
config.context,
|
|
966
|
-
'runner:failed_result',
|
|
967
|
-
{
|
|
968
|
-
sandboxId: sandbox.id,
|
|
969
|
-
exitCode: execution.exitCode,
|
|
970
|
-
outputBytes: runnerOutput.length,
|
|
971
|
-
error: result.error,
|
|
972
|
-
outputTail: getDaytonaOutputTail(runnerOutput),
|
|
973
|
-
elapsedMs: Date.now() - startedAt,
|
|
974
|
-
},
|
|
975
|
-
);
|
|
976
|
-
}
|
|
977
|
-
return withCleanup(result);
|
|
978
|
-
}
|
|
979
|
-
|
|
980
|
-
// executeCommand only returns the transport wrapper's captured-file
|
|
981
|
-
// path. The downloaded runner output contains the actual exception.
|
|
982
|
-
const outputTail = getDaytonaOutputTail(runnerOutput);
|
|
983
|
-
if (outputTail) {
|
|
984
|
-
emitDaytonaStage(
|
|
985
|
-
callbacks,
|
|
986
|
-
config.context,
|
|
987
|
-
'runner:missing_result',
|
|
988
|
-
{
|
|
989
|
-
sandboxId: sandbox.id,
|
|
990
|
-
exitCode: execution.exitCode,
|
|
991
|
-
outputTail,
|
|
992
|
-
elapsedMs: Date.now() - startedAt,
|
|
993
|
-
},
|
|
994
|
-
);
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
return withCleanup(
|
|
998
|
-
createDaytonaFailedResult({
|
|
999
|
-
config,
|
|
1000
|
-
error:
|
|
1001
|
-
typeof execution.exitCode === 'number' &&
|
|
1002
|
-
execution.exitCode !== 0
|
|
1003
|
-
? `Daytona play runner exited with code ${execution.exitCode}${
|
|
1004
|
-
execution.exitCode === 137
|
|
1005
|
-
? ' (sandbox OOM or forced SIGKILL)'
|
|
1006
|
-
: ''
|
|
1007
|
-
}.${outputTail ? ` Output tail: ${outputTail}` : ''}`
|
|
1008
|
-
: 'Daytona play runner produced no result.',
|
|
1009
|
-
runtimeTiming,
|
|
1010
|
-
}),
|
|
1011
|
-
);
|
|
729
|
+
sessionId: start.sessionId,
|
|
730
|
+
cmdId: start.cmdId,
|
|
731
|
+
outputPath: stagedPayload.outputPath,
|
|
732
|
+
exitCodePath: stagedPayload.exitCodePath,
|
|
733
|
+
startedAtMs: Date.now(),
|
|
734
|
+
ceilingMs: DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000,
|
|
735
|
+
},
|
|
736
|
+
logs: [],
|
|
737
|
+
stats: {},
|
|
738
|
+
steps: [],
|
|
739
|
+
checkpoint: config.checkpoint ?? {
|
|
740
|
+
completedBatches: {},
|
|
741
|
+
completedToolBatches: {},
|
|
742
|
+
resolvedWaterfalls: {},
|
|
743
|
+
resolvedBoundaries: {},
|
|
744
|
+
},
|
|
745
|
+
tableNamespace: null,
|
|
746
|
+
runtimeTiming,
|
|
747
|
+
};
|
|
1012
748
|
} catch (error) {
|
|
1013
749
|
if (cancellationRequested || callbacks?.cancellationSignal?.aborted) {
|
|
1014
750
|
onCancel();
|
|
@@ -1053,10 +789,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1053
789
|
emitDaytonaStage(callbacks, config.context, 'execute:error', {
|
|
1054
790
|
sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
|
|
1055
791
|
error: formatDaytonaError(error),
|
|
1056
|
-
operation:
|
|
1057
|
-
error instanceof DaytonaCommandTransportError
|
|
1058
|
-
? 'sandbox.process.executeCommand'
|
|
1059
|
-
: 'runner_backend',
|
|
792
|
+
operation: 'runner_backend',
|
|
1060
793
|
elapsedMs: Date.now() - startedAt,
|
|
1061
794
|
});
|
|
1062
795
|
return withCleanup(
|