deepline 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +4 -4
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +40 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +9 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +2 -21
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +75 -58
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +22 -14
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +26 -0
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.mjs +2 -2
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -4113,13 +4113,13 @@ export class DeeplineClient {
|
|
|
4113
4113
|
const deployed = await this.http.request<MonitorDeployResult>(
|
|
4114
4114
|
'/api/v2/monitors/deploy',
|
|
4115
4115
|
{
|
|
4116
|
-
|
|
4117
|
-
|
|
4116
|
+
method: 'POST',
|
|
4117
|
+
body: definition,
|
|
4118
4118
|
},
|
|
4119
4119
|
);
|
|
4120
|
-
if (definition.tool !== 'deepline.
|
|
4120
|
+
if (definition.tool !== 'deepline.analytics') return deployed;
|
|
4121
4121
|
|
|
4122
|
-
//
|
|
4122
|
+
// Analytics remains an ordinary monitor deploy. Its provider-specific
|
|
4123
4123
|
// post-deploy work creates/reuses the tracker and returns the artifact a
|
|
4124
4124
|
// caller needs to install it; there is intentionally no separate setup
|
|
4125
4125
|
// command in the public monitor lifecycle.
|
|
@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
|
|
|
160
160
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
161
161
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
162
162
|
// release keeps lazy paging semantics independent of row residency.
|
|
163
|
-
version: '0.2.
|
|
163
|
+
version: '0.2.9',
|
|
164
164
|
contracts: {
|
|
165
165
|
api: {
|
|
166
166
|
name: 'sdk-http-api',
|
|
@@ -178,6 +178,40 @@ export type PlayRunFailureDetails = {
|
|
|
178
178
|
causes?: string[];
|
|
179
179
|
};
|
|
180
180
|
|
|
181
|
+
function formatRuntimeLimitDuration(timeoutSeconds: number): string {
|
|
182
|
+
if (timeoutSeconds % 3_600 === 0) {
|
|
183
|
+
const hours = timeoutSeconds / 3_600;
|
|
184
|
+
return `${hours} ${hours === 1 ? 'hour' : 'hours'}`;
|
|
185
|
+
}
|
|
186
|
+
if (timeoutSeconds % 60 === 0) {
|
|
187
|
+
const minutes = timeoutSeconds / 60;
|
|
188
|
+
return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`;
|
|
189
|
+
}
|
|
190
|
+
return `${timeoutSeconds} ${timeoutSeconds === 1 ? 'second' : 'seconds'}`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function runtimeLimitExceededCause(timeoutSeconds: number): string {
|
|
194
|
+
return `RUNTIME_LIMIT_EXCEEDED: Play exceeded its configured ${timeoutSeconds} second runtime limit and was stopped.`;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function runtimeLimitExceededMessage(timeoutSeconds: number): string {
|
|
198
|
+
if (timeoutSeconds === 30 * 60) return RUNTIME_LIMIT_EXCEEDED_MESSAGE;
|
|
199
|
+
return `The play reached its configured runtime limit of ${formatRuntimeLimitDuration(timeoutSeconds)} and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export function runtimeLimitExceededFailure(
|
|
203
|
+
timeoutSeconds: number,
|
|
204
|
+
cause = runtimeLimitExceededCause(timeoutSeconds),
|
|
205
|
+
): PlayRunFailureDetails {
|
|
206
|
+
return {
|
|
207
|
+
code: 'RUNTIME_LIMIT_EXCEEDED',
|
|
208
|
+
phase: 'runtime',
|
|
209
|
+
message: runtimeLimitExceededMessage(timeoutSeconds),
|
|
210
|
+
retryable: false,
|
|
211
|
+
cause: boundedFailureText(cause),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
181
215
|
const PUBLIC_FAILURE_STACK_LINE_LIMIT = 12;
|
|
182
216
|
const PUBLIC_FAILURE_TEXT_BYTE_LIMIT = 64 * 1024;
|
|
183
217
|
const PUBLIC_FAILURE_CAUSE_LIMIT = 8;
|
|
@@ -261,13 +295,12 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
261
295
|
};
|
|
262
296
|
}
|
|
263
297
|
if (RUNTIME_LIMIT_EXCEEDED_RE.test(rawCause)) {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
cause
|
|
270
|
-
};
|
|
298
|
+
const configuredSeconds = Number(
|
|
299
|
+
/configured\s+(\d+)\s+second runtime limit/i.exec(rawCause)?.[1],
|
|
300
|
+
);
|
|
301
|
+
return Number.isSafeInteger(configuredSeconds) && configuredSeconds > 0
|
|
302
|
+
? runtimeLimitExceededFailure(configuredSeconds, cause)
|
|
303
|
+
: runtimeLimitExceededFailure(30 * 60, cause);
|
|
271
304
|
}
|
|
272
305
|
if (RUNTIME_RUNNER_LOST_RE.test(rawCause)) {
|
|
273
306
|
const stack = error instanceof Error ? boundedFailureStack(error) : null;
|
|
@@ -14,6 +14,10 @@ import {
|
|
|
14
14
|
PLAY_RUNNER_STARTUP_GRACE_SECONDS,
|
|
15
15
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
16
16
|
} from '@shared_libs/play-runtime/runtime-constants';
|
|
17
|
+
import {
|
|
18
|
+
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
19
|
+
validatePlaySandboxRuntimeLimits,
|
|
20
|
+
} from '@shared_libs/play-runtime/sandbox-runtime-limits';
|
|
17
21
|
import {
|
|
18
22
|
RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
|
|
19
23
|
RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
|
|
@@ -508,9 +512,11 @@ export async function stageRunnerPayload(input: {
|
|
|
508
512
|
materializedFiles: remoteMaterializedFiles,
|
|
509
513
|
context: remoteRuntimeContextForDaytona(input.config.context),
|
|
510
514
|
};
|
|
511
|
-
const runtimeLimitSeconds =
|
|
512
|
-
input.config.context.sandboxRuntimeLimits
|
|
513
|
-
|
|
515
|
+
const runtimeLimitSeconds = validatePlaySandboxRuntimeLimits(
|
|
516
|
+
input.config.context.sandboxRuntimeLimits ?? {
|
|
517
|
+
...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
518
|
+
},
|
|
519
|
+
).timeoutSeconds;
|
|
514
520
|
|
|
515
521
|
const envelopeUpload = input.bundlePromise.then((bundle) =>
|
|
516
522
|
timedRunnerPayloadUpload({
|
|
@@ -18,6 +18,7 @@ const limitMs = Math.max(1, Number.parseInt(process.argv[5] || '', 10) || 1);
|
|
|
18
18
|
const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) || 1);
|
|
19
19
|
const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
|
|
20
20
|
const runtimeLimitMarkerPath = process.argv[8] || '';
|
|
21
|
+
const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
|
|
21
22
|
const child = spawn(process.execPath, [runnerPath, configPath], {
|
|
22
23
|
env: process.env,
|
|
23
24
|
stdio: 'inherit',
|
|
@@ -54,7 +55,11 @@ function armRuntimeDeadline() {
|
|
|
54
55
|
}
|
|
55
56
|
timedOut = true;
|
|
56
57
|
markRuntimeLimit();
|
|
57
|
-
process.stderr.write(
|
|
58
|
+
process.stderr.write(
|
|
59
|
+
'RUNTIME_LIMIT_EXCEEDED: Play exceeded its configured ' +
|
|
60
|
+
runtimeLimitSeconds +
|
|
61
|
+
' second runtime limit and was stopped by the external watchdog.\\n',
|
|
62
|
+
);
|
|
58
63
|
stopChild();
|
|
59
64
|
}, limitMs);
|
|
60
65
|
}
|
|
@@ -113,5 +118,7 @@ function finish(code) {
|
|
|
113
118
|
|
|
114
119
|
child.once('error', () => finish(1));
|
|
115
120
|
child.once('exit', (code) => finish(code));
|
|
121
|
+
process.once('SIGTERM', stopChild);
|
|
122
|
+
process.once('SIGINT', stopChild);
|
|
116
123
|
`;
|
|
117
124
|
}
|
|
@@ -15,7 +15,6 @@ import type {
|
|
|
15
15
|
} from '@shared_libs/play-runtime/protocol';
|
|
16
16
|
import {
|
|
17
17
|
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
18
|
-
STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
|
|
19
18
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
20
19
|
} from '@shared_libs/play-runtime/runtime-constants';
|
|
21
20
|
import {
|
|
@@ -39,10 +38,6 @@ import {
|
|
|
39
38
|
prepareDetachedDaytonaRunner,
|
|
40
39
|
} from './daytona-session-execution';
|
|
41
40
|
|
|
42
|
-
const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
43
|
-
const DAYTONA_DETACHED_CEILING_SECONDS =
|
|
44
|
-
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS + PLAY_RUNNER_TERMINAL_GRACE_SECONDS;
|
|
45
|
-
const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
|
|
46
41
|
const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
|
|
47
42
|
const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
|
|
48
43
|
const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
|
|
@@ -459,13 +454,6 @@ export async function deleteDaytonaSandboxById(input: {
|
|
|
459
454
|
return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
|
|
460
455
|
}
|
|
461
456
|
|
|
462
|
-
function formatDaytonaExecutionError(error: unknown): string {
|
|
463
|
-
const message = formatDaytonaError(error);
|
|
464
|
-
return isConfiguredDaytonaRuntimeLimit(error)
|
|
465
|
-
? STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR
|
|
466
|
-
: message;
|
|
467
|
-
}
|
|
468
|
-
|
|
469
457
|
function formatDaytonaError(error: unknown): string {
|
|
470
458
|
if (!(error instanceof Error)) {
|
|
471
459
|
return String(error);
|
|
@@ -483,14 +471,6 @@ function formatDaytonaError(error: unknown): string {
|
|
|
483
471
|
return `${details || 'Error'}${code}${cause}`;
|
|
484
472
|
}
|
|
485
473
|
|
|
486
|
-
function isConfiguredDaytonaRuntimeLimit(error: unknown): boolean {
|
|
487
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
488
|
-
return new RegExp(
|
|
489
|
-
`(?:execution|command).{0,80}(?:timed out|timeout).{0,80}${DAYTONA_EXECUTE_TIMEOUT_SECONDS}|${DAYTONA_EXECUTE_TIMEOUT_SECONDS}.{0,80}(?:timed out|timeout)`,
|
|
490
|
-
'i',
|
|
491
|
-
).test(message);
|
|
492
|
-
}
|
|
493
|
-
|
|
494
474
|
function sleep(ms: number): Promise<void> {
|
|
495
475
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
496
476
|
}
|
|
@@ -1070,6 +1050,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1070
1050
|
runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
|
|
1071
1051
|
startedAtMs: Date.now(),
|
|
1072
1052
|
heartbeatTimeoutMs: push.leaseSeconds * 1_000,
|
|
1053
|
+
runtimeLimitSeconds: executionTimeoutSeconds,
|
|
1073
1054
|
ceilingMs:
|
|
1074
1055
|
(executionTimeoutSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
|
|
1075
1056
|
1_000,
|
|
@@ -1164,7 +1145,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1164
1145
|
return withCleanup(
|
|
1165
1146
|
createDaytonaFailedResult({
|
|
1166
1147
|
config,
|
|
1167
|
-
error:
|
|
1148
|
+
error: formatDaytonaError(error),
|
|
1168
1149
|
runtimeTiming,
|
|
1169
1150
|
}),
|
|
1170
1151
|
);
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
|
-
import { cp, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { cp, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import type { SpawnOptionsWithoutStdio } from 'node:child_process';
|
|
6
6
|
import {
|
|
@@ -20,7 +20,22 @@ import {
|
|
|
20
20
|
recordComputeBillingItemViaAppRuntime,
|
|
21
21
|
type WorkerRuntimeApiContext,
|
|
22
22
|
} from '@shared_libs/play-runtime/app-runtime-api';
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
PLAY_RUNNER_STARTUP_GRACE_SECONDS,
|
|
25
|
+
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
26
|
+
} from '@shared_libs/play-runtime/runtime-constants';
|
|
27
|
+
import {
|
|
28
|
+
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
29
|
+
validatePlaySandboxRuntimeLimits,
|
|
30
|
+
} from '@shared_libs/play-runtime/sandbox-runtime-limits';
|
|
31
|
+
import {
|
|
32
|
+
runtimeLimitExceededCause,
|
|
33
|
+
runtimeLimitExceededFailure,
|
|
34
|
+
} from '@shared_libs/play-runtime/run-failure';
|
|
35
|
+
import {
|
|
36
|
+
buildDaytonaRuntimeWatchdogSource,
|
|
37
|
+
DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE,
|
|
38
|
+
} from './daytona-runtime-watchdog';
|
|
24
39
|
|
|
25
40
|
interface CommandExecutionResult {
|
|
26
41
|
exitCode: number;
|
|
@@ -41,9 +56,6 @@ const LOCAL_PROCESS_COMPUTE_MEMORY_GIB = 1;
|
|
|
41
56
|
const LOCAL_PROCESS_COMPUTE_DISK_GIB = 3;
|
|
42
57
|
export const LOCAL_PROCESS_CANCEL_GRACE_MS = 2_000;
|
|
43
58
|
const LOCAL_PROCESS_CANCELLED_ERROR = 'Local play runner cancelled';
|
|
44
|
-
function localProcessRuntimeLimitError(runtimeLimitSeconds: number): string {
|
|
45
|
-
return `Configured max runtime is ${runtimeLimitSeconds} seconds. Use smaller batches or increase runtime.timeout.`;
|
|
46
|
-
}
|
|
47
59
|
|
|
48
60
|
export const LOCAL_PROCESS_COMPUTE_PROFILE = {
|
|
49
61
|
source: LOCAL_PROCESS_COMPUTE_SOURCE,
|
|
@@ -194,55 +206,11 @@ async function runCommand(
|
|
|
194
206
|
export function resolveLocalProcessRuntimeLimitSeconds(
|
|
195
207
|
context: PlayRunnerExecutionConfig['context'],
|
|
196
208
|
): number {
|
|
197
|
-
return (
|
|
198
|
-
context.sandboxRuntimeLimits
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
async function runCommandWithRuntimeLimit(
|
|
204
|
-
command: string,
|
|
205
|
-
args: string[],
|
|
206
|
-
runtimeLimitSeconds: number,
|
|
207
|
-
options?: SpawnOptionsWithoutStdio,
|
|
208
|
-
callbacks?: StreamedCommandCallbacks,
|
|
209
|
-
): Promise<CommandExecutionResult> {
|
|
210
|
-
const runtimeLimitController = new AbortController();
|
|
211
|
-
const runtimeDeadlineTimer = setTimeout(() => {
|
|
212
|
-
runtimeLimitController.abort(
|
|
213
|
-
localProcessRuntimeLimitError(runtimeLimitSeconds),
|
|
214
|
-
);
|
|
215
|
-
}, runtimeLimitSeconds * 1000);
|
|
216
|
-
runtimeDeadlineTimer.unref?.();
|
|
217
|
-
|
|
218
|
-
const onExternalCancel = () => {
|
|
219
|
-
if (!runtimeLimitController.signal.aborted) {
|
|
220
|
-
runtimeLimitController.abort(
|
|
221
|
-
callbacks?.cancellationSignal?.reason ?? LOCAL_PROCESS_CANCELLED_ERROR,
|
|
222
|
-
);
|
|
223
|
-
}
|
|
224
|
-
};
|
|
225
|
-
|
|
226
|
-
if (callbacks?.cancellationSignal?.aborted) {
|
|
227
|
-
onExternalCancel();
|
|
228
|
-
} else {
|
|
229
|
-
callbacks?.cancellationSignal?.addEventListener('abort', onExternalCancel, {
|
|
230
|
-
once: true,
|
|
231
|
-
});
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
try {
|
|
235
|
-
return await runCommand(command, args, options, {
|
|
236
|
-
...callbacks,
|
|
237
|
-
cancellationSignal: runtimeLimitController.signal,
|
|
238
|
-
});
|
|
239
|
-
} finally {
|
|
240
|
-
clearTimeout(runtimeDeadlineTimer);
|
|
241
|
-
callbacks?.cancellationSignal?.removeEventListener(
|
|
242
|
-
'abort',
|
|
243
|
-
onExternalCancel,
|
|
244
|
-
);
|
|
245
|
-
}
|
|
209
|
+
return validatePlaySandboxRuntimeLimits(
|
|
210
|
+
context.sandboxRuntimeLimits ?? {
|
|
211
|
+
...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
212
|
+
},
|
|
213
|
+
).timeoutSeconds;
|
|
246
214
|
}
|
|
247
215
|
|
|
248
216
|
function createFailedResult(config: PlayRunnerExecutionConfig, error: string) {
|
|
@@ -424,10 +392,34 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
424
392
|
},
|
|
425
393
|
},
|
|
426
394
|
async (span) => {
|
|
427
|
-
const
|
|
395
|
+
const runtimeLimitSeconds =
|
|
396
|
+
resolveLocalProcessRuntimeLimitSeconds(config.context);
|
|
397
|
+
const runtimeStartedPath = join(
|
|
398
|
+
workspaceRoot,
|
|
399
|
+
'runtime-started.json',
|
|
400
|
+
);
|
|
401
|
+
const runtimeCompletedPath = join(
|
|
402
|
+
workspaceRoot,
|
|
403
|
+
'runtime-completed.json',
|
|
404
|
+
);
|
|
405
|
+
const runtimeLimitMarkerPath = join(
|
|
406
|
+
workspaceRoot,
|
|
407
|
+
'runtime-limit.marker',
|
|
408
|
+
);
|
|
409
|
+
const result = await runCommand(
|
|
428
410
|
'node',
|
|
429
|
-
[
|
|
430
|
-
|
|
411
|
+
[
|
|
412
|
+
'-e',
|
|
413
|
+
buildDaytonaRuntimeWatchdogSource(),
|
|
414
|
+
runnerPath,
|
|
415
|
+
configPath,
|
|
416
|
+
runtimeStartedPath,
|
|
417
|
+
runtimeCompletedPath,
|
|
418
|
+
String(runtimeLimitSeconds * 1_000),
|
|
419
|
+
String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
|
|
420
|
+
String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
|
|
421
|
+
runtimeLimitMarkerPath,
|
|
422
|
+
],
|
|
431
423
|
{
|
|
432
424
|
cwd: workspaceRoot,
|
|
433
425
|
env: {
|
|
@@ -436,6 +428,10 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
436
428
|
TMPDIR: workspaceRoot,
|
|
437
429
|
TEMP: workspaceRoot,
|
|
438
430
|
TMP: workspaceRoot,
|
|
431
|
+
DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH:
|
|
432
|
+
runtimeStartedPath,
|
|
433
|
+
DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH:
|
|
434
|
+
runtimeCompletedPath,
|
|
439
435
|
},
|
|
440
436
|
},
|
|
441
437
|
{
|
|
@@ -463,12 +459,18 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
463
459
|
},
|
|
464
460
|
},
|
|
465
461
|
);
|
|
462
|
+
const runtimeLimitReached =
|
|
463
|
+
result.exitCode === DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE &&
|
|
464
|
+
(await readFile(runtimeLimitMarkerPath, 'utf-8')
|
|
465
|
+
.then((value) => value.trim() === 'runtime_limit')
|
|
466
|
+
.catch(() => false));
|
|
466
467
|
setSpanAttributes(span, {
|
|
467
468
|
'plays.child_exit_code': result.exitCode,
|
|
468
469
|
'plays.stdout_bytes': result.stdout.length,
|
|
469
470
|
'plays.stderr_bytes': result.stderr.length,
|
|
471
|
+
'plays.runtime_limit_reached': runtimeLimitReached,
|
|
470
472
|
});
|
|
471
|
-
return result;
|
|
473
|
+
return { ...result, runtimeLimitReached, runtimeLimitSeconds };
|
|
472
474
|
},
|
|
473
475
|
);
|
|
474
476
|
logLocalRunnerPerf(
|
|
@@ -514,6 +516,21 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
514
516
|
return result;
|
|
515
517
|
}
|
|
516
518
|
|
|
519
|
+
if (execution.runtimeLimitReached) {
|
|
520
|
+
const error = runtimeLimitExceededCause(
|
|
521
|
+
execution.runtimeLimitSeconds,
|
|
522
|
+
);
|
|
523
|
+
return {
|
|
524
|
+
...createFailedResult(config, error),
|
|
525
|
+
errors: [
|
|
526
|
+
runtimeLimitExceededFailure(
|
|
527
|
+
execution.runtimeLimitSeconds,
|
|
528
|
+
error,
|
|
529
|
+
),
|
|
530
|
+
],
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
517
534
|
return createFailedResult(
|
|
518
535
|
config,
|
|
519
536
|
execution.exitCode !== 0
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
loadModalClientConfig,
|
|
18
18
|
loadModalRequiredConfig,
|
|
19
19
|
} from '@shared_libs/play-runtime/modal-runtime-config';
|
|
20
|
+
import type { PlaySandboxRuntimeLimits } from '@shared_libs/play-runtime/sandbox-runtime-limits';
|
|
20
21
|
import { validateDaytonaExecutionContext } from './daytona-lifecycle';
|
|
21
22
|
import { stageRunnerPayload } from './daytona-payload-transport';
|
|
22
23
|
import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
|
|
@@ -24,6 +25,23 @@ import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session
|
|
|
24
25
|
const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
|
|
25
26
|
type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
|
|
26
27
|
|
|
28
|
+
export function modalSandboxLifetimeMs(
|
|
29
|
+
limits: PlaySandboxRuntimeLimits,
|
|
30
|
+
): number {
|
|
31
|
+
return (
|
|
32
|
+
(limits.timeoutSeconds +
|
|
33
|
+
PLAY_RUNNER_STARTUP_GRACE_SECONDS +
|
|
34
|
+
PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
|
|
35
|
+
1_000
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function modalDetachedRunnerCeilingMs(
|
|
40
|
+
limits: PlaySandboxRuntimeLimits,
|
|
41
|
+
): number {
|
|
42
|
+
return (limits.timeoutSeconds + PLAY_RUNNER_TERMINAL_GRACE_SECONDS) * 1_000;
|
|
43
|
+
}
|
|
44
|
+
|
|
27
45
|
function failed(
|
|
28
46
|
config: PlayRunnerExecutionConfig,
|
|
29
47
|
error: unknown,
|
|
@@ -126,16 +144,8 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
126
144
|
cpuLimit: MODAL_SANDBOX_CPU_CORES,
|
|
127
145
|
memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
|
|
128
146
|
memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
|
|
129
|
-
timeoutMs:
|
|
130
|
-
|
|
131
|
-
PLAY_RUNNER_STARTUP_GRACE_SECONDS +
|
|
132
|
-
PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
|
|
133
|
-
1_000,
|
|
134
|
-
idleTimeoutMs:
|
|
135
|
-
(modalConfig.limits.timeoutSeconds +
|
|
136
|
-
PLAY_RUNNER_STARTUP_GRACE_SECONDS +
|
|
137
|
-
PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
|
|
138
|
-
1_000,
|
|
147
|
+
timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
148
|
+
idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
139
149
|
workdir: modalConfig.workdir,
|
|
140
150
|
tags: {
|
|
141
151
|
source: 'deepline-play-runner',
|
|
@@ -265,10 +275,8 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
265
275
|
runtimeCompletedPath: payload.runtimeCompletedPath,
|
|
266
276
|
startedAtMs: Date.now(),
|
|
267
277
|
heartbeatTimeoutMs: push.leaseSeconds * 1_000,
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
PLAY_RUNNER_TERMINAL_GRACE_SECONDS) *
|
|
271
|
-
1_000,
|
|
278
|
+
runtimeLimitSeconds: modalConfig.limits.timeoutSeconds,
|
|
279
|
+
ceilingMs: modalDetachedRunnerCeilingMs(modalConfig.limits),
|
|
272
280
|
},
|
|
273
281
|
logs: [],
|
|
274
282
|
stats: {},
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import {
|
|
2
|
+
PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
|
|
3
|
+
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
4
|
+
} from './runtime-constants';
|
|
5
|
+
|
|
1
6
|
export type PlayExecutionSuspension =
|
|
2
7
|
| {
|
|
3
8
|
kind: 'sleep';
|
|
@@ -57,6 +62,8 @@ export type PlayExecutionSuspension =
|
|
|
57
62
|
heartbeatTimeoutMs: number;
|
|
58
63
|
/** Overall run ceiling; the park timeout. */
|
|
59
64
|
ceilingMs: number;
|
|
65
|
+
/** Launch-pinned user-code deadline. Missing on historical suspensions. */
|
|
66
|
+
runtimeLimitSeconds?: number;
|
|
60
67
|
};
|
|
61
68
|
|
|
62
69
|
export type DetachedRunnerSuspension = Extract<
|
|
@@ -64,6 +71,25 @@ export type DetachedRunnerSuspension = Extract<
|
|
|
64
71
|
{ kind: 'detached_runner' }
|
|
65
72
|
>;
|
|
66
73
|
|
|
74
|
+
export function resolveDetachedRunnerRuntimeLimitSeconds(
|
|
75
|
+
suspension: Pick<
|
|
76
|
+
DetachedRunnerSuspension,
|
|
77
|
+
'runtimeLimitSeconds' | 'ceilingMs'
|
|
78
|
+
>,
|
|
79
|
+
): number {
|
|
80
|
+
if (
|
|
81
|
+
Number.isSafeInteger(suspension.runtimeLimitSeconds) &&
|
|
82
|
+
Number(suspension.runtimeLimitSeconds) > 0
|
|
83
|
+
) {
|
|
84
|
+
return Number(suspension.runtimeLimitSeconds);
|
|
85
|
+
}
|
|
86
|
+
const legacyDerived =
|
|
87
|
+
(suspension.ceilingMs - PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000) / 1_000;
|
|
88
|
+
return Number.isSafeInteger(legacyDerived) && legacyDerived > 0
|
|
89
|
+
? legacyDerived
|
|
90
|
+
: STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
91
|
+
}
|
|
92
|
+
|
|
67
93
|
export class PlayExecutionSuspendedError extends Error {
|
|
68
94
|
readonly suspension: PlayExecutionSuspension;
|
|
69
95
|
|
package/dist/cli/index.js
CHANGED
|
@@ -1040,7 +1040,7 @@ var SDK_RELEASE = {
|
|
|
1040
1040
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1041
1041
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1042
1042
|
// release keeps lazy paging semantics independent of row residency.
|
|
1043
|
-
version: "0.2.
|
|
1043
|
+
version: "0.2.9",
|
|
1044
1044
|
contracts: {
|
|
1045
1045
|
api: {
|
|
1046
1046
|
name: "sdk-http-api",
|
|
@@ -5751,7 +5751,7 @@ var DeeplineClient = class {
|
|
|
5751
5751
|
body: definition
|
|
5752
5752
|
}
|
|
5753
5753
|
);
|
|
5754
|
-
if (definition.tool !== "deepline.
|
|
5754
|
+
if (definition.tool !== "deepline.analytics") return deployed;
|
|
5755
5755
|
const setup = await this.setupMonitor(
|
|
5756
5756
|
definition.tool,
|
|
5757
5757
|
definition.payload ?? {}
|
package/dist/cli/index.mjs
CHANGED
|
@@ -1025,7 +1025,7 @@ var SDK_RELEASE = {
|
|
|
1025
1025
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1026
1026
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1027
1027
|
// release keeps lazy paging semantics independent of row residency.
|
|
1028
|
-
version: "0.2.
|
|
1028
|
+
version: "0.2.9",
|
|
1029
1029
|
contracts: {
|
|
1030
1030
|
api: {
|
|
1031
1031
|
name: "sdk-http-api",
|
|
@@ -5736,7 +5736,7 @@ var DeeplineClient = class {
|
|
|
5736
5736
|
body: definition
|
|
5737
5737
|
}
|
|
5738
5738
|
);
|
|
5739
|
-
if (definition.tool !== "deepline.
|
|
5739
|
+
if (definition.tool !== "deepline.analytics") return deployed;
|
|
5740
5740
|
const setup = await this.setupMonitor(
|
|
5741
5741
|
definition.tool,
|
|
5742
5742
|
definition.payload ?? {}
|
package/dist/index.js
CHANGED
|
@@ -763,7 +763,7 @@ var SDK_RELEASE = {
|
|
|
763
763
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
764
764
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
765
765
|
// release keeps lazy paging semantics independent of row residency.
|
|
766
|
-
version: "0.2.
|
|
766
|
+
version: "0.2.9",
|
|
767
767
|
contracts: {
|
|
768
768
|
api: {
|
|
769
769
|
name: "sdk-http-api",
|
|
@@ -5474,7 +5474,7 @@ var DeeplineClient = class {
|
|
|
5474
5474
|
body: definition
|
|
5475
5475
|
}
|
|
5476
5476
|
);
|
|
5477
|
-
if (definition.tool !== "deepline.
|
|
5477
|
+
if (definition.tool !== "deepline.analytics") return deployed;
|
|
5478
5478
|
const setup = await this.setupMonitor(
|
|
5479
5479
|
definition.tool,
|
|
5480
5480
|
definition.payload ?? {}
|
package/dist/index.mjs
CHANGED
|
@@ -689,7 +689,7 @@ var SDK_RELEASE = {
|
|
|
689
689
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
690
690
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
691
691
|
// release keeps lazy paging semantics independent of row residency.
|
|
692
|
-
version: "0.2.
|
|
692
|
+
version: "0.2.9",
|
|
693
693
|
contracts: {
|
|
694
694
|
api: {
|
|
695
695
|
name: "sdk-http-api",
|
|
@@ -5400,7 +5400,7 @@ var DeeplineClient = class {
|
|
|
5400
5400
|
body: definition
|
|
5401
5401
|
}
|
|
5402
5402
|
);
|
|
5403
|
-
if (definition.tool !== "deepline.
|
|
5403
|
+
if (definition.tool !== "deepline.analytics") return deployed;
|
|
5404
5404
|
const setup = await this.setupMonitor(
|
|
5405
5405
|
definition.tool,
|
|
5406
5406
|
definition.payload ?? {}
|