deepline 0.1.320 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +13 -2
- package/dist/bundling-sources/sdk/src/release.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/backend.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/modal-runtime-config.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +106 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-modal-fallback.ts +218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +26 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +380 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +240 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +17 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +188 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +77 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +39 -0
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +43 -47
- package/dist/cli/index.js +55 -28
- package/dist/cli/index.mjs +55 -28
- package/dist/index.d.mts +8 -9
- package/dist/index.d.ts +8 -9
- package/dist/index.js +86 -38
- package/dist/index.mjs +86 -38
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +7 -1
- package/dist/{tool-execution-error-YDz7UMl-.d.mts → tool-execution-error-4-rhemLQ.d.mts} +7 -1
- package/dist/{tool-execution-error-YDz7UMl-.d.ts → tool-execution-error-4-rhemLQ.d.ts} +7 -1
- package/package.json +1 -1
|
@@ -187,7 +187,7 @@ function resolvePlayRunRuntimeSelection(
|
|
|
187
187
|
const runtime = normalizePlayRuntimeSelection(request.runtime);
|
|
188
188
|
if (!runtime) {
|
|
189
189
|
throw new DeeplineError(
|
|
190
|
-
'runtime must be
|
|
190
|
+
'runtime must be { environment: "preview", namespace, backend?: "daytona" | "modal" } with namespace matching ^[a-z][a-z0-9-]{0,30}$.',
|
|
191
191
|
undefined,
|
|
192
192
|
'INVALID_RUNTIME_SELECTION',
|
|
193
193
|
);
|
|
@@ -229,7 +229,18 @@ function resolvePlayRunRuntimeSelection(
|
|
|
229
229
|
'INVALID_RUNTIME_NAMESPACE',
|
|
230
230
|
);
|
|
231
231
|
}
|
|
232
|
-
|
|
232
|
+
const configuredBackend = process.env.DEEPLINE_PLAY_RUNNER_BACKEND?.trim();
|
|
233
|
+
if (!configuredBackend) {
|
|
234
|
+
return { environment, namespace };
|
|
235
|
+
}
|
|
236
|
+
if (configuredBackend !== 'daytona' && configuredBackend !== 'modal') {
|
|
237
|
+
throw new DeeplineError(
|
|
238
|
+
`DEEPLINE_PLAY_RUNNER_BACKEND must be daytona or modal for preview runtime selection. Received "${configuredBackend}".`,
|
|
239
|
+
undefined,
|
|
240
|
+
'INVALID_RUNTIME_BACKEND',
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
return { environment, namespace, backend: configuredBackend };
|
|
233
244
|
}
|
|
234
245
|
|
|
235
246
|
function runtimeSelectionHeaders(
|
|
@@ -157,7 +157,10 @@ export const SDK_RELEASE = {
|
|
|
157
157
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
158
158
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
159
159
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
160
|
-
|
|
160
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
161
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
162
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
163
|
+
version: '0.2.1',
|
|
161
164
|
contracts: {
|
|
162
165
|
api: {
|
|
163
166
|
name: 'sdk-http-api',
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export const PLAY_RUNTIME_BACKENDS = {
|
|
2
2
|
localProcess: 'local_process',
|
|
3
3
|
daytona: 'daytona',
|
|
4
|
+
modal: 'modal',
|
|
4
5
|
} as const;
|
|
5
6
|
|
|
6
7
|
export type PlayRuntimeBackendId =
|
|
@@ -39,6 +40,11 @@ export const PLAY_BACKEND_DESCRIPTORS: Record<
|
|
|
39
40
|
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
40
41
|
label: 'Daytona sandbox',
|
|
41
42
|
},
|
|
43
|
+
[PLAY_RUNTIME_BACKENDS.modal]: {
|
|
44
|
+
id: PLAY_RUNTIME_BACKENDS.modal,
|
|
45
|
+
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
46
|
+
label: 'Modal sandbox',
|
|
47
|
+
},
|
|
42
48
|
};
|
|
43
49
|
|
|
44
50
|
export function describePlayBackend(
|
|
@@ -65,6 +71,10 @@ export function normalizePlayRuntimeBackend(
|
|
|
65
71
|
return PLAY_RUNTIME_BACKENDS.daytona;
|
|
66
72
|
}
|
|
67
73
|
|
|
74
|
+
if (normalized === PLAY_RUNTIME_BACKENDS.modal) {
|
|
75
|
+
return PLAY_RUNTIME_BACKENDS.modal;
|
|
76
|
+
}
|
|
77
|
+
|
|
68
78
|
if (normalized === PLAY_RUNTIME_BACKENDS.localProcess) {
|
|
69
79
|
return PLAY_RUNTIME_BACKENDS.localProcess;
|
|
70
80
|
}
|
|
@@ -118,3 +128,12 @@ export function isPlayRuntimeBackendId(
|
|
|
118
128
|
Object.values(PLAY_RUNTIME_BACKENDS).includes(value as PlayRuntimeBackendId)
|
|
119
129
|
);
|
|
120
130
|
}
|
|
131
|
+
|
|
132
|
+
export function isManagedSandboxRuntimeBackend(
|
|
133
|
+
value: string | null | undefined,
|
|
134
|
+
): boolean {
|
|
135
|
+
return (
|
|
136
|
+
value === PLAY_RUNTIME_BACKENDS.daytona ||
|
|
137
|
+
value === PLAY_RUNTIME_BACKENDS.modal
|
|
138
|
+
);
|
|
139
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { ModalClient } from 'modal';
|
|
2
|
+
import { isIsolatedRuntimeSchedulerSchema } from './runtime-scheduler-topology';
|
|
3
|
+
import {
|
|
4
|
+
STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
|
|
5
|
+
validatePlaySandboxRuntimeLimits,
|
|
6
|
+
type PlaySandboxRuntimeLimits,
|
|
7
|
+
} from './sandbox-runtime-limits';
|
|
8
|
+
|
|
9
|
+
export const MODAL_RUNNER_APP_NAME = 'deepline-play-runner';
|
|
10
|
+
export const MODAL_RUNNER_IMAGE = 'node:20-bookworm-slim';
|
|
11
|
+
export const MODAL_RUNNER_WORKDIR = '/root/deepline';
|
|
12
|
+
|
|
13
|
+
// Modal bills one physical core as two vCPUs. This matches the standard
|
|
14
|
+
// Deepline one-vCPU / one-GiB sandbox.
|
|
15
|
+
export const MODAL_SANDBOX_CPU_CORES = 0.5;
|
|
16
|
+
export const MODAL_SANDBOX_MEMORY_MIB = 1024;
|
|
17
|
+
|
|
18
|
+
export type ModalRequiredConfig = {
|
|
19
|
+
client: ModalClient;
|
|
20
|
+
appName: string;
|
|
21
|
+
image: string;
|
|
22
|
+
workdir: string;
|
|
23
|
+
outboundCidrAllowlist: string[] | null;
|
|
24
|
+
limits: PlaySandboxRuntimeLimits;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type ModalClientConfig = Omit<
|
|
28
|
+
ModalRequiredConfig,
|
|
29
|
+
'outboundCidrAllowlist' | 'limits'
|
|
30
|
+
>;
|
|
31
|
+
|
|
32
|
+
function requiredEnv(
|
|
33
|
+
env: NodeJS.ProcessEnv,
|
|
34
|
+
name: 'MODAL_TOKEN_ID' | 'MODAL_TOKEN_SECRET',
|
|
35
|
+
): string {
|
|
36
|
+
const value = env[name]?.trim();
|
|
37
|
+
if (!value) throw new Error(`Missing required Modal configuration: ${name}.`);
|
|
38
|
+
return value;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function cidrAllowlist(raw: string | null | undefined): string[] | null {
|
|
42
|
+
const values = (raw ?? '')
|
|
43
|
+
.split(',')
|
|
44
|
+
.map((value) => value.trim())
|
|
45
|
+
.filter(Boolean)
|
|
46
|
+
.filter((value) => !value.includes(':'));
|
|
47
|
+
return values.length > 0 ? values : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function resolveModalSandboxNetworkPolicy(input: {
|
|
51
|
+
runtimeSchedulerSchema: string | null | undefined;
|
|
52
|
+
env?: NodeJS.ProcessEnv;
|
|
53
|
+
}): { outboundCidrAllowlist: string[] | null } {
|
|
54
|
+
const env = input.env ?? process.env;
|
|
55
|
+
const outboundCidrAllowlist = cidrAllowlist(
|
|
56
|
+
env.DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST,
|
|
57
|
+
);
|
|
58
|
+
if (
|
|
59
|
+
!outboundCidrAllowlist &&
|
|
60
|
+
env.NODE_ENV === 'production' &&
|
|
61
|
+
!isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
|
|
62
|
+
) {
|
|
63
|
+
throw new Error(
|
|
64
|
+
'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST is required for the Modal capacity fallback in production. Refusing to run customer code with unrestricted outbound network access.',
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
return { outboundCidrAllowlist };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function loadModalRequiredConfig(input: {
|
|
71
|
+
env?: NodeJS.ProcessEnv;
|
|
72
|
+
runtimeSchedulerSchema?: string | null;
|
|
73
|
+
limits?: PlaySandboxRuntimeLimits | null;
|
|
74
|
+
}): Promise<ModalRequiredConfig> {
|
|
75
|
+
const env = input.env ?? process.env;
|
|
76
|
+
const clientConfig = await loadModalClientConfig({ env });
|
|
77
|
+
const { outboundCidrAllowlist } = resolveModalSandboxNetworkPolicy({
|
|
78
|
+
env,
|
|
79
|
+
runtimeSchedulerSchema: input.runtimeSchedulerSchema,
|
|
80
|
+
});
|
|
81
|
+
return {
|
|
82
|
+
...clientConfig,
|
|
83
|
+
outboundCidrAllowlist,
|
|
84
|
+
limits: validatePlaySandboxRuntimeLimits(
|
|
85
|
+
input.limits ?? { ...STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS },
|
|
86
|
+
),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export async function loadModalClientConfig(
|
|
91
|
+
input: { env?: NodeJS.ProcessEnv } = {},
|
|
92
|
+
): Promise<ModalClientConfig> {
|
|
93
|
+
const env = input.env ?? process.env;
|
|
94
|
+
const tokenId = requiredEnv(env, 'MODAL_TOKEN_ID');
|
|
95
|
+
const tokenSecret = requiredEnv(env, 'MODAL_TOKEN_SECRET');
|
|
96
|
+
const { ModalClient } = await import('modal');
|
|
97
|
+
return {
|
|
98
|
+
client: new ModalClient({ tokenId, tokenSecret }),
|
|
99
|
+
appName:
|
|
100
|
+
env.DEEPLINE_MODAL_RUNNER_APP_NAME?.trim() || MODAL_RUNNER_APP_NAME,
|
|
101
|
+
image: env.DEEPLINE_MODAL_RUNNER_IMAGE?.trim() || MODAL_RUNNER_IMAGE,
|
|
102
|
+
workdir: env.DEEPLINE_MODAL_RUNNER_WORKDIR?.trim() || MODAL_RUNNER_WORKDIR,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -231,10 +231,13 @@ export type PlayRunnerEvent =
|
|
|
231
231
|
};
|
|
232
232
|
|
|
233
233
|
export type PlayRunnerRuntimeTiming = {
|
|
234
|
-
backend: 'daytona';
|
|
234
|
+
backend: 'daytona' | 'modal';
|
|
235
235
|
daytonaCreateMs?: number;
|
|
236
236
|
daytonaUploadMs?: number;
|
|
237
237
|
daytonaExecuteMs?: number;
|
|
238
|
+
modalCreateMs?: number;
|
|
239
|
+
modalUploadMs?: number;
|
|
240
|
+
modalExecuteMs?: number;
|
|
238
241
|
};
|
|
239
242
|
|
|
240
243
|
/**
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -12,6 +12,9 @@ import {
|
|
|
12
12
|
|
|
13
13
|
const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
|
|
14
14
|
const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
|
|
15
|
+
const DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS = [
|
|
16
|
+
0, 250, 750, 1_500, 3_000,
|
|
17
|
+
] as const;
|
|
15
18
|
// Explicit runner deadline + scheduler GC own the normal lifecycle. Daytona's
|
|
16
19
|
// inactivity stop is a wider crash backstop measured from sandbox creation, so
|
|
17
20
|
// setup time cannot consume the terminal-flush grace.
|
|
@@ -54,6 +57,47 @@ export type OneShotDaytonaSandboxLifecycle = {
|
|
|
54
57
|
dispose: () => Promise<void>;
|
|
55
58
|
};
|
|
56
59
|
|
|
60
|
+
export type DaytonaSandboxAcquisitionUnavailableReason =
|
|
61
|
+
| 'daytona_total_cpu_limit_exceeded'
|
|
62
|
+
| 'daytona_sandbox_start_timeout';
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* A typed, pre-customer-code acquisition rejection. Only this error may move
|
|
66
|
+
* managed placement to another provider.
|
|
67
|
+
*/
|
|
68
|
+
export class DaytonaSandboxAcquisitionUnavailableError extends Error {
|
|
69
|
+
readonly reason: DaytonaSandboxAcquisitionUnavailableReason;
|
|
70
|
+
|
|
71
|
+
constructor(
|
|
72
|
+
reason: DaytonaSandboxAcquisitionUnavailableReason,
|
|
73
|
+
message: string,
|
|
74
|
+
) {
|
|
75
|
+
super(message);
|
|
76
|
+
this.name = 'DaytonaSandboxAcquisitionUnavailableError';
|
|
77
|
+
this.reason = reason;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function resolveDaytonaSandboxAcquisitionUnavailableReason(
|
|
82
|
+
errors: readonly string[],
|
|
83
|
+
): DaytonaSandboxAcquisitionUnavailableReason | null {
|
|
84
|
+
if (
|
|
85
|
+
errors.length > 0 &&
|
|
86
|
+
errors.every((error) =>
|
|
87
|
+
/Total CPU limit exceeded\.\s*Maximum allowed:\s*\d+/i.test(error),
|
|
88
|
+
)
|
|
89
|
+
) {
|
|
90
|
+
return 'daytona_total_cpu_limit_exceeded';
|
|
91
|
+
}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function isDaytonaSandboxStartTimeout(error: string): boolean {
|
|
96
|
+
return /Failed to create and start sandbox within 10 seconds\. Operation timed out\./.test(
|
|
97
|
+
error,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
57
101
|
type DaytonaCreateResult = {
|
|
58
102
|
sandbox: DaytonaSandbox;
|
|
59
103
|
attempt: number;
|
|
@@ -177,6 +221,7 @@ async function createOneShotDaytonaSandbox(input: {
|
|
|
177
221
|
daytona: DaytonaClient;
|
|
178
222
|
orgId: string;
|
|
179
223
|
context: DaytonaExecutionContext;
|
|
224
|
+
sandboxName: string;
|
|
180
225
|
}): Promise<DaytonaSandbox> {
|
|
181
226
|
const limits = validatePlaySandboxRuntimeLimits(
|
|
182
227
|
input.context.sandboxRuntimeLimits ?? {
|
|
@@ -205,8 +250,13 @@ async function createOneShotDaytonaSandbox(input: {
|
|
|
205
250
|
});
|
|
206
251
|
|
|
207
252
|
const commonParams = {
|
|
253
|
+
name: input.sandboxName,
|
|
208
254
|
labels,
|
|
255
|
+
// This is also the provider-owned backstop for an HTTP create timeout
|
|
256
|
+
// whose named sandbox never becomes lookup-addressable to this process:
|
|
257
|
+
// Daytona deletes an ephemeral sandbox immediately when it stops.
|
|
209
258
|
ephemeral: true,
|
|
259
|
+
autoDeleteInterval: 0,
|
|
210
260
|
autoStopInterval: Math.ceil(
|
|
211
261
|
(limits.timeoutSeconds + PLAY_RUNNER_TIMEOUT_SECONDS - 30 * 60) / 60,
|
|
212
262
|
),
|
|
@@ -224,6 +274,34 @@ async function createOneShotDaytonaSandbox(input: {
|
|
|
224
274
|
});
|
|
225
275
|
}
|
|
226
276
|
|
|
277
|
+
async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
|
|
278
|
+
daytona: DaytonaClient;
|
|
279
|
+
sandboxName: string;
|
|
280
|
+
}): Promise<boolean> {
|
|
281
|
+
if (!input.daytona.get) return false;
|
|
282
|
+
for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
|
|
283
|
+
if (delayMs > 0) {
|
|
284
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
285
|
+
}
|
|
286
|
+
let sandbox: DaytonaSandbox;
|
|
287
|
+
try {
|
|
288
|
+
sandbox = await input.daytona.get(input.sandboxName);
|
|
289
|
+
} catch {
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
try {
|
|
293
|
+
await sandbox.delete(30);
|
|
294
|
+
} catch (error) {
|
|
295
|
+
throw new Error(
|
|
296
|
+
`Daytona timed-out sandbox ${sandbox.id} was reconciled by name but could not be deleted. Modal fallback suppressed.`,
|
|
297
|
+
{ cause: error },
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
return true;
|
|
301
|
+
}
|
|
302
|
+
return false;
|
|
303
|
+
}
|
|
304
|
+
|
|
227
305
|
async function createRetriedOneShotDaytonaSandbox(input: {
|
|
228
306
|
daytona: DaytonaClient;
|
|
229
307
|
orgId: string;
|
|
@@ -234,6 +312,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
234
312
|
const errors: string[] = [];
|
|
235
313
|
for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
|
|
236
314
|
const attempt = index + 1;
|
|
315
|
+
const sandboxName = `dl-${crypto.randomUUID()}`;
|
|
237
316
|
if (delayMs > 0) {
|
|
238
317
|
input.emitStage('create:retry', {
|
|
239
318
|
attempt: index,
|
|
@@ -249,6 +328,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
249
328
|
daytona: input.daytona,
|
|
250
329
|
orgId: input.orgId,
|
|
251
330
|
context: input.context,
|
|
331
|
+
sandboxName,
|
|
252
332
|
});
|
|
253
333
|
return {
|
|
254
334
|
sandbox,
|
|
@@ -270,11 +350,34 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
270
350
|
attempt,
|
|
271
351
|
error: message,
|
|
272
352
|
});
|
|
353
|
+
if (isDaytonaSandboxStartTimeout(message)) {
|
|
354
|
+
const deleted = await reconcileAndDeleteTimedOutDaytonaSandbox({
|
|
355
|
+
daytona: input.daytona,
|
|
356
|
+
sandboxName,
|
|
357
|
+
});
|
|
358
|
+
if (deleted) {
|
|
359
|
+
throw new DaytonaSandboxAcquisitionUnavailableError(
|
|
360
|
+
'daytona_sandbox_start_timeout',
|
|
361
|
+
`${message} Timed-out Daytona sandbox was reconciled and deleted before fallback.`,
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
throw new Error(
|
|
365
|
+
`${message} Daytona did not return a cleanup-addressable sandbox identity; Modal fallback suppressed. The named ephemeral sandbox retains Daytona's auto-delete-on-stop backstop.`,
|
|
366
|
+
{ cause: error },
|
|
367
|
+
);
|
|
368
|
+
}
|
|
273
369
|
}
|
|
274
370
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
371
|
+
const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`;
|
|
372
|
+
const fallbackReason =
|
|
373
|
+
resolveDaytonaSandboxAcquisitionUnavailableReason(errors);
|
|
374
|
+
if (fallbackReason) {
|
|
375
|
+
throw new DaytonaSandboxAcquisitionUnavailableError(
|
|
376
|
+
fallbackReason,
|
|
377
|
+
message,
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
throw new Error(message);
|
|
278
381
|
}
|
|
279
382
|
|
|
280
383
|
async function acquireOneShotDaytonaSandbox(input: {
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlayRunnerBackend,
|
|
3
|
+
PlayRunnerCallbacks,
|
|
4
|
+
PlayRunnerPreparedExecution,
|
|
5
|
+
PlayRunnerPrepareInput,
|
|
6
|
+
} from '../types';
|
|
7
|
+
import type {
|
|
8
|
+
PlayRunnerExecutionConfig,
|
|
9
|
+
PlayRunnerResult,
|
|
10
|
+
} from '@shared_libs/play-runtime/protocol';
|
|
11
|
+
import { daytonaPlayRunnerBackend } from './daytona';
|
|
12
|
+
import { DaytonaSandboxAcquisitionUnavailableError } from './daytona-lifecycle';
|
|
13
|
+
import { modalPlayRunnerBackend } from './modal';
|
|
14
|
+
import {
|
|
15
|
+
canTransitionRuntimeSandboxPlacement,
|
|
16
|
+
resolveRuntimeSandboxPlacementPolicy,
|
|
17
|
+
RUNTIME_SANDBOX_PLACEMENT_POLICIES,
|
|
18
|
+
type RuntimeSandboxPlacementPolicyId,
|
|
19
|
+
type RuntimeSandboxPlacementFailureReason,
|
|
20
|
+
type RuntimeSandboxPlacementPolicy,
|
|
21
|
+
type RuntimeSandboxProvider,
|
|
22
|
+
} from '@shared_libs/play-runtime/runtime-sandbox-placement-policy';
|
|
23
|
+
|
|
24
|
+
type DaytonaFallbackPreparedExecution = PlayRunnerPreparedExecution & {
|
|
25
|
+
kind: 'runtime_sandbox_placement';
|
|
26
|
+
policyId: string;
|
|
27
|
+
provider: RuntimeSandboxProvider;
|
|
28
|
+
providerPrepared: PlayRunnerPreparedExecution | undefined;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
function isPrepared(
|
|
32
|
+
prepared: PlayRunnerPreparedExecution | undefined,
|
|
33
|
+
): prepared is DaytonaFallbackPreparedExecution {
|
|
34
|
+
return prepared?.kind === 'runtime_sandbox_placement';
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type RuntimeSandboxProviderAdapter = {
|
|
38
|
+
backend: PlayRunnerBackend;
|
|
39
|
+
classifyAcquisitionFailure: (
|
|
40
|
+
error: unknown,
|
|
41
|
+
) => RuntimeSandboxPlacementFailureReason | null;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
function classifyDaytonaAcquisitionFailure(
|
|
45
|
+
error: unknown,
|
|
46
|
+
): RuntimeSandboxPlacementFailureReason | null {
|
|
47
|
+
if (!(error instanceof DaytonaSandboxAcquisitionUnavailableError)) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
if (error.reason === 'daytona_total_cpu_limit_exceeded') {
|
|
51
|
+
return 'provider_capacity_exhausted';
|
|
52
|
+
}
|
|
53
|
+
if (error.reason === 'daytona_sandbox_start_timeout') {
|
|
54
|
+
return 'provider_start_timeout_before_execution';
|
|
55
|
+
}
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Deep placement Implementation. Provider order alone never permits a retry:
|
|
61
|
+
* the policy must also name the exact pre-execution transition and the source
|
|
62
|
+
* Adapter must classify the thrown failure into that provider-neutral reason.
|
|
63
|
+
*/
|
|
64
|
+
export function createRuntimeSandboxPlacementBackend(input: {
|
|
65
|
+
policy: RuntimeSandboxPlacementPolicy;
|
|
66
|
+
adapters: Readonly<
|
|
67
|
+
Partial<Record<RuntimeSandboxProvider, RuntimeSandboxProviderAdapter>>
|
|
68
|
+
>;
|
|
69
|
+
}): PlayRunnerBackend {
|
|
70
|
+
const primaryProvider = input.policy.providers[0];
|
|
71
|
+
const primaryAdapter = input.adapters[primaryProvider];
|
|
72
|
+
if (!primaryAdapter) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${primaryProvider}.`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
async prepare(
|
|
79
|
+
prepareInput: PlayRunnerPrepareInput,
|
|
80
|
+
callbacks?: PlayRunnerCallbacks,
|
|
81
|
+
): Promise<PlayRunnerPreparedExecution> {
|
|
82
|
+
const providerPrepared = await primaryAdapter.backend.prepare?.(
|
|
83
|
+
prepareInput,
|
|
84
|
+
callbacks,
|
|
85
|
+
);
|
|
86
|
+
const prepared: DaytonaFallbackPreparedExecution = {
|
|
87
|
+
kind: 'runtime_sandbox_placement',
|
|
88
|
+
policyId: input.policy.id,
|
|
89
|
+
provider: primaryProvider,
|
|
90
|
+
providerPrepared,
|
|
91
|
+
dispose: async () => await providerPrepared?.dispose?.(),
|
|
92
|
+
};
|
|
93
|
+
return prepared;
|
|
94
|
+
},
|
|
95
|
+
|
|
96
|
+
async execute(
|
|
97
|
+
config: PlayRunnerExecutionConfig,
|
|
98
|
+
callbacks?: PlayRunnerCallbacks,
|
|
99
|
+
prepared?: PlayRunnerPreparedExecution,
|
|
100
|
+
): Promise<PlayRunnerResult> {
|
|
101
|
+
const policyPrepared =
|
|
102
|
+
isPrepared(prepared) && prepared.policyId === input.policy.id
|
|
103
|
+
? prepared
|
|
104
|
+
: null;
|
|
105
|
+
for (const [index, provider] of input.policy.providers.entries()) {
|
|
106
|
+
const adapter = input.adapters[provider];
|
|
107
|
+
if (!adapter) {
|
|
108
|
+
throw new Error(
|
|
109
|
+
`Runtime sandbox placement policy ${input.policy.id} has no Adapter for ${provider}.`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
return await adapter.backend.execute(
|
|
114
|
+
config,
|
|
115
|
+
callbacks,
|
|
116
|
+
index === 0 && policyPrepared?.provider === provider
|
|
117
|
+
? policyPrepared.providerPrepared
|
|
118
|
+
: undefined,
|
|
119
|
+
);
|
|
120
|
+
} catch (error) {
|
|
121
|
+
const nextProvider = input.policy.providers[index + 1];
|
|
122
|
+
const reason = adapter.classifyAcquisitionFailure(error);
|
|
123
|
+
if (
|
|
124
|
+
!nextProvider ||
|
|
125
|
+
!reason ||
|
|
126
|
+
callbacks?.cancellationSignal?.aborted ||
|
|
127
|
+
!canTransitionRuntimeSandboxPlacement({
|
|
128
|
+
policy: input.policy,
|
|
129
|
+
from: provider,
|
|
130
|
+
to: nextProvider,
|
|
131
|
+
stage: 'acquisition',
|
|
132
|
+
reason,
|
|
133
|
+
})
|
|
134
|
+
) {
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
console.warn('[play-runner.sandbox_provider_fallback]', {
|
|
138
|
+
runId: config.context.runId ?? null,
|
|
139
|
+
workflowId: config.context.workflowId ?? null,
|
|
140
|
+
policyId: input.policy.id,
|
|
141
|
+
from: provider,
|
|
142
|
+
to: nextProvider,
|
|
143
|
+
stage: 'acquisition',
|
|
144
|
+
reason,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
throw new Error(
|
|
149
|
+
`Runtime sandbox placement policy ${input.policy.id} exhausted without a result.`,
|
|
150
|
+
);
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function createRuntimeSandboxPlacementBackendForPolicy(
|
|
156
|
+
policyId: RuntimeSandboxPlacementPolicyId | string,
|
|
157
|
+
overrides: {
|
|
158
|
+
daytona?: PlayRunnerBackend;
|
|
159
|
+
modal?: PlayRunnerBackend;
|
|
160
|
+
} = {},
|
|
161
|
+
): PlayRunnerBackend {
|
|
162
|
+
return createRuntimeSandboxPlacementBackend({
|
|
163
|
+
policy: resolveRuntimeSandboxPlacementPolicy(policyId),
|
|
164
|
+
adapters: {
|
|
165
|
+
daytona: {
|
|
166
|
+
backend: overrides.daytona ?? daytonaPlayRunnerBackend,
|
|
167
|
+
classifyAcquisitionFailure: classifyDaytonaAcquisitionFailure,
|
|
168
|
+
},
|
|
169
|
+
modal: {
|
|
170
|
+
backend: overrides.modal ?? modalPlayRunnerBackend,
|
|
171
|
+
classifyAcquisitionFailure: () => null,
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Compatibility factory retained for existing imports and tests. */
|
|
178
|
+
export function createDaytonaModalFallbackBackend(
|
|
179
|
+
input: {
|
|
180
|
+
daytona?: PlayRunnerBackend;
|
|
181
|
+
modal?: PlayRunnerBackend;
|
|
182
|
+
} = {},
|
|
183
|
+
): PlayRunnerBackend {
|
|
184
|
+
return createRuntimeSandboxPlacementBackendForPolicy(
|
|
185
|
+
RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1,
|
|
186
|
+
input,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export const daytonaModalFallbackPlayRunnerBackend =
|
|
191
|
+
createDaytonaModalFallbackBackend();
|
|
192
|
+
|
|
193
|
+
export const daytonaOnlyPlayRunnerBackend =
|
|
194
|
+
createRuntimeSandboxPlacementBackendForPolicy(
|
|
195
|
+
RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1,
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
export const modalOnlyPlayRunnerBackend =
|
|
199
|
+
createRuntimeSandboxPlacementBackendForPolicy(
|
|
200
|
+
RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1,
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
const DEFAULT_POLICY_BACKENDS: Readonly<
|
|
204
|
+
Record<RuntimeSandboxPlacementPolicyId, PlayRunnerBackend>
|
|
205
|
+
> = {
|
|
206
|
+
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaOnlyV1]:
|
|
207
|
+
daytonaOnlyPlayRunnerBackend,
|
|
208
|
+
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.daytonaThenModalV1]:
|
|
209
|
+
daytonaModalFallbackPlayRunnerBackend,
|
|
210
|
+
[RUNTIME_SANDBOX_PLACEMENT_POLICIES.modalOnlyV1]: modalOnlyPlayRunnerBackend,
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
export function resolveDefaultRuntimeSandboxPlacementBackend(
|
|
214
|
+
policyId: RuntimeSandboxPlacementPolicyId | string,
|
|
215
|
+
): PlayRunnerBackend {
|
|
216
|
+
const policy = resolveRuntimeSandboxPlacementPolicy(policyId);
|
|
217
|
+
return DEFAULT_POLICY_BACKENDS[policy.id];
|
|
218
|
+
}
|
|
@@ -65,6 +65,11 @@ export type StagedDaytonaPayload = {
|
|
|
65
65
|
progressEventPath: string;
|
|
66
66
|
};
|
|
67
67
|
|
|
68
|
+
export type RemoteRunnerPayloadSandbox = {
|
|
69
|
+
id: string;
|
|
70
|
+
uploadFile(content: Buffer, path: string): Promise<void>;
|
|
71
|
+
};
|
|
72
|
+
|
|
68
73
|
function shellQuote(value: string): string {
|
|
69
74
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
70
75
|
}
|
|
@@ -390,23 +395,23 @@ function remoteRuntimeContextForDaytona(
|
|
|
390
395
|
};
|
|
391
396
|
}
|
|
392
397
|
|
|
393
|
-
async function
|
|
394
|
-
sandbox:
|
|
398
|
+
async function timedRunnerPayloadUpload(input: {
|
|
399
|
+
sandbox: RemoteRunnerPayloadSandbox;
|
|
395
400
|
emitStage: DaytonaStageEmitter;
|
|
396
401
|
label: string;
|
|
397
402
|
path: string;
|
|
398
403
|
content: Buffer;
|
|
399
404
|
}) {
|
|
400
405
|
const startedAt = Date.now();
|
|
401
|
-
await input.sandbox.
|
|
406
|
+
await input.sandbox.uploadFile(input.content, input.path);
|
|
402
407
|
input.emitStage(`upload:${input.label}:done`, {
|
|
403
408
|
bytes: input.content.byteLength,
|
|
404
409
|
elapsedMs: Date.now() - startedAt,
|
|
405
410
|
});
|
|
406
411
|
}
|
|
407
412
|
|
|
408
|
-
export async function
|
|
409
|
-
sandbox:
|
|
413
|
+
export async function stageRunnerPayload(input: {
|
|
414
|
+
sandbox: RemoteRunnerPayloadSandbox;
|
|
410
415
|
bundlePromise: Promise<string>;
|
|
411
416
|
config: PlayRunnerExecutionConfig;
|
|
412
417
|
workDir: string;
|
|
@@ -458,7 +463,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
458
463
|
|
|
459
464
|
await Promise.all(
|
|
460
465
|
[...uniqueLocalPaths.entries()].map(async ([localPath, remotePath]) => {
|
|
461
|
-
await
|
|
466
|
+
await timedRunnerPayloadUpload({
|
|
462
467
|
sandbox: input.sandbox,
|
|
463
468
|
emitStage: input.emitStage,
|
|
464
469
|
label:
|
|
@@ -508,7 +513,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
508
513
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
|
|
509
514
|
|
|
510
515
|
const envelopeUpload = input.bundlePromise.then((bundle) =>
|
|
511
|
-
|
|
516
|
+
timedRunnerPayloadUpload({
|
|
512
517
|
sandbox: input.sandbox,
|
|
513
518
|
emitStage: input.emitStage,
|
|
514
519
|
label: 'payload',
|
|
@@ -583,3 +588,17 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
583
588
|
progressEventPath,
|
|
584
589
|
};
|
|
585
590
|
}
|
|
591
|
+
|
|
592
|
+
export async function stageDaytonaRunnerPayload(
|
|
593
|
+
input: Omit<Parameters<typeof stageRunnerPayload>[0], 'sandbox'> & {
|
|
594
|
+
sandbox: DaytonaSandbox;
|
|
595
|
+
},
|
|
596
|
+
): Promise<StagedDaytonaPayload> {
|
|
597
|
+
return await stageRunnerPayload({
|
|
598
|
+
...input,
|
|
599
|
+
sandbox: {
|
|
600
|
+
id: input.sandbox.id,
|
|
601
|
+
uploadFile: (content, path) => input.sandbox.fs.uploadFile(content, path),
|
|
602
|
+
},
|
|
603
|
+
});
|
|
604
|
+
}
|