deepline 0.2.74 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +75 -4
- package/dist/bundling-sources/sdk/src/compat.ts +4 -0
- package/dist/bundling-sources/sdk/src/play.ts +7 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +15 -1
- package/dist/bundling-sources/sdk/src/release.ts +12 -1
- package/dist/bundling-sources/sdk/src/types.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +146 -29
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +57 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +20 -19
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +100 -25
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +25 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +0 -9
- package/dist/bundling-sources/shared_libs/play-runtime/tool-response-contract.ts +89 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +29 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +203 -16
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +6 -5
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +3 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +5 -2
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +14 -0
- package/dist/cli/index.js +123 -47
- package/dist/cli/index.mjs +86 -10
- package/dist/{compiler-manifest-DFuz9-0_.d.mts → compiler-manifest-BX85pKXW.d.mts} +7 -2
- package/dist/{compiler-manifest-DFuz9-0_.d.ts → compiler-manifest-BX85pKXW.d.ts} +7 -2
- package/dist/index.d.mts +14 -3
- package/dist/index.d.ts +14 -3
- package/dist/index.js +85 -4
- package/dist/index.mjs +85 -4
- package/dist/install-integrity.json +3 -2
- package/dist/plays/bundle-play-file.d.mts +12 -2
- package/dist/plays/bundle-play-file.d.ts +12 -2
- package/dist/plays/bundle-play-file.mjs +20 -4
- package/package.json +1 -1
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -51,6 +51,7 @@ export type AcquiredDaytonaSandbox = {
|
|
|
51
51
|
daytonaOrganizationId: string;
|
|
52
52
|
billingStartedAt: number;
|
|
53
53
|
billingEndedAt?: number;
|
|
54
|
+
sandboxCapacityLeaseId?: string;
|
|
54
55
|
};
|
|
55
56
|
|
|
56
57
|
export type OneShotDaytonaSandboxLifecycle = {
|
|
@@ -118,6 +119,7 @@ type DaytonaCreateResult = {
|
|
|
118
119
|
sandbox: DaytonaSandbox;
|
|
119
120
|
attempt: number;
|
|
120
121
|
attemptElapsedMs: number;
|
|
122
|
+
sandboxCapacityLeaseId?: string;
|
|
121
123
|
};
|
|
122
124
|
|
|
123
125
|
function daytonaCreateErrorClass(
|
|
@@ -133,6 +135,8 @@ function daytonaCreateErrorClass(
|
|
|
133
135
|
async function rejectAcquiredSandbox(
|
|
134
136
|
sandbox: DaytonaSandbox,
|
|
135
137
|
reason: string,
|
|
138
|
+
reservation?: { leaseId: string },
|
|
139
|
+
releaseSandboxCapacity?: (leaseId: string) => Promise<void>,
|
|
136
140
|
): Promise<never> {
|
|
137
141
|
try {
|
|
138
142
|
await sandbox.delete(30);
|
|
@@ -143,6 +147,9 @@ async function rejectAcquiredSandbox(
|
|
|
143
147
|
{ cause: error },
|
|
144
148
|
);
|
|
145
149
|
}
|
|
150
|
+
if (reservation) {
|
|
151
|
+
await releaseSandboxCapacity?.(reservation.leaseId);
|
|
152
|
+
}
|
|
146
153
|
throw new Error(reason);
|
|
147
154
|
}
|
|
148
155
|
|
|
@@ -350,6 +357,8 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
350
357
|
context: DaytonaExecutionContext;
|
|
351
358
|
emitStage: DaytonaStageEmitter;
|
|
352
359
|
observeCreateCall?: DaytonaCreateCallObserver;
|
|
360
|
+
reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
|
|
361
|
+
releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
|
|
353
362
|
nextProviderAttempt: () => number;
|
|
354
363
|
startedAt: number;
|
|
355
364
|
}): Promise<DaytonaCreateResult> {
|
|
@@ -372,6 +381,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
372
381
|
occurredAtMs: attemptStartedAt,
|
|
373
382
|
providerAttempt: attempt,
|
|
374
383
|
});
|
|
384
|
+
const reservation = await input.reserveSandboxCapacity?.();
|
|
375
385
|
let sandbox: DaytonaSandbox;
|
|
376
386
|
try {
|
|
377
387
|
sandbox = await createOneShotDaytonaSandbox({
|
|
@@ -381,6 +391,12 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
381
391
|
sandboxName,
|
|
382
392
|
});
|
|
383
393
|
} catch (error) {
|
|
394
|
+
// A normal rejected create has no durable provider resource. A timeout
|
|
395
|
+
// can be ambiguous, so retain its lease until the crash TTL rather than
|
|
396
|
+
// admitting another sandbox against an unknown provider outcome.
|
|
397
|
+
if (reservation && !isDaytonaSandboxStartTimeout(String(error))) {
|
|
398
|
+
await input.releaseSandboxCapacity?.(reservation.leaseId);
|
|
399
|
+
}
|
|
384
400
|
await input.observeCreateCall?.({
|
|
385
401
|
type: 'daytona_create_call_failed',
|
|
386
402
|
occurredAtMs: Date.now(),
|
|
@@ -407,6 +423,13 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
407
423
|
sandboxName,
|
|
408
424
|
});
|
|
409
425
|
if (deleted) {
|
|
426
|
+
// The provider-confirmed reconciliation means this reservation can
|
|
427
|
+
// no longer represent a live sandbox. Release before returning the
|
|
428
|
+
// acquisition-unavailable result so Modal fallback is not blocked
|
|
429
|
+
// behind the unbound crash TTL.
|
|
430
|
+
if (reservation) {
|
|
431
|
+
await input.releaseSandboxCapacity?.(reservation.leaseId);
|
|
432
|
+
}
|
|
410
433
|
throw new DaytonaSandboxAcquisitionUnavailableError(
|
|
411
434
|
'daytona_sandbox_start_timeout',
|
|
412
435
|
`${message} Timed-out Daytona sandbox was reconciled and deleted before fallback.`,
|
|
@@ -448,6 +471,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
448
471
|
sandbox,
|
|
449
472
|
attempt,
|
|
450
473
|
attemptElapsedMs: acquiredAt - attemptStartedAt,
|
|
474
|
+
...(reservation ? { sandboxCapacityLeaseId: reservation.leaseId } : {}),
|
|
451
475
|
};
|
|
452
476
|
}
|
|
453
477
|
const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`;
|
|
@@ -468,6 +492,8 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
468
492
|
context: DaytonaExecutionContext;
|
|
469
493
|
emitStage: DaytonaStageEmitter;
|
|
470
494
|
observeCreateCall?: DaytonaCreateCallObserver;
|
|
495
|
+
reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
|
|
496
|
+
releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
|
|
471
497
|
nextProviderAttempt: () => number;
|
|
472
498
|
startedAt: number;
|
|
473
499
|
}): Promise<AcquiredDaytonaSandbox> {
|
|
@@ -493,6 +519,10 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
493
519
|
await rejectAcquiredSandbox(
|
|
494
520
|
result.sandbox,
|
|
495
521
|
`Daytona sandbox resource boundary mismatch: expected cpu=${limits.cpu} memoryGiB=${limits.memoryGiB} diskGiB=${limits.diskGiB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
|
|
522
|
+
result.sandboxCapacityLeaseId
|
|
523
|
+
? { leaseId: result.sandboxCapacityLeaseId }
|
|
524
|
+
: undefined,
|
|
525
|
+
input.releaseSandboxCapacity,
|
|
496
526
|
);
|
|
497
527
|
}
|
|
498
528
|
const configuredOrganizationId =
|
|
@@ -506,6 +536,10 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
506
536
|
await rejectAcquiredSandbox(
|
|
507
537
|
result.sandbox,
|
|
508
538
|
'Daytona sandbox organization routing mismatch. Refusing to run customer code in a sandbox whose observed organization differs from the configured organization.',
|
|
539
|
+
result.sandboxCapacityLeaseId
|
|
540
|
+
? { leaseId: result.sandboxCapacityLeaseId }
|
|
541
|
+
: undefined,
|
|
542
|
+
input.releaseSandboxCapacity,
|
|
509
543
|
);
|
|
510
544
|
}
|
|
511
545
|
let lookupOrganizationId: string | null = null;
|
|
@@ -528,6 +562,10 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
528
562
|
return await rejectAcquiredSandbox(
|
|
529
563
|
result.sandbox,
|
|
530
564
|
'Daytona sandbox organization routing identity is missing. Refusing to run customer code without a durable cleanup routing domain.',
|
|
565
|
+
result.sandboxCapacityLeaseId
|
|
566
|
+
? { leaseId: result.sandboxCapacityLeaseId }
|
|
567
|
+
: undefined,
|
|
568
|
+
input.releaseSandboxCapacity,
|
|
531
569
|
);
|
|
532
570
|
}
|
|
533
571
|
const billingStartedAt = Date.now();
|
|
@@ -541,7 +579,14 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
541
579
|
memoryGiB: granted.memoryGiB,
|
|
542
580
|
diskGiB: granted.diskGiB,
|
|
543
581
|
});
|
|
544
|
-
return {
|
|
582
|
+
return {
|
|
583
|
+
sandbox,
|
|
584
|
+
daytonaOrganizationId,
|
|
585
|
+
billingStartedAt,
|
|
586
|
+
...(result.sandboxCapacityLeaseId
|
|
587
|
+
? { sandboxCapacityLeaseId: result.sandboxCapacityLeaseId }
|
|
588
|
+
: {}),
|
|
589
|
+
};
|
|
545
590
|
}
|
|
546
591
|
|
|
547
592
|
export function createOneShotDaytonaSandboxLifecycle(input: {
|
|
@@ -549,6 +594,8 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
|
|
|
549
594
|
context: DaytonaExecutionContext;
|
|
550
595
|
emitStage: DaytonaStageEmitter;
|
|
551
596
|
observeCreateCall?: DaytonaCreateCallObserver;
|
|
597
|
+
reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
|
|
598
|
+
releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
|
|
552
599
|
startedAt?: number;
|
|
553
600
|
}): OneShotDaytonaSandboxLifecycle {
|
|
554
601
|
const orgId = validateDaytonaExecutionContext(input.context);
|
|
@@ -564,6 +611,8 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
|
|
|
564
611
|
context: input.context,
|
|
565
612
|
emitStage: input.emitStage,
|
|
566
613
|
observeCreateCall: input.observeCreateCall,
|
|
614
|
+
reserveSandboxCapacity: input.reserveSandboxCapacity,
|
|
615
|
+
releaseSandboxCapacity: input.releaseSandboxCapacity,
|
|
567
616
|
nextProviderAttempt: () => {
|
|
568
617
|
providerAttempt += 1;
|
|
569
618
|
return providerAttempt;
|
|
@@ -594,6 +643,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
|
|
|
594
643
|
try {
|
|
595
644
|
const acquired = await latestAcquiredSandboxPromise;
|
|
596
645
|
await acquired.sandbox.delete(30);
|
|
646
|
+
// `dispose` has provider-confirmed that the sandbox is gone. This is
|
|
647
|
+
// normally the narrow cancellation window before the scheduler has
|
|
648
|
+
// durably recorded the resource, so no cleanup job will release an
|
|
649
|
+
// unbound lease on our behalf.
|
|
650
|
+
if (acquired.sandboxCapacityLeaseId) {
|
|
651
|
+
await input.releaseSandboxCapacity?.(acquired.sandboxCapacityLeaseId);
|
|
652
|
+
}
|
|
597
653
|
} catch (error) {
|
|
598
654
|
console.warn('[play-runner.daytona.dispose_failed_before_acquire]', {
|
|
599
655
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -5,7 +5,10 @@ import type {
|
|
|
5
5
|
PlayRunnerPreparedExecution,
|
|
6
6
|
PlayRunnerPrepareInput,
|
|
7
7
|
} from '../types';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
isRuntimeSandboxCapacityLimitError,
|
|
10
|
+
RuntimeResourceFenceLostError,
|
|
11
|
+
} from '../types';
|
|
9
12
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
10
13
|
import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
|
|
11
14
|
import type {
|
|
@@ -26,6 +29,7 @@ import {
|
|
|
26
29
|
DaytonaSandboxAcquisitionUnavailableError,
|
|
27
30
|
createDaytonaSandboxCleanupManager,
|
|
28
31
|
createOneShotDaytonaSandboxLifecycle,
|
|
32
|
+
type AcquiredDaytonaSandbox,
|
|
29
33
|
type DaytonaClient,
|
|
30
34
|
type DaytonaExecutionContext,
|
|
31
35
|
type DaytonaSandbox,
|
|
@@ -945,6 +949,10 @@ function prepareDaytonaExecution(
|
|
|
945
949
|
emitStage: (stage, extra) =>
|
|
946
950
|
emitDaytonaStage(callbacks, input.context, stage, extra),
|
|
947
951
|
observeCreateCall: callbacks?.onRuntimeLifecycleEvent,
|
|
952
|
+
reserveSandboxCapacity: callbacks?.reserveSandboxCapacity
|
|
953
|
+
? () => callbacks.reserveSandboxCapacity!('daytona')
|
|
954
|
+
: undefined,
|
|
955
|
+
releaseSandboxCapacity: callbacks?.releaseSandboxCapacity,
|
|
948
956
|
});
|
|
949
957
|
return {
|
|
950
958
|
kind: 'daytona',
|
|
@@ -1055,22 +1063,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1055
1063
|
};
|
|
1056
1064
|
let cancellationRequested = false;
|
|
1057
1065
|
let cancellationCleanupStarted = false;
|
|
1058
|
-
let activeAcquiredResource:
|
|
1059
|
-
sandbox: DaytonaSandbox;
|
|
1060
|
-
daytonaOrganizationId: string;
|
|
1061
|
-
billingStartedAt: number;
|
|
1062
|
-
billingEndedAt?: number;
|
|
1063
|
-
} | null = null;
|
|
1066
|
+
let activeAcquiredResource: AcquiredDaytonaSandbox | null = null;
|
|
1064
1067
|
const uploadDeadlineBreaches: DaytonaUploadAttemptTiming[] = [];
|
|
1065
1068
|
const reportedSandboxIds = new Set<string>();
|
|
1066
1069
|
const reportedSandboxEndTimes = new Map<string, number | null>();
|
|
1067
1070
|
const runtimeResourceReportErrors = new Set<unknown>();
|
|
1068
|
-
const reportRuntimeResource = async (acquired: {
|
|
1069
|
-
sandbox: DaytonaSandbox;
|
|
1070
|
-
daytonaOrganizationId: string;
|
|
1071
|
-
billingStartedAt: number;
|
|
1072
|
-
billingEndedAt?: number;
|
|
1073
|
-
}) => {
|
|
1071
|
+
const reportRuntimeResource = async (acquired: AcquiredDaytonaSandbox) => {
|
|
1074
1072
|
const sandbox = acquired.sandbox;
|
|
1075
1073
|
const billingEndedAt = acquired.billingEndedAt ?? null;
|
|
1076
1074
|
if (
|
|
@@ -1101,6 +1099,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1101
1099
|
cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
|
|
1102
1100
|
memoryGiB: typeof sandbox.memory === 'number' ? sandbox.memory : null,
|
|
1103
1101
|
diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
|
|
1102
|
+
...(acquired.sandboxCapacityLeaseId
|
|
1103
|
+
? { sandboxCapacityLeaseId: acquired.sandboxCapacityLeaseId }
|
|
1104
|
+
: {}),
|
|
1104
1105
|
});
|
|
1105
1106
|
} catch (error) {
|
|
1106
1107
|
runtimeResourceReportErrors.add(error);
|
|
@@ -1109,12 +1110,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1109
1110
|
reportedSandboxIds.add(sandbox.id);
|
|
1110
1111
|
reportedSandboxEndTimes.set(sandbox.id, billingEndedAt);
|
|
1111
1112
|
};
|
|
1112
|
-
const reportRetiringRuntimeResource = async (
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
billingStartedAt: number;
|
|
1116
|
-
billingEndedAt?: number;
|
|
1117
|
-
}) => {
|
|
1113
|
+
const reportRetiringRuntimeResource = async (
|
|
1114
|
+
acquired: AcquiredDaytonaSandbox,
|
|
1115
|
+
) => {
|
|
1118
1116
|
try {
|
|
1119
1117
|
await reportRuntimeResource(acquired);
|
|
1120
1118
|
} catch (error) {
|
|
@@ -1429,6 +1427,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1429
1427
|
}
|
|
1430
1428
|
throw error;
|
|
1431
1429
|
}
|
|
1430
|
+
if (isRuntimeSandboxCapacityLimitError(error)) {
|
|
1431
|
+
throw error;
|
|
1432
|
+
}
|
|
1432
1433
|
if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
|
|
1433
1434
|
throw error;
|
|
1434
1435
|
}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { ModalClient } from 'modal';
|
|
2
2
|
import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
isRuntimeSandboxCapacityLimitError,
|
|
5
|
+
RuntimeResourceFenceLostError,
|
|
6
|
+
} from '../types';
|
|
4
7
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
5
8
|
import type {
|
|
6
9
|
PlayRunnerExecutionConfig,
|
|
@@ -23,6 +26,7 @@ import { stageRunnerPayload } from './daytona-payload-transport';
|
|
|
23
26
|
import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
|
|
24
27
|
|
|
25
28
|
const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
|
|
29
|
+
const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
|
|
26
30
|
type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
|
|
27
31
|
|
|
28
32
|
export function modalSandboxLifetimeMs(
|
|
@@ -99,13 +103,38 @@ async function confirmDetachedModalRunnerReady(input: {
|
|
|
99
103
|
);
|
|
100
104
|
}
|
|
101
105
|
|
|
102
|
-
async function terminateModalSandbox(sandbox: ModalSandbox): Promise<
|
|
103
|
-
|
|
106
|
+
async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
|
|
107
|
+
try {
|
|
108
|
+
// The default terminate call only acknowledges the request. Capacity may
|
|
109
|
+
// be released only after Modal confirms the sandbox itself has stopped.
|
|
110
|
+
await sandbox.terminate({ wait: true });
|
|
111
|
+
return true;
|
|
112
|
+
} catch (error) {
|
|
104
113
|
console.warn('[play-runner.modal.terminate_failed]', {
|
|
105
114
|
sandboxId: sandbox.sandboxId,
|
|
106
115
|
error: error instanceof Error ? error.message : String(error),
|
|
107
116
|
});
|
|
108
|
-
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function releaseModalSandboxCapacityAfterConfirmedTermination(input: {
|
|
122
|
+
leaseId: string;
|
|
123
|
+
release: NonNullable<PlayRunnerCallbacks['releaseSandboxCapacity']>;
|
|
124
|
+
}): Promise<void> {
|
|
125
|
+
let lastError: unknown;
|
|
126
|
+
for (const delayMs of MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS) {
|
|
127
|
+
if (delayMs > 0) {
|
|
128
|
+
await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
|
|
129
|
+
}
|
|
130
|
+
try {
|
|
131
|
+
await input.release(input.leaseId);
|
|
132
|
+
return;
|
|
133
|
+
} catch (error) {
|
|
134
|
+
lastError = error;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
throw lastError;
|
|
109
138
|
}
|
|
110
139
|
|
|
111
140
|
export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
@@ -139,24 +168,35 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
139
168
|
createIfMissing: true,
|
|
140
169
|
});
|
|
141
170
|
const image = modalConfig.client.images.fromRegistry(modalConfig.image);
|
|
142
|
-
const
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
171
|
+
const capacityReservation =
|
|
172
|
+
await callbacks?.reserveSandboxCapacity?.('modal');
|
|
173
|
+
const sandbox = await modalConfig.client.sandboxes
|
|
174
|
+
.create(app, image, {
|
|
175
|
+
cpu: MODAL_SANDBOX_CPU_CORES,
|
|
176
|
+
cpuLimit: MODAL_SANDBOX_CPU_CORES,
|
|
177
|
+
memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
|
|
178
|
+
memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
|
|
179
|
+
timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
180
|
+
idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
|
|
181
|
+
workdir: modalConfig.workdir,
|
|
182
|
+
tags: {
|
|
183
|
+
source: 'deepline-play-runner',
|
|
184
|
+
orgId: config.context.orgId ?? 'unknown',
|
|
185
|
+
workflowId: config.context.workflowId ?? 'unknown',
|
|
186
|
+
runId: config.context.runId ?? 'unknown',
|
|
187
|
+
},
|
|
188
|
+
...(modalConfig.outboundCidrAllowlist
|
|
189
|
+
? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
|
|
190
|
+
: {}),
|
|
191
|
+
})
|
|
192
|
+
.catch(async (error) => {
|
|
193
|
+
if (capacityReservation) {
|
|
194
|
+
await callbacks?.releaseSandboxCapacity?.(
|
|
195
|
+
capacityReservation.leaseId,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
throw error;
|
|
199
|
+
});
|
|
160
200
|
const billingStartedAt = Date.now();
|
|
161
201
|
runtimeTiming.modalCreateMs = billingStartedAt - startedAt;
|
|
162
202
|
emitModalStage(config.context, 'create:done', {
|
|
@@ -164,6 +204,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
164
204
|
elapsedMs: runtimeTiming.modalCreateMs,
|
|
165
205
|
});
|
|
166
206
|
let detached = false;
|
|
207
|
+
let setupError: unknown;
|
|
167
208
|
let cancelExecution!: (error: Error) => void;
|
|
168
209
|
const cancellationPromise = new Promise<never>((_resolve, reject) => {
|
|
169
210
|
cancelExecution = reject;
|
|
@@ -192,6 +233,9 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
192
233
|
cpu: MODAL_SANDBOX_CPU_CORES,
|
|
193
234
|
memoryGiB: MODAL_SANDBOX_MEMORY_MIB / 1024,
|
|
194
235
|
diskGiB: 0,
|
|
236
|
+
...(capacityReservation
|
|
237
|
+
? { sandboxCapacityLeaseId: capacityReservation.leaseId }
|
|
238
|
+
: {}),
|
|
195
239
|
});
|
|
196
240
|
} catch (error) {
|
|
197
241
|
runtimeResourceRegistrationError = error;
|
|
@@ -290,14 +334,45 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
290
334
|
tableNamespace: null,
|
|
291
335
|
runtimeTiming,
|
|
292
336
|
};
|
|
337
|
+
} catch (error) {
|
|
338
|
+
setupError = error;
|
|
339
|
+
throw error;
|
|
293
340
|
} finally {
|
|
294
341
|
callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
|
|
295
|
-
if (!detached
|
|
342
|
+
if (!detached && (await terminateModalSandbox(sandbox))) {
|
|
343
|
+
// This branch synchronously confirms that the just-created resource
|
|
344
|
+
// is gone. Do not keep its admission lease until the cleanup loop
|
|
345
|
+
// sees a resource that no longer exists.
|
|
346
|
+
if (capacityReservation && callbacks?.releaseSandboxCapacity) {
|
|
347
|
+
try {
|
|
348
|
+
await releaseModalSandboxCapacityAfterConfirmedTermination({
|
|
349
|
+
leaseId: capacityReservation.leaseId,
|
|
350
|
+
release: callbacks.releaseSandboxCapacity,
|
|
351
|
+
});
|
|
352
|
+
} catch (releaseError) {
|
|
353
|
+
// The scheduler fence is the primary correctness signal. Keep
|
|
354
|
+
// it intact if its best-effort early capacity release is
|
|
355
|
+
// transiently unavailable; durable cleanup will retry release.
|
|
356
|
+
if (setupError) {
|
|
357
|
+
console.error('[play-runner.modal.capacity_release_failed]', {
|
|
358
|
+
sandboxId: sandbox.sandboxId,
|
|
359
|
+
error:
|
|
360
|
+
releaseError instanceof Error
|
|
361
|
+
? releaseError.message
|
|
362
|
+
: String(releaseError),
|
|
363
|
+
});
|
|
364
|
+
} else {
|
|
365
|
+
throw releaseError;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
296
370
|
}
|
|
297
371
|
} catch (error) {
|
|
298
372
|
if (
|
|
299
373
|
error === runtimeResourceRegistrationError ||
|
|
300
|
-
error instanceof RuntimeResourceFenceLostError
|
|
374
|
+
error instanceof RuntimeResourceFenceLostError ||
|
|
375
|
+
isRuntimeSandboxCapacityLimitError(error)
|
|
301
376
|
) {
|
|
302
377
|
throw error;
|
|
303
378
|
}
|
|
@@ -335,7 +410,7 @@ export async function deleteModalSandboxById(input: {
|
|
|
335
410
|
appId: expectedAppId,
|
|
336
411
|
})) {
|
|
337
412
|
if (sandbox.sandboxId !== input.sandboxId) continue;
|
|
338
|
-
await sandbox.terminate();
|
|
413
|
+
await sandbox.terminate({ wait: true });
|
|
339
414
|
return { kind: 'deleted', appId: expectedAppId };
|
|
340
415
|
}
|
|
341
416
|
// Cleanup means ensure absent. An exact app-scoped inventory proving the
|
|
@@ -28,6 +28,8 @@ export type PlayRunnerRuntimeResource = {
|
|
|
28
28
|
cpu?: number | null;
|
|
29
29
|
memoryGiB?: number | null;
|
|
30
30
|
diskGiB?: number | null;
|
|
31
|
+
/** Scheduler-only soft-capacity reservation made before provider create. */
|
|
32
|
+
sandboxCapacityLeaseId?: string;
|
|
31
33
|
};
|
|
32
34
|
|
|
33
35
|
export type RuntimeResourceTerminalReason =
|
|
@@ -64,6 +66,23 @@ export class RuntimeResourceFenceLostError extends Error {
|
|
|
64
66
|
}
|
|
65
67
|
}
|
|
66
68
|
|
|
69
|
+
/**
|
|
70
|
+
* A scheduler-owned admission result crosses the provider-backend boundary.
|
|
71
|
+
* Keep it typed by name here so the shared backend can rethrow it without an
|
|
72
|
+
* import from the app-owned scheduler implementation.
|
|
73
|
+
*/
|
|
74
|
+
export const RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME =
|
|
75
|
+
'RuntimeSandboxCapacityLimitError';
|
|
76
|
+
|
|
77
|
+
export function isRuntimeSandboxCapacityLimitError(
|
|
78
|
+
error: unknown,
|
|
79
|
+
): error is Error {
|
|
80
|
+
return (
|
|
81
|
+
error instanceof Error &&
|
|
82
|
+
error.name === RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
67
86
|
export interface PlayRunnerCallbacks {
|
|
68
87
|
onLog?: (event: PlayRunnerLogEvent) => void;
|
|
69
88
|
onCheckpoint?: (checkpoint: PlayCheckpoint) => void;
|
|
@@ -72,6 +91,12 @@ export interface PlayRunnerCallbacks {
|
|
|
72
91
|
onRuntimeResourceAcquired?: (
|
|
73
92
|
resource: PlayRunnerRuntimeResource,
|
|
74
93
|
) => void | Promise<void>;
|
|
94
|
+
/** Reserve one physical-sandbox slot immediately before a provider create. */
|
|
95
|
+
reserveSandboxCapacity?: (
|
|
96
|
+
provider: 'daytona' | 'modal',
|
|
97
|
+
) => Promise<{ leaseId: string }>;
|
|
98
|
+
/** Release an unbound reservation when provider create never produced a resource. */
|
|
99
|
+
releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
|
|
75
100
|
/**
|
|
76
101
|
* Durable scheduler-owned evidence for a provider create call. Backends
|
|
77
102
|
* await this callback at the create boundary; it is not best-effort logging.
|
|
@@ -13,7 +13,6 @@ export const RUNTIME_CAPACITY_POLICY = {
|
|
|
13
13
|
/** 32 Machines x 8 claim slots permits 256 concurrent launch/resume legs. */
|
|
14
14
|
maxActiveLaneMachines: 32,
|
|
15
15
|
workerSlotsPerMachine: 8,
|
|
16
|
-
perOrgConcurrency: 4,
|
|
17
16
|
/** Queue age is an escape hatch for small-but-stuck backlogs. */
|
|
18
17
|
scaleUpQueueAgeMs: 10_000,
|
|
19
18
|
/** Bound one Fly reconciliation without turning backlog into a stampede. */
|
|
@@ -113,14 +112,6 @@ export function assertRuntimeCapacityPolicy(): void {
|
|
|
113
112
|
'Receipt claim operations must leave at least five seconds for cancellation and response delivery.',
|
|
114
113
|
);
|
|
115
114
|
}
|
|
116
|
-
if (
|
|
117
|
-
RUNTIME_CAPACITY_POLICY.absurd.perOrgConcurrency >
|
|
118
|
-
ABSURD_GLOBAL_RUN_CONCURRENCY
|
|
119
|
-
) {
|
|
120
|
-
throw new Error(
|
|
121
|
-
'Per-org runtime concurrency cannot exceed global runtime concurrency.',
|
|
122
|
-
);
|
|
123
|
-
}
|
|
124
115
|
if (
|
|
125
116
|
RUNTIME_CAPACITY_POLICY.absurd.activeLaneMachines >
|
|
126
117
|
RUNTIME_CAPACITY_POLICY.absurd.maxActiveLaneMachines
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public tool-response contracts shared by the API, SDK, Play bundler, and
|
|
3
|
+
* runtime. A response contract is transport behavior, never Play authoring.
|
|
4
|
+
*/
|
|
5
|
+
export const V2_TOOL_RESPONSE_CONTRACT = 'v2-tool-response' as const;
|
|
6
|
+
export const RAW_V2_TOOL_RESPONSE_CONTRACT = 'raw-v2' as const;
|
|
7
|
+
|
|
8
|
+
export type ToolResponseContract =
|
|
9
|
+
| typeof V2_TOOL_RESPONSE_CONTRACT
|
|
10
|
+
| typeof RAW_V2_TOOL_RESPONSE_CONTRACT;
|
|
11
|
+
|
|
12
|
+
export type ToolResponseView = 'data' | 'rawV2';
|
|
13
|
+
|
|
14
|
+
export function isToolResponseContract(
|
|
15
|
+
value: unknown,
|
|
16
|
+
): value is ToolResponseContract {
|
|
17
|
+
return (
|
|
18
|
+
value === V2_TOOL_RESPONSE_CONTRACT ||
|
|
19
|
+
value === RAW_V2_TOOL_RESPONSE_CONTRACT
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class UnsupportedToolResponseContractError extends Error {
|
|
24
|
+
constructor(value: unknown) {
|
|
25
|
+
super(
|
|
26
|
+
`Unsupported tool response contract ${String(value)}. Supported contracts: ${V2_TOOL_RESPONSE_CONTRACT}, ${RAW_V2_TOOL_RESPONSE_CONTRACT}.`,
|
|
27
|
+
);
|
|
28
|
+
this.name = 'UnsupportedToolResponseContractError';
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Missing artifact compatibility predates canonical bodies and stays V2. */
|
|
33
|
+
export function normalizeToolResponseContract(
|
|
34
|
+
value: unknown,
|
|
35
|
+
): ToolResponseContract {
|
|
36
|
+
if (value == null) return V2_TOOL_RESPONSE_CONTRACT;
|
|
37
|
+
if (isToolResponseContract(value)) return value;
|
|
38
|
+
throw new UnsupportedToolResponseContractError(value);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function legacyRawFromToolResponseRawV2(
|
|
42
|
+
rawV2: unknown,
|
|
43
|
+
view: ToolResponseView,
|
|
44
|
+
responseMeta?: Record<string, unknown>,
|
|
45
|
+
): unknown {
|
|
46
|
+
const legacyRaw =
|
|
47
|
+
view === 'data' &&
|
|
48
|
+
rawV2 &&
|
|
49
|
+
typeof rawV2 === 'object' &&
|
|
50
|
+
!Array.isArray(rawV2)
|
|
51
|
+
? (rawV2 as Record<string, unknown>).data
|
|
52
|
+
: rawV2;
|
|
53
|
+
// Before raw-v2, an async launch without a data envelope exposed
|
|
54
|
+
// Deepline's billing summary at `toolResponse.raw.deepline_billing`.
|
|
55
|
+
// The canonical response keeps it separate from provider data, so reattach
|
|
56
|
+
// it only to this derived legacy view.
|
|
57
|
+
const deeplineBilling = responseMeta?.deepline_billing;
|
|
58
|
+
if (
|
|
59
|
+
view === 'rawV2' &&
|
|
60
|
+
deeplineBilling !== undefined &&
|
|
61
|
+
legacyRaw &&
|
|
62
|
+
typeof legacyRaw === 'object' &&
|
|
63
|
+
!Array.isArray(legacyRaw)
|
|
64
|
+
) {
|
|
65
|
+
return {
|
|
66
|
+
...(legacyRaw as Record<string, unknown>),
|
|
67
|
+
deepline_billing: deeplineBilling,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return legacyRaw;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function providerMetaFromToolResponseRawV2(
|
|
74
|
+
rawV2: unknown,
|
|
75
|
+
view: ToolResponseView,
|
|
76
|
+
): Record<string, unknown> | undefined {
|
|
77
|
+
if (
|
|
78
|
+
view !== 'data' ||
|
|
79
|
+
!rawV2 ||
|
|
80
|
+
typeof rawV2 !== 'object' ||
|
|
81
|
+
Array.isArray(rawV2)
|
|
82
|
+
) {
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
const meta = (rawV2 as Record<string, unknown>).meta;
|
|
86
|
+
return meta && typeof meta === 'object' && !Array.isArray(meta)
|
|
87
|
+
? (meta as Record<string, unknown>)
|
|
88
|
+
: undefined;
|
|
89
|
+
}
|
|
@@ -75,7 +75,7 @@ export type ToolResultEnvelope<
|
|
|
75
75
|
meta?: TMeta;
|
|
76
76
|
};
|
|
77
77
|
|
|
78
|
-
export type
|
|
78
|
+
export type SerializedToolExecuteResultV1 = {
|
|
79
79
|
__kind: 'deepline.tool_execute_result.v1';
|
|
80
80
|
status: string;
|
|
81
81
|
job_id?: string;
|
|
@@ -98,11 +98,36 @@ export type SerializedToolExecuteResult = {
|
|
|
98
98
|
execution: ToolResultExecutionMetadata;
|
|
99
99
|
};
|
|
100
100
|
|
|
101
|
+
/**
|
|
102
|
+
* Canonical receipt form. Provider data is stored once as `rawV2`; the legacy
|
|
103
|
+
* raw/meta view is recreated from the payload-free projection descriptor.
|
|
104
|
+
*/
|
|
105
|
+
export type SerializedToolExecuteResultV2 = Omit<
|
|
106
|
+
SerializedToolExecuteResultV1,
|
|
107
|
+
'__kind' | 'toolResponse'
|
|
108
|
+
> & {
|
|
109
|
+
__kind: 'deepline.tool_execute_result.v2';
|
|
110
|
+
toolResponse: {
|
|
111
|
+
rawV2: unknown;
|
|
112
|
+
view: 'data' | 'rawV2';
|
|
113
|
+
/** Deepline-owned additions not already represented in `rawV2.meta`. */
|
|
114
|
+
responseMeta?: Record<string, unknown>;
|
|
115
|
+
};
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
export type SerializedToolExecuteResult =
|
|
119
|
+
| SerializedToolExecuteResultV1
|
|
120
|
+
| SerializedToolExecuteResultV2;
|
|
121
|
+
|
|
101
122
|
export type ToolResponseEnvelope<
|
|
102
123
|
TData = unknown,
|
|
103
124
|
TMeta = Record<string, unknown>,
|
|
104
125
|
> = {
|
|
105
126
|
raw: TData;
|
|
127
|
+
/** Complete parsed and scrubbed provider response, materialized from raw-v2. */
|
|
128
|
+
rawV2?: unknown;
|
|
129
|
+
/** Durable descriptor for deriving the legacy raw view from `rawV2`. */
|
|
130
|
+
view?: 'data' | 'rawV2';
|
|
106
131
|
meta?: TMeta;
|
|
107
132
|
};
|
|
108
133
|
|
|
@@ -156,8 +181,9 @@ export type ToolExecuteResultAccessors<
|
|
|
156
181
|
* Canonical result returned by Deepline tool execution.
|
|
157
182
|
*
|
|
158
183
|
* The top-level object is Deepline-owned execution metadata and semantic
|
|
159
|
-
* extraction state.
|
|
160
|
-
*
|
|
184
|
+
* extraction state. The canonical provider response lives under
|
|
185
|
+
* `toolResponse.rawV2`; `toolResponse.raw` remains the legacy compatibility
|
|
186
|
+
* projection. Response metadata lives under `toolResponse.meta`. Semantic single-value
|
|
161
187
|
* getters live under `extractedValues.<name>.get()`, and list getters live
|
|
162
188
|
* under `extractedLists.<name>.get()`.
|
|
163
189
|
*
|