deepline 0.3.43 → 0.3.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/scheduled-work.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1540 -265
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +22 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +16 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +418 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/index.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +304 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/testkit.ts +83 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/always-fresh-adapter.ts +50 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/contract.ts +135 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +291 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +335 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +904 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +522 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/testkit.ts +165 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +11 -0
- package/package.json +1 -1
|
@@ -59,6 +59,11 @@ export interface RowState {
|
|
|
59
59
|
|
|
60
60
|
export interface ToolCallRequest {
|
|
61
61
|
callId: string;
|
|
62
|
+
/**
|
|
63
|
+
* Stable per-run call identity used to fence a forced provider operation
|
|
64
|
+
* when receipt coordination deliberately has no active lease.
|
|
65
|
+
*/
|
|
66
|
+
logicalCallId?: string;
|
|
62
67
|
cacheKey: string;
|
|
63
68
|
/** Stable external-operation identity, independent of receipt serialization. */
|
|
64
69
|
providerIdempotencyKeyBase?: string | null;
|
|
@@ -723,6 +728,12 @@ export interface ContextOptions {
|
|
|
723
728
|
}) => Promise<void | {
|
|
724
729
|
updated: number;
|
|
725
730
|
staleDroppedKeys?: string[];
|
|
731
|
+
/**
|
|
732
|
+
* Keys initially fenced by a terminal write but verified by the
|
|
733
|
+
* runner-owned sheet session as terminal rows for this same run. These are
|
|
734
|
+
* idempotent settlements, not output that belongs to a newer run.
|
|
735
|
+
*/
|
|
736
|
+
recoveredStaleKeys?: string[];
|
|
726
737
|
}>;
|
|
727
738
|
/**
|
|
728
739
|
* Persists coalesced, non-terminal row patches before a map suspends.
|
|
@@ -358,6 +358,14 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
358
358
|
sessionId: string;
|
|
359
359
|
readiness: DetachedRunnerReadinessPort;
|
|
360
360
|
cancellation?: Promise<never>;
|
|
361
|
+
/** Invoked immediately after Daytona accepts the detached command. */
|
|
362
|
+
onCommandAccepted?: (
|
|
363
|
+
state: Extract<DetachedDaytonaRunnerState, { phase: 'command_accepted' }>,
|
|
364
|
+
) => Promise<void> | void;
|
|
365
|
+
/** Invoked after scheduler-owned liveness is confirmed. */
|
|
366
|
+
onReady?: (
|
|
367
|
+
state: Extract<DetachedDaytonaRunnerState, { phase: 'ready' }>,
|
|
368
|
+
) => Promise<void> | void;
|
|
361
369
|
}): DetachedDaytonaRunnerSupervisor {
|
|
362
370
|
const cancellation = input.cancellation ?? new Promise<never>(() => {});
|
|
363
371
|
const sessionCreation = input.sandbox.process
|
|
@@ -409,6 +417,7 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
409
417
|
cmdId,
|
|
410
418
|
baselineHeartbeatAt: baseline.heartbeatAt,
|
|
411
419
|
};
|
|
420
|
+
await input.onCommandAccepted?.(state);
|
|
412
421
|
await Promise.race([
|
|
413
422
|
confirmDetachedDaytonaRunnerReady({
|
|
414
423
|
sandbox: input.sandbox,
|
|
@@ -422,6 +431,7 @@ export function prepareDetachedDaytonaRunner(input: {
|
|
|
422
431
|
cancellation,
|
|
423
432
|
]);
|
|
424
433
|
state = { ...state, phase: 'ready' };
|
|
434
|
+
await input.onReady?.(state);
|
|
425
435
|
return state;
|
|
426
436
|
},
|
|
427
437
|
};
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
import {
|
|
9
9
|
isRuntimeSandboxCapacityLimitError,
|
|
10
10
|
RuntimeResourceFenceLostError,
|
|
11
|
+
SandboxRunnerPreCodeDeferralError,
|
|
11
12
|
} from '../types';
|
|
12
13
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
13
14
|
import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
|
|
@@ -1197,11 +1198,29 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1197
1198
|
// Session allocation is independent of payload staging. Start it
|
|
1198
1199
|
// now so Daytona control-plane latency overlaps compression/upload.
|
|
1199
1200
|
const sessionId = `deepline-play-${randomUUID()}`;
|
|
1201
|
+
const runnerAttempt = Math.max(
|
|
1202
|
+
0,
|
|
1203
|
+
Math.floor(config.context.runAttempt ?? 0),
|
|
1204
|
+
);
|
|
1200
1205
|
const detachedRunner = prepareDetachedDaytonaRunner({
|
|
1201
1206
|
sandbox,
|
|
1202
1207
|
sessionId,
|
|
1203
1208
|
readiness: readRunnerReadiness,
|
|
1204
1209
|
cancellation: cancellationPromise,
|
|
1210
|
+
onCommandAccepted: async () => {
|
|
1211
|
+
await callbacks?.onRunnerCommandAccepted?.({
|
|
1212
|
+
provider: 'daytona',
|
|
1213
|
+
sandboxId: sandbox.id,
|
|
1214
|
+
});
|
|
1215
|
+
},
|
|
1216
|
+
onReady: async (ready) => {
|
|
1217
|
+
await callbacks?.onDetachedRunnerReady?.({
|
|
1218
|
+
provider: 'daytona',
|
|
1219
|
+
sandboxId: sandbox.id,
|
|
1220
|
+
boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
|
|
1221
|
+
baselineHeartbeatAt: ready.baselineHeartbeatAt,
|
|
1222
|
+
});
|
|
1223
|
+
},
|
|
1205
1224
|
});
|
|
1206
1225
|
|
|
1207
1226
|
const uploadStartedAt = Date.now();
|
|
@@ -1310,10 +1329,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1310
1329
|
// classification below decide fresh-sandbox retry vs loud failure.
|
|
1311
1330
|
throw error;
|
|
1312
1331
|
}
|
|
1313
|
-
const runnerAttempt = Math.max(
|
|
1314
|
-
0,
|
|
1315
|
-
Math.floor(config.context.runAttempt ?? 0),
|
|
1316
|
-
);
|
|
1317
1332
|
emitDaytonaStage(callbacks, config.context, 'execute:detached', {
|
|
1318
1333
|
sandboxId: sandbox.id,
|
|
1319
1334
|
sessionId: start.sessionId,
|
|
@@ -1435,6 +1450,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1435
1450
|
if (isRuntimeSandboxCapacityLimitError(error)) {
|
|
1436
1451
|
throw error;
|
|
1437
1452
|
}
|
|
1453
|
+
if (error instanceof SandboxRunnerPreCodeDeferralError) {
|
|
1454
|
+
throw error;
|
|
1455
|
+
}
|
|
1438
1456
|
if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
|
|
1439
1457
|
throw error;
|
|
1440
1458
|
}
|
|
@@ -3,6 +3,7 @@ import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
|
|
|
3
3
|
import {
|
|
4
4
|
isRuntimeSandboxCapacityLimitError,
|
|
5
5
|
RuntimeResourceFenceLostError,
|
|
6
|
+
SandboxRunnerPreCodeDeferralError,
|
|
6
7
|
} from '../types';
|
|
7
8
|
import { buildPlayRunnerBundle } from '../bundle';
|
|
8
9
|
import type {
|
|
@@ -618,6 +619,10 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
618
619
|
cancellationPromise,
|
|
619
620
|
]);
|
|
620
621
|
runtimeTiming.modalExecuteMs = Date.now() - executeStartedAt;
|
|
622
|
+
await callbacks?.onRunnerCommandAccepted?.({
|
|
623
|
+
provider: 'modal',
|
|
624
|
+
sandboxId: sandbox.sandboxId,
|
|
625
|
+
});
|
|
621
626
|
// A readiness observation after the command is accepted is ambiguous:
|
|
622
627
|
// the detached runner may already have received its durable park and
|
|
623
628
|
// started customer code while this worker cannot read the scheduler.
|
|
@@ -627,16 +632,22 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
627
632
|
baselineHeartbeatAt,
|
|
628
633
|
cancellation: cancellationPromise,
|
|
629
634
|
});
|
|
635
|
+
const runnerAttempt = Math.max(
|
|
636
|
+
0,
|
|
637
|
+
Math.floor(config.context.runAttempt ?? 0),
|
|
638
|
+
);
|
|
639
|
+
await callbacks?.onDetachedRunnerReady?.({
|
|
640
|
+
provider: 'modal',
|
|
641
|
+
sandboxId: sandbox.sandboxId,
|
|
642
|
+
boundaryId: `detached-runner:${push.runId}:${runnerAttempt}`,
|
|
643
|
+
baselineHeartbeatAt,
|
|
644
|
+
});
|
|
630
645
|
if (callbacks?.cancellationSignal?.aborted) {
|
|
631
646
|
throw new Error('Modal play runner cancelled');
|
|
632
647
|
}
|
|
633
648
|
|
|
634
649
|
detached = true;
|
|
635
650
|
sandbox.detach();
|
|
636
|
-
const runnerAttempt = Math.max(
|
|
637
|
-
0,
|
|
638
|
-
Math.floor(config.context.runAttempt ?? 0),
|
|
639
|
-
);
|
|
640
651
|
emitModalStage(config.context, 'execute:detached', {
|
|
641
652
|
sandboxId: sandbox.sandboxId,
|
|
642
653
|
runnerAttempt,
|
|
@@ -716,6 +727,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
716
727
|
if (
|
|
717
728
|
error === runtimeResourceRegistrationError ||
|
|
718
729
|
error instanceof RuntimeResourceFenceLostError ||
|
|
730
|
+
error instanceof SandboxRunnerPreCodeDeferralError ||
|
|
719
731
|
isRuntimeSandboxCapacityLimitError(error) ||
|
|
720
732
|
isPlayRunRecoveryError(error)
|
|
721
733
|
) {
|
|
@@ -79,6 +79,26 @@ export type PlayRunnerRuntimeLifecycleEvent = {
|
|
|
79
79
|
errorClass?: 'timeout' | 'capacity' | 'rate_limit' | 'other';
|
|
80
80
|
};
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* The provider has accepted the detached runner command. This is deliberately
|
|
84
|
+
* smaller than a runner payload: the scheduler needs only the physical
|
|
85
|
+
* resource identity to cross the no-replay boundary.
|
|
86
|
+
*/
|
|
87
|
+
export type PlayRunnerCommandAcceptance = {
|
|
88
|
+
provider: 'daytona' | 'modal';
|
|
89
|
+
sandboxId: string;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Scheduler-owned liveness was observed after a detached command was
|
|
94
|
+
* accepted. A backend must emit this only immediately before it returns a
|
|
95
|
+
* detached suspension.
|
|
96
|
+
*/
|
|
97
|
+
export type PlayRunnerDetachedRunnerReady = PlayRunnerCommandAcceptance & {
|
|
98
|
+
boundaryId: string;
|
|
99
|
+
baselineHeartbeatAt: string | null;
|
|
100
|
+
};
|
|
101
|
+
|
|
82
102
|
export class RuntimeResourceFenceLostError extends Error {
|
|
83
103
|
constructor(message: string) {
|
|
84
104
|
super(message);
|
|
@@ -94,6 +114,27 @@ export class RuntimeResourceFenceLostError extends Error {
|
|
|
94
114
|
export const RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME =
|
|
95
115
|
'RuntimeSandboxCapacityLimitError';
|
|
96
116
|
|
|
117
|
+
/**
|
|
118
|
+
* A scheduler admission decision reached the legacy runner protocol before a
|
|
119
|
+
* provider create. Backends rethrow it verbatim so the Sandbox Runs Adapter can
|
|
120
|
+
* preserve a retry-safe `pre_code_deferred` outcome instead of manufacturing a
|
|
121
|
+
* runner failure.
|
|
122
|
+
*/
|
|
123
|
+
export class SandboxRunnerPreCodeDeferralError extends Error {
|
|
124
|
+
constructor(
|
|
125
|
+
readonly reason:
|
|
126
|
+
| 'capacity'
|
|
127
|
+
| 'provider_unavailable'
|
|
128
|
+
| 'rate_limited'
|
|
129
|
+
| 'startup_timeout',
|
|
130
|
+
) {
|
|
131
|
+
super(
|
|
132
|
+
`Sandbox Runs deferred fresh execution before provider create: ${reason}.`,
|
|
133
|
+
);
|
|
134
|
+
this.name = 'SandboxRunnerPreCodeDeferralError';
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
97
138
|
export function isRuntimeSandboxCapacityLimitError(
|
|
98
139
|
error: unknown,
|
|
99
140
|
): error is Error {
|
|
@@ -155,6 +196,21 @@ export interface PlayRunnerCallbacks {
|
|
|
155
196
|
onRuntimeLifecycleEvent?: (
|
|
156
197
|
event: PlayRunnerRuntimeLifecycleEvent,
|
|
157
198
|
) => Promise<void>;
|
|
199
|
+
/**
|
|
200
|
+
* Marks the command acceptance no-replay boundary. This is separate from a
|
|
201
|
+
* resource acquisition because a recorded sandbox can still fail staging
|
|
202
|
+
* before customer code is accepted.
|
|
203
|
+
*/
|
|
204
|
+
onRunnerCommandAccepted?: (
|
|
205
|
+
acceptance: PlayRunnerCommandAcceptance,
|
|
206
|
+
) => void | Promise<void>;
|
|
207
|
+
/**
|
|
208
|
+
* Marks the scheduler-owned readiness that makes a detached suspension safe
|
|
209
|
+
* to park. It is never a provider heartbeat or a runner payload callback.
|
|
210
|
+
*/
|
|
211
|
+
onDetachedRunnerReady?: (
|
|
212
|
+
readiness: PlayRunnerDetachedRunnerReady,
|
|
213
|
+
) => void | Promise<void>;
|
|
158
214
|
/**
|
|
159
215
|
* Scheduler-owned readiness read for a detached runner attempt. Daytona uses
|
|
160
216
|
* this direct control-plane port before parking; it is never serialized into
|
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sandbox Runs is the portable lifecycle boundary around a physical execution
|
|
3
|
+
* resource. It is deliberately narrower than the scheduler attempt: it does
|
|
4
|
+
* not claim an attempt, choose retry policy, write a terminal, or delete a
|
|
5
|
+
* sandbox. Its one non-negotiable rule is that a resource fact is durable
|
|
6
|
+
* before the runner command can be accepted.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export type SandboxProvider = 'daytona' | 'modal' | 'local';
|
|
10
|
+
|
|
11
|
+
/** A portable, credential-free description of one execution resource. */
|
|
12
|
+
export type SandboxRunResource = {
|
|
13
|
+
provider: SandboxProvider;
|
|
14
|
+
resourceId: string;
|
|
15
|
+
billingStartedAtMs: number;
|
|
16
|
+
/** Provider ownership domain used later by the cleanup Adapter. */
|
|
17
|
+
routingDomain: string | null;
|
|
18
|
+
/** The scheduler's already-reserved soft-capacity slot, if managed. */
|
|
19
|
+
capacityLeaseId?: string;
|
|
20
|
+
maxBillingDurationSeconds?: number | null;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/** The scheduler's already claimed attempt; it is never a provider credential. */
|
|
24
|
+
export type SandboxRunInput = {
|
|
25
|
+
attempt: {
|
|
26
|
+
runId: string;
|
|
27
|
+
number: number;
|
|
28
|
+
};
|
|
29
|
+
/** Opaque scheduler authority that the resource-fact Adapter fences. */
|
|
30
|
+
authority: {
|
|
31
|
+
fenceId: string;
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/** A pre-code rejection is safe for the attempt policy to defer or retry. */
|
|
36
|
+
export type SandboxPreCodeDeferralReason =
|
|
37
|
+
| 'capacity'
|
|
38
|
+
| 'provider_unavailable'
|
|
39
|
+
| 'rate_limited'
|
|
40
|
+
| 'startup_timeout';
|
|
41
|
+
|
|
42
|
+
export type SandboxAcquisition<Handle> =
|
|
43
|
+
| {
|
|
44
|
+
kind: 'acquired';
|
|
45
|
+
resource: SandboxRunResource;
|
|
46
|
+
/** Private to the acquire/start Adapters; never persisted by this job. */
|
|
47
|
+
handle: Handle;
|
|
48
|
+
}
|
|
49
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
50
|
+
/**
|
|
51
|
+
* A provider create may have crossed a local timeout. Its Adapter has kept
|
|
52
|
+
* a durable reconciliation obligation; this job must not acquire again.
|
|
53
|
+
*/
|
|
54
|
+
| { kind: 'acquisition_unknown'; reason: string };
|
|
55
|
+
|
|
56
|
+
/** The durable resource writer also binds capacity in the same fence. */
|
|
57
|
+
export type SandboxResourceFacts = {
|
|
58
|
+
record(input: {
|
|
59
|
+
attempt: SandboxRunInput['attempt'];
|
|
60
|
+
authority: SandboxRunInput['authority'];
|
|
61
|
+
resource: SandboxRunResource;
|
|
62
|
+
}): Promise<{ kind: 'recorded' } | { kind: 'fence_lost' }>;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** A scheduler-owned observation target, not a provider command or payload. */
|
|
66
|
+
export type SandboxRunnerObservation = {
|
|
67
|
+
boundaryId: string;
|
|
68
|
+
baselineHeartbeatAt: string | null;
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
export type SandboxRunnerStart<Handle> = {
|
|
72
|
+
start(input: {
|
|
73
|
+
attempt: SandboxRunInput['attempt'];
|
|
74
|
+
resource: SandboxRunResource;
|
|
75
|
+
handle: Handle;
|
|
76
|
+
}): Promise<
|
|
77
|
+
| { kind: 'command_accepted'; observation: SandboxRunnerObservation }
|
|
78
|
+
/** The command is known not to have been accepted; cleanup owns resource release. */
|
|
79
|
+
| { kind: 'not_started'; reason: string }
|
|
80
|
+
/** Command acceptance cannot be proven either way; never acquire/replay. */
|
|
81
|
+
| { kind: 'start_unknown'; reason: string }
|
|
82
|
+
>;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export type SandboxRunnerObserver = {
|
|
86
|
+
observe(input: {
|
|
87
|
+
attempt: SandboxRunInput['attempt'];
|
|
88
|
+
resource: SandboxRunResource;
|
|
89
|
+
observation: SandboxRunnerObservation;
|
|
90
|
+
}): Promise<
|
|
91
|
+
| { kind: 'heartbeat_observed'; heartbeatAt: string }
|
|
92
|
+
| {
|
|
93
|
+
kind: 'terminal_observed';
|
|
94
|
+
terminal: 'completed' | 'failed' | 'cancelled';
|
|
95
|
+
}
|
|
96
|
+
/** The command could already be executing; caller must not replay it. */
|
|
97
|
+
| { kind: 'unconfirmed'; reason: string }
|
|
98
|
+
>;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
export type SandboxRunDependencies<Handle> = {
|
|
102
|
+
acquire: {
|
|
103
|
+
acquire(input: SandboxRunInput): Promise<SandboxAcquisition<Handle>>;
|
|
104
|
+
};
|
|
105
|
+
resourceFacts: SandboxResourceFacts;
|
|
106
|
+
runner: SandboxRunnerStart<Handle>;
|
|
107
|
+
observer: SandboxRunnerObserver;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/** Canonical lifecycle outcomes consumed by the attempt policy. */
|
|
111
|
+
export type SandboxRunOutcome =
|
|
112
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
113
|
+
| { kind: 'acquisition_unknown'; reason: string }
|
|
114
|
+
/** A physical resource exists but the scheduler fence rejected its fact. */
|
|
115
|
+
| { kind: 'resource_recording_rejected'; resource: SandboxRunResource }
|
|
116
|
+
/** Resource fact exists; no command was accepted; cleanup is now authoritative. */
|
|
117
|
+
| {
|
|
118
|
+
kind: 'post_record_start_failed';
|
|
119
|
+
resource: SandboxRunResource;
|
|
120
|
+
reason: string;
|
|
121
|
+
}
|
|
122
|
+
/** Resource fact exists and command acceptance is ambiguous; no replay is safe. */
|
|
123
|
+
| {
|
|
124
|
+
kind: 'runner_start_unknown';
|
|
125
|
+
resource: SandboxRunResource;
|
|
126
|
+
reason: string;
|
|
127
|
+
}
|
|
128
|
+
| {
|
|
129
|
+
kind: 'parked';
|
|
130
|
+
resource: SandboxRunResource;
|
|
131
|
+
observation: SandboxRunnerObservation;
|
|
132
|
+
heartbeatAt: string;
|
|
133
|
+
}
|
|
134
|
+
| {
|
|
135
|
+
kind: 'terminal_observed';
|
|
136
|
+
resource: SandboxRunResource;
|
|
137
|
+
terminal: 'completed' | 'failed' | 'cancelled';
|
|
138
|
+
}
|
|
139
|
+
| {
|
|
140
|
+
kind: 'runner_start_unconfirmed';
|
|
141
|
+
resource: SandboxRunResource;
|
|
142
|
+
observation: SandboxRunnerObservation;
|
|
143
|
+
reason: string;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
export type SandboxRunsJob = {
|
|
147
|
+
execute(input: SandboxRunInput): Promise<SandboxRunOutcome>;
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The rich Sandbox Runs boundary used by the real runner adapters.
|
|
152
|
+
*
|
|
153
|
+
* The legacy four-port contract above remains exported while callers migrate,
|
|
154
|
+
* but it cannot describe a production sandbox launch: Daytona may retire and
|
|
155
|
+
* recreate sandboxes, and Modal must record an ambiguity fence before create
|
|
156
|
+
* can begin. This contract keeps those facts explicit without importing a
|
|
157
|
+
* scheduler, provider SDK, or runner payload into the portable library.
|
|
158
|
+
*/
|
|
159
|
+
|
|
160
|
+
export type SandboxExecutionMode<Checkpoint, Suspension> =
|
|
161
|
+
| { kind: 'fresh'; checkpoint: Checkpoint | null }
|
|
162
|
+
| {
|
|
163
|
+
kind: 'continuation';
|
|
164
|
+
checkpoint: Checkpoint;
|
|
165
|
+
/** A scheduler continuation can resume after a non-sandbox boundary. */
|
|
166
|
+
suspension: Suspension | null;
|
|
167
|
+
}
|
|
168
|
+
| {
|
|
169
|
+
/** Read persisted runner state; never create another sandbox first. */
|
|
170
|
+
kind: 'detached_observation';
|
|
171
|
+
checkpoint: Checkpoint;
|
|
172
|
+
suspension: Suspension;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
/** The launch is opaque here; only the runner/provider Adapter may interpret it. */
|
|
176
|
+
export type SandboxExecutionInput<Launch, Checkpoint, Suspension> =
|
|
177
|
+
SandboxRunInput & {
|
|
178
|
+
launch: Launch;
|
|
179
|
+
mode: SandboxExecutionMode<Checkpoint, Suspension>;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* A resource fact has enough identity to be recovered and deleted safely. It
|
|
184
|
+
* deliberately excludes provider handles, commands, credentials, and payloads.
|
|
185
|
+
*/
|
|
186
|
+
export type SandboxExecutionResource = SandboxRunResource & {
|
|
187
|
+
kind: 'daytona_sandbox' | 'modal_sandbox' | 'local_process';
|
|
188
|
+
runtimeEnvironment?: 'preview' | 'production';
|
|
189
|
+
billingEndedAtMs?: number | null;
|
|
190
|
+
cpu?: number | null;
|
|
191
|
+
memoryGiB?: number | null;
|
|
192
|
+
diskGiB?: number | null;
|
|
193
|
+
/** A create settled after the launching attempt had already timed out. */
|
|
194
|
+
lateAcquired?: boolean;
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
export type SandboxCreateIntent = {
|
|
198
|
+
provider: 'modal';
|
|
199
|
+
routingDomain: string;
|
|
200
|
+
runtimeEnvironment: 'preview' | 'production';
|
|
201
|
+
/** Stable, non-secret provider correlation attributes only. */
|
|
202
|
+
correlation: Record<string, string>;
|
|
203
|
+
capacityLeaseId?: string;
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
export type SandboxProviderLifecycleEvent = {
|
|
207
|
+
provider: 'daytona' | 'modal';
|
|
208
|
+
type: 'create_call_started' | 'create_call_succeeded' | 'create_call_failed';
|
|
209
|
+
providerAttempt: number;
|
|
210
|
+
occurredAtMs: number;
|
|
211
|
+
resourceId?: string;
|
|
212
|
+
errorClass?: 'timeout' | 'capacity' | 'rate_limit' | 'other';
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
export type SandboxExecutionAdmission = {
|
|
216
|
+
beforeFirstProviderCreate(input: {
|
|
217
|
+
attempt: SandboxRunInput['attempt'];
|
|
218
|
+
authority: SandboxRunInput['authority'];
|
|
219
|
+
}): Promise<
|
|
220
|
+
| { kind: 'allowed' }
|
|
221
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
222
|
+
>;
|
|
223
|
+
reserveCapacity(input: {
|
|
224
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
225
|
+
/** Distinguishes provider retries within one scheduler attempt. */
|
|
226
|
+
providerAttempt?: number;
|
|
227
|
+
}): Promise<
|
|
228
|
+
| {
|
|
229
|
+
kind: 'reserved';
|
|
230
|
+
leaseId: string;
|
|
231
|
+
/** Stable provider correlation for operation-first managed creates. */
|
|
232
|
+
providerOperationToken?: string | null;
|
|
233
|
+
}
|
|
234
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
235
|
+
>;
|
|
236
|
+
releaseCapacity(input: { leaseId: string }): Promise<void>;
|
|
237
|
+
claimProviderCircuit(input: {
|
|
238
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
239
|
+
}): Promise<{ allowed: boolean; probeToken: string | null }>;
|
|
240
|
+
markProviderHealthy(input: {
|
|
241
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
242
|
+
probeToken: string | null;
|
|
243
|
+
}): Promise<void>;
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
export type SandboxExecutionResourceFacts = {
|
|
247
|
+
record(input: {
|
|
248
|
+
attempt: SandboxRunInput['attempt'];
|
|
249
|
+
authority: SandboxRunInput['authority'];
|
|
250
|
+
resource: SandboxExecutionResource;
|
|
251
|
+
}): Promise<{ kind: 'recorded' } | { kind: 'fence_lost' }>;
|
|
252
|
+
/** A late create must become a cleanup obligation, never an invisible resource. */
|
|
253
|
+
recordLate(input: {
|
|
254
|
+
attempt: SandboxRunInput['attempt'];
|
|
255
|
+
authority: SandboxRunInput['authority'];
|
|
256
|
+
resource: SandboxExecutionResource;
|
|
257
|
+
}): Promise<{ kind: 'recorded' } | { kind: 'fence_lost' }>;
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export type SandboxExecutionCreateIntents = {
|
|
261
|
+
record(input: {
|
|
262
|
+
attempt: SandboxRunInput['attempt'];
|
|
263
|
+
authority: SandboxRunInput['authority'];
|
|
264
|
+
intent: SandboxCreateIntent;
|
|
265
|
+
}): Promise<{ kind: 'recorded' } | { kind: 'fence_lost' }>;
|
|
266
|
+
resolve(input: {
|
|
267
|
+
attempt: SandboxRunInput['attempt'];
|
|
268
|
+
authority: SandboxRunInput['authority'];
|
|
269
|
+
}): Promise<{ kind: 'resolved' } | { kind: 'fence_lost' }>;
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
export type SandboxExecutionLifecycleEvents = {
|
|
273
|
+
record(input: {
|
|
274
|
+
attempt: SandboxRunInput['attempt'];
|
|
275
|
+
authority: SandboxRunInput['authority'];
|
|
276
|
+
event: SandboxProviderLifecycleEvent;
|
|
277
|
+
}): Promise<{ kind: 'recorded' } | { kind: 'fence_lost' }>;
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
/** The only capability a provider Adapter receives while executing a launch. */
|
|
281
|
+
export type SandboxExecutionLifecycle = {
|
|
282
|
+
beforeFirstProviderCreate(): Promise<
|
|
283
|
+
| { kind: 'allowed' }
|
|
284
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
285
|
+
>;
|
|
286
|
+
reserveCapacity(input: {
|
|
287
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
288
|
+
/** Preserves one provider-create retry's durable operation identity. */
|
|
289
|
+
providerAttempt?: number;
|
|
290
|
+
}): Promise<
|
|
291
|
+
| {
|
|
292
|
+
kind: 'reserved';
|
|
293
|
+
leaseId: string;
|
|
294
|
+
/** Stable provider correlation for operation-first managed creates. */
|
|
295
|
+
providerOperationToken?: string | null;
|
|
296
|
+
}
|
|
297
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
298
|
+
>;
|
|
299
|
+
releaseCapacity(input: { leaseId: string }): Promise<void>;
|
|
300
|
+
claimProviderCircuit(input: {
|
|
301
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
302
|
+
}): Promise<{ allowed: boolean; probeToken: string | null }>;
|
|
303
|
+
markProviderHealthy(input: {
|
|
304
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
305
|
+
probeToken: string | null;
|
|
306
|
+
}): Promise<void>;
|
|
307
|
+
recordCreateIntent(input: SandboxCreateIntent): Promise<void>;
|
|
308
|
+
resolveCreateIntent(): Promise<void>;
|
|
309
|
+
recordProviderLifecycleEvent(
|
|
310
|
+
input: SandboxProviderLifecycleEvent,
|
|
311
|
+
): Promise<void>;
|
|
312
|
+
/** Must resolve before the Adapter can accept a command for this resource. */
|
|
313
|
+
recordResource(input: SandboxExecutionResource): Promise<void>;
|
|
314
|
+
/** Same durable rule for a create that settles after the caller timed out. */
|
|
315
|
+
recordLateResource(input: SandboxExecutionResource): Promise<void>;
|
|
316
|
+
/**
|
|
317
|
+
* Marks the non-replayable boundary. Remote providers require a previously
|
|
318
|
+
* recorded resource; local execution has no physical sandbox resource.
|
|
319
|
+
*/
|
|
320
|
+
commandAccepted(input: {
|
|
321
|
+
provider: SandboxProvider;
|
|
322
|
+
resource?: SandboxExecutionResource;
|
|
323
|
+
}): void;
|
|
324
|
+
/** A detached runner cannot be parked until scheduler-owned readiness exists. */
|
|
325
|
+
runnerReady(input: {
|
|
326
|
+
provider: Exclude<SandboxProvider, 'local'>;
|
|
327
|
+
resource: SandboxExecutionResource;
|
|
328
|
+
observation: SandboxRunnerObservation;
|
|
329
|
+
}): void;
|
|
330
|
+
};
|
|
331
|
+
|
|
332
|
+
/** Outcome returned by the provider/runner Adapter after exactly one execution. */
|
|
333
|
+
export type SandboxExecutionPortOutcome<RunnerResult> =
|
|
334
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
335
|
+
| { kind: 'acquisition_unknown'; reason: string }
|
|
336
|
+
/** Provider or customer code may have started. The attempt must not replay. */
|
|
337
|
+
| { kind: 'post_code_unknown'; reason: string }
|
|
338
|
+
| {
|
|
339
|
+
kind: 'suspended';
|
|
340
|
+
runnerResult: RunnerResult;
|
|
341
|
+
/**
|
|
342
|
+
* Only a detached runner needs a scheduler-owned liveness proof before
|
|
343
|
+
* the worker parks. Ordinary Play boundaries (sleep, signal, etc.) do
|
|
344
|
+
* not create or observe a long-lived sandbox command.
|
|
345
|
+
*/
|
|
346
|
+
requiresDetachedRunnerReadiness?: boolean;
|
|
347
|
+
}
|
|
348
|
+
| { kind: 'terminal'; runnerResult: RunnerResult };
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* One invocation of the existing runner backend, not separate acquire/start
|
|
352
|
+
* calls. This preserves the provider boundary while moving its durable
|
|
353
|
+
* lifecycle callbacks behind the Sandbox Runs Module.
|
|
354
|
+
*/
|
|
355
|
+
export type SandboxExecutionPort<Launch, Checkpoint, Suspension, RunnerResult> =
|
|
356
|
+
{
|
|
357
|
+
execute(input: {
|
|
358
|
+
execution: SandboxExecutionInput<Launch, Checkpoint, Suspension>;
|
|
359
|
+
lifecycle: SandboxExecutionLifecycle;
|
|
360
|
+
}): Promise<SandboxExecutionPortOutcome<RunnerResult>>;
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
export type SandboxExecutionDependencies<
|
|
364
|
+
Launch,
|
|
365
|
+
Checkpoint,
|
|
366
|
+
Suspension,
|
|
367
|
+
RunnerResult,
|
|
368
|
+
> = {
|
|
369
|
+
execution: SandboxExecutionPort<Launch, Checkpoint, Suspension, RunnerResult>;
|
|
370
|
+
/** Every resource record binds its capacity lease in the scheduler fence. */
|
|
371
|
+
resourceFacts: SandboxExecutionResourceFacts;
|
|
372
|
+
/** Required when the Adapter crosses Modal's ambiguous-create boundary. */
|
|
373
|
+
createIntents?: SandboxExecutionCreateIntents;
|
|
374
|
+
/** Required when the Adapter emits provider-edge lifecycle evidence. */
|
|
375
|
+
lifecycleEvents?: SandboxExecutionLifecycleEvents;
|
|
376
|
+
/** Required when the Adapter opts into admission, capacity, or circuit policy. */
|
|
377
|
+
admission?: SandboxExecutionAdmission;
|
|
378
|
+
};
|
|
379
|
+
|
|
380
|
+
export type SandboxExecutionOutcome<RunnerResult> =
|
|
381
|
+
| { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
|
|
382
|
+
| { kind: 'acquisition_unknown'; reason: string }
|
|
383
|
+
| {
|
|
384
|
+
kind: 'post_code_unknown';
|
|
385
|
+
reason: string;
|
|
386
|
+
resources: readonly SandboxExecutionResource[];
|
|
387
|
+
}
|
|
388
|
+
| {
|
|
389
|
+
kind: 'resource_recording_rejected';
|
|
390
|
+
resources: readonly SandboxExecutionResource[];
|
|
391
|
+
}
|
|
392
|
+
| {
|
|
393
|
+
kind: 'suspended';
|
|
394
|
+
runnerResult: RunnerResult;
|
|
395
|
+
resources: readonly SandboxExecutionResource[];
|
|
396
|
+
}
|
|
397
|
+
| {
|
|
398
|
+
kind: 'terminal';
|
|
399
|
+
runnerResult: RunnerResult;
|
|
400
|
+
resources: readonly SandboxExecutionResource[];
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
export type SandboxExecutionJob<Launch, Checkpoint, Suspension, RunnerResult> =
|
|
404
|
+
{
|
|
405
|
+
execute(
|
|
406
|
+
input: SandboxExecutionInput<Launch, Checkpoint, Suspension>,
|
|
407
|
+
): Promise<SandboxExecutionOutcome<RunnerResult>>;
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
/** A fence loss must stop the Adapter before it can accept a provider command. */
|
|
411
|
+
export class SandboxResourceFenceLostError extends Error {
|
|
412
|
+
constructor() {
|
|
413
|
+
super(
|
|
414
|
+
'Sandbox Runs lost its scheduler fence while recording a durable lifecycle fact.',
|
|
415
|
+
);
|
|
416
|
+
this.name = 'SandboxResourceFenceLostError';
|
|
417
|
+
}
|
|
418
|
+
}
|