deepline 0.1.201 → 0.1.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +30 -9
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +25 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +139 -124
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +17 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +154 -23
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +121 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +86 -19
- package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +141 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/budget-state-backend.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +129 -41
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +40 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +92 -33
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +13 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +122 -120
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +10 -8
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +140 -32
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +225 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +21 -0
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +32 -10
- package/dist/cli/index.js +335 -77
- package/dist/cli/index.mjs +341 -83
- package/dist/index.js +3 -2
- package/dist/index.mjs +3 -2
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +0 -355
|
@@ -32,12 +32,14 @@ export type PlayRunnerRateStateAcquireInput = {
|
|
|
32
32
|
export type PlayRunnerRateStateAcquireResult = {
|
|
33
33
|
granted: number;
|
|
34
34
|
waitMs: number;
|
|
35
|
+
/** One independently-expiring concurrency reservation per granted permit. */
|
|
36
|
+
leaseIds?: string[];
|
|
35
37
|
};
|
|
36
38
|
|
|
37
39
|
export type PlayRunnerRateStateReleaseInput = {
|
|
38
40
|
bucketId: string;
|
|
39
41
|
rules: PacingRule[];
|
|
40
|
-
|
|
42
|
+
leaseIds: string[];
|
|
41
43
|
schedulerSchema?: string | null;
|
|
42
44
|
};
|
|
43
45
|
|
|
@@ -47,6 +49,23 @@ export type PlayRunnerRateStatePenalizeInput = {
|
|
|
47
49
|
schedulerSchema?: string | null;
|
|
48
50
|
};
|
|
49
51
|
|
|
52
|
+
export type PlayRunnerBudgetCharge = {
|
|
53
|
+
key: string;
|
|
54
|
+
amount: number;
|
|
55
|
+
limit: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export type PlayRunnerBudgetChargeInput = {
|
|
59
|
+
rootRunId: string;
|
|
60
|
+
reservationId: string;
|
|
61
|
+
charges: PlayRunnerBudgetCharge[];
|
|
62
|
+
schedulerSchema?: string | null;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
export type PlayRunnerBudgetChargeResult = {
|
|
66
|
+
counters: Record<string, number>;
|
|
67
|
+
};
|
|
68
|
+
|
|
50
69
|
export interface PlayRunnerContextConfig {
|
|
51
70
|
executorToken?: string;
|
|
52
71
|
baseUrl?: string;
|
|
@@ -70,7 +89,7 @@ export interface PlayRunnerContextConfig {
|
|
|
70
89
|
* dispatch; `sandbox_public_key` means the runner mints missing sessions
|
|
71
90
|
* through the app runtime API with a per-request public key.
|
|
72
91
|
*/
|
|
73
|
-
dbSessionStrategy?: 'preloaded' | 'sandbox_public_key';
|
|
92
|
+
dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | 'gateway_only';
|
|
74
93
|
/**
|
|
75
94
|
* Short-lived scoped Postgres sessions minted by the worker before the runner
|
|
76
95
|
* starts. They are only an initial fast path: the runner still renews through
|
|
@@ -78,6 +97,18 @@ export interface PlayRunnerContextConfig {
|
|
|
78
97
|
* exact-enough match for the requested play/table/operation scope.
|
|
79
98
|
*/
|
|
80
99
|
preloadedDbSessions?: PreloadedRuntimeDbSession[];
|
|
100
|
+
/**
|
|
101
|
+
* Secret that decrypts `preloadedDbSessions` postgres URLs. The preloaded
|
|
102
|
+
* sessions are AES-GCM sealed at submit time with the run's original executor
|
|
103
|
+
* (authority) token, whose string is the key material. Durable schedulers
|
|
104
|
+
* re-mint a fresh per-attempt executor token (with `run_attempt`/capabilities
|
|
105
|
+
* claims) for API auth, which changes the token string and would break unwrap
|
|
106
|
+
* if used as the decryption key. This field pins the stable submit-time secret
|
|
107
|
+
* so unwrap always uses the exact token that encrypted the sessions,
|
|
108
|
+
* independent of per-attempt re-minting. Absent means fall back to
|
|
109
|
+
* `executorToken` (legacy/no-remint paths where they are equal).
|
|
110
|
+
*/
|
|
111
|
+
postgresSessionUnwrapKey?: string | null;
|
|
81
112
|
rateStateBackend?: PlayRunnerRateStateBackendConfig;
|
|
82
113
|
runtimeSchedulerSchema?: string | null;
|
|
83
114
|
/**
|
|
@@ -87,6 +118,13 @@ export interface PlayRunnerContextConfig {
|
|
|
87
118
|
* of defaulting away from the parent scheduler. Absent means `absurd`.
|
|
88
119
|
*/
|
|
89
120
|
childRunProfile?: string | null;
|
|
121
|
+
/**
|
|
122
|
+
* The parent worker's absurd release lane id. Posted as the
|
|
123
|
+
* `x-deepline-absurd-release` header on child `ctx.runPlay` launches so the
|
|
124
|
+
* child pins to the parent's lane (defense-in-depth). Absent off the absurd
|
|
125
|
+
* path or on a pre-release parent.
|
|
126
|
+
*/
|
|
127
|
+
absurdReleaseId?: string | null;
|
|
90
128
|
governance?: GovernanceSnapshot | null;
|
|
91
129
|
}
|
|
92
130
|
|
|
@@ -40,6 +40,11 @@ interface PendingReceiptCompletion<TInput, TOutput> {
|
|
|
40
40
|
reject: (error: unknown) => void;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
type CapacityWaiter = {
|
|
44
|
+
resolve: () => void;
|
|
45
|
+
reject: (error: unknown) => void;
|
|
46
|
+
};
|
|
47
|
+
|
|
43
48
|
function normalizePositiveInteger(
|
|
44
49
|
value: number | undefined,
|
|
45
50
|
fallback: number,
|
|
@@ -81,10 +86,14 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
81
86
|
readonly maxInputBytes: number;
|
|
82
87
|
|
|
83
88
|
#pending: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
|
|
84
|
-
#
|
|
89
|
+
#pendingHead = 0;
|
|
90
|
+
/** Pending plus in-flight bytes. Released only after transport settles. */
|
|
91
|
+
#residentBytes = 0;
|
|
92
|
+
#capacityWaiters: CapacityWaiter[] = [];
|
|
85
93
|
#flushTimer: ReceiptCompletionSinkTimer | null = null;
|
|
86
94
|
#flushPromise: Promise<void> | null = null;
|
|
87
95
|
#closed = false;
|
|
96
|
+
#failure: Error | null = null;
|
|
88
97
|
#completeMany: (inputs: TInput[]) => Promise<TOutput[]>;
|
|
89
98
|
#estimateBytes: (input: TInput) => number;
|
|
90
99
|
#onFlushStart?: (event: ReceiptCompletionSinkFlushEvent) => void;
|
|
@@ -124,20 +133,16 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
124
133
|
return this.addMany([input]).then((outputs) => outputs[0]!);
|
|
125
134
|
}
|
|
126
135
|
|
|
127
|
-
addMany(inputs: readonly TInput[]): Promise<TOutput[]> {
|
|
136
|
+
async addMany(inputs: readonly TInput[]): Promise<TOutput[]> {
|
|
128
137
|
if (inputs.length === 0) return Promise.resolve([]);
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
new Error('Runtime receipt completion sink is already closed.'),
|
|
132
|
-
);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const outputPromises = inputs.map((input) => this.#enqueue(input));
|
|
138
|
+
this.#throwIfUnavailable();
|
|
139
|
+
const outputPromises = inputs.map((input) => this.#admit(input));
|
|
136
140
|
this.#scheduleFlush();
|
|
137
|
-
return Promise.all(outputPromises);
|
|
141
|
+
return await Promise.all(outputPromises);
|
|
138
142
|
}
|
|
139
143
|
|
|
140
144
|
async flush(): Promise<void> {
|
|
145
|
+
if (this.#failure) throw this.#failure;
|
|
141
146
|
this.#clearFlushTimer();
|
|
142
147
|
this.#flushPromise ??= this.#runFlushLoop().finally(() => {
|
|
143
148
|
this.#flushPromise = null;
|
|
@@ -147,18 +152,19 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
147
152
|
|
|
148
153
|
async close(): Promise<void> {
|
|
149
154
|
this.#closed = true;
|
|
155
|
+
this.#wakeCapacityWaiters();
|
|
150
156
|
await this.flush();
|
|
151
157
|
}
|
|
152
158
|
|
|
153
159
|
getStats(): { pending: number; pendingBytes: number; flushing: boolean } {
|
|
154
160
|
return {
|
|
155
|
-
pending: this.#
|
|
156
|
-
pendingBytes: this.#
|
|
161
|
+
pending: this.#pendingCount(),
|
|
162
|
+
pendingBytes: this.#residentBytes,
|
|
157
163
|
flushing: Boolean(this.#flushPromise),
|
|
158
164
|
};
|
|
159
165
|
}
|
|
160
166
|
|
|
161
|
-
#
|
|
167
|
+
async #admit(input: TInput): Promise<TOutput> {
|
|
162
168
|
const bytes = Math.max(1, Math.ceil(this.#estimateBytes(input)));
|
|
163
169
|
if (bytes > this.maxInputBytes) {
|
|
164
170
|
return Promise.reject(
|
|
@@ -167,25 +173,32 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
167
173
|
),
|
|
168
174
|
);
|
|
169
175
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
176
|
+
if (bytes > this.maxBufferedBytes) {
|
|
177
|
+
throw new ReceiptCompletionBufferLimitError(
|
|
178
|
+
`Runtime receipt completion payload is ${bytes} bytes, above the ${this.maxBufferedBytes} byte sink capacity.`,
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
while (this.#residentBytes + bytes > this.maxBufferedBytes) {
|
|
182
|
+
this.#throwIfUnavailable();
|
|
175
183
|
void this.flush().catch(() => undefined);
|
|
184
|
+
await new Promise<void>((resolve, reject) => {
|
|
185
|
+
this.#capacityWaiters.push({ resolve, reject });
|
|
186
|
+
});
|
|
176
187
|
}
|
|
177
|
-
|
|
178
|
-
|
|
188
|
+
this.#throwIfUnavailable();
|
|
189
|
+
this.#residentBytes += bytes;
|
|
190
|
+
const completion = new Promise<TOutput>((resolve, reject) => {
|
|
179
191
|
this.#pending.push({ input, bytes, resolve, reject });
|
|
180
|
-
this.#
|
|
181
|
-
if (this.#pending.length >= this.maxBatchSize) {
|
|
192
|
+
if (this.#pendingCount() >= this.maxBatchSize) {
|
|
182
193
|
void this.flush().catch(() => undefined);
|
|
183
194
|
}
|
|
184
195
|
});
|
|
196
|
+
this.#scheduleFlush();
|
|
197
|
+
return await completion;
|
|
185
198
|
}
|
|
186
199
|
|
|
187
200
|
#scheduleFlush(): void {
|
|
188
|
-
if (this.#flushTimer || this.#flushPromise || this.#
|
|
201
|
+
if (this.#flushTimer || this.#flushPromise || this.#pendingCount() === 0) {
|
|
189
202
|
return;
|
|
190
203
|
}
|
|
191
204
|
this.#flushTimer = setTimeout(() => {
|
|
@@ -201,28 +214,36 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
201
214
|
}
|
|
202
215
|
|
|
203
216
|
async #runFlushLoop(): Promise<void> {
|
|
204
|
-
while (this.#
|
|
217
|
+
while (this.#pendingCount() > 0) {
|
|
205
218
|
const batch = this.#takeBatch();
|
|
206
|
-
|
|
219
|
+
try {
|
|
220
|
+
await this.#flushBatch(batch);
|
|
221
|
+
} catch (error) {
|
|
222
|
+
this.#fail(error);
|
|
223
|
+
throw this.#failure;
|
|
224
|
+
}
|
|
207
225
|
}
|
|
208
226
|
}
|
|
209
227
|
|
|
210
228
|
#takeBatch(): Array<PendingReceiptCompletion<TInput, TOutput>> {
|
|
211
229
|
const batch: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
|
|
212
230
|
let batchBytes = 0;
|
|
213
|
-
while (this.#
|
|
214
|
-
const next = this.#pending[
|
|
215
|
-
if (
|
|
216
|
-
batch.length > 0 &&
|
|
217
|
-
batchBytes + next.bytes > this.maxBufferedBytes
|
|
218
|
-
) {
|
|
231
|
+
while (this.#pendingCount() > 0 && batch.length < this.maxBatchSize) {
|
|
232
|
+
const next = this.#pending[this.#pendingHead]!;
|
|
233
|
+
if (batch.length > 0 && batchBytes + next.bytes > this.maxBufferedBytes) {
|
|
219
234
|
break;
|
|
220
235
|
}
|
|
221
|
-
this.#
|
|
222
|
-
this.#pendingBytes -= next.bytes;
|
|
236
|
+
this.#pendingHead += 1;
|
|
223
237
|
batch.push(next);
|
|
224
238
|
batchBytes += next.bytes;
|
|
225
239
|
}
|
|
240
|
+
if (
|
|
241
|
+
this.#pendingHead > 1_024 &&
|
|
242
|
+
this.#pendingHead * 2 >= this.#pending.length
|
|
243
|
+
) {
|
|
244
|
+
this.#pending = this.#pending.slice(this.#pendingHead);
|
|
245
|
+
this.#pendingHead = 0;
|
|
246
|
+
}
|
|
226
247
|
return batch;
|
|
227
248
|
}
|
|
228
249
|
|
|
@@ -257,6 +278,44 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
257
278
|
error,
|
|
258
279
|
});
|
|
259
280
|
throw error;
|
|
281
|
+
} finally {
|
|
282
|
+
this.#residentBytes = Math.max(0, this.#residentBytes - bytes);
|
|
283
|
+
this.#wakeCapacityWaiters();
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
#pendingCount(): number {
|
|
288
|
+
return this.#pending.length - this.#pendingHead;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
#throwIfUnavailable(): void {
|
|
292
|
+
if (this.#failure) throw this.#failure;
|
|
293
|
+
if (this.#closed) {
|
|
294
|
+
throw new Error('Runtime receipt completion sink is already closed.');
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
#wakeCapacityWaiters(): void {
|
|
299
|
+
const waiters = this.#capacityWaiters;
|
|
300
|
+
this.#capacityWaiters = [];
|
|
301
|
+
for (const waiter of waiters) {
|
|
302
|
+
if (this.#failure) waiter.reject(this.#failure);
|
|
303
|
+
else if (this.#closed) {
|
|
304
|
+
waiter.reject(
|
|
305
|
+
new Error('Runtime receipt completion sink is already closed.'),
|
|
306
|
+
);
|
|
307
|
+
} else waiter.resolve();
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
#fail(error: unknown): void {
|
|
312
|
+
this.#failure =
|
|
313
|
+
error instanceof Error ? error : new Error(String(error ?? 'Unknown'));
|
|
314
|
+
while (this.#pendingCount() > 0) {
|
|
315
|
+
const pending = this.#pending[this.#pendingHead++]!;
|
|
316
|
+
this.#residentBytes = Math.max(0, this.#residentBytes - pending.bytes);
|
|
317
|
+
pending.reject(this.#failure);
|
|
260
318
|
}
|
|
319
|
+
this.#wakeCapacityWaiters();
|
|
261
320
|
}
|
|
262
321
|
}
|
|
@@ -3,6 +3,7 @@ import type { PlayExecutionGovernor, WorkLease } from './governor/governor';
|
|
|
3
3
|
export type RuntimeResourceLease = WorkLease;
|
|
4
4
|
|
|
5
5
|
export type RuntimeResourceObservation = {
|
|
6
|
+
toolId?: string | null;
|
|
6
7
|
providerResourceKey?: string | null;
|
|
7
8
|
rowEstimatedBytes?: number | null;
|
|
8
9
|
rowAdmissionWaitMs?: number | null;
|
|
@@ -172,12 +173,19 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
172
173
|
}
|
|
173
174
|
if (
|
|
174
175
|
observation.providerSuccess === true &&
|
|
175
|
-
observation.providerResourceKey
|
|
176
|
+
(observation.toolId || observation.providerResourceKey)
|
|
176
177
|
) {
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
178
|
+
if (observation.toolId) {
|
|
179
|
+
input.executionGovernor.observeToolSuccess({
|
|
180
|
+
toolId: observation.toolId,
|
|
181
|
+
latencyMs: observation.providerLatencyMs,
|
|
182
|
+
});
|
|
183
|
+
} else {
|
|
184
|
+
input.executionGovernor.observeProviderSuccess({
|
|
185
|
+
provider: observation.providerResourceKey!,
|
|
186
|
+
latencyMs: observation.providerLatencyMs,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
181
189
|
}
|
|
182
190
|
}
|
|
183
191
|
};
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -5,13 +5,26 @@ import type {
|
|
|
5
5
|
PlayRunnerResult,
|
|
6
6
|
} from '@shared_libs/play-runtime/protocol';
|
|
7
7
|
import { shouldCleanupSandboxInBackground } from '@shared_libs/play-runtime/daytona-runtime-config';
|
|
8
|
+
import { isIsolatedRuntimeSchedulerSchema } from '@shared_libs/play-runtime/runtime-scheduler-topology';
|
|
8
9
|
|
|
9
10
|
const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
|
|
10
|
-
const
|
|
11
|
-
const DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS = 3_500;
|
|
11
|
+
const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
|
|
12
12
|
const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = 5;
|
|
13
13
|
const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
|
|
14
14
|
const DAYTONA_SANDBOX_LABEL_SOURCE = 'deepline-play-runner';
|
|
15
|
+
// The hosted Daytona API pins sandbox resources to the creating snapshot: it
|
|
16
|
+
// rejects create-time resource overrides for snapshot sandboxes (400 "Cannot
|
|
17
|
+
// specify Sandbox resources when using a snapshot") and does not serve
|
|
18
|
+
// POST /sandbox/:id/resize (404 "Cannot POST"). The resource boundary is
|
|
19
|
+
// therefore the default snapshot's grant, verified against these pins after
|
|
20
|
+
// create. If Daytona changes the default snapshot's resources, sandbox
|
|
21
|
+
// acquisition fails loudly instead of silently running plays on a different
|
|
22
|
+
// compute boundary; bump these pins deliberately when that happens.
|
|
23
|
+
export const DAYTONA_SANDBOX_CPU = 1;
|
|
24
|
+
export const DAYTONA_SANDBOX_MEMORY_GIB = 1;
|
|
25
|
+
export const DAYTONA_SANDBOX_DISK_GIB = 3;
|
|
26
|
+
export const DAYTONA_SANDBOX_GPU = 0;
|
|
27
|
+
const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
|
|
15
28
|
|
|
16
29
|
export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
|
|
17
30
|
|
|
@@ -35,7 +48,7 @@ export type OneShotDaytonaSandboxLifecycle = {
|
|
|
35
48
|
dispose: () => Promise<void>;
|
|
36
49
|
};
|
|
37
50
|
|
|
38
|
-
type
|
|
51
|
+
type DaytonaCreateResult = {
|
|
39
52
|
sandbox: DaytonaSandbox;
|
|
40
53
|
attempt: number;
|
|
41
54
|
attemptElapsedMs: number;
|
|
@@ -64,6 +77,48 @@ export function validateDaytonaExecutionContext(
|
|
|
64
77
|
return orgId;
|
|
65
78
|
}
|
|
66
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Outbound-network policy for a one-shot sandbox. The invariant this protects:
|
|
82
|
+
* CUSTOMER code in PRODUCTION never runs with unrestricted egress — the
|
|
83
|
+
* production worker must provision `DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST`
|
|
84
|
+
* (comma-separated CIDRs, per the Daytona `networkAllowList` contract) and
|
|
85
|
+
* every prod sandbox is created `networkBlockAll: true` + that allow-list.
|
|
86
|
+
*
|
|
87
|
+
* "Production" here is the RUN'S topology, not the build flag: a run whose
|
|
88
|
+
* scheduler schema is isolated (per-PR preview CI, per-worktree dev — see
|
|
89
|
+
* `isIsolatedRuntimeSchedulerSchema`) executes synthetic internal test plays
|
|
90
|
+
* against dynamic per-PR hosts (Vercel preview app, per-PR Fly gateway) whose
|
|
91
|
+
* IPs cannot be enumerated as stable CIDRs, so the allow-list requirement
|
|
92
|
+
* cannot apply there. `NODE_ENV=production` alone is the wrong key — the Fly
|
|
93
|
+
* worker image bakes it for BOTH prod and preview fleets (build optimization),
|
|
94
|
+
* which made every preview sandbox create fail. Fail-closed: an absent schema
|
|
95
|
+
* is the production topology, never "unknown".
|
|
96
|
+
*
|
|
97
|
+
* An explicitly provisioned allow-list is always honored (preview included),
|
|
98
|
+
* so a future preview fleet with enumerable egress can opt in to full parity.
|
|
99
|
+
*/
|
|
100
|
+
export function resolveDaytonaSandboxNetworkPolicy(input: {
|
|
101
|
+
runtimeSchedulerSchema: string | null | undefined;
|
|
102
|
+
env?: NodeJS.ProcessEnv;
|
|
103
|
+
}): { networkAllowList: string | null } {
|
|
104
|
+
const env = input.env ?? process.env;
|
|
105
|
+
const networkAllowList = env[DAYTONA_NETWORK_ALLOW_LIST_ENV]?.trim() || null;
|
|
106
|
+
if (
|
|
107
|
+
!networkAllowList &&
|
|
108
|
+
env.NODE_ENV === 'production' &&
|
|
109
|
+
!isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
|
|
110
|
+
) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
`${DAYTONA_NETWORK_ALLOW_LIST_ENV} is required for production runs. ` +
|
|
113
|
+
`Refusing to start customer code with unrestricted outbound network ` +
|
|
114
|
+
`access. Provision a comma-separated CIDR allow-list on the production ` +
|
|
115
|
+
`worker. (Isolated-schema runs — preview CI / worktrees — are exempt: ` +
|
|
116
|
+
`they execute synthetic internal plays against dynamic per-PR hosts.)`,
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
return { networkAllowList };
|
|
120
|
+
}
|
|
121
|
+
|
|
67
122
|
async function createOneShotDaytonaSandbox(input: {
|
|
68
123
|
daytona: DaytonaClient;
|
|
69
124
|
orgId: string;
|
|
@@ -86,6 +141,9 @@ async function createOneShotDaytonaSandbox(input: {
|
|
|
86
141
|
if (workflowId) labels.workflowId = workflowId;
|
|
87
142
|
if (playId) labels.playId = playId;
|
|
88
143
|
if (runId) labels.runId = runId;
|
|
144
|
+
const { networkAllowList } = resolveDaytonaSandboxNetworkPolicy({
|
|
145
|
+
runtimeSchedulerSchema: input.context.runtimeSchedulerSchema ?? null,
|
|
146
|
+
});
|
|
89
147
|
|
|
90
148
|
// Intentionally omit image/snapshot so Daytona uses its default fast sandbox
|
|
91
149
|
// image; custom images would add build/pull tax to one-shot cold starts.
|
|
@@ -95,136 +153,60 @@ async function createOneShotDaytonaSandbox(input: {
|
|
|
95
153
|
ephemeral: true,
|
|
96
154
|
autoStopInterval: DAYTONA_AUTO_STOP_INTERVAL_MINUTES,
|
|
97
155
|
autoArchiveInterval: DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES,
|
|
156
|
+
...(networkAllowList
|
|
157
|
+
? {
|
|
158
|
+
networkBlockAll: true,
|
|
159
|
+
networkAllowList,
|
|
160
|
+
}
|
|
161
|
+
: {}),
|
|
98
162
|
},
|
|
99
163
|
{ timeout: DAYTONA_CREATE_TIMEOUT_SECONDS },
|
|
100
164
|
);
|
|
101
165
|
}
|
|
102
166
|
|
|
103
|
-
function
|
|
104
|
-
sandbox: DaytonaSandbox;
|
|
105
|
-
attempt: number;
|
|
106
|
-
emitStage: DaytonaStageEmitter;
|
|
107
|
-
}): void {
|
|
108
|
-
void input.sandbox.delete(30).then(
|
|
109
|
-
() =>
|
|
110
|
-
input.emitStage('create:late_cleanup_done', {
|
|
111
|
-
sandboxId: input.sandbox.id,
|
|
112
|
-
attempt: input.attempt,
|
|
113
|
-
}),
|
|
114
|
-
(error: unknown) => {
|
|
115
|
-
console.warn('[play-runner.daytona.create_late_cleanup_failed]', {
|
|
116
|
-
sandboxId: input.sandbox.id,
|
|
117
|
-
attempt: input.attempt,
|
|
118
|
-
error: error instanceof Error ? error.message : String(error),
|
|
119
|
-
});
|
|
120
|
-
},
|
|
121
|
-
);
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async function createHedgedOneShotDaytonaSandbox(input: {
|
|
167
|
+
async function createRetriedOneShotDaytonaSandbox(input: {
|
|
125
168
|
daytona: DaytonaClient;
|
|
126
169
|
orgId: string;
|
|
127
170
|
context: DaytonaExecutionContext;
|
|
128
171
|
emitStage: DaytonaStageEmitter;
|
|
129
172
|
startedAt: number;
|
|
130
|
-
}): Promise<
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if (
|
|
147
|
-
settled ||
|
|
148
|
-
rejectedAttempts !== DAYTONA_CREATE_HEDGE_DELAYS_MS.length
|
|
149
|
-
) {
|
|
150
|
-
return;
|
|
151
|
-
}
|
|
152
|
-
settled = true;
|
|
153
|
-
clearTimers();
|
|
154
|
-
reject(
|
|
155
|
-
new Error(
|
|
156
|
-
`Daytona sandbox create failed across ${rejectedAttempts} attempts: ${errors.join('; ')}`,
|
|
157
|
-
),
|
|
158
|
-
);
|
|
159
|
-
};
|
|
160
|
-
|
|
161
|
-
const startAttempt = (attempt: number) => {
|
|
162
|
-
if (settled) return;
|
|
163
|
-
startedAttempts = Math.max(startedAttempts, attempt);
|
|
164
|
-
if (attempt > 1) {
|
|
165
|
-
input.emitStage('create:retry', {
|
|
166
|
-
attempt: attempt - 1,
|
|
167
|
-
elapsedMs: Date.now() - input.startedAt,
|
|
168
|
-
reason: 'hedge_delay',
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
const attemptStartedAt = Date.now();
|
|
172
|
-
void createOneShotDaytonaSandbox({
|
|
173
|
+
}): Promise<DaytonaCreateResult> {
|
|
174
|
+
const errors: string[] = [];
|
|
175
|
+
for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
|
|
176
|
+
const attempt = index + 1;
|
|
177
|
+
if (delayMs > 0) {
|
|
178
|
+
input.emitStage('create:retry', {
|
|
179
|
+
attempt: index,
|
|
180
|
+
elapsedMs: Date.now() - input.startedAt,
|
|
181
|
+
reason: 'bounded_backoff',
|
|
182
|
+
delayMs,
|
|
183
|
+
});
|
|
184
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
185
|
+
}
|
|
186
|
+
const attemptStartedAt = Date.now();
|
|
187
|
+
try {
|
|
188
|
+
const sandbox = await createOneShotDaytonaSandbox({
|
|
173
189
|
daytona: input.daytona,
|
|
174
190
|
orgId: input.orgId,
|
|
175
191
|
context: input.context,
|
|
176
|
-
})
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
sandbox,
|
|
190
|
-
attempt,
|
|
191
|
-
attemptElapsedMs: Date.now() - attemptStartedAt,
|
|
192
|
-
});
|
|
193
|
-
},
|
|
194
|
-
(error: unknown) => {
|
|
195
|
-
rejectedAttempts += 1;
|
|
196
|
-
errors.push(error instanceof Error ? error.message : String(error));
|
|
197
|
-
console.warn('[play-runner.daytona.create_attempt_failed]', {
|
|
198
|
-
attempt,
|
|
199
|
-
error: error instanceof Error ? error.message : String(error),
|
|
200
|
-
});
|
|
201
|
-
rejectIfExhausted();
|
|
202
|
-
},
|
|
203
|
-
);
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
for (const [index, delayMs] of DAYTONA_CREATE_HEDGE_DELAYS_MS.entries()) {
|
|
207
|
-
const attempt = index + 1;
|
|
208
|
-
if (delayMs === 0) {
|
|
209
|
-
startAttempt(attempt);
|
|
210
|
-
} else {
|
|
211
|
-
timers.push(setTimeout(() => startAttempt(attempt), delayMs));
|
|
212
|
-
}
|
|
192
|
+
});
|
|
193
|
+
return {
|
|
194
|
+
sandbox,
|
|
195
|
+
attempt,
|
|
196
|
+
attemptElapsedMs: Date.now() - attemptStartedAt,
|
|
197
|
+
};
|
|
198
|
+
} catch (error) {
|
|
199
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
200
|
+
errors.push(message);
|
|
201
|
+
console.warn('[play-runner.daytona.create_attempt_failed]', {
|
|
202
|
+
attempt,
|
|
203
|
+
error: message,
|
|
204
|
+
});
|
|
213
205
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
settled = true;
|
|
219
|
-
clearTimers();
|
|
220
|
-
reject(
|
|
221
|
-
new Error(
|
|
222
|
-
`Daytona sandbox create did not complete within ${DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS}ms after starting ${startedAttempts} attempts.`,
|
|
223
|
-
),
|
|
224
|
-
);
|
|
225
|
-
}, DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS),
|
|
226
|
-
);
|
|
227
|
-
});
|
|
206
|
+
}
|
|
207
|
+
throw new Error(
|
|
208
|
+
`Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`,
|
|
209
|
+
);
|
|
228
210
|
}
|
|
229
211
|
|
|
230
212
|
async function acquireOneShotDaytonaSandbox(input: {
|
|
@@ -235,13 +217,33 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
235
217
|
startedAt: number;
|
|
236
218
|
}): Promise<AcquiredDaytonaSandbox> {
|
|
237
219
|
input.emitStage('create:start');
|
|
238
|
-
const result = await
|
|
220
|
+
const result = await createRetriedOneShotDaytonaSandbox(input);
|
|
221
|
+
const granted = {
|
|
222
|
+
cpu: result.sandbox.cpu,
|
|
223
|
+
memoryGiB: result.sandbox.memory,
|
|
224
|
+
diskGiB: result.sandbox.disk,
|
|
225
|
+
gpu: result.sandbox.gpu ?? 0,
|
|
226
|
+
};
|
|
227
|
+
if (
|
|
228
|
+
granted.cpu !== DAYTONA_SANDBOX_CPU ||
|
|
229
|
+
granted.memoryGiB !== DAYTONA_SANDBOX_MEMORY_GIB ||
|
|
230
|
+
granted.diskGiB !== DAYTONA_SANDBOX_DISK_GIB ||
|
|
231
|
+
granted.gpu !== DAYTONA_SANDBOX_GPU
|
|
232
|
+
) {
|
|
233
|
+
await result.sandbox.delete(30).catch(() => undefined);
|
|
234
|
+
throw new Error(
|
|
235
|
+
`Daytona sandbox resource boundary mismatch: expected cpu=${DAYTONA_SANDBOX_CPU} memoryGiB=${DAYTONA_SANDBOX_MEMORY_GIB} diskGiB=${DAYTONA_SANDBOX_DISK_GIB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
|
|
236
|
+
);
|
|
237
|
+
}
|
|
239
238
|
const billingStartedAt = Date.now();
|
|
240
239
|
input.emitStage('create:done', {
|
|
241
240
|
sandboxId: result.sandbox.id,
|
|
242
241
|
attempt: result.attempt,
|
|
243
242
|
elapsedMs: billingStartedAt - input.startedAt,
|
|
244
243
|
attemptElapsedMs: result.attemptElapsedMs,
|
|
244
|
+
cpu: granted.cpu,
|
|
245
|
+
memoryGiB: granted.memoryGiB,
|
|
246
|
+
diskGiB: granted.diskGiB,
|
|
245
247
|
});
|
|
246
248
|
return {
|
|
247
249
|
sandbox: result.sandbox,
|
|
@@ -60,16 +60,18 @@ function nodeMaterializePayloadCommand(input: {
|
|
|
60
60
|
function remoteRuntimeContextForDaytona(
|
|
61
61
|
context: DaytonaExecutionContext,
|
|
62
62
|
): DaytonaRemoteRuntimeContext {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
};
|
|
63
|
+
const gatewayBaseUrl = context.receiptGatewayBaseUrl?.trim();
|
|
64
|
+
if (!gatewayBaseUrl) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
'Daytona execution requires DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL. Refusing to put app credentials or database sessions in the customer-code sandbox.',
|
|
67
|
+
);
|
|
69
68
|
}
|
|
70
69
|
return {
|
|
71
70
|
...context,
|
|
72
|
-
|
|
71
|
+
baseUrl: gatewayBaseUrl,
|
|
72
|
+
receiptGatewayBaseUrl: gatewayBaseUrl,
|
|
73
|
+
vercelProtectionBypassToken: null,
|
|
74
|
+
dbSessionStrategy: 'gateway_only',
|
|
73
75
|
preloadedDbSessions: undefined,
|
|
74
76
|
};
|
|
75
77
|
}
|
|
@@ -215,7 +217,7 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
215
217
|
configPath,
|
|
216
218
|
artifactCodePath,
|
|
217
219
|
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
218
|
-
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
220
|
+
const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
|
|
219
221
|
|
|
220
222
|
return {
|
|
221
223
|
workDir: input.workDir,
|