deepline 0.1.201 → 0.1.202

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +30 -9
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +25 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +139 -124
  4. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +17 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-dispatch.ts +52 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +152 -23
  8. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +10 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +121 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +6 -1
  12. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-budget-state-backend.ts +22 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +86 -19
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +141 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/budget-state-backend.ts +43 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +129 -41
  17. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +40 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +92 -33
  19. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +13 -5
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +122 -120
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +10 -8
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +140 -32
  23. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +225 -29
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-scheduler-topology.ts +26 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +21 -0
  26. package/dist/bundling-sources/shared_libs/plays/dataset.ts +32 -10
  27. package/dist/cli/index.js +198 -76
  28. package/dist/cli/index.mjs +204 -82
  29. package/dist/index.js +3 -2
  30. package/dist/index.mjs +3 -2
  31. package/package.json +1 -1
  32. package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +0 -355
@@ -32,12 +32,14 @@ export type PlayRunnerRateStateAcquireInput = {
32
32
  export type PlayRunnerRateStateAcquireResult = {
33
33
  granted: number;
34
34
  waitMs: number;
35
+ /** One independently-expiring concurrency reservation per granted permit. */
36
+ leaseIds?: string[];
35
37
  };
36
38
 
37
39
  export type PlayRunnerRateStateReleaseInput = {
38
40
  bucketId: string;
39
41
  rules: PacingRule[];
40
- count: number;
42
+ leaseIds: string[];
41
43
  schedulerSchema?: string | null;
42
44
  };
43
45
 
@@ -47,6 +49,23 @@ export type PlayRunnerRateStatePenalizeInput = {
47
49
  schedulerSchema?: string | null;
48
50
  };
49
51
 
52
+ export type PlayRunnerBudgetCharge = {
53
+ key: string;
54
+ amount: number;
55
+ limit: number;
56
+ };
57
+
58
+ export type PlayRunnerBudgetChargeInput = {
59
+ rootRunId: string;
60
+ reservationId: string;
61
+ charges: PlayRunnerBudgetCharge[];
62
+ schedulerSchema?: string | null;
63
+ };
64
+
65
+ export type PlayRunnerBudgetChargeResult = {
66
+ counters: Record<string, number>;
67
+ };
68
+
50
69
  export interface PlayRunnerContextConfig {
51
70
  executorToken?: string;
52
71
  baseUrl?: string;
@@ -70,7 +89,7 @@ export interface PlayRunnerContextConfig {
70
89
  * dispatch; `sandbox_public_key` means the runner mints missing sessions
71
90
  * through the app runtime API with a per-request public key.
72
91
  */
73
- dbSessionStrategy?: 'preloaded' | 'sandbox_public_key';
92
+ dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | 'gateway_only';
74
93
  /**
75
94
  * Short-lived scoped Postgres sessions minted by the worker before the runner
76
95
  * starts. They are only an initial fast path: the runner still renews through
@@ -78,6 +97,18 @@ export interface PlayRunnerContextConfig {
78
97
  * exact-enough match for the requested play/table/operation scope.
79
98
  */
80
99
  preloadedDbSessions?: PreloadedRuntimeDbSession[];
100
+ /**
101
+ * Secret that decrypts `preloadedDbSessions` postgres URLs. The preloaded
102
+ * sessions are AES-GCM sealed at submit time with the run's original executor
103
+ * (authority) token, whose string is the key material. Durable schedulers
104
+ * re-mint a fresh per-attempt executor token (with `run_attempt`/capabilities
105
+ * claims) for API auth, which changes the token string and would break unwrap
106
+ * if used as the decryption key. This field pins the stable submit-time secret
107
+ * so unwrap always uses the exact token that encrypted the sessions,
108
+ * independent of per-attempt re-minting. Absent means fall back to
109
+ * `executorToken` (legacy/no-remint paths where they are equal).
110
+ */
111
+ postgresSessionUnwrapKey?: string | null;
81
112
  rateStateBackend?: PlayRunnerRateStateBackendConfig;
82
113
  runtimeSchedulerSchema?: string | null;
83
114
  /**
@@ -87,6 +118,13 @@ export interface PlayRunnerContextConfig {
87
118
  * of defaulting away from the parent scheduler. Absent means `absurd`.
88
119
  */
89
120
  childRunProfile?: string | null;
121
+ /**
122
+ * The parent worker's absurd release lane id. Posted as the
123
+ * `x-deepline-absurd-release` header on child `ctx.runPlay` launches so the
124
+ * child pins to the parent's lane (defense-in-depth). Absent off the absurd
125
+ * path or on a pre-release parent.
126
+ */
127
+ absurdReleaseId?: string | null;
90
128
  governance?: GovernanceSnapshot | null;
91
129
  }
92
130
 
@@ -40,6 +40,11 @@ interface PendingReceiptCompletion<TInput, TOutput> {
40
40
  reject: (error: unknown) => void;
41
41
  }
42
42
 
43
+ type CapacityWaiter = {
44
+ resolve: () => void;
45
+ reject: (error: unknown) => void;
46
+ };
47
+
43
48
  function normalizePositiveInteger(
44
49
  value: number | undefined,
45
50
  fallback: number,
@@ -81,10 +86,14 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
81
86
  readonly maxInputBytes: number;
82
87
 
83
88
  #pending: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
84
- #pendingBytes = 0;
89
+ #pendingHead = 0;
90
+ /** Pending plus in-flight bytes. Released only after transport settles. */
91
+ #residentBytes = 0;
92
+ #capacityWaiters: CapacityWaiter[] = [];
85
93
  #flushTimer: ReceiptCompletionSinkTimer | null = null;
86
94
  #flushPromise: Promise<void> | null = null;
87
95
  #closed = false;
96
+ #failure: Error | null = null;
88
97
  #completeMany: (inputs: TInput[]) => Promise<TOutput[]>;
89
98
  #estimateBytes: (input: TInput) => number;
90
99
  #onFlushStart?: (event: ReceiptCompletionSinkFlushEvent) => void;
@@ -124,20 +133,16 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
124
133
  return this.addMany([input]).then((outputs) => outputs[0]!);
125
134
  }
126
135
 
127
- addMany(inputs: readonly TInput[]): Promise<TOutput[]> {
136
+ async addMany(inputs: readonly TInput[]): Promise<TOutput[]> {
128
137
  if (inputs.length === 0) return Promise.resolve([]);
129
- if (this.#closed) {
130
- return Promise.reject(
131
- new Error('Runtime receipt completion sink is already closed.'),
132
- );
133
- }
134
-
135
- const outputPromises = inputs.map((input) => this.#enqueue(input));
138
+ this.#throwIfUnavailable();
139
+ const outputPromises = inputs.map((input) => this.#admit(input));
136
140
  this.#scheduleFlush();
137
- return Promise.all(outputPromises);
141
+ return await Promise.all(outputPromises);
138
142
  }
139
143
 
140
144
  async flush(): Promise<void> {
145
+ if (this.#failure) throw this.#failure;
141
146
  this.#clearFlushTimer();
142
147
  this.#flushPromise ??= this.#runFlushLoop().finally(() => {
143
148
  this.#flushPromise = null;
@@ -147,18 +152,19 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
147
152
 
148
153
  async close(): Promise<void> {
149
154
  this.#closed = true;
155
+ this.#wakeCapacityWaiters();
150
156
  await this.flush();
151
157
  }
152
158
 
153
159
  getStats(): { pending: number; pendingBytes: number; flushing: boolean } {
154
160
  return {
155
- pending: this.#pending.length,
156
- pendingBytes: this.#pendingBytes,
161
+ pending: this.#pendingCount(),
162
+ pendingBytes: this.#residentBytes,
157
163
  flushing: Boolean(this.#flushPromise),
158
164
  };
159
165
  }
160
166
 
161
- #enqueue(input: TInput): Promise<TOutput> {
167
+ async #admit(input: TInput): Promise<TOutput> {
162
168
  const bytes = Math.max(1, Math.ceil(this.#estimateBytes(input)));
163
169
  if (bytes > this.maxInputBytes) {
164
170
  return Promise.reject(
@@ -167,25 +173,32 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
167
173
  ),
168
174
  );
169
175
  }
170
-
171
- if (
172
- this.#pending.length >= this.maxBatchSize ||
173
- this.#pendingBytes + bytes > this.maxBufferedBytes
174
- ) {
176
+ if (bytes > this.maxBufferedBytes) {
177
+ throw new ReceiptCompletionBufferLimitError(
178
+ `Runtime receipt completion payload is ${bytes} bytes, above the ${this.maxBufferedBytes} byte sink capacity.`,
179
+ );
180
+ }
181
+ while (this.#residentBytes + bytes > this.maxBufferedBytes) {
182
+ this.#throwIfUnavailable();
175
183
  void this.flush().catch(() => undefined);
184
+ await new Promise<void>((resolve, reject) => {
185
+ this.#capacityWaiters.push({ resolve, reject });
186
+ });
176
187
  }
177
-
178
- return new Promise<TOutput>((resolve, reject) => {
188
+ this.#throwIfUnavailable();
189
+ this.#residentBytes += bytes;
190
+ const completion = new Promise<TOutput>((resolve, reject) => {
179
191
  this.#pending.push({ input, bytes, resolve, reject });
180
- this.#pendingBytes += bytes;
181
- if (this.#pending.length >= this.maxBatchSize) {
192
+ if (this.#pendingCount() >= this.maxBatchSize) {
182
193
  void this.flush().catch(() => undefined);
183
194
  }
184
195
  });
196
+ this.#scheduleFlush();
197
+ return await completion;
185
198
  }
186
199
 
187
200
  #scheduleFlush(): void {
188
- if (this.#flushTimer || this.#flushPromise || this.#pending.length === 0) {
201
+ if (this.#flushTimer || this.#flushPromise || this.#pendingCount() === 0) {
189
202
  return;
190
203
  }
191
204
  this.#flushTimer = setTimeout(() => {
@@ -201,28 +214,36 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
201
214
  }
202
215
 
203
216
  async #runFlushLoop(): Promise<void> {
204
- while (this.#pending.length > 0) {
217
+ while (this.#pendingCount() > 0) {
205
218
  const batch = this.#takeBatch();
206
- await this.#flushBatch(batch);
219
+ try {
220
+ await this.#flushBatch(batch);
221
+ } catch (error) {
222
+ this.#fail(error);
223
+ throw this.#failure;
224
+ }
207
225
  }
208
226
  }
209
227
 
210
228
  #takeBatch(): Array<PendingReceiptCompletion<TInput, TOutput>> {
211
229
  const batch: Array<PendingReceiptCompletion<TInput, TOutput>> = [];
212
230
  let batchBytes = 0;
213
- while (this.#pending.length > 0 && batch.length < this.maxBatchSize) {
214
- const next = this.#pending[0]!;
215
- if (
216
- batch.length > 0 &&
217
- batchBytes + next.bytes > this.maxBufferedBytes
218
- ) {
231
+ while (this.#pendingCount() > 0 && batch.length < this.maxBatchSize) {
232
+ const next = this.#pending[this.#pendingHead]!;
233
+ if (batch.length > 0 && batchBytes + next.bytes > this.maxBufferedBytes) {
219
234
  break;
220
235
  }
221
- this.#pending.shift();
222
- this.#pendingBytes -= next.bytes;
236
+ this.#pendingHead += 1;
223
237
  batch.push(next);
224
238
  batchBytes += next.bytes;
225
239
  }
240
+ if (
241
+ this.#pendingHead > 1_024 &&
242
+ this.#pendingHead * 2 >= this.#pending.length
243
+ ) {
244
+ this.#pending = this.#pending.slice(this.#pendingHead);
245
+ this.#pendingHead = 0;
246
+ }
226
247
  return batch;
227
248
  }
228
249
 
@@ -257,6 +278,44 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
257
278
  error,
258
279
  });
259
280
  throw error;
281
+ } finally {
282
+ this.#residentBytes = Math.max(0, this.#residentBytes - bytes);
283
+ this.#wakeCapacityWaiters();
284
+ }
285
+ }
286
+
287
+ #pendingCount(): number {
288
+ return this.#pending.length - this.#pendingHead;
289
+ }
290
+
291
+ #throwIfUnavailable(): void {
292
+ if (this.#failure) throw this.#failure;
293
+ if (this.#closed) {
294
+ throw new Error('Runtime receipt completion sink is already closed.');
295
+ }
296
+ }
297
+
298
+ #wakeCapacityWaiters(): void {
299
+ const waiters = this.#capacityWaiters;
300
+ this.#capacityWaiters = [];
301
+ for (const waiter of waiters) {
302
+ if (this.#failure) waiter.reject(this.#failure);
303
+ else if (this.#closed) {
304
+ waiter.reject(
305
+ new Error('Runtime receipt completion sink is already closed.'),
306
+ );
307
+ } else waiter.resolve();
308
+ }
309
+ }
310
+
311
+ #fail(error: unknown): void {
312
+ this.#failure =
313
+ error instanceof Error ? error : new Error(String(error ?? 'Unknown'));
314
+ while (this.#pendingCount() > 0) {
315
+ const pending = this.#pending[this.#pendingHead++]!;
316
+ this.#residentBytes = Math.max(0, this.#residentBytes - pending.bytes);
317
+ pending.reject(this.#failure);
260
318
  }
319
+ this.#wakeCapacityWaiters();
261
320
  }
262
321
  }
@@ -3,6 +3,7 @@ import type { PlayExecutionGovernor, WorkLease } from './governor/governor';
3
3
  export type RuntimeResourceLease = WorkLease;
4
4
 
5
5
  export type RuntimeResourceObservation = {
6
+ toolId?: string | null;
6
7
  providerResourceKey?: string | null;
7
8
  rowEstimatedBytes?: number | null;
8
9
  rowAdmissionWaitMs?: number | null;
@@ -172,12 +173,19 @@ export function createRuntimeResourceGovernor(input: {
172
173
  }
173
174
  if (
174
175
  observation.providerSuccess === true &&
175
- observation.providerResourceKey
176
+ (observation.toolId || observation.providerResourceKey)
176
177
  ) {
177
- input.executionGovernor.observeProviderSuccess({
178
- provider: observation.providerResourceKey,
179
- latencyMs: observation.providerLatencyMs,
180
- });
178
+ if (observation.toolId) {
179
+ input.executionGovernor.observeToolSuccess({
180
+ toolId: observation.toolId,
181
+ latencyMs: observation.providerLatencyMs,
182
+ });
183
+ } else {
184
+ input.executionGovernor.observeProviderSuccess({
185
+ provider: observation.providerResourceKey!,
186
+ latencyMs: observation.providerLatencyMs,
187
+ });
188
+ }
181
189
  }
182
190
  }
183
191
  };
@@ -5,13 +5,26 @@ import type {
5
5
  PlayRunnerResult,
6
6
  } from '@shared_libs/play-runtime/protocol';
7
7
  import { shouldCleanupSandboxInBackground } from '@shared_libs/play-runtime/daytona-runtime-config';
8
+ import { isIsolatedRuntimeSchedulerSchema } from '@shared_libs/play-runtime/runtime-scheduler-topology';
8
9
 
9
10
  const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
10
- const DAYTONA_CREATE_HEDGE_DELAYS_MS = [0, 200, 400, 700] as const;
11
- const DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS = 3_500;
11
+ const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
12
12
  const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = 5;
13
13
  const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
14
14
  const DAYTONA_SANDBOX_LABEL_SOURCE = 'deepline-play-runner';
15
+ // The hosted Daytona API pins sandbox resources to the creating snapshot: it
16
+ // rejects create-time resource overrides for snapshot sandboxes (400 "Cannot
17
+ // specify Sandbox resources when using a snapshot") and does not serve
18
+ // POST /sandbox/:id/resize (404 "Cannot POST"). The resource boundary is
19
+ // therefore the default snapshot's grant, verified against these pins after
20
+ // create. If Daytona changes the default snapshot's resources, sandbox
21
+ // acquisition fails loudly instead of silently running plays on a different
22
+ // compute boundary; bump these pins deliberately when that happens.
23
+ export const DAYTONA_SANDBOX_CPU = 1;
24
+ export const DAYTONA_SANDBOX_MEMORY_GIB = 1;
25
+ export const DAYTONA_SANDBOX_DISK_GIB = 3;
26
+ export const DAYTONA_SANDBOX_GPU = 0;
27
+ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
15
28
 
16
29
  export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
17
30
 
@@ -35,7 +48,7 @@ export type OneShotDaytonaSandboxLifecycle = {
35
48
  dispose: () => Promise<void>;
36
49
  };
37
50
 
38
- type HedgedDaytonaCreateResult = {
51
+ type DaytonaCreateResult = {
39
52
  sandbox: DaytonaSandbox;
40
53
  attempt: number;
41
54
  attemptElapsedMs: number;
@@ -64,6 +77,48 @@ export function validateDaytonaExecutionContext(
64
77
  return orgId;
65
78
  }
66
79
 
80
+ /**
81
+ * Outbound-network policy for a one-shot sandbox. The invariant this protects:
82
+ * CUSTOMER code in PRODUCTION never runs with unrestricted egress — the
83
+ * production worker must provision `DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST`
84
+ * (comma-separated CIDRs, per the Daytona `networkAllowList` contract) and
85
+ * every prod sandbox is created `networkBlockAll: true` + that allow-list.
86
+ *
87
+ * "Production" here is the RUN'S topology, not the build flag: a run whose
88
+ * scheduler schema is isolated (per-PR preview CI, per-worktree dev — see
89
+ * `isIsolatedRuntimeSchedulerSchema`) executes synthetic internal test plays
90
+ * against dynamic per-PR hosts (Vercel preview app, per-PR Fly gateway) whose
91
+ * IPs cannot be enumerated as stable CIDRs, so the allow-list requirement
92
+ * cannot apply there. `NODE_ENV=production` alone is the wrong key — the Fly
93
+ * worker image bakes it for BOTH prod and preview fleets (build optimization),
94
+ * which made every preview sandbox create fail. Fail-closed: an absent schema
95
+ * is the production topology, never "unknown".
96
+ *
97
+ * An explicitly provisioned allow-list is always honored (preview included),
98
+ * so a future preview fleet with enumerable egress can opt in to full parity.
99
+ */
100
+ export function resolveDaytonaSandboxNetworkPolicy(input: {
101
+ runtimeSchedulerSchema: string | null | undefined;
102
+ env?: NodeJS.ProcessEnv;
103
+ }): { networkAllowList: string | null } {
104
+ const env = input.env ?? process.env;
105
+ const networkAllowList = env[DAYTONA_NETWORK_ALLOW_LIST_ENV]?.trim() || null;
106
+ if (
107
+ !networkAllowList &&
108
+ env.NODE_ENV === 'production' &&
109
+ !isIsolatedRuntimeSchedulerSchema(input.runtimeSchedulerSchema)
110
+ ) {
111
+ throw new Error(
112
+ `${DAYTONA_NETWORK_ALLOW_LIST_ENV} is required for production runs. ` +
113
+ `Refusing to start customer code with unrestricted outbound network ` +
114
+ `access. Provision a comma-separated CIDR allow-list on the production ` +
115
+ `worker. (Isolated-schema runs — preview CI / worktrees — are exempt: ` +
116
+ `they execute synthetic internal plays against dynamic per-PR hosts.)`,
117
+ );
118
+ }
119
+ return { networkAllowList };
120
+ }
121
+
67
122
  async function createOneShotDaytonaSandbox(input: {
68
123
  daytona: DaytonaClient;
69
124
  orgId: string;
@@ -86,6 +141,9 @@ async function createOneShotDaytonaSandbox(input: {
86
141
  if (workflowId) labels.workflowId = workflowId;
87
142
  if (playId) labels.playId = playId;
88
143
  if (runId) labels.runId = runId;
144
+ const { networkAllowList } = resolveDaytonaSandboxNetworkPolicy({
145
+ runtimeSchedulerSchema: input.context.runtimeSchedulerSchema ?? null,
146
+ });
89
147
 
90
148
  // Intentionally omit image/snapshot so Daytona uses its default fast sandbox
91
149
  // image; custom images would add build/pull tax to one-shot cold starts.
@@ -95,136 +153,60 @@ async function createOneShotDaytonaSandbox(input: {
95
153
  ephemeral: true,
96
154
  autoStopInterval: DAYTONA_AUTO_STOP_INTERVAL_MINUTES,
97
155
  autoArchiveInterval: DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES,
156
+ ...(networkAllowList
157
+ ? {
158
+ networkBlockAll: true,
159
+ networkAllowList,
160
+ }
161
+ : {}),
98
162
  },
99
163
  { timeout: DAYTONA_CREATE_TIMEOUT_SECONDS },
100
164
  );
101
165
  }
102
166
 
103
- function cleanupLateDaytonaCreate(input: {
104
- sandbox: DaytonaSandbox;
105
- attempt: number;
106
- emitStage: DaytonaStageEmitter;
107
- }): void {
108
- void input.sandbox.delete(30).then(
109
- () =>
110
- input.emitStage('create:late_cleanup_done', {
111
- sandboxId: input.sandbox.id,
112
- attempt: input.attempt,
113
- }),
114
- (error: unknown) => {
115
- console.warn('[play-runner.daytona.create_late_cleanup_failed]', {
116
- sandboxId: input.sandbox.id,
117
- attempt: input.attempt,
118
- error: error instanceof Error ? error.message : String(error),
119
- });
120
- },
121
- );
122
- }
123
-
124
- async function createHedgedOneShotDaytonaSandbox(input: {
167
+ async function createRetriedOneShotDaytonaSandbox(input: {
125
168
  daytona: DaytonaClient;
126
169
  orgId: string;
127
170
  context: DaytonaExecutionContext;
128
171
  emitStage: DaytonaStageEmitter;
129
172
  startedAt: number;
130
- }): Promise<HedgedDaytonaCreateResult> {
131
- return await new Promise<HedgedDaytonaCreateResult>((resolve, reject) => {
132
- let settled = false;
133
- let startedAttempts = 0;
134
- let rejectedAttempts = 0;
135
- const errors: string[] = [];
136
- const timers: ReturnType<typeof setTimeout>[] = [];
137
-
138
- const clearTimers = () => {
139
- while (timers.length > 0) {
140
- const timer = timers.pop();
141
- if (timer) clearTimeout(timer);
142
- }
143
- };
144
-
145
- const rejectIfExhausted = () => {
146
- if (
147
- settled ||
148
- rejectedAttempts !== DAYTONA_CREATE_HEDGE_DELAYS_MS.length
149
- ) {
150
- return;
151
- }
152
- settled = true;
153
- clearTimers();
154
- reject(
155
- new Error(
156
- `Daytona sandbox create failed across ${rejectedAttempts} attempts: ${errors.join('; ')}`,
157
- ),
158
- );
159
- };
160
-
161
- const startAttempt = (attempt: number) => {
162
- if (settled) return;
163
- startedAttempts = Math.max(startedAttempts, attempt);
164
- if (attempt > 1) {
165
- input.emitStage('create:retry', {
166
- attempt: attempt - 1,
167
- elapsedMs: Date.now() - input.startedAt,
168
- reason: 'hedge_delay',
169
- });
170
- }
171
- const attemptStartedAt = Date.now();
172
- void createOneShotDaytonaSandbox({
173
+ }): Promise<DaytonaCreateResult> {
174
+ const errors: string[] = [];
175
+ for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
176
+ const attempt = index + 1;
177
+ if (delayMs > 0) {
178
+ input.emitStage('create:retry', {
179
+ attempt: index,
180
+ elapsedMs: Date.now() - input.startedAt,
181
+ reason: 'bounded_backoff',
182
+ delayMs,
183
+ });
184
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
185
+ }
186
+ const attemptStartedAt = Date.now();
187
+ try {
188
+ const sandbox = await createOneShotDaytonaSandbox({
173
189
  daytona: input.daytona,
174
190
  orgId: input.orgId,
175
191
  context: input.context,
176
- }).then(
177
- (sandbox) => {
178
- if (settled) {
179
- cleanupLateDaytonaCreate({
180
- sandbox,
181
- attempt,
182
- emitStage: input.emitStage,
183
- });
184
- return;
185
- }
186
- settled = true;
187
- clearTimers();
188
- resolve({
189
- sandbox,
190
- attempt,
191
- attemptElapsedMs: Date.now() - attemptStartedAt,
192
- });
193
- },
194
- (error: unknown) => {
195
- rejectedAttempts += 1;
196
- errors.push(error instanceof Error ? error.message : String(error));
197
- console.warn('[play-runner.daytona.create_attempt_failed]', {
198
- attempt,
199
- error: error instanceof Error ? error.message : String(error),
200
- });
201
- rejectIfExhausted();
202
- },
203
- );
204
- };
205
-
206
- for (const [index, delayMs] of DAYTONA_CREATE_HEDGE_DELAYS_MS.entries()) {
207
- const attempt = index + 1;
208
- if (delayMs === 0) {
209
- startAttempt(attempt);
210
- } else {
211
- timers.push(setTimeout(() => startAttempt(attempt), delayMs));
212
- }
192
+ });
193
+ return {
194
+ sandbox,
195
+ attempt,
196
+ attemptElapsedMs: Date.now() - attemptStartedAt,
197
+ };
198
+ } catch (error) {
199
+ const message = error instanceof Error ? error.message : String(error);
200
+ errors.push(message);
201
+ console.warn('[play-runner.daytona.create_attempt_failed]', {
202
+ attempt,
203
+ error: message,
204
+ });
213
205
  }
214
-
215
- timers.push(
216
- setTimeout(() => {
217
- if (settled) return;
218
- settled = true;
219
- clearTimers();
220
- reject(
221
- new Error(
222
- `Daytona sandbox create did not complete within ${DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS}ms after starting ${startedAttempts} attempts.`,
223
- ),
224
- );
225
- }, DAYTONA_CREATE_ACQUIRE_TIMEOUT_MS),
226
- );
227
- });
206
+ }
207
+ throw new Error(
208
+ `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`,
209
+ );
228
210
  }
229
211
 
230
212
  async function acquireOneShotDaytonaSandbox(input: {
@@ -235,13 +217,33 @@ async function acquireOneShotDaytonaSandbox(input: {
235
217
  startedAt: number;
236
218
  }): Promise<AcquiredDaytonaSandbox> {
237
219
  input.emitStage('create:start');
238
- const result = await createHedgedOneShotDaytonaSandbox(input);
220
+ const result = await createRetriedOneShotDaytonaSandbox(input);
221
+ const granted = {
222
+ cpu: result.sandbox.cpu,
223
+ memoryGiB: result.sandbox.memory,
224
+ diskGiB: result.sandbox.disk,
225
+ gpu: result.sandbox.gpu ?? 0,
226
+ };
227
+ if (
228
+ granted.cpu !== DAYTONA_SANDBOX_CPU ||
229
+ granted.memoryGiB !== DAYTONA_SANDBOX_MEMORY_GIB ||
230
+ granted.diskGiB !== DAYTONA_SANDBOX_DISK_GIB ||
231
+ granted.gpu !== DAYTONA_SANDBOX_GPU
232
+ ) {
233
+ await result.sandbox.delete(30).catch(() => undefined);
234
+ throw new Error(
235
+ `Daytona sandbox resource boundary mismatch: expected cpu=${DAYTONA_SANDBOX_CPU} memoryGiB=${DAYTONA_SANDBOX_MEMORY_GIB} diskGiB=${DAYTONA_SANDBOX_DISK_GIB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
236
+ );
237
+ }
239
238
  const billingStartedAt = Date.now();
240
239
  input.emitStage('create:done', {
241
240
  sandboxId: result.sandbox.id,
242
241
  attempt: result.attempt,
243
242
  elapsedMs: billingStartedAt - input.startedAt,
244
243
  attemptElapsedMs: result.attemptElapsedMs,
244
+ cpu: granted.cpu,
245
+ memoryGiB: granted.memoryGiB,
246
+ diskGiB: granted.diskGiB,
245
247
  });
246
248
  return {
247
249
  sandbox: result.sandbox,
@@ -60,16 +60,18 @@ function nodeMaterializePayloadCommand(input: {
60
60
  function remoteRuntimeContextForDaytona(
61
61
  context: DaytonaExecutionContext,
62
62
  ): DaytonaRemoteRuntimeContext {
63
- if ((context.preloadedDbSessions?.length ?? 0) > 0) {
64
- return {
65
- ...context,
66
- dbSessionStrategy: 'preloaded',
67
- preloadedDbSessions: context.preloadedDbSessions,
68
- };
63
+ const gatewayBaseUrl = context.receiptGatewayBaseUrl?.trim();
64
+ if (!gatewayBaseUrl) {
65
+ throw new Error(
66
+ 'Daytona execution requires DEEPLINE_RUNTIME_RECEIPT_GATEWAY_URL. Refusing to put app credentials or database sessions in the customer-code sandbox.',
67
+ );
69
68
  }
70
69
  return {
71
70
  ...context,
72
- dbSessionStrategy: 'sandbox_public_key',
71
+ baseUrl: gatewayBaseUrl,
72
+ receiptGatewayBaseUrl: gatewayBaseUrl,
73
+ vercelProtectionBypassToken: null,
74
+ dbSessionStrategy: 'gateway_only',
73
75
  preloadedDbSessions: undefined,
74
76
  };
75
77
  }
@@ -215,7 +217,7 @@ export async function stageDaytonaRunnerPayload(input: {
215
217
  configPath,
216
218
  artifactCodePath,
217
219
  })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
218
- const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
220
+ const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
219
221
 
220
222
  return {
221
223
  workDir: input.workDir,