deepline 0.2.74 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +75 -4
  2. package/dist/bundling-sources/sdk/src/compat.ts +4 -0
  3. package/dist/bundling-sources/sdk/src/play.ts +7 -0
  4. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +15 -1
  5. package/dist/bundling-sources/sdk/src/release.ts +12 -1
  6. package/dist/bundling-sources/sdk/src/types.ts +7 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +146 -29
  8. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +57 -1
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +20 -19
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +100 -25
  13. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +25 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +0 -9
  15. package/dist/bundling-sources/shared_libs/play-runtime/tool-response-contract.ts +89 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +29 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +203 -16
  18. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +6 -5
  19. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +3 -0
  20. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +1 -0
  21. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +5 -2
  22. package/dist/bundling-sources/shared_libs/plays/contracts.ts +14 -0
  23. package/dist/cli/index.js +123 -47
  24. package/dist/cli/index.mjs +86 -10
  25. package/dist/{compiler-manifest-DFuz9-0_.d.mts → compiler-manifest-BX85pKXW.d.mts} +7 -2
  26. package/dist/{compiler-manifest-DFuz9-0_.d.ts → compiler-manifest-BX85pKXW.d.ts} +7 -2
  27. package/dist/index.d.mts +14 -3
  28. package/dist/index.d.ts +14 -3
  29. package/dist/index.js +85 -4
  30. package/dist/index.mjs +85 -4
  31. package/dist/install-integrity.json +3 -2
  32. package/dist/plays/bundle-play-file.d.mts +12 -2
  33. package/dist/plays/bundle-play-file.d.ts +12 -2
  34. package/dist/plays/bundle-play-file.mjs +20 -4
  35. package/package.json +1 -1
@@ -51,6 +51,7 @@ export type AcquiredDaytonaSandbox = {
51
51
  daytonaOrganizationId: string;
52
52
  billingStartedAt: number;
53
53
  billingEndedAt?: number;
54
+ sandboxCapacityLeaseId?: string;
54
55
  };
55
56
 
56
57
  export type OneShotDaytonaSandboxLifecycle = {
@@ -118,6 +119,7 @@ type DaytonaCreateResult = {
118
119
  sandbox: DaytonaSandbox;
119
120
  attempt: number;
120
121
  attemptElapsedMs: number;
122
+ sandboxCapacityLeaseId?: string;
121
123
  };
122
124
 
123
125
  function daytonaCreateErrorClass(
@@ -133,6 +135,8 @@ function daytonaCreateErrorClass(
133
135
  async function rejectAcquiredSandbox(
134
136
  sandbox: DaytonaSandbox,
135
137
  reason: string,
138
+ reservation?: { leaseId: string },
139
+ releaseSandboxCapacity?: (leaseId: string) => Promise<void>,
136
140
  ): Promise<never> {
137
141
  try {
138
142
  await sandbox.delete(30);
@@ -143,6 +147,9 @@ async function rejectAcquiredSandbox(
143
147
  { cause: error },
144
148
  );
145
149
  }
150
+ if (reservation) {
151
+ await releaseSandboxCapacity?.(reservation.leaseId);
152
+ }
146
153
  throw new Error(reason);
147
154
  }
148
155
 
@@ -350,6 +357,8 @@ async function createRetriedOneShotDaytonaSandbox(input: {
350
357
  context: DaytonaExecutionContext;
351
358
  emitStage: DaytonaStageEmitter;
352
359
  observeCreateCall?: DaytonaCreateCallObserver;
360
+ reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
361
+ releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
353
362
  nextProviderAttempt: () => number;
354
363
  startedAt: number;
355
364
  }): Promise<DaytonaCreateResult> {
@@ -372,6 +381,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
372
381
  occurredAtMs: attemptStartedAt,
373
382
  providerAttempt: attempt,
374
383
  });
384
+ const reservation = await input.reserveSandboxCapacity?.();
375
385
  let sandbox: DaytonaSandbox;
376
386
  try {
377
387
  sandbox = await createOneShotDaytonaSandbox({
@@ -381,6 +391,12 @@ async function createRetriedOneShotDaytonaSandbox(input: {
381
391
  sandboxName,
382
392
  });
383
393
  } catch (error) {
394
+ // A normal rejected create has no durable provider resource. A timeout
395
+ // can be ambiguous, so retain its lease until the crash TTL rather than
396
+ // admitting another sandbox against an unknown provider outcome.
397
+ if (reservation && !isDaytonaSandboxStartTimeout(String(error))) {
398
+ await input.releaseSandboxCapacity?.(reservation.leaseId);
399
+ }
384
400
  await input.observeCreateCall?.({
385
401
  type: 'daytona_create_call_failed',
386
402
  occurredAtMs: Date.now(),
@@ -407,6 +423,13 @@ async function createRetriedOneShotDaytonaSandbox(input: {
407
423
  sandboxName,
408
424
  });
409
425
  if (deleted) {
426
+ // The provider-confirmed reconciliation means this reservation can
427
+ // no longer represent a live sandbox. Release before returning the
428
+ // acquisition-unavailable result so Modal fallback is not blocked
429
+ // behind the unbound crash TTL.
430
+ if (reservation) {
431
+ await input.releaseSandboxCapacity?.(reservation.leaseId);
432
+ }
410
433
  throw new DaytonaSandboxAcquisitionUnavailableError(
411
434
  'daytona_sandbox_start_timeout',
412
435
  `${message} Timed-out Daytona sandbox was reconciled and deleted before fallback.`,
@@ -448,6 +471,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
448
471
  sandbox,
449
472
  attempt,
450
473
  attemptElapsedMs: acquiredAt - attemptStartedAt,
474
+ ...(reservation ? { sandboxCapacityLeaseId: reservation.leaseId } : {}),
451
475
  };
452
476
  }
453
477
  const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${errors.join('; ')}`;
@@ -468,6 +492,8 @@ async function acquireOneShotDaytonaSandbox(input: {
468
492
  context: DaytonaExecutionContext;
469
493
  emitStage: DaytonaStageEmitter;
470
494
  observeCreateCall?: DaytonaCreateCallObserver;
495
+ reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
496
+ releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
471
497
  nextProviderAttempt: () => number;
472
498
  startedAt: number;
473
499
  }): Promise<AcquiredDaytonaSandbox> {
@@ -493,6 +519,10 @@ async function acquireOneShotDaytonaSandbox(input: {
493
519
  await rejectAcquiredSandbox(
494
520
  result.sandbox,
495
521
  `Daytona sandbox resource boundary mismatch: expected cpu=${limits.cpu} memoryGiB=${limits.memoryGiB} diskGiB=${limits.diskGiB} gpu=${DAYTONA_SANDBOX_GPU}, granted cpu=${granted.cpu} memoryGiB=${granted.memoryGiB} diskGiB=${granted.diskGiB} gpu=${granted.gpu}`,
522
+ result.sandboxCapacityLeaseId
523
+ ? { leaseId: result.sandboxCapacityLeaseId }
524
+ : undefined,
525
+ input.releaseSandboxCapacity,
496
526
  );
497
527
  }
498
528
  const configuredOrganizationId =
@@ -506,6 +536,10 @@ async function acquireOneShotDaytonaSandbox(input: {
506
536
  await rejectAcquiredSandbox(
507
537
  result.sandbox,
508
538
  'Daytona sandbox organization routing mismatch. Refusing to run customer code in a sandbox whose observed organization differs from the configured organization.',
539
+ result.sandboxCapacityLeaseId
540
+ ? { leaseId: result.sandboxCapacityLeaseId }
541
+ : undefined,
542
+ input.releaseSandboxCapacity,
509
543
  );
510
544
  }
511
545
  let lookupOrganizationId: string | null = null;
@@ -528,6 +562,10 @@ async function acquireOneShotDaytonaSandbox(input: {
528
562
  return await rejectAcquiredSandbox(
529
563
  result.sandbox,
530
564
  'Daytona sandbox organization routing identity is missing. Refusing to run customer code without a durable cleanup routing domain.',
565
+ result.sandboxCapacityLeaseId
566
+ ? { leaseId: result.sandboxCapacityLeaseId }
567
+ : undefined,
568
+ input.releaseSandboxCapacity,
531
569
  );
532
570
  }
533
571
  const billingStartedAt = Date.now();
@@ -541,7 +579,14 @@ async function acquireOneShotDaytonaSandbox(input: {
541
579
  memoryGiB: granted.memoryGiB,
542
580
  diskGiB: granted.diskGiB,
543
581
  });
544
- return { sandbox, daytonaOrganizationId, billingStartedAt };
582
+ return {
583
+ sandbox,
584
+ daytonaOrganizationId,
585
+ billingStartedAt,
586
+ ...(result.sandboxCapacityLeaseId
587
+ ? { sandboxCapacityLeaseId: result.sandboxCapacityLeaseId }
588
+ : {}),
589
+ };
545
590
  }
546
591
 
547
592
  export function createOneShotDaytonaSandboxLifecycle(input: {
@@ -549,6 +594,8 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
549
594
  context: DaytonaExecutionContext;
550
595
  emitStage: DaytonaStageEmitter;
551
596
  observeCreateCall?: DaytonaCreateCallObserver;
597
+ reserveSandboxCapacity?: () => Promise<{ leaseId: string }>;
598
+ releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
552
599
  startedAt?: number;
553
600
  }): OneShotDaytonaSandboxLifecycle {
554
601
  const orgId = validateDaytonaExecutionContext(input.context);
@@ -564,6 +611,8 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
564
611
  context: input.context,
565
612
  emitStage: input.emitStage,
566
613
  observeCreateCall: input.observeCreateCall,
614
+ reserveSandboxCapacity: input.reserveSandboxCapacity,
615
+ releaseSandboxCapacity: input.releaseSandboxCapacity,
567
616
  nextProviderAttempt: () => {
568
617
  providerAttempt += 1;
569
618
  return providerAttempt;
@@ -594,6 +643,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
594
643
  try {
595
644
  const acquired = await latestAcquiredSandboxPromise;
596
645
  await acquired.sandbox.delete(30);
646
+ // `dispose` has provider-confirmed that the sandbox is gone. This is
647
+ // normally the narrow cancellation window before the scheduler has
648
+ // durably recorded the resource, so no cleanup job will release an
649
+ // unbound lease on our behalf.
650
+ if (acquired.sandboxCapacityLeaseId) {
651
+ await input.releaseSandboxCapacity?.(acquired.sandboxCapacityLeaseId);
652
+ }
597
653
  } catch (error) {
598
654
  console.warn('[play-runner.daytona.dispose_failed_before_acquire]', {
599
655
  error: error instanceof Error ? error.message : String(error),
@@ -5,7 +5,10 @@ import type {
5
5
  PlayRunnerPreparedExecution,
6
6
  PlayRunnerPrepareInput,
7
7
  } from '../types';
8
- import { RuntimeResourceFenceLostError } from '../types';
8
+ import {
9
+ isRuntimeSandboxCapacityLimitError,
10
+ RuntimeResourceFenceLostError,
11
+ } from '../types';
9
12
  import { buildPlayRunnerBundle } from '../bundle';
10
13
  import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
11
14
  import type {
@@ -26,6 +29,7 @@ import {
26
29
  DaytonaSandboxAcquisitionUnavailableError,
27
30
  createDaytonaSandboxCleanupManager,
28
31
  createOneShotDaytonaSandboxLifecycle,
32
+ type AcquiredDaytonaSandbox,
29
33
  type DaytonaClient,
30
34
  type DaytonaExecutionContext,
31
35
  type DaytonaSandbox,
@@ -945,6 +949,10 @@ function prepareDaytonaExecution(
945
949
  emitStage: (stage, extra) =>
946
950
  emitDaytonaStage(callbacks, input.context, stage, extra),
947
951
  observeCreateCall: callbacks?.onRuntimeLifecycleEvent,
952
+ reserveSandboxCapacity: callbacks?.reserveSandboxCapacity
953
+ ? () => callbacks.reserveSandboxCapacity!('daytona')
954
+ : undefined,
955
+ releaseSandboxCapacity: callbacks?.releaseSandboxCapacity,
948
956
  });
949
957
  return {
950
958
  kind: 'daytona',
@@ -1055,22 +1063,12 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1055
1063
  };
1056
1064
  let cancellationRequested = false;
1057
1065
  let cancellationCleanupStarted = false;
1058
- let activeAcquiredResource: {
1059
- sandbox: DaytonaSandbox;
1060
- daytonaOrganizationId: string;
1061
- billingStartedAt: number;
1062
- billingEndedAt?: number;
1063
- } | null = null;
1066
+ let activeAcquiredResource: AcquiredDaytonaSandbox | null = null;
1064
1067
  const uploadDeadlineBreaches: DaytonaUploadAttemptTiming[] = [];
1065
1068
  const reportedSandboxIds = new Set<string>();
1066
1069
  const reportedSandboxEndTimes = new Map<string, number | null>();
1067
1070
  const runtimeResourceReportErrors = new Set<unknown>();
1068
- const reportRuntimeResource = async (acquired: {
1069
- sandbox: DaytonaSandbox;
1070
- daytonaOrganizationId: string;
1071
- billingStartedAt: number;
1072
- billingEndedAt?: number;
1073
- }) => {
1071
+ const reportRuntimeResource = async (acquired: AcquiredDaytonaSandbox) => {
1074
1072
  const sandbox = acquired.sandbox;
1075
1073
  const billingEndedAt = acquired.billingEndedAt ?? null;
1076
1074
  if (
@@ -1101,6 +1099,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1101
1099
  cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : null,
1102
1100
  memoryGiB: typeof sandbox.memory === 'number' ? sandbox.memory : null,
1103
1101
  diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : null,
1102
+ ...(acquired.sandboxCapacityLeaseId
1103
+ ? { sandboxCapacityLeaseId: acquired.sandboxCapacityLeaseId }
1104
+ : {}),
1104
1105
  });
1105
1106
  } catch (error) {
1106
1107
  runtimeResourceReportErrors.add(error);
@@ -1109,12 +1110,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1109
1110
  reportedSandboxIds.add(sandbox.id);
1110
1111
  reportedSandboxEndTimes.set(sandbox.id, billingEndedAt);
1111
1112
  };
1112
- const reportRetiringRuntimeResource = async (acquired: {
1113
- sandbox: DaytonaSandbox;
1114
- daytonaOrganizationId: string;
1115
- billingStartedAt: number;
1116
- billingEndedAt?: number;
1117
- }) => {
1113
+ const reportRetiringRuntimeResource = async (
1114
+ acquired: AcquiredDaytonaSandbox,
1115
+ ) => {
1118
1116
  try {
1119
1117
  await reportRuntimeResource(acquired);
1120
1118
  } catch (error) {
@@ -1429,6 +1427,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1429
1427
  }
1430
1428
  throw error;
1431
1429
  }
1430
+ if (isRuntimeSandboxCapacityLimitError(error)) {
1431
+ throw error;
1432
+ }
1432
1433
  if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
1433
1434
  throw error;
1434
1435
  }
@@ -1,6 +1,9 @@
1
1
  import type { ModalClient } from 'modal';
2
2
  import type { PlayRunnerBackend, PlayRunnerCallbacks } from '../types';
3
- import { RuntimeResourceFenceLostError } from '../types';
3
+ import {
4
+ isRuntimeSandboxCapacityLimitError,
5
+ RuntimeResourceFenceLostError,
6
+ } from '../types';
4
7
  import { buildPlayRunnerBundle } from '../bundle';
5
8
  import type {
6
9
  PlayRunnerExecutionConfig,
@@ -23,6 +26,7 @@ import { stageRunnerPayload } from './daytona-payload-transport';
23
26
  import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
24
27
 
25
28
  const MODAL_RUNNER_READY_TIMEOUT_MS = 30_000;
29
+ const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
26
30
  type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
27
31
 
28
32
  export function modalSandboxLifetimeMs(
@@ -99,13 +103,38 @@ async function confirmDetachedModalRunnerReady(input: {
99
103
  );
100
104
  }
101
105
 
102
- async function terminateModalSandbox(sandbox: ModalSandbox): Promise<void> {
103
- await sandbox.terminate().catch((error) => {
106
+ async function terminateModalSandbox(sandbox: ModalSandbox): Promise<boolean> {
107
+ try {
108
+ // The default terminate call only acknowledges the request. Capacity may
109
+ // be released only after Modal confirms the sandbox itself has stopped.
110
+ await sandbox.terminate({ wait: true });
111
+ return true;
112
+ } catch (error) {
104
113
  console.warn('[play-runner.modal.terminate_failed]', {
105
114
  sandboxId: sandbox.sandboxId,
106
115
  error: error instanceof Error ? error.message : String(error),
107
116
  });
108
- });
117
+ return false;
118
+ }
119
+ }
120
+
121
+ async function releaseModalSandboxCapacityAfterConfirmedTermination(input: {
122
+ leaseId: string;
123
+ release: NonNullable<PlayRunnerCallbacks['releaseSandboxCapacity']>;
124
+ }): Promise<void> {
125
+ let lastError: unknown;
126
+ for (const delayMs of MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS) {
127
+ if (delayMs > 0) {
128
+ await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
129
+ }
130
+ try {
131
+ await input.release(input.leaseId);
132
+ return;
133
+ } catch (error) {
134
+ lastError = error;
135
+ }
136
+ }
137
+ throw lastError;
109
138
  }
110
139
 
111
140
  export const modalPlayRunnerBackend: PlayRunnerBackend = {
@@ -139,24 +168,35 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
139
168
  createIfMissing: true,
140
169
  });
141
170
  const image = modalConfig.client.images.fromRegistry(modalConfig.image);
142
- const sandbox = await modalConfig.client.sandboxes.create(app, image, {
143
- cpu: MODAL_SANDBOX_CPU_CORES,
144
- cpuLimit: MODAL_SANDBOX_CPU_CORES,
145
- memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
146
- memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
147
- timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
148
- idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
149
- workdir: modalConfig.workdir,
150
- tags: {
151
- source: 'deepline-play-runner',
152
- orgId: config.context.orgId ?? 'unknown',
153
- workflowId: config.context.workflowId ?? 'unknown',
154
- runId: config.context.runId ?? 'unknown',
155
- },
156
- ...(modalConfig.outboundCidrAllowlist
157
- ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
158
- : {}),
159
- });
171
+ const capacityReservation =
172
+ await callbacks?.reserveSandboxCapacity?.('modal');
173
+ const sandbox = await modalConfig.client.sandboxes
174
+ .create(app, image, {
175
+ cpu: MODAL_SANDBOX_CPU_CORES,
176
+ cpuLimit: MODAL_SANDBOX_CPU_CORES,
177
+ memoryMiB: MODAL_SANDBOX_MEMORY_MIB,
178
+ memoryLimitMiB: MODAL_SANDBOX_MEMORY_MIB,
179
+ timeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
180
+ idleTimeoutMs: modalSandboxLifetimeMs(modalConfig.limits),
181
+ workdir: modalConfig.workdir,
182
+ tags: {
183
+ source: 'deepline-play-runner',
184
+ orgId: config.context.orgId ?? 'unknown',
185
+ workflowId: config.context.workflowId ?? 'unknown',
186
+ runId: config.context.runId ?? 'unknown',
187
+ },
188
+ ...(modalConfig.outboundCidrAllowlist
189
+ ? { outboundCidrAllowlist: modalConfig.outboundCidrAllowlist }
190
+ : {}),
191
+ })
192
+ .catch(async (error) => {
193
+ if (capacityReservation) {
194
+ await callbacks?.releaseSandboxCapacity?.(
195
+ capacityReservation.leaseId,
196
+ );
197
+ }
198
+ throw error;
199
+ });
160
200
  const billingStartedAt = Date.now();
161
201
  runtimeTiming.modalCreateMs = billingStartedAt - startedAt;
162
202
  emitModalStage(config.context, 'create:done', {
@@ -164,6 +204,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
164
204
  elapsedMs: runtimeTiming.modalCreateMs,
165
205
  });
166
206
  let detached = false;
207
+ let setupError: unknown;
167
208
  let cancelExecution!: (error: Error) => void;
168
209
  const cancellationPromise = new Promise<never>((_resolve, reject) => {
169
210
  cancelExecution = reject;
@@ -192,6 +233,9 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
192
233
  cpu: MODAL_SANDBOX_CPU_CORES,
193
234
  memoryGiB: MODAL_SANDBOX_MEMORY_MIB / 1024,
194
235
  diskGiB: 0,
236
+ ...(capacityReservation
237
+ ? { sandboxCapacityLeaseId: capacityReservation.leaseId }
238
+ : {}),
195
239
  });
196
240
  } catch (error) {
197
241
  runtimeResourceRegistrationError = error;
@@ -290,14 +334,45 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
290
334
  tableNamespace: null,
291
335
  runtimeTiming,
292
336
  };
337
+ } catch (error) {
338
+ setupError = error;
339
+ throw error;
293
340
  } finally {
294
341
  callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
295
- if (!detached) await terminateModalSandbox(sandbox);
342
+ if (!detached && (await terminateModalSandbox(sandbox))) {
343
+ // This branch synchronously confirms that the just-created resource
344
+ // is gone. Do not keep its admission lease until the cleanup loop
345
+ // sees a resource that no longer exists.
346
+ if (capacityReservation && callbacks?.releaseSandboxCapacity) {
347
+ try {
348
+ await releaseModalSandboxCapacityAfterConfirmedTermination({
349
+ leaseId: capacityReservation.leaseId,
350
+ release: callbacks.releaseSandboxCapacity,
351
+ });
352
+ } catch (releaseError) {
353
+ // The scheduler fence is the primary correctness signal. Keep
354
+ // it intact if its best-effort early capacity release is
355
+ // transiently unavailable; durable cleanup will retry release.
356
+ if (setupError) {
357
+ console.error('[play-runner.modal.capacity_release_failed]', {
358
+ sandboxId: sandbox.sandboxId,
359
+ error:
360
+ releaseError instanceof Error
361
+ ? releaseError.message
362
+ : String(releaseError),
363
+ });
364
+ } else {
365
+ throw releaseError;
366
+ }
367
+ }
368
+ }
369
+ }
296
370
  }
297
371
  } catch (error) {
298
372
  if (
299
373
  error === runtimeResourceRegistrationError ||
300
- error instanceof RuntimeResourceFenceLostError
374
+ error instanceof RuntimeResourceFenceLostError ||
375
+ isRuntimeSandboxCapacityLimitError(error)
301
376
  ) {
302
377
  throw error;
303
378
  }
@@ -335,7 +410,7 @@ export async function deleteModalSandboxById(input: {
335
410
  appId: expectedAppId,
336
411
  })) {
337
412
  if (sandbox.sandboxId !== input.sandboxId) continue;
338
- await sandbox.terminate();
413
+ await sandbox.terminate({ wait: true });
339
414
  return { kind: 'deleted', appId: expectedAppId };
340
415
  }
341
416
  // Cleanup means ensure absent. An exact app-scoped inventory proving the
@@ -28,6 +28,8 @@ export type PlayRunnerRuntimeResource = {
28
28
  cpu?: number | null;
29
29
  memoryGiB?: number | null;
30
30
  diskGiB?: number | null;
31
+ /** Scheduler-only soft-capacity reservation made before provider create. */
32
+ sandboxCapacityLeaseId?: string;
31
33
  };
32
34
 
33
35
  export type RuntimeResourceTerminalReason =
@@ -64,6 +66,23 @@ export class RuntimeResourceFenceLostError extends Error {
64
66
  }
65
67
  }
66
68
 
69
+ /**
70
+ * A scheduler-owned admission result crosses the provider-backend boundary.
71
+ * Keep it typed by name here so the shared backend can rethrow it without an
72
+ * import from the app-owned scheduler implementation.
73
+ */
74
+ export const RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME =
75
+ 'RuntimeSandboxCapacityLimitError';
76
+
77
+ export function isRuntimeSandboxCapacityLimitError(
78
+ error: unknown,
79
+ ): error is Error {
80
+ return (
81
+ error instanceof Error &&
82
+ error.name === RUNTIME_SANDBOX_CAPACITY_LIMIT_ERROR_NAME
83
+ );
84
+ }
85
+
67
86
  export interface PlayRunnerCallbacks {
68
87
  onLog?: (event: PlayRunnerLogEvent) => void;
69
88
  onCheckpoint?: (checkpoint: PlayCheckpoint) => void;
@@ -72,6 +91,12 @@ export interface PlayRunnerCallbacks {
72
91
  onRuntimeResourceAcquired?: (
73
92
  resource: PlayRunnerRuntimeResource,
74
93
  ) => void | Promise<void>;
94
+ /** Reserve one physical-sandbox slot immediately before a provider create. */
95
+ reserveSandboxCapacity?: (
96
+ provider: 'daytona' | 'modal',
97
+ ) => Promise<{ leaseId: string }>;
98
+ /** Release an unbound reservation when provider create never produced a resource. */
99
+ releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
75
100
  /**
76
101
  * Durable scheduler-owned evidence for a provider create call. Backends
77
102
  * await this callback at the create boundary; it is not best-effort logging.
@@ -13,7 +13,6 @@ export const RUNTIME_CAPACITY_POLICY = {
13
13
  /** 32 Machines x 8 claim slots permits 256 concurrent launch/resume legs. */
14
14
  maxActiveLaneMachines: 32,
15
15
  workerSlotsPerMachine: 8,
16
- perOrgConcurrency: 4,
17
16
  /** Queue age is an escape hatch for small-but-stuck backlogs. */
18
17
  scaleUpQueueAgeMs: 10_000,
19
18
  /** Bound one Fly reconciliation without turning backlog into a stampede. */
@@ -113,14 +112,6 @@ export function assertRuntimeCapacityPolicy(): void {
113
112
  'Receipt claim operations must leave at least five seconds for cancellation and response delivery.',
114
113
  );
115
114
  }
116
- if (
117
- RUNTIME_CAPACITY_POLICY.absurd.perOrgConcurrency >
118
- ABSURD_GLOBAL_RUN_CONCURRENCY
119
- ) {
120
- throw new Error(
121
- 'Per-org runtime concurrency cannot exceed global runtime concurrency.',
122
- );
123
- }
124
115
  if (
125
116
  RUNTIME_CAPACITY_POLICY.absurd.activeLaneMachines >
126
117
  RUNTIME_CAPACITY_POLICY.absurd.maxActiveLaneMachines
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Public tool-response contracts shared by the API, SDK, Play bundler, and
3
+ * runtime. A response contract is transport behavior, never Play authoring.
4
+ */
5
+ export const V2_TOOL_RESPONSE_CONTRACT = 'v2-tool-response' as const;
6
+ export const RAW_V2_TOOL_RESPONSE_CONTRACT = 'raw-v2' as const;
7
+
8
+ export type ToolResponseContract =
9
+ | typeof V2_TOOL_RESPONSE_CONTRACT
10
+ | typeof RAW_V2_TOOL_RESPONSE_CONTRACT;
11
+
12
+ export type ToolResponseView = 'data' | 'rawV2';
13
+
14
+ export function isToolResponseContract(
15
+ value: unknown,
16
+ ): value is ToolResponseContract {
17
+ return (
18
+ value === V2_TOOL_RESPONSE_CONTRACT ||
19
+ value === RAW_V2_TOOL_RESPONSE_CONTRACT
20
+ );
21
+ }
22
+
23
+ export class UnsupportedToolResponseContractError extends Error {
24
+ constructor(value: unknown) {
25
+ super(
26
+ `Unsupported tool response contract ${String(value)}. Supported contracts: ${V2_TOOL_RESPONSE_CONTRACT}, ${RAW_V2_TOOL_RESPONSE_CONTRACT}.`,
27
+ );
28
+ this.name = 'UnsupportedToolResponseContractError';
29
+ }
30
+ }
31
+
32
+ /** Missing artifact compatibility predates canonical bodies and stays V2. */
33
+ export function normalizeToolResponseContract(
34
+ value: unknown,
35
+ ): ToolResponseContract {
36
+ if (value == null) return V2_TOOL_RESPONSE_CONTRACT;
37
+ if (isToolResponseContract(value)) return value;
38
+ throw new UnsupportedToolResponseContractError(value);
39
+ }
40
+
41
+ export function legacyRawFromToolResponseRawV2(
42
+ rawV2: unknown,
43
+ view: ToolResponseView,
44
+ responseMeta?: Record<string, unknown>,
45
+ ): unknown {
46
+ const legacyRaw =
47
+ view === 'data' &&
48
+ rawV2 &&
49
+ typeof rawV2 === 'object' &&
50
+ !Array.isArray(rawV2)
51
+ ? (rawV2 as Record<string, unknown>).data
52
+ : rawV2;
53
+ // Before raw-v2, an async launch without a data envelope exposed
54
+ // Deepline's billing summary at `toolResponse.raw.deepline_billing`.
55
+ // The canonical response keeps it separate from provider data, so reattach
56
+ // it only to this derived legacy view.
57
+ const deeplineBilling = responseMeta?.deepline_billing;
58
+ if (
59
+ view === 'rawV2' &&
60
+ deeplineBilling !== undefined &&
61
+ legacyRaw &&
62
+ typeof legacyRaw === 'object' &&
63
+ !Array.isArray(legacyRaw)
64
+ ) {
65
+ return {
66
+ ...(legacyRaw as Record<string, unknown>),
67
+ deepline_billing: deeplineBilling,
68
+ };
69
+ }
70
+ return legacyRaw;
71
+ }
72
+
73
+ export function providerMetaFromToolResponseRawV2(
74
+ rawV2: unknown,
75
+ view: ToolResponseView,
76
+ ): Record<string, unknown> | undefined {
77
+ if (
78
+ view !== 'data' ||
79
+ !rawV2 ||
80
+ typeof rawV2 !== 'object' ||
81
+ Array.isArray(rawV2)
82
+ ) {
83
+ return undefined;
84
+ }
85
+ const meta = (rawV2 as Record<string, unknown>).meta;
86
+ return meta && typeof meta === 'object' && !Array.isArray(meta)
87
+ ? (meta as Record<string, unknown>)
88
+ : undefined;
89
+ }
@@ -75,7 +75,7 @@ export type ToolResultEnvelope<
75
75
  meta?: TMeta;
76
76
  };
77
77
 
78
- export type SerializedToolExecuteResult = {
78
+ export type SerializedToolExecuteResultV1 = {
79
79
  __kind: 'deepline.tool_execute_result.v1';
80
80
  status: string;
81
81
  job_id?: string;
@@ -98,11 +98,36 @@ export type SerializedToolExecuteResult = {
98
98
  execution: ToolResultExecutionMetadata;
99
99
  };
100
100
 
101
+ /**
102
+ * Canonical receipt form. Provider data is stored once as `rawV2`; the legacy
103
+ * raw/meta view is recreated from the payload-free projection descriptor.
104
+ */
105
+ export type SerializedToolExecuteResultV2 = Omit<
106
+ SerializedToolExecuteResultV1,
107
+ '__kind' | 'toolResponse'
108
+ > & {
109
+ __kind: 'deepline.tool_execute_result.v2';
110
+ toolResponse: {
111
+ rawV2: unknown;
112
+ view: 'data' | 'rawV2';
113
+ /** Deepline-owned additions not already represented in `rawV2.meta`. */
114
+ responseMeta?: Record<string, unknown>;
115
+ };
116
+ };
117
+
118
+ export type SerializedToolExecuteResult =
119
+ | SerializedToolExecuteResultV1
120
+ | SerializedToolExecuteResultV2;
121
+
101
122
  export type ToolResponseEnvelope<
102
123
  TData = unknown,
103
124
  TMeta = Record<string, unknown>,
104
125
  > = {
105
126
  raw: TData;
127
+ /** Complete parsed and scrubbed provider response, materialized from raw-v2. */
128
+ rawV2?: unknown;
129
+ /** Durable descriptor for deriving the legacy raw view from `rawV2`. */
130
+ view?: 'data' | 'rawV2';
106
131
  meta?: TMeta;
107
132
  };
108
133
 
@@ -156,8 +181,9 @@ export type ToolExecuteResultAccessors<
156
181
  * Canonical result returned by Deepline tool execution.
157
182
  *
158
183
  * The top-level object is Deepline-owned execution metadata and semantic
159
- * extraction state. Raw tool/provider data lives under `toolResponse.raw`;
160
- * response metadata lives under `toolResponse.meta`. Semantic single-value
184
+ * extraction state. The canonical provider response lives under
185
+ * `toolResponse.rawV2`; `toolResponse.raw` remains the legacy compatibility
186
+ * projection. Response metadata lives under `toolResponse.meta`. Semantic single-value
161
187
  * getters live under `extractedValues.<name>.get()`, and list getters live
162
188
  * under `extractedLists.<name>.get()`.
163
189
  *