deepline 0.3.453 → 0.3.455

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/bundling-sources/sdk/src/http.ts +17 -3
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/sdk/src/tool-calls.ts +13 -1
  4. package/dist/bundling-sources/sdk/src/types.ts +4 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +13 -1
  7. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +83 -4
  8. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +11 -3
  9. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +16 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +20 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +77 -12
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +5 -2
  13. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +254 -64
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +11 -0
  15. package/dist/cli/{auth-6CXS55CN.mjs → auth-BGPCPVHW.mjs} +4 -4
  16. package/dist/cli/auth-SRR4DW75.js +40 -0
  17. package/dist/cli/{billing-SWLTV6PV.mjs → billing-6BLEXHCR.mjs} +4 -4
  18. package/dist/cli/{billing-QC52XTAD.js → billing-FCC2YOKB.js} +73 -73
  19. package/dist/cli/{chunk-RTR2KUU6.js → chunk-2R7JD2NY.js} +8 -6
  20. package/dist/cli/{chunk-V6AZQ4JH.js → chunk-2SN2EEQW.js} +16 -16
  21. package/dist/cli/{chunk-2LDU5BMG.mjs → chunk-2VHR5DXQ.mjs} +3 -3
  22. package/dist/cli/{chunk-43HOSM2F.mjs → chunk-3E4WSS35.mjs} +1 -1
  23. package/dist/cli/{chunk-KQD6EFPD.mjs → chunk-3RQY2OG7.mjs} +2 -2
  24. package/dist/cli/{chunk-IFBLLSGH.mjs → chunk-4BZFRLKU.mjs} +5 -5
  25. package/dist/cli/{chunk-BX343KU7.mjs → chunk-4Z3X5CWI.mjs} +1 -1
  26. package/dist/cli/{chunk-YU6IE234.js → chunk-6F7TD4M4.js} +6 -6
  27. package/dist/cli/{chunk-RG644LJU.js → chunk-6TOMXZX3.js} +1 -1
  28. package/dist/cli/{chunk-6CTVXF6X.mjs → chunk-B7K3QD5W.mjs} +10 -9
  29. package/dist/cli/{chunk-HUDUN4AR.js → chunk-BCB2QNG4.js} +2 -2
  30. package/dist/cli/{chunk-V7S2RUVE.js → chunk-CZREA5NE.js} +8 -8
  31. package/dist/cli/{chunk-IXVABCNA.mjs → chunk-DPTEABFU.mjs} +1 -1
  32. package/dist/cli/{chunk-NOHA3SZJ.js → chunk-EXZOUJCD.js} +3 -3
  33. package/dist/cli/{chunk-7557D4FM.mjs → chunk-FCQ52RUB.mjs} +2 -2
  34. package/dist/cli/{chunk-XH4QC5ZX.mjs → chunk-IUSGWMGK.mjs} +2 -2
  35. package/dist/cli/{chunk-WTTHH2K3.js → chunk-J7WYWQF4.js} +6 -6
  36. package/dist/cli/{chunk-NSQZUUYB.js → chunk-LQH2KJ53.js} +93 -92
  37. package/dist/cli/{chunk-X2LUT22I.mjs → chunk-LRA5YJVV.mjs} +5 -3
  38. package/dist/cli/{chunk-PJWXEUUH.js → chunk-MCMLQ7GD.js} +8 -8
  39. package/dist/cli/{chunk-VG6WESY7.js → chunk-MFKTYIBJ.js} +51 -51
  40. package/dist/cli/{chunk-XV4463XW.mjs → chunk-PQ4JU2MV.mjs} +4 -2
  41. package/dist/cli/{chunk-P4UQCYKH.mjs → chunk-R5VMRVRN.mjs} +4 -4
  42. package/dist/cli/{chunk-WWZ4YJ6J.js → chunk-RQMEYYN4.js} +35 -35
  43. package/dist/cli/{chunk-FS2RMEJ7.mjs → chunk-RYGDM5BG.mjs} +1 -1
  44. package/dist/cli/{chunk-KFKM5J2N.js → chunk-SMZUU4NY.js} +2 -2
  45. package/dist/cli/{chunk-RNCIPLU5.js → chunk-UVJLG5KD.js} +50 -50
  46. package/dist/cli/{chunk-ZVBLECZX.mjs → chunk-VIGEE67Z.mjs} +3 -3
  47. package/dist/cli/{chunk-JOF3B22H.js → chunk-VMGTYM3A.js} +20 -20
  48. package/dist/cli/{chunk-DMTAFWMS.js → chunk-W3DIXRDT.js} +241 -239
  49. package/dist/cli/{chunk-SQUJHT7O.mjs → chunk-WBULEVM4.mjs} +1 -1
  50. package/dist/cli/{chunk-IWFI6FTJ.mjs → chunk-WVGPAP7A.mjs} +3 -3
  51. package/dist/cli/{chunk-XNN3OTGT.mjs → chunk-YIFCV77L.mjs} +2 -2
  52. package/dist/cli/{chunk-MP35NE55.js → chunk-YKEWDVZC.js} +26 -26
  53. package/dist/cli/{csv-FON4ZQKC.mjs → csv-27T3L4RC.mjs} +3 -3
  54. package/dist/cli/{csv-WIQB5XWZ.js → csv-TEAA2YV5.js} +12 -12
  55. package/dist/cli/{db-VMDDODCN.js → db-J3PPU7GX.js} +25 -25
  56. package/dist/cli/{db-3KA62DOQ.mjs → db-N3NRYYI4.mjs} +4 -4
  57. package/dist/cli/{describe-NTAAHCCX.js → describe-PX7XBLZF.js} +17 -17
  58. package/dist/cli/{describe-LGDK77QI.mjs → describe-SJBCCTR7.mjs} +13 -13
  59. package/dist/cli/describe-core.d.mts +1 -1
  60. package/dist/cli/describe-core.d.ts +1 -1
  61. package/dist/cli/{executions-OT2KNJDO.js → executions-WXIL4C5M.js} +6 -6
  62. package/dist/cli/{executions-TM6AMTQ7.mjs → executions-XN376RR4.mjs} +4 -4
  63. package/dist/cli/{feedback-HWLUVFE2.mjs → feedback-KHJXJ2K7.mjs} +3 -3
  64. package/dist/cli/{feedback-4HQGY3TO.js → feedback-LG5NWE3R.js} +7 -7
  65. package/dist/cli/{git-TDJVSPE6.mjs → git-726A5HHA.mjs} +4 -4
  66. package/dist/cli/git-PXYOFDQH.js +24 -0
  67. package/dist/cli/index.js +132 -132
  68. package/dist/cli/index.mjs +37 -37
  69. package/dist/cli/{legacy-noops-XH3FDLWT.js → legacy-noops-QI6VN34P.js} +7 -7
  70. package/dist/cli/{legacy-noops-YB2PES43.mjs → legacy-noops-V4WS63SD.mjs} +4 -4
  71. package/dist/cli/{monitors-KQDDMSDR.mjs → monitors-AAWRYPHY.mjs} +5 -5
  72. package/dist/cli/{monitors-E3QFEJKF.js → monitors-ICQ2Q4XT.js} +89 -89
  73. package/dist/cli/{org-PMTLR7VP.js → org-5JV6KYY2.js} +66 -66
  74. package/dist/cli/{org-7X2C7YKA.mjs → org-7VJKBTZX.mjs} +4 -4
  75. package/dist/cli/{play-7FNEPSVR.mjs → play-2D6ZIJEV.mjs} +11 -11
  76. package/dist/cli/{play-RH22Z3D6.js → play-HDI2MLJL.js} +284 -284
  77. package/dist/cli/{providers-X5NY5IQH.js → providers-GIE2W4ZG.js} +7 -7
  78. package/dist/cli/{providers-HOEDJNUK.mjs → providers-KSKVBQFP.mjs} +4 -4
  79. package/dist/cli/{quickstart-NCL7P2US.mjs → quickstart-7OGLDF3M.mjs} +3 -3
  80. package/dist/cli/{quickstart-FI4A6VTV.js → quickstart-JKNKNDXG.js} +5 -5
  81. package/dist/cli/{search-UHT7ZNPO.mjs → search-CPKXOGV4.mjs} +5 -5
  82. package/dist/cli/search-LOWDQS5X.js +30 -0
  83. package/dist/cli/{secrets-TT6J3SZJ.js → secrets-LFQXTVIE.js} +13 -13
  84. package/dist/cli/{secrets-D4ZB5RKB.mjs → secrets-QHGLI7O6.mjs} +4 -4
  85. package/dist/cli/{sessions-SRQQZMM7.js → sessions-HVMTS7N5.js} +5 -5
  86. package/dist/cli/{sessions-HRLJ6SLL.mjs → sessions-M4OT6NPP.mjs} +4 -4
  87. package/dist/cli/{settings-EJIKXLC7.js → settings-42WLEHAZ.js} +76 -76
  88. package/dist/cli/{settings-PJJYPWU3.mjs → settings-BJBOMHL2.mjs} +9 -9
  89. package/dist/cli/{setup-3FL77Z33.js → setup-6TZ4L5LT.js} +43 -43
  90. package/dist/cli/{setup-DNI6ENFD.mjs → setup-ILUF7GCP.mjs} +8 -8
  91. package/dist/cli/{skills-FO7MY2OE.mjs → skills-7RIWQDD6.mjs} +6 -6
  92. package/dist/cli/skills-I5OWZZ2Z.js +30 -0
  93. package/dist/cli/{tools-F72WONN3.mjs → tools-J3VDLDF2.mjs} +13 -13
  94. package/dist/cli/tools-Y4SPWXFJ.js +45 -0
  95. package/dist/cli/{update-JZU33TC5.mjs → update-GMXJF4WT.mjs} +8 -8
  96. package/dist/cli/update-HQ5WT3G6.js +32 -0
  97. package/dist/cli/{workflow-BGKC73V2.js → workflow-LVVK2N3T.js} +15 -15
  98. package/dist/cli/{workflow-PQYQXQPH.mjs → workflow-W7NQS26K.mjs} +5 -5
  99. package/dist/{client-mRA_esYI.d.mts → client-BbU5_j7n.d.mts} +9 -0
  100. package/dist/{client-D6XhrO0f.d.ts → client-CYqG23K1.d.ts} +9 -0
  101. package/dist/index.d.mts +2 -2
  102. package/dist/index.d.ts +2 -2
  103. package/dist/index.js +6 -6
  104. package/dist/index.mjs +3 -3
  105. package/dist/install-integrity.json +82 -82
  106. package/dist/plays/play-description.d.mts +1 -1
  107. package/dist/plays/play-description.d.ts +1 -1
  108. package/dist/plays/run-request.d.mts +1 -1
  109. package/dist/plays/run-request.d.ts +1 -1
  110. package/dist/release.d.mts +1 -1
  111. package/dist/release.d.ts +1 -1
  112. package/dist/release.js +2 -2
  113. package/dist/release.mjs +1 -1
  114. package/package.json +1 -1
  115. package/dist/cli/auth-AICIA232.js +0 -40
  116. package/dist/cli/git-QIZWO5BA.js +0 -24
  117. package/dist/cli/search-6FUMJBIY.js +0 -30
  118. package/dist/cli/skills-DEKOSL7F.js +0 -30
  119. package/dist/cli/tools-E6XRW6PW.js +0 -45
  120. package/dist/cli/update-6M3CAS4T.js +0 -32
@@ -80,8 +80,16 @@ interface RequestOptions {
80
80
  * most mutating API calls must fail loudly after the first server response.
81
81
  */
82
82
  retryApiErrors?: boolean;
83
- /** Only explicit server-authorized responses may retry; transport failures never retry. */
83
+ /**
84
+ * Only explicit server-authorized responses may retry; transport failures
85
+ * never retry unless `retryAmbiguousDelivery` is also set.
86
+ */
84
87
  retryFromExecutePolicy?: boolean;
88
+ /**
89
+ * The server declared this tool safe to resend after a timeout or lost
90
+ * response, so those failures retry and stay retryable when exhausted.
91
+ */
92
+ retryAmbiguousDelivery?: boolean;
85
93
  /** Retry budget enabled only by an explicit safe execute-policy response. */
86
94
  executePolicyMaxRetries?: number;
87
95
  /** Per-request retry override. Use 0 for possibly delivered mutating calls. */
@@ -707,7 +715,12 @@ export class HttpClient {
707
715
  timeoutMs,
708
716
  timedOut: requestTimedOut,
709
717
  });
710
- if (options?.retryFromExecutePolicy) break requestAttempts;
718
+ if (
719
+ options?.retryFromExecutePolicy &&
720
+ options.retryAmbiguousDelivery !== true
721
+ ) {
722
+ break requestAttempts;
723
+ }
711
724
  } finally {
712
725
  clearTimeout(timeoutId);
713
726
  options?.signal?.removeEventListener('abort', abortRequest);
@@ -753,7 +766,7 @@ export class HttpClient {
753
766
  category: 'network',
754
767
  // A direct client timeout has ambiguous delivery. Fail closed unless
755
768
  // the server returns the authoritative action delivery contract.
756
- retryable: false,
769
+ retryable: options.retryAmbiguousDelivery === true,
757
770
  statusCode: null,
758
771
  requestId: null,
759
772
  retryAfterMs: null,
@@ -934,6 +947,7 @@ export class HttpClient {
934
947
  RequestOptions,
935
948
  | 'retryApiErrors'
936
949
  | 'retryFromExecutePolicy'
950
+ | 'retryAmbiguousDelivery'
937
951
  | 'executePolicyMaxRetries'
938
952
  | 'timeout'
939
953
  | 'signal'
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
205
205
  // getters keep their established compatibility behavior.
206
206
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
207
207
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
208
- version: '0.3.453',
208
+ version: '0.3.455',
209
209
  updateSummary: {
210
210
  introducedIn: '0.3.90',
211
211
  summary:
@@ -130,6 +130,11 @@ export type ExecuteToolRawOptions = {
130
130
  maxRetries?: number;
131
131
  /** Retry transient HTTP API responses when the caller has confirmed safety. */
132
132
  retryApiErrors?: boolean;
133
+ /**
134
+ * Resend after a timeout or lost response. Pass only for a tool whose
135
+ * metadata declares `retrySafeAmbiguousDelivery`. Needs `atomicToolOnly`.
136
+ */
137
+ retryAmbiguousDelivery?: boolean;
133
138
  /** Reuse this stable key to recover or retry an ambiguous execution. */
134
139
  idempotencyKey?: string;
135
140
  /** Generate a key and enable recoverable execution before dispatch. */
@@ -457,6 +462,10 @@ export class ToolCalls {
457
462
  const legacyEndpoint = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
458
463
  let useLegacyEndpoint = idempotencyKey === undefined;
459
464
  const atomicToolOnly = options?.atomicToolOnly === true;
465
+ const retryAmbiguousDelivery =
466
+ atomicToolOnly &&
467
+ !idempotencyKey &&
468
+ options?.retryAmbiguousDelivery === true;
460
469
  const dispatchRequest = async (
461
470
  requestTimeoutMs: number | undefined,
462
471
  ): Promise<ToolExecution<TData, TMeta>> => {
@@ -476,7 +485,9 @@ export class ToolCalls {
476
485
  headers,
477
486
  {
478
487
  timeout: requestTimeoutMs,
479
- maxRetries: idempotencyKey ? 0 : (options?.maxRetries ?? 0),
488
+ maxRetries: idempotencyKey
489
+ ? 0
490
+ : (options?.maxRetries ?? (retryAmbiguousDelivery ? 2 : 0)),
480
491
  ...(atomicToolOnly &&
481
492
  !idempotencyKey &&
482
493
  options?.maxRetries === undefined
@@ -484,6 +495,7 @@ export class ToolCalls {
484
495
  : {}),
485
496
  retryApiErrors: atomicToolOnly || options?.retryApiErrors === true,
486
497
  retryFromExecutePolicy: atomicToolOnly,
498
+ retryAmbiguousDelivery,
487
499
  exactUrlOnly: true,
488
500
  toolId,
489
501
  idempotencyKey,
@@ -434,6 +434,8 @@ export type ToolExecutionMetadata = {
434
434
  sourceOperation: string;
435
435
  effectiveOperation: string;
436
436
  retrySafeTransientHttp?: true;
437
+ /** A client timeout or lost response may be resent without a second effect or charge. */
438
+ retrySafeAmbiguousDelivery?: true;
437
439
  batch: {
438
440
  batchOperation: string | null;
439
441
  maxBatchSize: number;
@@ -882,6 +884,8 @@ export interface PlayRunPackage {
882
884
  startedAt?: number | null;
883
885
  finishedAt?: number | null;
884
886
  durationMs?: number | null;
887
+ /** The provider that ran the sandbox, known once the run has ended. */
888
+ sandboxProvider?: 'daytona' | 'modal';
885
889
  outcome?: PlayRunOutcome;
886
890
  recovery?: {
887
891
  mode: 'replayed' | 'forced' | 'recovered' | 'joined';
@@ -309,6 +309,9 @@ export interface RunnerPushExecutionConfig {
309
309
  leaseSeconds: number;
310
310
  /** The run the runner heartbeats. Must match the executor token's run scope. */
311
311
  runId: string;
312
+ /** The provider that acquired this sandbox, which a Daytona-selected launch
313
+ * can differ from after fallback. Absent from older workers. */
314
+ sandboxProvider?: 'daytona' | 'modal';
312
315
  }
313
316
 
314
317
  export interface PlayRunnerExecutionConfig {
@@ -4076,6 +4076,12 @@ async function main() {
4076
4076
  (
4077
4077
  result as PlayRunnerResult & { workCompletedAtMs?: number }
4078
4078
  ).workCompletedAtMs = workCompletedAtMs;
4079
+ if (runnerTerminalPushContext?.sandboxProvider) {
4080
+ result.stats = {
4081
+ ...result.stats,
4082
+ sandboxProvider: runnerTerminalPushContext.sandboxProvider,
4083
+ };
4084
+ }
4079
4085
  const terminalResult = runnerTerminalPushContext
4080
4086
  ? runnerTerminalResultForTransport(result)
4081
4087
  : null;
@@ -4260,6 +4266,7 @@ let runnerTerminalPushContext: {
4260
4266
  gatewayBaseUrl: string;
4261
4267
  executorToken: string;
4262
4268
  runId: string;
4269
+ sandboxProvider?: 'daytona' | 'modal';
4263
4270
  } | null = null;
4264
4271
 
4265
4272
  function terminalPushContextFromConfig(
@@ -4271,7 +4278,12 @@ function terminalPushContextFromConfig(
4271
4278
  if (!push || !gatewayBaseUrl || !executorToken) {
4272
4279
  return null;
4273
4280
  }
4274
- return { gatewayBaseUrl, executorToken, runId: push.runId };
4281
+ return {
4282
+ gatewayBaseUrl,
4283
+ executorToken,
4284
+ runId: push.runId,
4285
+ ...(push.sandboxProvider ? { sandboxProvider: push.sandboxProvider } : {}),
4286
+ };
4275
4287
  }
4276
4288
 
4277
4289
  async function pushRunnerTerminalIfConfigured(
@@ -16,7 +16,9 @@ import {
16
16
  } from '@daytonaio/sdk';
17
17
  import axios, { type InternalAxiosRequestConfig } from 'axios';
18
18
  import { Readable } from 'node:stream';
19
+ import { setTimeout as delay } from 'node:timers/promises';
19
20
  import type { DaytonaClientOptions } from '../../daytona-runtime-config';
21
+ import type { DaytonaDeleteRetryAuthorization } from './daytona-lifecycle';
20
22
  import {
21
23
  DAYTONA_CONTROL_TIMEOUT_MS,
22
24
  DaytonaControlTimeoutError,
@@ -83,6 +85,10 @@ const DAYTONA_SDK_ERROR_NAMES: ReadonlySet<string> = new Set([
83
85
  ]);
84
86
  const DAYTONA_PROVIDER_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_.-]{1,63}$/;
85
87
  const DAYTONA_PROVIDER_DETAIL_PATTERN = / \[provider [^\]]*\]/g;
88
+ // A pass costs two provider calls and one fenced scheduler write, so a sandbox
89
+ // stuck in transition must not repeat it every 100 ms for the whole phase.
90
+ const DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS = 100;
91
+ const DAYTONA_RETIREMENT_BACKOFF_MAX_MS = 5_000;
86
92
 
87
93
  /** A provider state from Daytona's published enum, or `unrecognized`. */
88
94
  export function daytonaSandboxStateForLog(state: unknown): string | null {
@@ -599,10 +605,15 @@ export function createDaytonaLaunchClient(
599
605
  phase.dispose();
600
606
  }
601
607
  },
602
- async delete(sandbox: Pick<Sandbox, 'id'>, timeoutSeconds = 30) {
608
+ async delete(
609
+ sandbox: Pick<Sandbox, 'id'>,
610
+ timeoutSeconds = 30,
611
+ retry?: DaytonaDeleteRetryAuthorization,
612
+ ) {
603
613
  // Cleanup must survive execution cancellation and must not trigger the
604
614
  // SDK's unawaited refreshDataSafe after a confirmed delete response.
605
615
  const phase = deadline(timeoutSeconds * 1_000);
616
+ let absenceOnlyConflict: DaytonaConflictError | null = null;
606
617
  try {
607
618
  const api = new SandboxApi(
608
619
  new Configuration({
@@ -610,10 +621,78 @@ export function createDaytonaLaunchClient(
610
621
  baseOptions: { ...configuration.baseOptions, signal: phase.signal },
611
622
  }),
612
623
  );
613
- await api.deleteSandbox(sandbox.id, undefined, {
614
- timeout: timeoutSeconds * 1_000,
615
- });
624
+ let backoffMs = DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS;
625
+ for (;;) {
626
+ phase.signal.throwIfAborted();
627
+ if (!absenceOnlyConflict) {
628
+ try {
629
+ await api.deleteSandbox(sandbox.id, undefined, {
630
+ timeout: timeoutSeconds * 1_000,
631
+ });
632
+ return;
633
+ } catch (caught) {
634
+ const error = providerError(caught);
635
+ // A prior accepted delete can already have removed this exact ID.
636
+ if (retry && error instanceof DaytonaNotFoundError) return;
637
+ if (
638
+ !retry ||
639
+ !(error instanceof DaytonaConflictError) ||
640
+ error.statusCode !== 409
641
+ )
642
+ throw error;
643
+ if (
644
+ error.message.trim() ===
645
+ 'Sandbox was modified by another operation'
646
+ ) {
647
+ // This conflict does not authorize mutation replay. A concurrent
648
+ // cleanup is deleting this exact ID, so only prove its absence.
649
+ absenceOnlyConflict = error;
650
+ } else if (
651
+ !/^Sandbox state change in progress[.!]?$/i.test(
652
+ error.message.trim(),
653
+ )
654
+ ) {
655
+ throw error;
656
+ }
657
+ }
658
+ }
659
+ // Give the in-progress transition (including durable cleanup) its
660
+ // backoff before observing absence, so a closed resource need not
661
+ // be re-reported just to learn that deletion succeeded.
662
+ await delay(backoffMs, undefined, { signal: phase.signal });
663
+ backoffMs = Math.min(backoffMs * 2, DAYTONA_RETIREMENT_BACKOFF_MAX_MS);
664
+ // Reconcile only the acquired ID. Never resolve a name, start it, or
665
+ // infer absence from a failed/ambiguous provider read.
666
+ try {
667
+ await api.getSandbox(sandbox.id, undefined, undefined, {
668
+ signal: phase.signal,
669
+ });
670
+ } catch (caught) {
671
+ const error = providerError(caught);
672
+ if (error instanceof DaytonaNotFoundError) return;
673
+ throw error;
674
+ }
675
+ if (absenceOnlyConflict) continue;
676
+ // Authorization and every provider call share this ONE phase budget.
677
+ // A delayed ACK after expiry must never authorize another DELETE.
678
+ const signal = phase.signal;
679
+ signal.throwIfAborted();
680
+ await new Promise<void>((resolve, reject) => {
681
+ const abort = () => reject(signal.reason);
682
+ signal.addEventListener('abort', abort, { once: true });
683
+ Promise.resolve()
684
+ // Only managed callers get here: the DELETE branch rethrows
685
+ // every failure when retry is absent.
686
+ .then(retry!.authorizeRetry)
687
+ .then(resolve, reject)
688
+ .finally(() => signal.removeEventListener('abort', abort));
689
+ });
690
+ phase.signal.throwIfAborted();
691
+ }
616
692
  } catch (error) {
693
+ // Absence never arrived, so the provider's conflict is the real cause.
694
+ if (absenceOnlyConflict && phase.signal.aborted)
695
+ throw absenceOnlyConflict;
617
696
  throw providerError(error);
618
697
  } finally {
619
698
  phase.dispose();
@@ -59,13 +59,21 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
59
59
 
60
60
  export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
61
61
 
62
+ export type DaytonaDeleteRetryAuthorization = {
63
+ authorizeRetry: () => Promise<void>;
64
+ };
65
+
62
66
  export type DaytonaClient = {
63
67
  create(
64
68
  params: CreateSandboxFromSnapshotParams,
65
69
  options: { timeout: number },
66
70
  ): Promise<Sandbox>;
67
71
  get?(name: string): Promise<Sandbox>;
68
- delete(sandbox: Pick<Sandbox, 'id'>, timeoutSeconds?: number): Promise<void>;
72
+ delete(
73
+ sandbox: Pick<Sandbox, 'id'>,
74
+ timeoutSeconds?: number,
75
+ retry?: DaytonaDeleteRetryAuthorization,
76
+ ): Promise<void>;
69
77
  scope(
70
78
  sandbox: Sandbox,
71
79
  timeoutMs: number,
@@ -84,7 +92,7 @@ type DaytonaCreateCallObserver = (
84
92
 
85
93
  export type AcquiredDaytonaSandbox = {
86
94
  sandbox: DaytonaSandbox;
87
- deleteSandbox: () => Promise<void>;
95
+ deleteSandbox: (retry?: DaytonaDeleteRetryAuthorization) => Promise<void>;
88
96
  daytonaOrganizationId: string;
89
97
  billingStartedAt: number;
90
98
  billingEndedAt?: number;
@@ -1117,7 +1125,7 @@ async function acquireOneShotDaytonaSandbox(input: {
1117
1125
  });
1118
1126
  return {
1119
1127
  sandbox,
1120
- deleteSandbox: () => input.daytona.delete(sandbox, 30),
1128
+ deleteSandbox: (retry) => input.daytona.delete(sandbox, 30, retry),
1121
1129
  daytonaOrganizationId,
1122
1130
  billingStartedAt,
1123
1131
  ...(result.sandboxCapacityLeaseId
@@ -560,6 +560,22 @@ async function timedRunnerPayloadUpload(input: {
560
560
  });
561
561
  }
562
562
 
563
+ /** Records the acquired provider in the config the sandbox reads. */
564
+ export function withRunnerSandboxProvider(
565
+ config: PlayRunnerExecutionConfig,
566
+ sandboxProvider: 'daytona' | 'modal',
567
+ ): PlayRunnerExecutionConfig {
568
+ const push = config.context.runnerPushExecution;
569
+ if (!push) return config;
570
+ return {
571
+ ...config,
572
+ context: {
573
+ ...config.context,
574
+ runnerPushExecution: { ...push, sandboxProvider },
575
+ },
576
+ };
577
+ }
578
+
563
579
  export async function stageRunnerPayload(input: {
564
580
  sandbox: RemoteRunnerPayloadSandbox;
565
581
  bundlePromise: Promise<string>;
@@ -189,6 +189,8 @@ let forceKillTimer = null;
189
189
  let runtimeTimer = null;
190
190
  let terminalGraceTimer = null;
191
191
  let heartbeatTimer = null;
192
+ let heartbeatRetryTimer = null;
193
+ let heartbeatCadenceMs = 0;
192
194
  let heartbeatController = null;
193
195
  let heartbeatInFlight = false;
194
196
  let heartbeatStopped = false;
@@ -236,6 +238,10 @@ function stopRunnerHeartbeat() {
236
238
  clearInterval(heartbeatTimer);
237
239
  heartbeatTimer = null;
238
240
  }
241
+ if (heartbeatRetryTimer) {
242
+ clearTimeout(heartbeatRetryTimer);
243
+ heartbeatRetryTimer = null;
244
+ }
239
245
  if (heartbeatController) {
240
246
  heartbeatController.abort();
241
247
  heartbeatController = null;
@@ -266,6 +272,7 @@ function startRunnerHeartbeatSupervisor() {
266
272
  JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
267
273
  '\\n',
268
274
  );
275
+ heartbeatCadenceMs = cadenceMs;
269
276
  void runnerHeartbeatTick();
270
277
  heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
271
278
  heartbeatTimer.unref?.();
@@ -341,11 +348,24 @@ async function runnerHeartbeatTick() {
341
348
  '[play-runner.heartbeat] watchdog_owner_rejected ' + JSON.stringify(diagnostic) + '\\n',
342
349
  );
343
350
  stopChild('runner_heartbeat_rejected', diagnostic);
351
+ return;
344
352
  }
345
353
  // Otherwise keep renewing. An unreachable or failing control plane is not
346
354
  // evidence that this run is over: the scheduler owns the liveness verdict
347
355
  // and confirms it only after it can accept heartbeats again. The runtime
348
356
  // limit, the terminal channel above, and sandbox cleanup bound this process.
357
+ // The lease is only a few cadences long, so one shed renewal must not wait
358
+ // a whole cadence. Only a 429 retries early: it says nothing happened and
359
+ // to try again, whereas retrying a failing store would add to its load.
360
+ // Jitter keeps a fleet shed together from retrying together, and the
361
+ // in-flight guard stops a retry overlapping a tick.
362
+ if (errorCode === 'http_429' && !heartbeatRetryTimer) {
363
+ heartbeatRetryTimer = setTimeout(() => {
364
+ heartbeatRetryTimer = null;
365
+ void runnerHeartbeatTick();
366
+ }, (heartbeatCadenceMs / 4) * (1 + Math.random() / 2));
367
+ heartbeatRetryTimer.unref?.();
368
+ }
349
369
  } finally {
350
370
  clearTimeout(timeout);
351
371
  if (heartbeatController === controller) heartbeatController = null;
@@ -58,7 +58,10 @@ import {
58
58
  isPlayRunRecoveryError,
59
59
  recoveryForDaytonaSandboxTransitionPending,
60
60
  } from '../../play-run-recovery-policy';
61
- import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
61
+ import {
62
+ stageDaytonaRunnerPayload,
63
+ withRunnerSandboxProvider,
64
+ } from './daytona-payload-transport';
62
65
  import {
63
66
  DaytonaRunnerInitializationError,
64
67
  prepareDetachedDaytonaRunner,
@@ -819,6 +822,20 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
819
822
  * `{ exitCode: null, result: null }` and the attempt fails through the normal
820
823
  * engine retry/fencing path.
821
824
  */
825
+ /**
826
+ * Daytona states in which no process in the sandbox can still be running.
827
+ * `error` is left out: nothing here shows it means the sandbox stopped rather
828
+ * than that Daytona lost track of it, and a live run must not fail on that.
829
+ */
830
+ const DAYTONA_SANDBOX_STOPPED_STATES = new Set([
831
+ 'stopped',
832
+ 'stopping',
833
+ 'archived',
834
+ 'archiving',
835
+ 'destroyed',
836
+ 'destroying',
837
+ ]);
838
+
822
839
  export async function inspectDetachedDaytonaRunner(input: {
823
840
  sandboxId: string;
824
841
  sessionId: string;
@@ -836,6 +853,7 @@ export async function inspectDetachedDaytonaRunner(input: {
836
853
  | 'command_running'
837
854
  | 'result_missing'
838
855
  | 'command_inspection_failed'
856
+ | 'sandbox_stopped'
839
857
  | 'sandbox_missing'
840
858
  | 'sandbox_lookup_failed'
841
859
  | 'wrong_routing_domain';
@@ -882,6 +900,19 @@ export async function inspectDetachedDaytonaRunner(input: {
882
900
  error: error instanceof Error ? error.message : String(error),
883
901
  });
884
902
  }
903
+ const stopped = DAYTONA_SANDBOX_STOPPED_STATES.has(
904
+ String(sandbox.state ?? '').toLowerCase(),
905
+ );
906
+ if (!stopped && commandFound && exitCode === null) {
907
+ // A running command has no exit-code file to recover. Polling for one
908
+ // would hold a worker slot for the whole recovery budget, on every
909
+ // re-inspection of a run that is only quiet.
910
+ return {
911
+ exitCode,
912
+ result: null,
913
+ diagnosis: { stage: 'command_running', detail: null },
914
+ };
915
+ }
885
916
  // The full captured stdout file can scale with customer output. Metadata
886
917
  // time must not consume its existing independent file-recovery budget.
887
918
  const recoveryFiles = control.legacyOutputRecovery(sandbox);
@@ -896,13 +927,13 @@ export async function inspectDetachedDaytonaRunner(input: {
896
927
  return {
897
928
  exitCode,
898
929
  result: null,
899
- diagnosis: commandInspectionError
900
- ? {
901
- stage: 'command_inspection_failed',
902
- detail: commandInspectionError,
903
- }
904
- : commandFound && exitCode === null
905
- ? { stage: 'command_running', detail: null }
930
+ diagnosis: stopped
931
+ ? { stage: 'sandbox_stopped', detail: String(sandbox.state) }
932
+ : commandInspectionError
933
+ ? {
934
+ stage: 'command_inspection_failed',
935
+ detail: commandInspectionError,
936
+ }
906
937
  : { stage: 'result_missing', detail: null },
907
938
  };
908
939
  }
@@ -1491,10 +1522,14 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1491
1522
  const reportedSandboxIds = new Set<string>();
1492
1523
  const reportedSandboxEndTimes = new Map<string, number | null>();
1493
1524
  const runtimeResourceReportErrors = new Set<unknown>();
1494
- const reportRuntimeResource = async (acquired: AcquiredDaytonaSandbox) => {
1525
+ const reportRuntimeResource = async (
1526
+ acquired: AcquiredDaytonaSandbox,
1527
+ revalidate = false,
1528
+ ) => {
1495
1529
  const sandbox = acquired.sandbox;
1496
1530
  const billingEndedAt = acquired.billingEndedAt ?? null;
1497
1531
  if (
1532
+ !revalidate &&
1498
1533
  reportedSandboxIds.has(sandbox.id) &&
1499
1534
  reportedSandboxEndTimes.get(sandbox.id) === billingEndedAt
1500
1535
  ) {
@@ -1598,7 +1633,21 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1598
1633
  cancellationPromise,
1599
1634
  ]);
1600
1635
  activeAcquiredResource = acquiredSandbox;
1601
- sandboxCleanup.activate(acquiredSandbox);
1636
+ sandboxCleanup.activate({
1637
+ ...acquiredSandbox,
1638
+ deleteSandbox: () =>
1639
+ acquiredSandbox.deleteSandbox(
1640
+ callbacks?.onRuntimeResourceAcquired &&
1641
+ acquiredSandbox.sandboxCapacityLeaseId
1642
+ ? {
1643
+ // Bypass the local report cache: the scheduler must prove this
1644
+ // exact attempt still owns deletion before each mutating retry.
1645
+ authorizeRetry: () =>
1646
+ reportRuntimeResource(acquiredSandbox, true),
1647
+ }
1648
+ : undefined,
1649
+ ),
1650
+ });
1602
1651
  runtimeTiming.daytonaCreateMs =
1603
1652
  acquiredSandbox.billingStartedAt - startedAt;
1604
1653
  const sandbox = acquiredSandbox.sandbox;
@@ -1730,7 +1779,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1730
1779
  stageDaytonaRunnerPayload({
1731
1780
  sandbox: uploadScope.sandbox,
1732
1781
  bundlePromise,
1733
- config,
1782
+ config: withRunnerSandboxProvider(config, 'daytona'),
1734
1783
  workDir,
1735
1784
  startedAt,
1736
1785
  emitStage: (stage, extra) =>
@@ -1890,8 +1939,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
1890
1939
  if (sandboxForAttempt) {
1891
1940
  activeAcquiredResource!.billingEndedAt = Date.now();
1892
1941
  await reportRetiringRuntimeResource(activeAcquiredResource!);
1893
- await sandboxCleanup.stashActiveSandboxForRetry();
1942
+ try {
1943
+ await sandboxCleanup.stashActiveSandboxForRetry();
1944
+ } catch (retirementError) {
1945
+ if (runtimeResourceReportErrors.has(retirementError))
1946
+ throw retirementError;
1947
+ // Preserve both causes without feeding a retryable startup
1948
+ // cause back into replacement admission. The cleanup manager
1949
+ // retains the active resource and its single retirement promise.
1950
+ const failure = new Error(
1951
+ `Daytona sandbox retirement unconfirmed: ${formatDaytonaError(retirementError)}. Original startup failure: ${formatDaytonaError(error)}`,
1952
+ { cause: error },
1953
+ );
1954
+ failure.name = 'DaytonaSandboxRetirementUnconfirmedError';
1955
+ Object.assign(failure, { retirementError });
1956
+ throw failure;
1957
+ }
1894
1958
  }
1959
+ throwIfCancellationRequested();
1895
1960
  acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
1896
1961
  continue;
1897
1962
  }
@@ -24,7 +24,10 @@ import {
24
24
  } from '../../modal-runtime-config';
25
25
  import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
26
26
  import { validateDaytonaExecutionContext } from './daytona-lifecycle';
27
- import { stageRunnerPayload } from './daytona-payload-transport';
27
+ import {
28
+ stageRunnerPayload,
29
+ withRunnerSandboxProvider,
30
+ } from './daytona-payload-transport';
28
31
  import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
29
32
  import {
30
33
  isPlayRunRecoveryError,
@@ -594,7 +597,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
594
597
  }),
595
598
  },
596
599
  bundlePromise: buildPlayRunnerBundle(),
597
- config,
600
+ config: withRunnerSandboxProvider(config, 'modal'),
598
601
  workDir: modalConfig.workdir,
599
602
  startedAt,
600
603
  emitStage: (stage, extra) =>