deepline 0.3.453 → 0.3.455
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/http.ts +17 -3
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/tool-calls.ts +13 -1
- package/dist/bundling-sources/sdk/src/types.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +13 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +83 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +11 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +20 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +77 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +254 -64
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +11 -0
- package/dist/cli/{auth-6CXS55CN.mjs → auth-BGPCPVHW.mjs} +4 -4
- package/dist/cli/auth-SRR4DW75.js +40 -0
- package/dist/cli/{billing-SWLTV6PV.mjs → billing-6BLEXHCR.mjs} +4 -4
- package/dist/cli/{billing-QC52XTAD.js → billing-FCC2YOKB.js} +73 -73
- package/dist/cli/{chunk-RTR2KUU6.js → chunk-2R7JD2NY.js} +8 -6
- package/dist/cli/{chunk-V6AZQ4JH.js → chunk-2SN2EEQW.js} +16 -16
- package/dist/cli/{chunk-2LDU5BMG.mjs → chunk-2VHR5DXQ.mjs} +3 -3
- package/dist/cli/{chunk-43HOSM2F.mjs → chunk-3E4WSS35.mjs} +1 -1
- package/dist/cli/{chunk-KQD6EFPD.mjs → chunk-3RQY2OG7.mjs} +2 -2
- package/dist/cli/{chunk-IFBLLSGH.mjs → chunk-4BZFRLKU.mjs} +5 -5
- package/dist/cli/{chunk-BX343KU7.mjs → chunk-4Z3X5CWI.mjs} +1 -1
- package/dist/cli/{chunk-YU6IE234.js → chunk-6F7TD4M4.js} +6 -6
- package/dist/cli/{chunk-RG644LJU.js → chunk-6TOMXZX3.js} +1 -1
- package/dist/cli/{chunk-6CTVXF6X.mjs → chunk-B7K3QD5W.mjs} +10 -9
- package/dist/cli/{chunk-HUDUN4AR.js → chunk-BCB2QNG4.js} +2 -2
- package/dist/cli/{chunk-V7S2RUVE.js → chunk-CZREA5NE.js} +8 -8
- package/dist/cli/{chunk-IXVABCNA.mjs → chunk-DPTEABFU.mjs} +1 -1
- package/dist/cli/{chunk-NOHA3SZJ.js → chunk-EXZOUJCD.js} +3 -3
- package/dist/cli/{chunk-7557D4FM.mjs → chunk-FCQ52RUB.mjs} +2 -2
- package/dist/cli/{chunk-XH4QC5ZX.mjs → chunk-IUSGWMGK.mjs} +2 -2
- package/dist/cli/{chunk-WTTHH2K3.js → chunk-J7WYWQF4.js} +6 -6
- package/dist/cli/{chunk-NSQZUUYB.js → chunk-LQH2KJ53.js} +93 -92
- package/dist/cli/{chunk-X2LUT22I.mjs → chunk-LRA5YJVV.mjs} +5 -3
- package/dist/cli/{chunk-PJWXEUUH.js → chunk-MCMLQ7GD.js} +8 -8
- package/dist/cli/{chunk-VG6WESY7.js → chunk-MFKTYIBJ.js} +51 -51
- package/dist/cli/{chunk-XV4463XW.mjs → chunk-PQ4JU2MV.mjs} +4 -2
- package/dist/cli/{chunk-P4UQCYKH.mjs → chunk-R5VMRVRN.mjs} +4 -4
- package/dist/cli/{chunk-WWZ4YJ6J.js → chunk-RQMEYYN4.js} +35 -35
- package/dist/cli/{chunk-FS2RMEJ7.mjs → chunk-RYGDM5BG.mjs} +1 -1
- package/dist/cli/{chunk-KFKM5J2N.js → chunk-SMZUU4NY.js} +2 -2
- package/dist/cli/{chunk-RNCIPLU5.js → chunk-UVJLG5KD.js} +50 -50
- package/dist/cli/{chunk-ZVBLECZX.mjs → chunk-VIGEE67Z.mjs} +3 -3
- package/dist/cli/{chunk-JOF3B22H.js → chunk-VMGTYM3A.js} +20 -20
- package/dist/cli/{chunk-DMTAFWMS.js → chunk-W3DIXRDT.js} +241 -239
- package/dist/cli/{chunk-SQUJHT7O.mjs → chunk-WBULEVM4.mjs} +1 -1
- package/dist/cli/{chunk-IWFI6FTJ.mjs → chunk-WVGPAP7A.mjs} +3 -3
- package/dist/cli/{chunk-XNN3OTGT.mjs → chunk-YIFCV77L.mjs} +2 -2
- package/dist/cli/{chunk-MP35NE55.js → chunk-YKEWDVZC.js} +26 -26
- package/dist/cli/{csv-FON4ZQKC.mjs → csv-27T3L4RC.mjs} +3 -3
- package/dist/cli/{csv-WIQB5XWZ.js → csv-TEAA2YV5.js} +12 -12
- package/dist/cli/{db-VMDDODCN.js → db-J3PPU7GX.js} +25 -25
- package/dist/cli/{db-3KA62DOQ.mjs → db-N3NRYYI4.mjs} +4 -4
- package/dist/cli/{describe-NTAAHCCX.js → describe-PX7XBLZF.js} +17 -17
- package/dist/cli/{describe-LGDK77QI.mjs → describe-SJBCCTR7.mjs} +13 -13
- package/dist/cli/describe-core.d.mts +1 -1
- package/dist/cli/describe-core.d.ts +1 -1
- package/dist/cli/{executions-OT2KNJDO.js → executions-WXIL4C5M.js} +6 -6
- package/dist/cli/{executions-TM6AMTQ7.mjs → executions-XN376RR4.mjs} +4 -4
- package/dist/cli/{feedback-HWLUVFE2.mjs → feedback-KHJXJ2K7.mjs} +3 -3
- package/dist/cli/{feedback-4HQGY3TO.js → feedback-LG5NWE3R.js} +7 -7
- package/dist/cli/{git-TDJVSPE6.mjs → git-726A5HHA.mjs} +4 -4
- package/dist/cli/git-PXYOFDQH.js +24 -0
- package/dist/cli/index.js +132 -132
- package/dist/cli/index.mjs +37 -37
- package/dist/cli/{legacy-noops-XH3FDLWT.js → legacy-noops-QI6VN34P.js} +7 -7
- package/dist/cli/{legacy-noops-YB2PES43.mjs → legacy-noops-V4WS63SD.mjs} +4 -4
- package/dist/cli/{monitors-KQDDMSDR.mjs → monitors-AAWRYPHY.mjs} +5 -5
- package/dist/cli/{monitors-E3QFEJKF.js → monitors-ICQ2Q4XT.js} +89 -89
- package/dist/cli/{org-PMTLR7VP.js → org-5JV6KYY2.js} +66 -66
- package/dist/cli/{org-7X2C7YKA.mjs → org-7VJKBTZX.mjs} +4 -4
- package/dist/cli/{play-7FNEPSVR.mjs → play-2D6ZIJEV.mjs} +11 -11
- package/dist/cli/{play-RH22Z3D6.js → play-HDI2MLJL.js} +284 -284
- package/dist/cli/{providers-X5NY5IQH.js → providers-GIE2W4ZG.js} +7 -7
- package/dist/cli/{providers-HOEDJNUK.mjs → providers-KSKVBQFP.mjs} +4 -4
- package/dist/cli/{quickstart-NCL7P2US.mjs → quickstart-7OGLDF3M.mjs} +3 -3
- package/dist/cli/{quickstart-FI4A6VTV.js → quickstart-JKNKNDXG.js} +5 -5
- package/dist/cli/{search-UHT7ZNPO.mjs → search-CPKXOGV4.mjs} +5 -5
- package/dist/cli/search-LOWDQS5X.js +30 -0
- package/dist/cli/{secrets-TT6J3SZJ.js → secrets-LFQXTVIE.js} +13 -13
- package/dist/cli/{secrets-D4ZB5RKB.mjs → secrets-QHGLI7O6.mjs} +4 -4
- package/dist/cli/{sessions-SRQQZMM7.js → sessions-HVMTS7N5.js} +5 -5
- package/dist/cli/{sessions-HRLJ6SLL.mjs → sessions-M4OT6NPP.mjs} +4 -4
- package/dist/cli/{settings-EJIKXLC7.js → settings-42WLEHAZ.js} +76 -76
- package/dist/cli/{settings-PJJYPWU3.mjs → settings-BJBOMHL2.mjs} +9 -9
- package/dist/cli/{setup-3FL77Z33.js → setup-6TZ4L5LT.js} +43 -43
- package/dist/cli/{setup-DNI6ENFD.mjs → setup-ILUF7GCP.mjs} +8 -8
- package/dist/cli/{skills-FO7MY2OE.mjs → skills-7RIWQDD6.mjs} +6 -6
- package/dist/cli/skills-I5OWZZ2Z.js +30 -0
- package/dist/cli/{tools-F72WONN3.mjs → tools-J3VDLDF2.mjs} +13 -13
- package/dist/cli/tools-Y4SPWXFJ.js +45 -0
- package/dist/cli/{update-JZU33TC5.mjs → update-GMXJF4WT.mjs} +8 -8
- package/dist/cli/update-HQ5WT3G6.js +32 -0
- package/dist/cli/{workflow-BGKC73V2.js → workflow-LVVK2N3T.js} +15 -15
- package/dist/cli/{workflow-PQYQXQPH.mjs → workflow-W7NQS26K.mjs} +5 -5
- package/dist/{client-mRA_esYI.d.mts → client-BbU5_j7n.d.mts} +9 -0
- package/dist/{client-D6XhrO0f.d.ts → client-CYqG23K1.d.ts} +9 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +6 -6
- package/dist/index.mjs +3 -3
- package/dist/install-integrity.json +82 -82
- package/dist/plays/play-description.d.mts +1 -1
- package/dist/plays/play-description.d.ts +1 -1
- package/dist/plays/run-request.d.mts +1 -1
- package/dist/plays/run-request.d.ts +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-AICIA232.js +0 -40
- package/dist/cli/git-QIZWO5BA.js +0 -24
- package/dist/cli/search-6FUMJBIY.js +0 -30
- package/dist/cli/skills-DEKOSL7F.js +0 -30
- package/dist/cli/tools-E6XRW6PW.js +0 -45
- package/dist/cli/update-6M3CAS4T.js +0 -32
|
@@ -80,8 +80,16 @@ interface RequestOptions {
|
|
|
80
80
|
* most mutating API calls must fail loudly after the first server response.
|
|
81
81
|
*/
|
|
82
82
|
retryApiErrors?: boolean;
|
|
83
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* Only explicit server-authorized responses may retry; transport failures
|
|
85
|
+
* never retry unless `retryAmbiguousDelivery` is also set.
|
|
86
|
+
*/
|
|
84
87
|
retryFromExecutePolicy?: boolean;
|
|
88
|
+
/**
|
|
89
|
+
* The server declared this tool safe to resend after a timeout or lost
|
|
90
|
+
* response, so those failures retry and stay retryable when exhausted.
|
|
91
|
+
*/
|
|
92
|
+
retryAmbiguousDelivery?: boolean;
|
|
85
93
|
/** Retry budget enabled only by an explicit safe execute-policy response. */
|
|
86
94
|
executePolicyMaxRetries?: number;
|
|
87
95
|
/** Per-request retry override. Use 0 for possibly delivered mutating calls. */
|
|
@@ -707,7 +715,12 @@ export class HttpClient {
|
|
|
707
715
|
timeoutMs,
|
|
708
716
|
timedOut: requestTimedOut,
|
|
709
717
|
});
|
|
710
|
-
if (
|
|
718
|
+
if (
|
|
719
|
+
options?.retryFromExecutePolicy &&
|
|
720
|
+
options.retryAmbiguousDelivery !== true
|
|
721
|
+
) {
|
|
722
|
+
break requestAttempts;
|
|
723
|
+
}
|
|
711
724
|
} finally {
|
|
712
725
|
clearTimeout(timeoutId);
|
|
713
726
|
options?.signal?.removeEventListener('abort', abortRequest);
|
|
@@ -753,7 +766,7 @@ export class HttpClient {
|
|
|
753
766
|
category: 'network',
|
|
754
767
|
// A direct client timeout has ambiguous delivery. Fail closed unless
|
|
755
768
|
// the server returns the authoritative action delivery contract.
|
|
756
|
-
retryable:
|
|
769
|
+
retryable: options.retryAmbiguousDelivery === true,
|
|
757
770
|
statusCode: null,
|
|
758
771
|
requestId: null,
|
|
759
772
|
retryAfterMs: null,
|
|
@@ -934,6 +947,7 @@ export class HttpClient {
|
|
|
934
947
|
RequestOptions,
|
|
935
948
|
| 'retryApiErrors'
|
|
936
949
|
| 'retryFromExecutePolicy'
|
|
950
|
+
| 'retryAmbiguousDelivery'
|
|
937
951
|
| 'executePolicyMaxRetries'
|
|
938
952
|
| 'timeout'
|
|
939
953
|
| 'signal'
|
|
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
|
|
|
205
205
|
// getters keep their established compatibility behavior.
|
|
206
206
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
207
207
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
208
|
-
version: '0.3.
|
|
208
|
+
version: '0.3.455',
|
|
209
209
|
updateSummary: {
|
|
210
210
|
introducedIn: '0.3.90',
|
|
211
211
|
summary:
|
|
@@ -130,6 +130,11 @@ export type ExecuteToolRawOptions = {
|
|
|
130
130
|
maxRetries?: number;
|
|
131
131
|
/** Retry transient HTTP API responses when the caller has confirmed safety. */
|
|
132
132
|
retryApiErrors?: boolean;
|
|
133
|
+
/**
|
|
134
|
+
* Resend after a timeout or lost response. Pass only for a tool whose
|
|
135
|
+
* metadata declares `retrySafeAmbiguousDelivery`. Needs `atomicToolOnly`.
|
|
136
|
+
*/
|
|
137
|
+
retryAmbiguousDelivery?: boolean;
|
|
133
138
|
/** Reuse this stable key to recover or retry an ambiguous execution. */
|
|
134
139
|
idempotencyKey?: string;
|
|
135
140
|
/** Generate a key and enable recoverable execution before dispatch. */
|
|
@@ -457,6 +462,10 @@ export class ToolCalls {
|
|
|
457
462
|
const legacyEndpoint = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
458
463
|
let useLegacyEndpoint = idempotencyKey === undefined;
|
|
459
464
|
const atomicToolOnly = options?.atomicToolOnly === true;
|
|
465
|
+
const retryAmbiguousDelivery =
|
|
466
|
+
atomicToolOnly &&
|
|
467
|
+
!idempotencyKey &&
|
|
468
|
+
options?.retryAmbiguousDelivery === true;
|
|
460
469
|
const dispatchRequest = async (
|
|
461
470
|
requestTimeoutMs: number | undefined,
|
|
462
471
|
): Promise<ToolExecution<TData, TMeta>> => {
|
|
@@ -476,7 +485,9 @@ export class ToolCalls {
|
|
|
476
485
|
headers,
|
|
477
486
|
{
|
|
478
487
|
timeout: requestTimeoutMs,
|
|
479
|
-
maxRetries: idempotencyKey
|
|
488
|
+
maxRetries: idempotencyKey
|
|
489
|
+
? 0
|
|
490
|
+
: (options?.maxRetries ?? (retryAmbiguousDelivery ? 2 : 0)),
|
|
480
491
|
...(atomicToolOnly &&
|
|
481
492
|
!idempotencyKey &&
|
|
482
493
|
options?.maxRetries === undefined
|
|
@@ -484,6 +495,7 @@ export class ToolCalls {
|
|
|
484
495
|
: {}),
|
|
485
496
|
retryApiErrors: atomicToolOnly || options?.retryApiErrors === true,
|
|
486
497
|
retryFromExecutePolicy: atomicToolOnly,
|
|
498
|
+
retryAmbiguousDelivery,
|
|
487
499
|
exactUrlOnly: true,
|
|
488
500
|
toolId,
|
|
489
501
|
idempotencyKey,
|
|
@@ -434,6 +434,8 @@ export type ToolExecutionMetadata = {
|
|
|
434
434
|
sourceOperation: string;
|
|
435
435
|
effectiveOperation: string;
|
|
436
436
|
retrySafeTransientHttp?: true;
|
|
437
|
+
/** A client timeout or lost response may be resent without a second effect or charge. */
|
|
438
|
+
retrySafeAmbiguousDelivery?: true;
|
|
437
439
|
batch: {
|
|
438
440
|
batchOperation: string | null;
|
|
439
441
|
maxBatchSize: number;
|
|
@@ -882,6 +884,8 @@ export interface PlayRunPackage {
|
|
|
882
884
|
startedAt?: number | null;
|
|
883
885
|
finishedAt?: number | null;
|
|
884
886
|
durationMs?: number | null;
|
|
887
|
+
/** The provider that ran the sandbox, known once the run has ended. */
|
|
888
|
+
sandboxProvider?: 'daytona' | 'modal';
|
|
885
889
|
outcome?: PlayRunOutcome;
|
|
886
890
|
recovery?: {
|
|
887
891
|
mode: 'replayed' | 'forced' | 'recovered' | 'joined';
|
|
@@ -309,6 +309,9 @@ export interface RunnerPushExecutionConfig {
|
|
|
309
309
|
leaseSeconds: number;
|
|
310
310
|
/** The run the runner heartbeats. Must match the executor token's run scope. */
|
|
311
311
|
runId: string;
|
|
312
|
+
/** The provider that acquired this sandbox, which a Daytona-selected launch
|
|
313
|
+
* can differ from after fallback. Absent from older workers. */
|
|
314
|
+
sandboxProvider?: 'daytona' | 'modal';
|
|
312
315
|
}
|
|
313
316
|
|
|
314
317
|
export interface PlayRunnerExecutionConfig {
|
|
@@ -4076,6 +4076,12 @@ async function main() {
|
|
|
4076
4076
|
(
|
|
4077
4077
|
result as PlayRunnerResult & { workCompletedAtMs?: number }
|
|
4078
4078
|
).workCompletedAtMs = workCompletedAtMs;
|
|
4079
|
+
if (runnerTerminalPushContext?.sandboxProvider) {
|
|
4080
|
+
result.stats = {
|
|
4081
|
+
...result.stats,
|
|
4082
|
+
sandboxProvider: runnerTerminalPushContext.sandboxProvider,
|
|
4083
|
+
};
|
|
4084
|
+
}
|
|
4079
4085
|
const terminalResult = runnerTerminalPushContext
|
|
4080
4086
|
? runnerTerminalResultForTransport(result)
|
|
4081
4087
|
: null;
|
|
@@ -4260,6 +4266,7 @@ let runnerTerminalPushContext: {
|
|
|
4260
4266
|
gatewayBaseUrl: string;
|
|
4261
4267
|
executorToken: string;
|
|
4262
4268
|
runId: string;
|
|
4269
|
+
sandboxProvider?: 'daytona' | 'modal';
|
|
4263
4270
|
} | null = null;
|
|
4264
4271
|
|
|
4265
4272
|
function terminalPushContextFromConfig(
|
|
@@ -4271,7 +4278,12 @@ function terminalPushContextFromConfig(
|
|
|
4271
4278
|
if (!push || !gatewayBaseUrl || !executorToken) {
|
|
4272
4279
|
return null;
|
|
4273
4280
|
}
|
|
4274
|
-
return {
|
|
4281
|
+
return {
|
|
4282
|
+
gatewayBaseUrl,
|
|
4283
|
+
executorToken,
|
|
4284
|
+
runId: push.runId,
|
|
4285
|
+
...(push.sandboxProvider ? { sandboxProvider: push.sandboxProvider } : {}),
|
|
4286
|
+
};
|
|
4275
4287
|
}
|
|
4276
4288
|
|
|
4277
4289
|
async function pushRunnerTerminalIfConfigured(
|
|
@@ -16,7 +16,9 @@ import {
|
|
|
16
16
|
} from '@daytonaio/sdk';
|
|
17
17
|
import axios, { type InternalAxiosRequestConfig } from 'axios';
|
|
18
18
|
import { Readable } from 'node:stream';
|
|
19
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
19
20
|
import type { DaytonaClientOptions } from '../../daytona-runtime-config';
|
|
21
|
+
import type { DaytonaDeleteRetryAuthorization } from './daytona-lifecycle';
|
|
20
22
|
import {
|
|
21
23
|
DAYTONA_CONTROL_TIMEOUT_MS,
|
|
22
24
|
DaytonaControlTimeoutError,
|
|
@@ -83,6 +85,10 @@ const DAYTONA_SDK_ERROR_NAMES: ReadonlySet<string> = new Set([
|
|
|
83
85
|
]);
|
|
84
86
|
const DAYTONA_PROVIDER_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_.-]{1,63}$/;
|
|
85
87
|
const DAYTONA_PROVIDER_DETAIL_PATTERN = / \[provider [^\]]*\]/g;
|
|
88
|
+
// A pass costs two provider calls and one fenced scheduler write, so a sandbox
|
|
89
|
+
// stuck in transition must not repeat it every 100 ms for the whole phase.
|
|
90
|
+
const DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS = 100;
|
|
91
|
+
const DAYTONA_RETIREMENT_BACKOFF_MAX_MS = 5_000;
|
|
86
92
|
|
|
87
93
|
/** A provider state from Daytona's published enum, or `unrecognized`. */
|
|
88
94
|
export function daytonaSandboxStateForLog(state: unknown): string | null {
|
|
@@ -599,10 +605,15 @@ export function createDaytonaLaunchClient(
|
|
|
599
605
|
phase.dispose();
|
|
600
606
|
}
|
|
601
607
|
},
|
|
602
|
-
async delete(
|
|
608
|
+
async delete(
|
|
609
|
+
sandbox: Pick<Sandbox, 'id'>,
|
|
610
|
+
timeoutSeconds = 30,
|
|
611
|
+
retry?: DaytonaDeleteRetryAuthorization,
|
|
612
|
+
) {
|
|
603
613
|
// Cleanup must survive execution cancellation and must not trigger the
|
|
604
614
|
// SDK's unawaited refreshDataSafe after a confirmed delete response.
|
|
605
615
|
const phase = deadline(timeoutSeconds * 1_000);
|
|
616
|
+
let absenceOnlyConflict: DaytonaConflictError | null = null;
|
|
606
617
|
try {
|
|
607
618
|
const api = new SandboxApi(
|
|
608
619
|
new Configuration({
|
|
@@ -610,10 +621,78 @@ export function createDaytonaLaunchClient(
|
|
|
610
621
|
baseOptions: { ...configuration.baseOptions, signal: phase.signal },
|
|
611
622
|
}),
|
|
612
623
|
);
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
624
|
+
let backoffMs = DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS;
|
|
625
|
+
for (;;) {
|
|
626
|
+
phase.signal.throwIfAborted();
|
|
627
|
+
if (!absenceOnlyConflict) {
|
|
628
|
+
try {
|
|
629
|
+
await api.deleteSandbox(sandbox.id, undefined, {
|
|
630
|
+
timeout: timeoutSeconds * 1_000,
|
|
631
|
+
});
|
|
632
|
+
return;
|
|
633
|
+
} catch (caught) {
|
|
634
|
+
const error = providerError(caught);
|
|
635
|
+
// A prior accepted delete can already have removed this exact ID.
|
|
636
|
+
if (retry && error instanceof DaytonaNotFoundError) return;
|
|
637
|
+
if (
|
|
638
|
+
!retry ||
|
|
639
|
+
!(error instanceof DaytonaConflictError) ||
|
|
640
|
+
error.statusCode !== 409
|
|
641
|
+
)
|
|
642
|
+
throw error;
|
|
643
|
+
if (
|
|
644
|
+
error.message.trim() ===
|
|
645
|
+
'Sandbox was modified by another operation'
|
|
646
|
+
) {
|
|
647
|
+
// This conflict does not authorize mutation replay. A concurrent
|
|
648
|
+
// cleanup is deleting this exact ID, so only prove its absence.
|
|
649
|
+
absenceOnlyConflict = error;
|
|
650
|
+
} else if (
|
|
651
|
+
!/^Sandbox state change in progress[.!]?$/i.test(
|
|
652
|
+
error.message.trim(),
|
|
653
|
+
)
|
|
654
|
+
) {
|
|
655
|
+
throw error;
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
// Give the in-progress transition (including durable cleanup) its
|
|
660
|
+
// backoff before observing absence, so a closed resource need not
|
|
661
|
+
// be re-reported just to learn that deletion succeeded.
|
|
662
|
+
await delay(backoffMs, undefined, { signal: phase.signal });
|
|
663
|
+
backoffMs = Math.min(backoffMs * 2, DAYTONA_RETIREMENT_BACKOFF_MAX_MS);
|
|
664
|
+
// Reconcile only the acquired ID. Never resolve a name, start it, or
|
|
665
|
+
// infer absence from a failed/ambiguous provider read.
|
|
666
|
+
try {
|
|
667
|
+
await api.getSandbox(sandbox.id, undefined, undefined, {
|
|
668
|
+
signal: phase.signal,
|
|
669
|
+
});
|
|
670
|
+
} catch (caught) {
|
|
671
|
+
const error = providerError(caught);
|
|
672
|
+
if (error instanceof DaytonaNotFoundError) return;
|
|
673
|
+
throw error;
|
|
674
|
+
}
|
|
675
|
+
if (absenceOnlyConflict) continue;
|
|
676
|
+
// Authorization and every provider call share this ONE phase budget.
|
|
677
|
+
// A delayed ACK after expiry must never authorize another DELETE.
|
|
678
|
+
const signal = phase.signal;
|
|
679
|
+
signal.throwIfAborted();
|
|
680
|
+
await new Promise<void>((resolve, reject) => {
|
|
681
|
+
const abort = () => reject(signal.reason);
|
|
682
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
683
|
+
Promise.resolve()
|
|
684
|
+
// Only managed callers get here: the DELETE branch rethrows
|
|
685
|
+
// every failure when retry is absent.
|
|
686
|
+
.then(retry!.authorizeRetry)
|
|
687
|
+
.then(resolve, reject)
|
|
688
|
+
.finally(() => signal.removeEventListener('abort', abort));
|
|
689
|
+
});
|
|
690
|
+
phase.signal.throwIfAborted();
|
|
691
|
+
}
|
|
616
692
|
} catch (error) {
|
|
693
|
+
// Absence never arrived, so the provider's conflict is the real cause.
|
|
694
|
+
if (absenceOnlyConflict && phase.signal.aborted)
|
|
695
|
+
throw absenceOnlyConflict;
|
|
617
696
|
throw providerError(error);
|
|
618
697
|
} finally {
|
|
619
698
|
phase.dispose();
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -59,13 +59,21 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
|
|
|
59
59
|
|
|
60
60
|
export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
|
|
61
61
|
|
|
62
|
+
export type DaytonaDeleteRetryAuthorization = {
|
|
63
|
+
authorizeRetry: () => Promise<void>;
|
|
64
|
+
};
|
|
65
|
+
|
|
62
66
|
export type DaytonaClient = {
|
|
63
67
|
create(
|
|
64
68
|
params: CreateSandboxFromSnapshotParams,
|
|
65
69
|
options: { timeout: number },
|
|
66
70
|
): Promise<Sandbox>;
|
|
67
71
|
get?(name: string): Promise<Sandbox>;
|
|
68
|
-
delete(
|
|
72
|
+
delete(
|
|
73
|
+
sandbox: Pick<Sandbox, 'id'>,
|
|
74
|
+
timeoutSeconds?: number,
|
|
75
|
+
retry?: DaytonaDeleteRetryAuthorization,
|
|
76
|
+
): Promise<void>;
|
|
69
77
|
scope(
|
|
70
78
|
sandbox: Sandbox,
|
|
71
79
|
timeoutMs: number,
|
|
@@ -84,7 +92,7 @@ type DaytonaCreateCallObserver = (
|
|
|
84
92
|
|
|
85
93
|
export type AcquiredDaytonaSandbox = {
|
|
86
94
|
sandbox: DaytonaSandbox;
|
|
87
|
-
deleteSandbox: () => Promise<void>;
|
|
95
|
+
deleteSandbox: (retry?: DaytonaDeleteRetryAuthorization) => Promise<void>;
|
|
88
96
|
daytonaOrganizationId: string;
|
|
89
97
|
billingStartedAt: number;
|
|
90
98
|
billingEndedAt?: number;
|
|
@@ -1117,7 +1125,7 @@ async function acquireOneShotDaytonaSandbox(input: {
|
|
|
1117
1125
|
});
|
|
1118
1126
|
return {
|
|
1119
1127
|
sandbox,
|
|
1120
|
-
deleteSandbox: () => input.daytona.delete(sandbox, 30),
|
|
1128
|
+
deleteSandbox: (retry) => input.daytona.delete(sandbox, 30, retry),
|
|
1121
1129
|
daytonaOrganizationId,
|
|
1122
1130
|
billingStartedAt,
|
|
1123
1131
|
...(result.sandboxCapacityLeaseId
|
|
@@ -560,6 +560,22 @@ async function timedRunnerPayloadUpload(input: {
|
|
|
560
560
|
});
|
|
561
561
|
}
|
|
562
562
|
|
|
563
|
+
/** Records the acquired provider in the config the sandbox reads. */
|
|
564
|
+
export function withRunnerSandboxProvider(
|
|
565
|
+
config: PlayRunnerExecutionConfig,
|
|
566
|
+
sandboxProvider: 'daytona' | 'modal',
|
|
567
|
+
): PlayRunnerExecutionConfig {
|
|
568
|
+
const push = config.context.runnerPushExecution;
|
|
569
|
+
if (!push) return config;
|
|
570
|
+
return {
|
|
571
|
+
...config,
|
|
572
|
+
context: {
|
|
573
|
+
...config.context,
|
|
574
|
+
runnerPushExecution: { ...push, sandboxProvider },
|
|
575
|
+
},
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
|
|
563
579
|
export async function stageRunnerPayload(input: {
|
|
564
580
|
sandbox: RemoteRunnerPayloadSandbox;
|
|
565
581
|
bundlePromise: Promise<string>;
|
|
@@ -189,6 +189,8 @@ let forceKillTimer = null;
|
|
|
189
189
|
let runtimeTimer = null;
|
|
190
190
|
let terminalGraceTimer = null;
|
|
191
191
|
let heartbeatTimer = null;
|
|
192
|
+
let heartbeatRetryTimer = null;
|
|
193
|
+
let heartbeatCadenceMs = 0;
|
|
192
194
|
let heartbeatController = null;
|
|
193
195
|
let heartbeatInFlight = false;
|
|
194
196
|
let heartbeatStopped = false;
|
|
@@ -236,6 +238,10 @@ function stopRunnerHeartbeat() {
|
|
|
236
238
|
clearInterval(heartbeatTimer);
|
|
237
239
|
heartbeatTimer = null;
|
|
238
240
|
}
|
|
241
|
+
if (heartbeatRetryTimer) {
|
|
242
|
+
clearTimeout(heartbeatRetryTimer);
|
|
243
|
+
heartbeatRetryTimer = null;
|
|
244
|
+
}
|
|
239
245
|
if (heartbeatController) {
|
|
240
246
|
heartbeatController.abort();
|
|
241
247
|
heartbeatController = null;
|
|
@@ -266,6 +272,7 @@ function startRunnerHeartbeatSupervisor() {
|
|
|
266
272
|
JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
|
|
267
273
|
'\\n',
|
|
268
274
|
);
|
|
275
|
+
heartbeatCadenceMs = cadenceMs;
|
|
269
276
|
void runnerHeartbeatTick();
|
|
270
277
|
heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
|
|
271
278
|
heartbeatTimer.unref?.();
|
|
@@ -341,11 +348,24 @@ async function runnerHeartbeatTick() {
|
|
|
341
348
|
'[play-runner.heartbeat] watchdog_owner_rejected ' + JSON.stringify(diagnostic) + '\\n',
|
|
342
349
|
);
|
|
343
350
|
stopChild('runner_heartbeat_rejected', diagnostic);
|
|
351
|
+
return;
|
|
344
352
|
}
|
|
345
353
|
// Otherwise keep renewing. An unreachable or failing control plane is not
|
|
346
354
|
// evidence that this run is over: the scheduler owns the liveness verdict
|
|
347
355
|
// and confirms it only after it can accept heartbeats again. The runtime
|
|
348
356
|
// limit, the terminal channel above, and sandbox cleanup bound this process.
|
|
357
|
+
// The lease is only a few cadences long, so one shed renewal must not wait
|
|
358
|
+
// a whole cadence. Only a 429 retries early: it says nothing happened and
|
|
359
|
+
// to try again, whereas retrying a failing store would add to its load.
|
|
360
|
+
// Jitter keeps a fleet shed together from retrying together, and the
|
|
361
|
+
// in-flight guard stops a retry overlapping a tick.
|
|
362
|
+
if (errorCode === 'http_429' && !heartbeatRetryTimer) {
|
|
363
|
+
heartbeatRetryTimer = setTimeout(() => {
|
|
364
|
+
heartbeatRetryTimer = null;
|
|
365
|
+
void runnerHeartbeatTick();
|
|
366
|
+
}, (heartbeatCadenceMs / 4) * (1 + Math.random() / 2));
|
|
367
|
+
heartbeatRetryTimer.unref?.();
|
|
368
|
+
}
|
|
349
369
|
} finally {
|
|
350
370
|
clearTimeout(timeout);
|
|
351
371
|
if (heartbeatController === controller) heartbeatController = null;
|
|
@@ -58,7 +58,10 @@ import {
|
|
|
58
58
|
isPlayRunRecoveryError,
|
|
59
59
|
recoveryForDaytonaSandboxTransitionPending,
|
|
60
60
|
} from '../../play-run-recovery-policy';
|
|
61
|
-
import {
|
|
61
|
+
import {
|
|
62
|
+
stageDaytonaRunnerPayload,
|
|
63
|
+
withRunnerSandboxProvider,
|
|
64
|
+
} from './daytona-payload-transport';
|
|
62
65
|
import {
|
|
63
66
|
DaytonaRunnerInitializationError,
|
|
64
67
|
prepareDetachedDaytonaRunner,
|
|
@@ -819,6 +822,20 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
|
|
|
819
822
|
* `{ exitCode: null, result: null }` and the attempt fails through the normal
|
|
820
823
|
* engine retry/fencing path.
|
|
821
824
|
*/
|
|
825
|
+
/**
|
|
826
|
+
* Daytona states in which no process in the sandbox can still be running.
|
|
827
|
+
* `error` is left out: nothing here shows it means the sandbox stopped rather
|
|
828
|
+
* than that Daytona lost track of it, and a live run must not fail on that.
|
|
829
|
+
*/
|
|
830
|
+
const DAYTONA_SANDBOX_STOPPED_STATES = new Set([
|
|
831
|
+
'stopped',
|
|
832
|
+
'stopping',
|
|
833
|
+
'archived',
|
|
834
|
+
'archiving',
|
|
835
|
+
'destroyed',
|
|
836
|
+
'destroying',
|
|
837
|
+
]);
|
|
838
|
+
|
|
822
839
|
export async function inspectDetachedDaytonaRunner(input: {
|
|
823
840
|
sandboxId: string;
|
|
824
841
|
sessionId: string;
|
|
@@ -836,6 +853,7 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
836
853
|
| 'command_running'
|
|
837
854
|
| 'result_missing'
|
|
838
855
|
| 'command_inspection_failed'
|
|
856
|
+
| 'sandbox_stopped'
|
|
839
857
|
| 'sandbox_missing'
|
|
840
858
|
| 'sandbox_lookup_failed'
|
|
841
859
|
| 'wrong_routing_domain';
|
|
@@ -882,6 +900,19 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
882
900
|
error: error instanceof Error ? error.message : String(error),
|
|
883
901
|
});
|
|
884
902
|
}
|
|
903
|
+
const stopped = DAYTONA_SANDBOX_STOPPED_STATES.has(
|
|
904
|
+
String(sandbox.state ?? '').toLowerCase(),
|
|
905
|
+
);
|
|
906
|
+
if (!stopped && commandFound && exitCode === null) {
|
|
907
|
+
// A running command has no exit-code file to recover. Polling for one
|
|
908
|
+
// would hold a worker slot for the whole recovery budget, on every
|
|
909
|
+
// re-inspection of a run that is only quiet.
|
|
910
|
+
return {
|
|
911
|
+
exitCode,
|
|
912
|
+
result: null,
|
|
913
|
+
diagnosis: { stage: 'command_running', detail: null },
|
|
914
|
+
};
|
|
915
|
+
}
|
|
885
916
|
// The full captured stdout file can scale with customer output. Metadata
|
|
886
917
|
// time must not consume its existing independent file-recovery budget.
|
|
887
918
|
const recoveryFiles = control.legacyOutputRecovery(sandbox);
|
|
@@ -896,13 +927,13 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
896
927
|
return {
|
|
897
928
|
exitCode,
|
|
898
929
|
result: null,
|
|
899
|
-
diagnosis:
|
|
900
|
-
? {
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
930
|
+
diagnosis: stopped
|
|
931
|
+
? { stage: 'sandbox_stopped', detail: String(sandbox.state) }
|
|
932
|
+
: commandInspectionError
|
|
933
|
+
? {
|
|
934
|
+
stage: 'command_inspection_failed',
|
|
935
|
+
detail: commandInspectionError,
|
|
936
|
+
}
|
|
906
937
|
: { stage: 'result_missing', detail: null },
|
|
907
938
|
};
|
|
908
939
|
}
|
|
@@ -1491,10 +1522,14 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1491
1522
|
const reportedSandboxIds = new Set<string>();
|
|
1492
1523
|
const reportedSandboxEndTimes = new Map<string, number | null>();
|
|
1493
1524
|
const runtimeResourceReportErrors = new Set<unknown>();
|
|
1494
|
-
const reportRuntimeResource = async (
|
|
1525
|
+
const reportRuntimeResource = async (
|
|
1526
|
+
acquired: AcquiredDaytonaSandbox,
|
|
1527
|
+
revalidate = false,
|
|
1528
|
+
) => {
|
|
1495
1529
|
const sandbox = acquired.sandbox;
|
|
1496
1530
|
const billingEndedAt = acquired.billingEndedAt ?? null;
|
|
1497
1531
|
if (
|
|
1532
|
+
!revalidate &&
|
|
1498
1533
|
reportedSandboxIds.has(sandbox.id) &&
|
|
1499
1534
|
reportedSandboxEndTimes.get(sandbox.id) === billingEndedAt
|
|
1500
1535
|
) {
|
|
@@ -1598,7 +1633,21 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1598
1633
|
cancellationPromise,
|
|
1599
1634
|
]);
|
|
1600
1635
|
activeAcquiredResource = acquiredSandbox;
|
|
1601
|
-
sandboxCleanup.activate(
|
|
1636
|
+
sandboxCleanup.activate({
|
|
1637
|
+
...acquiredSandbox,
|
|
1638
|
+
deleteSandbox: () =>
|
|
1639
|
+
acquiredSandbox.deleteSandbox(
|
|
1640
|
+
callbacks?.onRuntimeResourceAcquired &&
|
|
1641
|
+
acquiredSandbox.sandboxCapacityLeaseId
|
|
1642
|
+
? {
|
|
1643
|
+
// Bypass the local report cache: the scheduler must prove this
|
|
1644
|
+
// exact attempt still owns deletion before each mutating retry.
|
|
1645
|
+
authorizeRetry: () =>
|
|
1646
|
+
reportRuntimeResource(acquiredSandbox, true),
|
|
1647
|
+
}
|
|
1648
|
+
: undefined,
|
|
1649
|
+
),
|
|
1650
|
+
});
|
|
1602
1651
|
runtimeTiming.daytonaCreateMs =
|
|
1603
1652
|
acquiredSandbox.billingStartedAt - startedAt;
|
|
1604
1653
|
const sandbox = acquiredSandbox.sandbox;
|
|
@@ -1730,7 +1779,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1730
1779
|
stageDaytonaRunnerPayload({
|
|
1731
1780
|
sandbox: uploadScope.sandbox,
|
|
1732
1781
|
bundlePromise,
|
|
1733
|
-
config,
|
|
1782
|
+
config: withRunnerSandboxProvider(config, 'daytona'),
|
|
1734
1783
|
workDir,
|
|
1735
1784
|
startedAt,
|
|
1736
1785
|
emitStage: (stage, extra) =>
|
|
@@ -1890,8 +1939,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1890
1939
|
if (sandboxForAttempt) {
|
|
1891
1940
|
activeAcquiredResource!.billingEndedAt = Date.now();
|
|
1892
1941
|
await reportRetiringRuntimeResource(activeAcquiredResource!);
|
|
1893
|
-
|
|
1942
|
+
try {
|
|
1943
|
+
await sandboxCleanup.stashActiveSandboxForRetry();
|
|
1944
|
+
} catch (retirementError) {
|
|
1945
|
+
if (runtimeResourceReportErrors.has(retirementError))
|
|
1946
|
+
throw retirementError;
|
|
1947
|
+
// Preserve both causes without feeding a retryable startup
|
|
1948
|
+
// cause back into replacement admission. The cleanup manager
|
|
1949
|
+
// retains the active resource and its single retirement promise.
|
|
1950
|
+
const failure = new Error(
|
|
1951
|
+
`Daytona sandbox retirement unconfirmed: ${formatDaytonaError(retirementError)}. Original startup failure: ${formatDaytonaError(error)}`,
|
|
1952
|
+
{ cause: error },
|
|
1953
|
+
);
|
|
1954
|
+
failure.name = 'DaytonaSandboxRetirementUnconfirmedError';
|
|
1955
|
+
Object.assign(failure, { retirementError });
|
|
1956
|
+
throw failure;
|
|
1957
|
+
}
|
|
1894
1958
|
}
|
|
1959
|
+
throwIfCancellationRequested();
|
|
1895
1960
|
acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
|
|
1896
1961
|
continue;
|
|
1897
1962
|
}
|
|
@@ -24,7 +24,10 @@ import {
|
|
|
24
24
|
} from '../../modal-runtime-config';
|
|
25
25
|
import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
|
|
26
26
|
import { validateDaytonaExecutionContext } from './daytona-lifecycle';
|
|
27
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
stageRunnerPayload,
|
|
29
|
+
withRunnerSandboxProvider,
|
|
30
|
+
} from './daytona-payload-transport';
|
|
28
31
|
import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
|
|
29
32
|
import {
|
|
30
33
|
isPlayRunRecoveryError,
|
|
@@ -594,7 +597,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
594
597
|
}),
|
|
595
598
|
},
|
|
596
599
|
bundlePromise: buildPlayRunnerBundle(),
|
|
597
|
-
config,
|
|
600
|
+
config: withRunnerSandboxProvider(config, 'modal'),
|
|
598
601
|
workDir: modalConfig.workdir,
|
|
599
602
|
startedAt,
|
|
600
603
|
emitStage: (stage, extra) =>
|