deepline 0.3.453 → 0.3.455

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/bundling-sources/sdk/src/http.ts +17 -3
  2. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  3. package/dist/bundling-sources/sdk/src/tool-calls.ts +13 -1
  4. package/dist/bundling-sources/sdk/src/types.ts +4 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +13 -1
  7. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +83 -4
  8. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +11 -3
  9. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +16 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +20 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +77 -12
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +5 -2
  13. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +254 -64
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +11 -0
  15. package/dist/cli/{auth-6CXS55CN.mjs → auth-BGPCPVHW.mjs} +4 -4
  16. package/dist/cli/auth-SRR4DW75.js +40 -0
  17. package/dist/cli/{billing-SWLTV6PV.mjs → billing-6BLEXHCR.mjs} +4 -4
  18. package/dist/cli/{billing-QC52XTAD.js → billing-FCC2YOKB.js} +73 -73
  19. package/dist/cli/{chunk-RTR2KUU6.js → chunk-2R7JD2NY.js} +8 -6
  20. package/dist/cli/{chunk-V6AZQ4JH.js → chunk-2SN2EEQW.js} +16 -16
  21. package/dist/cli/{chunk-2LDU5BMG.mjs → chunk-2VHR5DXQ.mjs} +3 -3
  22. package/dist/cli/{chunk-43HOSM2F.mjs → chunk-3E4WSS35.mjs} +1 -1
  23. package/dist/cli/{chunk-KQD6EFPD.mjs → chunk-3RQY2OG7.mjs} +2 -2
  24. package/dist/cli/{chunk-IFBLLSGH.mjs → chunk-4BZFRLKU.mjs} +5 -5
  25. package/dist/cli/{chunk-BX343KU7.mjs → chunk-4Z3X5CWI.mjs} +1 -1
  26. package/dist/cli/{chunk-YU6IE234.js → chunk-6F7TD4M4.js} +6 -6
  27. package/dist/cli/{chunk-RG644LJU.js → chunk-6TOMXZX3.js} +1 -1
  28. package/dist/cli/{chunk-6CTVXF6X.mjs → chunk-B7K3QD5W.mjs} +10 -9
  29. package/dist/cli/{chunk-HUDUN4AR.js → chunk-BCB2QNG4.js} +2 -2
  30. package/dist/cli/{chunk-V7S2RUVE.js → chunk-CZREA5NE.js} +8 -8
  31. package/dist/cli/{chunk-IXVABCNA.mjs → chunk-DPTEABFU.mjs} +1 -1
  32. package/dist/cli/{chunk-NOHA3SZJ.js → chunk-EXZOUJCD.js} +3 -3
  33. package/dist/cli/{chunk-7557D4FM.mjs → chunk-FCQ52RUB.mjs} +2 -2
  34. package/dist/cli/{chunk-XH4QC5ZX.mjs → chunk-IUSGWMGK.mjs} +2 -2
  35. package/dist/cli/{chunk-WTTHH2K3.js → chunk-J7WYWQF4.js} +6 -6
  36. package/dist/cli/{chunk-NSQZUUYB.js → chunk-LQH2KJ53.js} +93 -92
  37. package/dist/cli/{chunk-X2LUT22I.mjs → chunk-LRA5YJVV.mjs} +5 -3
  38. package/dist/cli/{chunk-PJWXEUUH.js → chunk-MCMLQ7GD.js} +8 -8
  39. package/dist/cli/{chunk-VG6WESY7.js → chunk-MFKTYIBJ.js} +51 -51
  40. package/dist/cli/{chunk-XV4463XW.mjs → chunk-PQ4JU2MV.mjs} +4 -2
  41. package/dist/cli/{chunk-P4UQCYKH.mjs → chunk-R5VMRVRN.mjs} +4 -4
  42. package/dist/cli/{chunk-WWZ4YJ6J.js → chunk-RQMEYYN4.js} +35 -35
  43. package/dist/cli/{chunk-FS2RMEJ7.mjs → chunk-RYGDM5BG.mjs} +1 -1
  44. package/dist/cli/{chunk-KFKM5J2N.js → chunk-SMZUU4NY.js} +2 -2
  45. package/dist/cli/{chunk-RNCIPLU5.js → chunk-UVJLG5KD.js} +50 -50
  46. package/dist/cli/{chunk-ZVBLECZX.mjs → chunk-VIGEE67Z.mjs} +3 -3
  47. package/dist/cli/{chunk-JOF3B22H.js → chunk-VMGTYM3A.js} +20 -20
  48. package/dist/cli/{chunk-DMTAFWMS.js → chunk-W3DIXRDT.js} +241 -239
  49. package/dist/cli/{chunk-SQUJHT7O.mjs → chunk-WBULEVM4.mjs} +1 -1
  50. package/dist/cli/{chunk-IWFI6FTJ.mjs → chunk-WVGPAP7A.mjs} +3 -3
  51. package/dist/cli/{chunk-XNN3OTGT.mjs → chunk-YIFCV77L.mjs} +2 -2
  52. package/dist/cli/{chunk-MP35NE55.js → chunk-YKEWDVZC.js} +26 -26
  53. package/dist/cli/{csv-FON4ZQKC.mjs → csv-27T3L4RC.mjs} +3 -3
  54. package/dist/cli/{csv-WIQB5XWZ.js → csv-TEAA2YV5.js} +12 -12
  55. package/dist/cli/{db-VMDDODCN.js → db-J3PPU7GX.js} +25 -25
  56. package/dist/cli/{db-3KA62DOQ.mjs → db-N3NRYYI4.mjs} +4 -4
  57. package/dist/cli/{describe-NTAAHCCX.js → describe-PX7XBLZF.js} +17 -17
  58. package/dist/cli/{describe-LGDK77QI.mjs → describe-SJBCCTR7.mjs} +13 -13
  59. package/dist/cli/describe-core.d.mts +1 -1
  60. package/dist/cli/describe-core.d.ts +1 -1
  61. package/dist/cli/{executions-OT2KNJDO.js → executions-WXIL4C5M.js} +6 -6
  62. package/dist/cli/{executions-TM6AMTQ7.mjs → executions-XN376RR4.mjs} +4 -4
  63. package/dist/cli/{feedback-HWLUVFE2.mjs → feedback-KHJXJ2K7.mjs} +3 -3
  64. package/dist/cli/{feedback-4HQGY3TO.js → feedback-LG5NWE3R.js} +7 -7
  65. package/dist/cli/{git-TDJVSPE6.mjs → git-726A5HHA.mjs} +4 -4
  66. package/dist/cli/git-PXYOFDQH.js +24 -0
  67. package/dist/cli/index.js +132 -132
  68. package/dist/cli/index.mjs +37 -37
  69. package/dist/cli/{legacy-noops-XH3FDLWT.js → legacy-noops-QI6VN34P.js} +7 -7
  70. package/dist/cli/{legacy-noops-YB2PES43.mjs → legacy-noops-V4WS63SD.mjs} +4 -4
  71. package/dist/cli/{monitors-KQDDMSDR.mjs → monitors-AAWRYPHY.mjs} +5 -5
  72. package/dist/cli/{monitors-E3QFEJKF.js → monitors-ICQ2Q4XT.js} +89 -89
  73. package/dist/cli/{org-PMTLR7VP.js → org-5JV6KYY2.js} +66 -66
  74. package/dist/cli/{org-7X2C7YKA.mjs → org-7VJKBTZX.mjs} +4 -4
  75. package/dist/cli/{play-7FNEPSVR.mjs → play-2D6ZIJEV.mjs} +11 -11
  76. package/dist/cli/{play-RH22Z3D6.js → play-HDI2MLJL.js} +284 -284
  77. package/dist/cli/{providers-X5NY5IQH.js → providers-GIE2W4ZG.js} +7 -7
  78. package/dist/cli/{providers-HOEDJNUK.mjs → providers-KSKVBQFP.mjs} +4 -4
  79. package/dist/cli/{quickstart-NCL7P2US.mjs → quickstart-7OGLDF3M.mjs} +3 -3
  80. package/dist/cli/{quickstart-FI4A6VTV.js → quickstart-JKNKNDXG.js} +5 -5
  81. package/dist/cli/{search-UHT7ZNPO.mjs → search-CPKXOGV4.mjs} +5 -5
  82. package/dist/cli/search-LOWDQS5X.js +30 -0
  83. package/dist/cli/{secrets-TT6J3SZJ.js → secrets-LFQXTVIE.js} +13 -13
  84. package/dist/cli/{secrets-D4ZB5RKB.mjs → secrets-QHGLI7O6.mjs} +4 -4
  85. package/dist/cli/{sessions-SRQQZMM7.js → sessions-HVMTS7N5.js} +5 -5
  86. package/dist/cli/{sessions-HRLJ6SLL.mjs → sessions-M4OT6NPP.mjs} +4 -4
  87. package/dist/cli/{settings-EJIKXLC7.js → settings-42WLEHAZ.js} +76 -76
  88. package/dist/cli/{settings-PJJYPWU3.mjs → settings-BJBOMHL2.mjs} +9 -9
  89. package/dist/cli/{setup-3FL77Z33.js → setup-6TZ4L5LT.js} +43 -43
  90. package/dist/cli/{setup-DNI6ENFD.mjs → setup-ILUF7GCP.mjs} +8 -8
  91. package/dist/cli/{skills-FO7MY2OE.mjs → skills-7RIWQDD6.mjs} +6 -6
  92. package/dist/cli/skills-I5OWZZ2Z.js +30 -0
  93. package/dist/cli/{tools-F72WONN3.mjs → tools-J3VDLDF2.mjs} +13 -13
  94. package/dist/cli/tools-Y4SPWXFJ.js +45 -0
  95. package/dist/cli/{update-JZU33TC5.mjs → update-GMXJF4WT.mjs} +8 -8
  96. package/dist/cli/update-HQ5WT3G6.js +32 -0
  97. package/dist/cli/{workflow-BGKC73V2.js → workflow-LVVK2N3T.js} +15 -15
  98. package/dist/cli/{workflow-PQYQXQPH.mjs → workflow-W7NQS26K.mjs} +5 -5
  99. package/dist/{client-mRA_esYI.d.mts → client-BbU5_j7n.d.mts} +9 -0
  100. package/dist/{client-D6XhrO0f.d.ts → client-CYqG23K1.d.ts} +9 -0
  101. package/dist/index.d.mts +2 -2
  102. package/dist/index.d.ts +2 -2
  103. package/dist/index.js +6 -6
  104. package/dist/index.mjs +3 -3
  105. package/dist/install-integrity.json +82 -82
  106. package/dist/plays/play-description.d.mts +1 -1
  107. package/dist/plays/play-description.d.ts +1 -1
  108. package/dist/plays/run-request.d.mts +1 -1
  109. package/dist/plays/run-request.d.ts +1 -1
  110. package/dist/release.d.mts +1 -1
  111. package/dist/release.d.ts +1 -1
  112. package/dist/release.js +2 -2
  113. package/dist/release.mjs +1 -1
  114. package/package.json +1 -1
  115. package/dist/cli/auth-AICIA232.js +0 -40
  116. package/dist/cli/git-QIZWO5BA.js +0 -24
  117. package/dist/cli/search-6FUMJBIY.js +0 -30
  118. package/dist/cli/skills-DEKOSL7F.js +0 -30
  119. package/dist/cli/tools-E6XRW6PW.js +0 -45
  120. package/dist/cli/update-6M3CAS4T.js +0 -32
@@ -839,12 +839,9 @@ export type AbsurdRuntimeWorkerDeps = {
839
839
  * detached runner that lost its gateway. This never executes customer code:
840
840
  * it can only recover the result the sandbox already emitted before exit.
841
841
  */
842
- inspectDetachedRunner?: (suspension: DetachedRunnerSuspension) => Promise<{
843
- exitCode: number | null;
844
- result: PlayRunnerResult | null;
845
- // Written by the runner, possibly an older build: readers default it.
846
- diagnosis?: { stage: string; detail: string | null };
847
- } | null>;
842
+ inspectDetachedRunner?: (
843
+ suspension: DetachedRunnerSuspension,
844
+ ) => Promise<DetachedRunnerInspection | null>;
848
845
  readSandboxCrashDiagnostic?: (
849
846
  request: PostgresSchedulerSandboxCrashDiagnosticRequest,
850
847
  routingDomain: string | null,
@@ -1332,6 +1329,49 @@ function signalData(
1332
1329
  }
1333
1330
 
1334
1331
  const LIVENESS_CONFIRMATION_SLICES_PER_CADENCE = 4;
1332
+ const LIVENESS_RECHECK_INTERVAL_CADENCES = 2;
1333
+ const LIVENESS_RECHECK_MIN_INTERVAL_MS = 1_000;
1334
+ const LIVENESS_PARK_MAX_REARMS_PER_WAKE = 8;
1335
+ const LIVENESS_REPLAY_READ_REUSE_MS = 100;
1336
+
1337
+ /**
1338
+ * Suspensions the park proved gone in this handler invocation, with the
1339
+ * inspection that proved it when this leg made one. The resolver reuses it
1340
+ * rather than asking the provider twice, and honors it rather than re-deciding:
1341
+ * re-parking on a disagreeing inspection would meet the same cached proof and
1342
+ * spin.
1343
+ */
1344
+ const detachedRunnerProvenGone = new WeakMap<
1345
+ DetachedRunnerSuspension,
1346
+ DetachedRunnerInspection | null
1347
+ >();
1348
+
1349
+ type DetachedRunnerInspection = {
1350
+ exitCode: number | null;
1351
+ result: PlayRunnerResult | null;
1352
+ // Written by the runner, possibly an older build: readers default it.
1353
+ diagnosis?: { stage: string; detail: string | null };
1354
+ };
1355
+
1356
+ /**
1357
+ * Whether a provider inspection proves the runner can no longer act. A stale
1358
+ * heartbeat alone never does: the gateway can lose heartbeats from a healthy
1359
+ * runner, and the provider can be unreachable while the sandbox still runs.
1360
+ * Only a recoverable result, a recorded process exit, or a missing or stopped
1361
+ * sandbox counts. Anything else leaves the run parked, bounded by its hard
1362
+ * ceiling.
1363
+ */
1364
+ export function detachedRunnerInspectionProvesGone(
1365
+ inspection: DetachedRunnerInspection | null,
1366
+ ): boolean {
1367
+ if (!inspection) return false;
1368
+ if (inspection.result && inspection.result.status !== 'suspended') {
1369
+ return true;
1370
+ }
1371
+ if (inspection.exitCode !== null) return true;
1372
+ const stage = inspection.diagnosis?.stage;
1373
+ return stage === 'sandbox_missing' || stage === 'sandbox_stopped';
1374
+ }
1335
1375
 
1336
1376
  /**
1337
1377
  * A stale heartbeat proves a lost runner only if the runner could have renewed
@@ -1340,8 +1380,11 @@ const LIVENESS_CONFIRMATION_SLICES_PER_CADENCE = 4;
1340
1380
  * counts only short sleep slices that woke on time, and decides after two
1341
1381
  * heartbeat cadences of such observed availability. A late slice counts for
1342
1382
  * nothing. Slice times and lateness are checkpointed, so replay is
1343
- * deterministic. Returns true when the runner stayed silent or the hard
1344
- * ceiling arrived, false as soon as it renewed or pushed its terminal.
1383
+ * deterministic.
1384
+ *
1385
+ * Silence is still not loss, so the worker then asks the provider, and keeps
1386
+ * asking on a paced schedule until the provider proves the runner gone, the
1387
+ * runner responds, or the hard ceiling arrives.
1345
1388
  */
1346
1389
  async function confirmDetachedRunnerSilence(input: {
1347
1390
  ctx: AbsurdSuspensionContext;
@@ -1349,6 +1392,79 @@ async function confirmDetachedRunnerSilence(input: {
1349
1392
  cadenceMs: number;
1350
1393
  ceilingDeadlineMs: number;
1351
1394
  runnerResponded: () => Promise<boolean>;
1395
+ inspectDetachedRunner?: (
1396
+ suspension: DetachedRunnerSuspension,
1397
+ ) => Promise<DetachedRunnerInspection | null>;
1398
+ suspension: DetachedRunnerSuspension;
1399
+ }): Promise<'responded' | 'gone' | 'ceiling'> {
1400
+ if (!(await confirmSilenceWindow(input))) return 'responded';
1401
+ if (Date.now() >= input.ceilingDeadlineMs) return 'ceiling';
1402
+ const inspect = input.inspectDetachedRunner;
1403
+ if (!inspect) {
1404
+ detachedRunnerProvenGone.set(input.suspension, null);
1405
+ return 'gone';
1406
+ }
1407
+ const recheckMs = Math.max(
1408
+ LIVENESS_RECHECK_MIN_INTERVAL_MS,
1409
+ LIVENESS_RECHECK_INTERVAL_CADENCES * input.cadenceMs,
1410
+ );
1411
+ for (let round = 1; ; round += 1) {
1412
+ const stepName = `${input.stepPrefix}:evidence:${round}`;
1413
+ let inspected: DetachedRunnerInspection | null = null;
1414
+ const gone = await input.ctx.step(`${stepName}:gone`, async () => {
1415
+ // Deferred so a synchronous throw, such as an unreadable persisted
1416
+ // sandbox ref, is an unproven inspection rather than a failed park.
1417
+ const inspection = await Promise.resolve()
1418
+ .then(() => inspect(input.suspension))
1419
+ .catch((error: unknown) => {
1420
+ console.warn(
1421
+ '[absurd-runtime.worker] detached_runner_liveness_evidence_failed',
1422
+ JSON.stringify({
1423
+ sandboxId: input.suspension.sandboxId,
1424
+ round,
1425
+ error: error instanceof Error ? error.message : String(error),
1426
+ }),
1427
+ );
1428
+ return null;
1429
+ });
1430
+ inspected = inspection;
1431
+ const proved = detachedRunnerInspectionProvesGone(inspection);
1432
+ if (!proved) {
1433
+ console.warn(
1434
+ '[absurd-runtime.worker] detached_runner_liveness_unproven',
1435
+ JSON.stringify({
1436
+ runnerAttempt: input.suspension.runnerAttempt,
1437
+ sandboxId: input.suspension.sandboxId,
1438
+ round,
1439
+ diagnosis: inspection?.diagnosis?.stage ?? 'unavailable',
1440
+ }),
1441
+ );
1442
+ }
1443
+ return proved;
1444
+ });
1445
+ if (gone) {
1446
+ detachedRunnerProvenGone.set(input.suspension, inspected);
1447
+ return 'gone';
1448
+ }
1449
+ // Jitter stops runners silenced together from being inspected together.
1450
+ const wakeAtMs = await input.ctx.step(`${stepName}:at`, async () =>
1451
+ Math.min(
1452
+ input.ceilingDeadlineMs,
1453
+ Date.now() + recheckMs * (1 + Math.random() / 2),
1454
+ ),
1455
+ );
1456
+ await input.ctx.sleepUntil(stepName, new Date(wakeAtMs));
1457
+ if (await input.runnerResponded()) return 'responded';
1458
+ if (wakeAtMs >= input.ceilingDeadlineMs) return 'ceiling';
1459
+ }
1460
+ }
1461
+
1462
+ async function confirmSilenceWindow(input: {
1463
+ ctx: AbsurdSuspensionContext;
1464
+ stepPrefix: string;
1465
+ cadenceMs: number;
1466
+ ceilingDeadlineMs: number;
1467
+ runnerResponded: () => Promise<boolean>;
1352
1468
  }): Promise<boolean> {
1353
1469
  const sliceMs = Math.floor(
1354
1470
  input.cadenceMs / LIVENESS_CONFIRMATION_SLICES_PER_CADENCE,
@@ -1398,6 +1514,11 @@ export async function parkAndResolveSuspension(input: {
1398
1514
  updatedAt: string | null;
1399
1515
  status: string;
1400
1516
  } | null>;
1517
+ /** Provider evidence the park requires before it reports a silent runner as
1518
+ * gone. Without it, silence alone ends the park. */
1519
+ inspectDetachedRunner?: (
1520
+ suspension: DetachedRunnerSuspension,
1521
+ ) => Promise<DetachedRunnerInspection | null>;
1401
1522
  }): Promise<PlayCheckpoint> {
1402
1523
  const { ctx, suspension, checkpoint, runId } = input;
1403
1524
 
@@ -1430,8 +1551,11 @@ export async function parkAndResolveSuspension(input: {
1430
1551
  ? configuredHeartbeatTimeoutMs
1431
1552
  : RUNTIME_CLAIM_LEASE_SECONDS * 1000;
1432
1553
  const ceilingDeadlineMs = suspension.startedAtMs + suspension.ceilingMs;
1554
+ const readRunnerTerminal = input.readRunnerTerminal;
1555
+ const readRunnerLiveness = input.readRunnerLiveness;
1556
+ let rearms = 0;
1433
1557
  for (;;) {
1434
- const liveness = await input.readRunnerLiveness(suspension.runnerAttempt);
1558
+ const liveness = await readRunnerLiveness(suspension.runnerAttempt);
1435
1559
  const heartbeatAtMs = liveness?.heartbeatAt
1436
1560
  ? Date.parse(liveness.heartbeatAt)
1437
1561
  : Number.NaN;
@@ -1441,26 +1565,33 @@ export async function parkAndResolveSuspension(input: {
1441
1565
  : suspension.startedAtMs) + heartbeatTimeoutMs;
1442
1566
  const deadlineMs = Math.min(ceilingDeadlineMs, heartbeatDeadlineMs);
1443
1567
  const remainingMs = Math.max(0, deadlineMs - Date.now());
1568
+ let timedOut = false;
1444
1569
  try {
1445
1570
  // The deadline is part of the durable step identity. A new persisted
1446
1571
  // heartbeat produces a new deadline/step and parks again without
1447
1572
  // allocating a new scheduler attempt; the live sandbox keeps its
1448
1573
  // original attempt-scoped executor token and heartbeat identity.
1574
+ // Absurd floors this to whole seconds, so round up: a timer that
1575
+ // fires early must not reach the liveness decision below.
1449
1576
  await ctx.awaitEvent(
1450
1577
  absurdRunnerTerminalEventName(runId, suspension.runnerAttempt),
1451
1578
  {
1452
1579
  stepName: `${suspension.boundaryId}:deadline:${deadlineMs}`,
1453
- timeout: remainingMs / 1000,
1580
+ timeout: Math.ceil(remainingMs / 1000),
1454
1581
  },
1455
1582
  );
1456
- break;
1457
1583
  } catch (error) {
1458
1584
  if (!(error instanceof TimeoutError)) {
1459
1585
  // SuspendTask (park) and any real error propagate.
1460
1586
  throw error;
1461
1587
  }
1588
+ timedOut = true;
1462
1589
  }
1463
- const renewed = await input.readRunnerLiveness(suspension.runnerAttempt);
1590
+ // A returned wait proves nothing. After a timed-out wake the run row
1591
+ // keeps its wake_event, so the next wait on this event returns null
1592
+ // instead of suspending. Only the persisted terminal ends the park.
1593
+ if (await readRunnerTerminal(suspension.runnerAttempt)) break;
1594
+ const renewed = await readRunnerLiveness(suspension.runnerAttempt);
1464
1595
  const renewedAtMs = renewed?.heartbeatAt
1465
1596
  ? Date.parse(renewed.heartbeatAt)
1466
1597
  : Number.NaN;
@@ -1470,29 +1601,45 @@ export async function parkAndResolveSuspension(input: {
1470
1601
  heartbeatTimeoutMs,
1471
1602
  );
1472
1603
  if (renewedDeadlineMs > Date.now()) {
1473
- if (renewedDeadlineMs === deadlineMs) break;
1604
+ // This deadline's step has already timed out, so waiting on it again
1605
+ // here would be a different step. The resolver re-parks from a new leg.
1606
+ if (timedOut && renewedDeadlineMs === deadlineMs) break;
1607
+ rearms += 1;
1608
+ if (rearms > LIVENESS_PARK_MAX_REARMS_PER_WAKE) {
1609
+ throw new Error(
1610
+ `Detached runner park for ${runId}/${suspension.runnerAttempt} woke ${rearms} times without a terminal or an expired deadline.`,
1611
+ );
1612
+ }
1474
1613
  continue;
1475
1614
  }
1476
1615
  if (ceilingDeadlineMs <= Date.now()) break;
1477
- const readRunnerTerminal = input.readRunnerTerminal;
1478
- const readRunnerLiveness = input.readRunnerLiveness;
1479
- const silent = await confirmDetachedRunnerSilence({
1616
+ let quietUntilMs = 0;
1617
+ const verdict = await confirmDetachedRunnerSilence({
1480
1618
  ctx,
1481
1619
  stepPrefix: `${suspension.boundaryId}:liveness-confirm:${Number.isFinite(renewedAtMs) ? renewedAtMs : 'none'}`,
1482
1620
  cadenceMs: Math.floor(
1483
1621
  heartbeatTimeoutMs / RUNTIME_CLAIM_HEARTBEAT_DIVISOR,
1484
1622
  ),
1485
1623
  ceilingDeadlineMs,
1624
+ inspectDetachedRunner: input.inspectDetachedRunner,
1625
+ suspension,
1486
1626
  runnerResponded: async () => {
1627
+ // A wake replays every finished slice. One read per replay is enough.
1628
+ if (Date.now() < quietUntilMs) return false;
1487
1629
  if (await readRunnerTerminal(suspension.runnerAttempt)) return true;
1488
1630
  const current = await readRunnerLiveness(suspension.runnerAttempt);
1489
1631
  const currentAtMs = current?.heartbeatAt
1490
1632
  ? Date.parse(current.heartbeatAt)
1491
1633
  : Number.NaN;
1492
- return Number.isFinite(currentAtMs) && !(currentAtMs <= renewedAtMs);
1634
+ if (Number.isFinite(currentAtMs) && !(currentAtMs <= renewedAtMs)) {
1635
+ return true;
1636
+ }
1637
+ quietUntilMs = Date.now() + LIVENESS_REPLAY_READ_REUSE_MS;
1638
+ return false;
1493
1639
  },
1494
1640
  });
1495
- if (silent) break;
1641
+ if (verdict === 'responded') continue;
1642
+ break;
1496
1643
  }
1497
1644
  // Truth is the persisted terminal (read by the caller's wake path); the
1498
1645
  // absurd event carries no payload we trust.
@@ -1594,6 +1741,9 @@ export function createAbsurdEngineSuspensionAdapter(input: {
1594
1741
  updatedAt: string | null;
1595
1742
  status: string;
1596
1743
  } | null>;
1744
+ inspectDetachedRunner?: (
1745
+ suspension: DetachedRunnerSuspension,
1746
+ ) => Promise<DetachedRunnerInspection | null>;
1597
1747
  }): PlayRunAttemptEngineSuspensionPort {
1598
1748
  let parkedSignal: SuspendTask | null = null;
1599
1749
  return {
@@ -1609,6 +1759,7 @@ export function createAbsurdEngineSuspensionAdapter(input: {
1609
1759
  claimSignal: input.claimSignal,
1610
1760
  readRunnerTerminal: input.readRunnerTerminal,
1611
1761
  readRunnerLiveness: input.readRunnerLiveness,
1762
+ inspectDetachedRunner: input.inspectDetachedRunner,
1612
1763
  }),
1613
1764
  };
1614
1765
  } catch (error) {
@@ -3376,6 +3527,7 @@ async function executeAssignedAbsurdAttempt(input: {
3376
3527
  options,
3377
3528
  ),
3378
3529
  ),
3530
+ inspectDetachedRunner: deps.inspectDetachedRunner,
3379
3531
  });
3380
3532
  const finalizeSuspensionProgress = () =>
3381
3533
  runWithAbsurdExecutionLeaseHeartbeat({
@@ -3582,6 +3734,7 @@ async function executeAssignedAbsurdAttempt(input: {
3582
3734
  options,
3583
3735
  ),
3584
3736
  ),
3737
+ inspectDetachedRunner: deps.inspectDetachedRunner,
3585
3738
  });
3586
3739
  const running = await deps.runWithClient((client) =>
3587
3740
  markPostgresSchedulerAttemptRunning(
@@ -5015,9 +5168,10 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
5015
5168
  * Truth ladder:
5016
5169
  * 1. The runner's pushed terminal persisted on its attempt row.
5017
5170
  * 2. A fresh heartbeat re-parks the same detached runner.
5018
- * 3. A stale heartbeat or hard ceiling writes one failed terminal without
5019
- * asking Daytona. Cleanup observes Daytona only after durable terminal
5020
- * truth exists.
5171
+ * 3. A captured result recovered from the sandbox.
5172
+ * 4. A hard ceiling, or a stale heartbeat the provider proves is final,
5173
+ * writes one failed terminal. A stale heartbeat without that proof
5174
+ * re-parks: a heartbeat gap alone never ends a run.
5021
5175
  *
5022
5176
  * A detached runner terminal is always terminal for its scheduler attempt.
5023
5177
  * By the time a sandbox can push an outcome, customer code may have started
@@ -5035,7 +5189,7 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
5035
5189
  * `resolveRunnerSuspension` persists its obligation before typed waiting.
5036
5190
  * Provider deletion never delays publishing the user's wait.
5037
5191
  */
5038
- async function resolveDetachedRunnerOutcome(input: {
5192
+ export async function resolveDetachedRunnerOutcome(input: {
5039
5193
  runId: string;
5040
5194
  suspension: DetachedRunnerSuspension;
5041
5195
  checkpoint: PlayCheckpoint;
@@ -5080,7 +5234,16 @@ async function resolveDetachedRunnerOutcome(input: {
5080
5234
  result: PlayRunnerResult,
5081
5235
  ): Promise<PlayRunnerResult> => {
5082
5236
  await recordWorkCompletion(result);
5083
- return result;
5237
+ // `runtimeBackend` is the launch's selector and fallback can leave it
5238
+ // pointing at a provider that never ran this sandbox. Record the one that
5239
+ // did, including on terminals the runner never got to write.
5240
+ return {
5241
+ ...result,
5242
+ stats: {
5243
+ ...result.stats,
5244
+ sandboxProvider: suspension.sandboxProvider ?? 'daytona',
5245
+ },
5246
+ };
5084
5247
  };
5085
5248
  const settleFirstWriteWinner = async (
5086
5249
  candidate: PlayRunnerResult,
@@ -5112,9 +5275,9 @@ async function resolveDetachedRunnerOutcome(input: {
5112
5275
  return settle(stored);
5113
5276
  }
5114
5277
 
5115
- // No pushed terminal. The scheduler owns this decision: never make a
5116
- // terminal depend on a Daytona lookup, because Daytona may be the failed
5117
- // component. A concurrent terminal remains allowed to win below.
5278
+ // No pushed terminal. A provider outage must not fail a run, so only a
5279
+ // positive provider answer does; an unavailable provider leaves the hard
5280
+ // ceiling to end it. A concurrent terminal remains allowed to win below.
5118
5281
  const now = Date.now();
5119
5282
  // A liveness-store outage is not evidence that the sandbox died. Let the
5120
5283
  // worker retry this read rather than producing a false terminal failure.
@@ -5138,43 +5301,47 @@ async function resolveDetachedRunnerOutcome(input: {
5138
5301
  configuredHeartbeatTimeoutMs >= 0
5139
5302
  ? configuredHeartbeatTimeoutMs
5140
5303
  : RUNTIME_CLAIM_LEASE_SECONDS * 1000;
5304
+ const reparkDetachedRunner = (): PlayRunnerResult => ({
5305
+ status: 'suspended',
5306
+ suspension,
5307
+ logs: [],
5308
+ stats: {},
5309
+ steps: [],
5310
+ checkpoint: input.checkpoint,
5311
+ tableNamespace: null,
5312
+ });
5141
5313
  if (!reachedCeiling && heartbeatAgeMs !== null) {
5142
5314
  if (heartbeatAgeMs < heartbeatTimeoutMs) {
5143
5315
  // We woke on an older deadline while the runner renewed liveness. Park
5144
5316
  // again from the new durable timestamp; do not touch Daytona.
5145
- return {
5146
- status: 'suspended',
5147
- suspension,
5148
- logs: [],
5149
- stats: {},
5150
- steps: [],
5151
- checkpoint: input.checkpoint,
5152
- tableNamespace: null,
5153
- };
5317
+ return reparkDetachedRunner();
5154
5318
  }
5155
5319
  }
5156
- // This event is consumed directly by the operational-health dashboard.
5157
- // Keep its second argument strict JSON so Fly's text log stream remains an
5158
- // executable producer contract, independent of Bun's object-inspect format.
5159
- console.error(
5160
- '[absurd-runtime.worker] detached_runner_liveness_expired',
5161
- JSON.stringify({
5162
- runId: input.runId,
5163
- runnerAttempt: suspension.runnerAttempt,
5164
- sandboxId: suspension.sandboxId,
5165
- sessionId: suspension.sessionId,
5166
- cmdId: suspension.cmdId,
5167
- startedAtMs: suspension.startedAtMs,
5168
- heartbeatTimeoutMs,
5169
- ceilingMs: suspension.ceilingMs,
5170
- observedAtMs: now,
5171
- reachedCeiling,
5172
- liveness,
5173
- heartbeatAgeMs,
5174
- pushedTerminalPresent: false,
5175
- }),
5176
- );
5320
+ const reportLivenessExpired = () => {
5321
+ // This event is consumed directly by the operational-health dashboard.
5322
+ // Keep its second argument strict JSON so Fly's text log stream remains an
5323
+ // executable producer contract, independent of Bun's object-inspect format.
5324
+ console.error(
5325
+ '[absurd-runtime.worker] detached_runner_liveness_expired',
5326
+ JSON.stringify({
5327
+ runId: input.runId,
5328
+ runnerAttempt: suspension.runnerAttempt,
5329
+ sandboxId: suspension.sandboxId,
5330
+ sessionId: suspension.sessionId,
5331
+ cmdId: suspension.cmdId,
5332
+ startedAtMs: suspension.startedAtMs,
5333
+ heartbeatTimeoutMs,
5334
+ ceilingMs: suspension.ceilingMs,
5335
+ observedAtMs: now,
5336
+ reachedCeiling,
5337
+ liveness,
5338
+ heartbeatAgeMs,
5339
+ pushedTerminalPresent: false,
5340
+ }),
5341
+ );
5342
+ };
5177
5343
  if (reachedCeiling) {
5344
+ reportLivenessExpired();
5178
5345
  return settleFirstWriteWinner({
5179
5346
  status: 'failed',
5180
5347
  error: `${runtimeLimitExceededCause(runtimeLimitSeconds)} Detached runner reached its hard execution ceiling without a terminal result. Sandbox ${suspension.sandboxId}. The run was stopped and was not retried; completed row state remains persisted.`,
@@ -5189,17 +5356,14 @@ async function resolveDetachedRunnerOutcome(input: {
5189
5356
  // bounded terminal push. A receipt-gateway outage can exhaust that push and
5190
5357
  // expire liveness even though the sandbox cleanly stopped itself. Recover
5191
5358
  // only that already-written result before synthesizing RUNNER_LOST: this
5192
- // does not execute customer code, and an unavailable inspection preserves
5193
- // the existing conservative failure below.
5194
- let inspection: {
5195
- exitCode: number | null;
5196
- result: PlayRunnerResult | null;
5197
- diagnosis?: { stage: string; detail: string | null };
5198
- } | null = null;
5359
+ // does not execute customer code.
5360
+ const provenGone = detachedRunnerProvenGone.has(suspension);
5361
+ let inspection = detachedRunnerProvenGone.get(suspension) ?? null;
5362
+ detachedRunnerProvenGone.delete(suspension);
5199
5363
  let inspectionFailed = false;
5200
5364
  if (input.inspectDetachedRunner) {
5201
5365
  try {
5202
- inspection = await input.inspectDetachedRunner(suspension);
5366
+ inspection ??= await input.inspectDetachedRunner(suspension);
5203
5367
  if (inspection?.result && inspection.result.status !== 'suspended') {
5204
5368
  console.info(
5205
5369
  '[absurd-runtime.worker] detached_runner_liveness_salvaged',
@@ -5240,6 +5404,32 @@ async function resolveDetachedRunnerOutcome(input: {
5240
5404
  );
5241
5405
  }
5242
5406
  }
5407
+ // Inspection takes seconds. A terminal or heartbeat that landed meanwhile
5408
+ // outranks everything read before it.
5409
+ const storedDuringInspection = parseDetachedRunnerResult(
5410
+ await input.readRunnerTerminal(suspension.runnerAttempt),
5411
+ );
5412
+ if (storedDuringInspection) {
5413
+ return settle(storedDuringInspection);
5414
+ }
5415
+ const latestLiveness = await input.readRunnerLiveness();
5416
+ const latestHeartbeatAtMs = latestLiveness?.heartbeatAt
5417
+ ? Date.parse(latestLiveness.heartbeatAt)
5418
+ : Number.NaN;
5419
+ if (
5420
+ Number.isFinite(latestHeartbeatAtMs) &&
5421
+ !(latestHeartbeatAtMs <= heartbeatAtMs)
5422
+ ) {
5423
+ return reparkDetachedRunner();
5424
+ }
5425
+ if (
5426
+ input.inspectDetachedRunner &&
5427
+ !provenGone &&
5428
+ !detachedRunnerInspectionProvesGone(inspection)
5429
+ ) {
5430
+ return reparkDetachedRunner();
5431
+ }
5432
+ reportLivenessExpired();
5243
5433
  const safeDiagnosticLabel = (value: unknown): string =>
5244
5434
  typeof value === 'string' && /^[a-z0-9_]{1,80}$/i.test(value)
5245
5435
  ? value
@@ -15346,9 +15346,17 @@ export function attachRunnerWorkProgressMetadata(
15346
15346
  ...outputWarnings,
15347
15347
  ...ctxFetchHttpWarnings,
15348
15348
  ].slice(0, 16);
15349
+ const sandboxProvider = (
15350
+ runnerResult as { stats?: { sandboxProvider?: unknown } }
15351
+ ).stats?.sandboxProvider;
15352
+ const provenSandboxProvider =
15353
+ sandboxProvider === 'daytona' || sandboxProvider === 'modal'
15354
+ ? sandboxProvider
15355
+ : null;
15349
15356
  if (
15350
15357
  (total === null || skipped === null || metadata?.workProgress) &&
15351
15358
  !runLogTail &&
15359
+ !provenSandboxProvider &&
15352
15360
  !runtimeResources &&
15353
15361
  !runtimeCounters &&
15354
15362
  !metadata?.runtimeCounters &&
@@ -15390,6 +15398,9 @@ export function attachRunnerWorkProgressMetadata(
15390
15398
  ...(rowOutcomes ? { rowOutcomes } : {}),
15391
15399
  ...(runLogTail ? { runLogTail } : {}),
15392
15400
  ...(runtimeResources ? { runtimeResources } : {}),
15401
+ ...(provenSandboxProvider
15402
+ ? { sandboxProvider: provenSandboxProvider }
15403
+ : {}),
15393
15404
  ...(mergedOutputWarnings.length > 0
15394
15405
  ? { outputWarnings: mergedOutputWarnings }
15395
15406
  : {}),
@@ -13,13 +13,13 @@ import {
13
13
  reportSetupCompleted,
14
14
  resolvePendingAuthClaimForResume,
15
15
  statusBannerLine
16
- } from "./chunk-7557D4FM.mjs";
17
- import "./chunk-FS2RMEJ7.mjs";
18
- import "./chunk-X2LUT22I.mjs";
16
+ } from "./chunk-FCQ52RUB.mjs";
17
+ import "./chunk-RYGDM5BG.mjs";
18
+ import "./chunk-LRA5YJVV.mjs";
19
19
  import "./chunk-4QF2K7TM.mjs";
20
20
  import "./chunk-DLIRSHV6.mjs";
21
21
  import "./chunk-HBL4PM5Q.mjs";
22
- import "./chunk-IXVABCNA.mjs";
22
+ import "./chunk-DPTEABFU.mjs";
23
23
  import "./chunk-WZYKRD7T.mjs";
24
24
  import "./chunk-KRD6H754.mjs";
25
25
  export {
@@ -0,0 +1,40 @@
1
+ "use strict";Object.defineProperty(exports, "__esModule", {value: true});
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+ var _chunkUVJLG5KDjs = require('./chunk-UVJLG5KD.js');
17
+ require('./chunk-J7WYWQF4.js');
18
+ require('./chunk-2R7JD2NY.js');
19
+ require('./chunk-ALJJRZNJ.js');
20
+ require('./chunk-XNYIZMNQ.js');
21
+ require('./chunk-DIAWYWT5.js');
22
+ require('./chunk-6TOMXZX3.js');
23
+ require('./chunk-OG64RBAC.js');
24
+ require('./chunk-COGTRVWW.js');
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+ exports.EXIT_WORKSPACE_MISMATCH = _chunkUVJLG5KDjs.EXIT_WORKSPACE_MISMATCH; exports.authStatusLine = _chunkUVJLG5KDjs.authStatusLine; exports.clearPendingClaim = _chunkUVJLG5KDjs.clearPendingClaim; exports.clearSatisfiedPendingClaim = _chunkUVJLG5KDjs.clearSatisfiedPendingClaim; exports.handleRegister = _chunkUVJLG5KDjs.handleRegister; exports.handleStatus = _chunkUVJLG5KDjs.handleStatus; exports.handleWait = _chunkUVJLG5KDjs.handleWait; exports.httpJson = _chunkUVJLG5KDjs.httpJson; exports.readPendingAuthClaim = _chunkUVJLG5KDjs.readPendingAuthClaim; exports.readStatusBanner = _chunkUVJLG5KDjs.readStatusBanner; exports.registerAuthCommands = _chunkUVJLG5KDjs.registerAuthCommands; exports.reportSetupCompleted = _chunkUVJLG5KDjs.reportSetupCompleted; exports.resolvePendingAuthClaimForResume = _chunkUVJLG5KDjs.resolvePendingAuthClaimForResume; exports.statusBannerLine = _chunkUVJLG5KDjs.statusBannerLine;
@@ -2,7 +2,7 @@ import {
2
2
  DeeplineClient,
3
3
  iterateCursorPages,
4
4
  requireBillingIdempotencyKey
5
- } from "./chunk-XV4463XW.mjs";
5
+ } from "./chunk-PQ4JU2MV.mjs";
6
6
  import "./chunk-L2VKAASU.mjs";
7
7
  import "./chunk-FAG4DFUC.mjs";
8
8
  import "./chunk-EIMG75BT.mjs";
@@ -13,8 +13,8 @@ import {
13
13
  openInBrowser,
14
14
  printCommandEnvelope,
15
15
  shouldEmitJson
16
- } from "./chunk-FS2RMEJ7.mjs";
17
- import "./chunk-X2LUT22I.mjs";
16
+ } from "./chunk-RYGDM5BG.mjs";
17
+ import "./chunk-LRA5YJVV.mjs";
18
18
  import {
19
19
  AuthError,
20
20
  ConfigError
@@ -23,7 +23,7 @@ import "./chunk-DLIRSHV6.mjs";
23
23
  import {
24
24
  DeeplineError
25
25
  } from "./chunk-HBL4PM5Q.mjs";
26
- import "./chunk-IXVABCNA.mjs";
26
+ import "./chunk-DPTEABFU.mjs";
27
27
  import "./chunk-WZYKRD7T.mjs";
28
28
  import "./chunk-KRD6H754.mjs";
29
29