@outerlayer/cli 0.38.0-next.20261111004457 → 0.38.0-next.20261111030822

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/README.md +8 -0
  2. package/dist/{agent-setup-OU7CGYSN.js → agent-setup-T2VITZUY.js} +9 -9
  3. package/dist/{agent-subcommand-GG4PSHYT.js → agent-subcommand-KV3JO5YD.js} +1 -1
  4. package/dist/{bridge-MVW5RNQD.js → bridge-7MCKZXCR.js} +4 -4
  5. package/dist/{build-D4Y3LBPO.js → build-N2MRLYFW.js} +10 -10
  6. package/dist/{build-disk-VDMWH3C3.js → build-disk-IJZYZ2K5.js} +8 -8
  7. package/dist/build-info.json +1 -1
  8. package/dist/{builder-6L5ZMXBE.js → builder-ATKIRCDJ.js} +7 -7
  9. package/dist/{caller-FFSXVMLB.js → caller-OWXJDYPI.js} +10 -10
  10. package/dist/{check-U5DOQU64.js → check-F7RN65GG.js} +40 -40
  11. package/dist/{chunk-IWJFARST.js → chunk-2FUJ5PSV.js} +1 -1
  12. package/dist/{chunk-EKV7QATX.js → chunk-2PRDLE4L.js} +1 -1
  13. package/dist/{chunk-IZHI643L.js → chunk-3VEPVZV4.js} +5 -5
  14. package/dist/{chunk-3IL6H7A4.js → chunk-4HUYH65P.js} +1 -1
  15. package/dist/{chunk-4S6ML2YX.js → chunk-4YAFIOIV.js} +17 -2
  16. package/dist/{chunk-GELUV567.js → chunk-5NJ367OD.js} +2 -2
  17. package/dist/{chunk-R6UNNKV3.js → chunk-64ROJ5LD.js} +1 -1
  18. package/dist/{chunk-5U4IKVF5.js → chunk-6T2X2JI6.js} +3 -3
  19. package/dist/{chunk-7ILAGC3U.js → chunk-6U3UTAWQ.js} +1 -1
  20. package/dist/{chunk-WVJDXOIG.js → chunk-7G2HRTSG.js} +3 -3
  21. package/dist/{chunk-2PYQ6BZJ.js → chunk-7HYXJVU6.js} +1 -1
  22. package/dist/{chunk-X7BVDOS2.js → chunk-7LWASV77.js} +3 -3
  23. package/dist/{chunk-MX5XVAOI.js → chunk-7U43B2UB.js} +1 -1
  24. package/dist/{chunk-JASHMKDN.js → chunk-AB4GMYIH.js} +1 -1
  25. package/dist/{chunk-NBJPY3OC.js → chunk-ASCXEXYJ.js} +5 -5
  26. package/dist/{chunk-UF3X5ECJ.js → chunk-BY2YRJSC.js} +2 -2
  27. package/dist/{chunk-U43K2JS5.js → chunk-ECMQNH2H.js} +2 -2
  28. package/dist/{chunk-YWQ3S3F5.js → chunk-EWGHEMEO.js} +2 -2
  29. package/dist/{chunk-F5ONFHHU.js → chunk-FB2UT5XA.js} +31 -27
  30. package/dist/{chunk-BZSGU2WC.js → chunk-FT5NDSSG.js} +4 -4
  31. package/dist/{chunk-MV5WJEIB.js → chunk-GGKXYHT5.js} +2 -2
  32. package/dist/{chunk-7NRULS6C.js → chunk-GPP5EZV7.js} +3 -3
  33. package/dist/{chunk-2ZPXDOCU.js → chunk-I5C667TJ.js} +9 -9
  34. package/dist/{chunk-QB5MU2WS.js → chunk-IB2UHXZ7.js} +2 -2
  35. package/dist/{chunk-EN3OWMED.js → chunk-ICW75TS5.js} +4 -4
  36. package/dist/{chunk-CKA4JQKR.js → chunk-IEX5FXEU.js} +2 -2
  37. package/dist/{chunk-OZDUVL5L.js → chunk-JPAO6HRN.js} +2 -2
  38. package/dist/{chunk-DDEJP7YT.js → chunk-K6KUWUUT.js} +1 -1
  39. package/dist/{chunk-JNEK36QK.js → chunk-KGJAYD7F.js} +1239 -1100
  40. package/dist/{chunk-OWBNADRQ.js → chunk-KM2IMFAG.js} +1 -1
  41. package/dist/{chunk-3AHABZMA.js → chunk-LXREVZX7.js} +2 -2
  42. package/dist/{chunk-A6CCWLUP.js → chunk-M5DYOU5G.js} +3 -3
  43. package/dist/{chunk-YE3KPAT5.js → chunk-MPF7CVQM.js} +1 -1
  44. package/dist/{chunk-W5OAKC2W.js → chunk-NC6HUEMI.js} +9 -9
  45. package/dist/{chunk-P5XX7VZ6.js → chunk-NWSKP5VI.js} +2 -2
  46. package/dist/{chunk-4JKCTCCC.js → chunk-O53PVAUY.js} +4 -4
  47. package/dist/{chunk-D7OVS2LL.js → chunk-OBSWWJJ3.js} +1 -1
  48. package/dist/{chunk-ONZNFKCH.js → chunk-OBXRHBOM.js} +2 -2
  49. package/dist/{chunk-JP5D37RZ.js → chunk-PP7MPUX7.js} +3 -3
  50. package/dist/{chunk-JPO6RJEW.js → chunk-QM3M7QN3.js} +1 -1
  51. package/dist/{chunk-JRM35NTG.js → chunk-R35KRBTS.js} +5 -5
  52. package/dist/{chunk-NIQXLC6A.js → chunk-R7BS36HP.js} +1 -1
  53. package/dist/{chunk-J3KW7GPE.js → chunk-REZNW2YV.js} +6 -6
  54. package/dist/{chunk-LNTNFSRI.js → chunk-RPIYENP2.js} +2 -2
  55. package/dist/{chunk-6Z5AC4ZK.js → chunk-T5CN56UE.js} +1 -1
  56. package/dist/{chunk-WWGXWKLT.js → chunk-U5YHNSSI.js} +5 -5
  57. package/dist/{chunk-DNCF3XOC.js → chunk-W2H6JURI.js} +3 -3
  58. package/dist/{chunk-BYWVHMBR.js → chunk-XMIL73Y2.js} +1 -1
  59. package/dist/{chunk-JDXYBQBZ.js → chunk-XSVZFWZ4.js} +7 -7
  60. package/dist/{chunk-ZNQRFRBW.js → chunk-YFMVZBLO.js} +5 -5
  61. package/dist/{chunk-NCPZP6PQ.js → chunk-YFQIK6LZ.js} +7 -7
  62. package/dist/{chunk-5TODASD5.js → chunk-YI2XPK7A.js} +1 -1
  63. package/dist/{chunk-RI37FFPL.js → chunk-ZGCIFD5Y.js} +2 -2
  64. package/dist/{claude-launcher-GNRTFQMI.js → claude-launcher-ZC664P33.js} +1 -1
  65. package/dist/{cli-SNHD2D6X.js → cli-GXIDG2NL.js} +141 -131
  66. package/dist/{command-NTKAT7KB.js → command-D656K4AE.js} +35 -35
  67. package/dist/{config-LUIWZESD.js → config-U2OR7MQW.js} +9 -9
  68. package/dist/{connect-cmd-4M4DN6GO.js → connect-cmd-73BBTTU5.js} +7 -7
  69. package/dist/{container-start-BVEDUA7N.js → container-start-JFJYC4GQ.js} +33 -33
  70. package/dist/{context-show-cmd-DAMQJOAL.js → context-show-cmd-ETWKBHXZ.js} +8 -8
  71. package/dist/{control-plane-launch-SD45JYKF.js → control-plane-launch-JOH7FS5U.js} +7 -7
  72. package/dist/{demo-record-cmd-CFOXCR42.js → demo-record-cmd-GBHPIYJA.js} +5 -5
  73. package/dist/{demo-voice-cmd-4YZJPUIP.js → demo-voice-cmd-GUYFA2NG.js} +13 -13
  74. package/dist/{docker-2DLKQOK3.js → docker-RC4CFNKI.js} +7 -7
  75. package/dist/{doctor-VFCDAKM4.js → doctor-TW6VREAT.js} +35 -35
  76. package/dist/{emit-artifact-cmd-IWZDXJBY.js → emit-artifact-cmd-CQLCPM4R.js} +12 -12
  77. package/dist/{emit-criteria-cmd-UVCLLYEZ.js → emit-criteria-cmd-VM7AO3E3.js} +9 -9
  78. package/dist/{emit-demo-cmd-BHKNT36Y.js → emit-demo-cmd-3NSODQTO.js} +13 -13
  79. package/dist/{emit-finding-cmd-Y3XX26B4.js → emit-finding-cmd-4CRLSMOK.js} +9 -9
  80. package/dist/{emit-result-cmd-TTQ3MLY4.js → emit-result-cmd-GM43QMCM.js} +11 -11
  81. package/dist/{emit-step-cmd-EASDIM5Q.js → emit-step-cmd-J7KS2QXL.js} +9 -9
  82. package/dist/{exec-client-MEIRWXLR.js → exec-client-DY7673SD.js} +5 -5
  83. package/dist/find-unit-ZVHHFTL2.js +24 -0
  84. package/dist/{findings-cmd-KQOL6NJG.js → findings-cmd-25F44AB4.js} +9 -9
  85. package/dist/{fleet-readiness-R2BMWLXQ.js → fleet-readiness-Z4C6W7XN.js} +7 -7
  86. package/dist/{governing-source-ARSS7R7L.js → governing-source-IURJZXUR.js} +11 -11
  87. package/dist/{guest-init-UGBFOWNC.js → guest-init-4B5MMXIQ.js} +6 -6
  88. package/dist/{hook-fast-DC556GE2.js → hook-fast-LB3VTEIW.js} +1 -1
  89. package/dist/{host-checks-UTZBDSWV.js → host-checks-AQTZ7IKK.js} +31 -31
  90. package/dist/{pending-COVCIH4Q.js → host-key-ZWMN7VIH.js} +5 -5
  91. package/dist/{image-tag-RU4RHPIT.js → image-tag-4DKLSTO6.js} +5 -5
  92. package/dist/index.js +4 -4
  93. package/dist/{init-I5HVGOQC.js → init-P2PDGLTU.js} +31 -31
  94. package/dist/{init-QFTRSBBT.js → init-XYFDIEOA.js} +31 -31
  95. package/dist/{init-agent-TKQ67BS4.js → init-agent-EGNFBPM5.js} +31 -31
  96. package/dist/{init-cmd-I36LJ6YC.js → init-cmd-C2I22ZJA.js} +12 -12
  97. package/dist/{init-update-KG7V3VJF.js → init-update-H4LSR7H4.js} +10 -10
  98. package/dist/{install-cmd-66Z7TY52.js → install-cmd-BI4C3MGF.js} +7 -7
  99. package/dist/{logs-XWMVEAUP.js → logs-7DD6J3M6.js} +4 -4
  100. package/dist/{loop-YCOSJRLW.js → loop-KEGBBAV4.js} +39 -39
  101. package/dist/{machine-BJEAYCHT.js → machine-JK27TGT6.js} +31 -31
  102. package/dist/{mcp-serve-cmd-M7CX4RO3.js → mcp-serve-cmd-HK2ODMHU.js} +6 -6
  103. package/dist/{paths-GQ66KHNT.js → paths-EHWU5QWB.js} +4 -4
  104. package/dist/{host-key-CTRFLP2W.js → pending-K4A23JIC.js} +5 -5
  105. package/dist/{pidfile-RBG6FQ7K.js → pidfile-PG3L2CVV.js} +5 -5
  106. package/dist/{record-6IAZ26H3.js → record-PXCRTDQU.js} +5 -5
  107. package/dist/{relay-VEBGGPYV.js → relay-2MJY25BV.js} +2 -2
  108. package/dist/{relay-protocol-2AJCUAMJ.js → relay-protocol-VBCRA347.js} +2 -2
  109. package/dist/{run-cmd-EJPHCENL.js → run-cmd-F7ANZPSX.js} +9 -9
  110. package/dist/{runner-agents-P562WZJB.js → runner-agents-HZ5FIFPU.js} +1 -1
  111. package/dist/{runner-env-XPZ3FPQL.js → runner-env-5AM2XYEN.js} +11 -11
  112. package/dist/{runner-identity-TAJAY5M5.js → runner-identity-CL2KMFGC.js} +7 -7
  113. package/dist/{runner-key-I3M7GEJE.js → runner-key-CJ66VZE3.js} +31 -31
  114. package/dist/{session-sender-OIOE4QR2.js → session-sender-RSAHCLL6.js} +12 -12
  115. package/dist/{spec-cmd-2J43XP3L.js → spec-cmd-ED6DKB54.js} +10 -10
  116. package/dist/{state-NAZKB77R.js → state-MV53BATE.js} +5 -5
  117. package/dist/{status-BFRDAZQP.js → status-CZTCYZLU.js} +31 -31
  118. package/dist/{sync-cmd-KWTFVQCR.js → sync-cmd-PQZD77PE.js} +11 -11
  119. package/dist/{update-steps-LINCNPGV.js → update-steps-DCFEKNW2.js} +32 -32
  120. package/dist/{watch-3IBFF2SQ.js → watch-FAQXAAUA.js} +11 -11
  121. package/dist/{work-claim-cmd-C2WFSOKE.js → work-claim-cmd-XPXHRV74.js} +15 -15
  122. package/dist/{work-cmd-DO2QNBKA.js → work-cmd-ME7DWBI6.js} +9 -9
  123. package/dist/{work-comment-cmd-P5VJHCX7.js → work-comment-cmd-ZT4O56QR.js} +11 -11
  124. package/dist/{work-demo-cmd-UJBZLOMQ.js → work-demo-cmd-YV22MZXG.js} +9 -9
  125. package/dist/{work-logs-cmd-OVE5JUMH.js → work-logs-cmd-LUK7THB2.js} +9 -9
  126. package/dist/{work-needs-attention-cmd-QNCOUXMT.js → work-needs-attention-cmd-KTLWMBJV.js} +9 -9
  127. package/dist/{work-no-change-cmd-KQSONMJX.js → work-no-change-cmd-TWGW5ZOO.js} +9 -9
  128. package/dist/{work-open-pr-cmd-HBE74CZU.js → work-open-pr-cmd-VEM5MEUS.js} +10 -10
  129. package/dist/{work-pr-cmd-62BGKUJM.js → work-pr-cmd-M4G35ZIV.js} +9 -9
  130. package/dist/{work-proofs-cmd-SLUFPOEN.js → work-proofs-cmd-7EZ4QZUR.js} +10 -10
  131. package/dist/{work-proofs-record-6KGPMUHD.js → work-proofs-record-AY72YKNC.js} +16 -16
  132. package/dist/{work-stop-cmd-7WEHA365.js → work-stop-cmd-FX5QTUYJ.js} +9 -9
  133. package/package.json +1 -1
  134. package/dist/find-unit-2RZ4NCXQ.js +0 -24
@@ -4,37 +4,37 @@ import { itemsSentence, criteriaOf, hostItemsOf, loadCitationFinder, testFileMat
4
4
  import { RUNNER_PROTOCOL } from './chunk-6XMC6R4W.js';
5
5
  import { sha256OfFile, downloadPinned, VoiceDownloadError } from './chunk-2PGI6RDE.js';
6
6
  import { zodProblem } from './chunk-I3LTIAOR.js';
7
- import { hasStoredCredential, storedCredentialVariables, storeAgentCredential, isAgentName, credentialFromStdin } from './chunk-EN3OWMED.js';
7
+ import { hasStoredCredential, storedCredentialVariables, storeAgentCredential, isAgentName, credentialFromStdin } from './chunk-ICW75TS5.js';
8
8
  import { upgradedStarterText } from './chunk-OUKSF3LK.js';
9
- import { HELPER_PATH, HELPER_RECORD, DOCKERD_LOG } from './chunk-X7BVDOS2.js';
10
- import { writeDockerClientConfig } from './chunk-EKV7QATX.js';
11
- import { senderPidfile } from './chunk-P5XX7VZ6.js';
12
- import { installScrubContext } from './chunk-WWGXWKLT.js';
9
+ import { HELPER_PATH, HELPER_RECORD, DOCKERD_LOG } from './chunk-7LWASV77.js';
10
+ import { writeDockerClientConfig } from './chunk-2PRDLE4L.js';
11
+ import { senderPidfile } from './chunk-NWSKP5VI.js';
12
+ import { installScrubContext } from './chunk-U5YHNSSI.js';
13
13
  import { readsJobState, READ_JOB_STATE_FORMATS, jobStateFormatOf } from './chunk-ZVB57WI6.js';
14
- import { runnerImageVersion } from './chunk-R6UNNKV3.js';
15
- import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey, configForJob } from './chunk-OZDUVL5L.js';
14
+ import { runnerImageVersion } from './chunk-64ROJ5LD.js';
15
+ import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey, configForJob } from './chunk-JPAO6HRN.js';
16
16
  import { receiveFiles } from './chunk-NN33FYXG.js';
17
- import { readDefaultBranchFiles, readDefaultBranchHead, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-NCPZP6PQ.js';
18
- import { FETCHED_DIR, toolsDockerfile, FETCHED_FFMPEG, FETCHED_GH, FETCHED_TINI, FETCHED_HEADLESS_SHELL, FETCHED_PLAYWRIGHT_FFMPEG, FETCHED_VOICE_DIR } from './chunk-DNCF3XOC.js';
17
+ import { readDefaultBranchFiles, readDefaultBranchHead, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-YFQIK6LZ.js';
18
+ import { FETCHED_DIR, toolsDockerfile, FETCHED_FFMPEG, FETCHED_GH, FETCHED_TINI, FETCHED_HEADLESS_SHELL, FETCHED_PLAYWRIGHT_FFMPEG, FETCHED_VOICE_DIR } from './chunk-W2H6JURI.js';
19
19
  import { isSameProcess, startTimeOf } from './chunk-JBJTC3UU.js';
20
20
  import { currentCliBuild } from './chunk-OGFGZQA3.js';
21
- import { vmExecArgs, runExec, execCapture, waitForAgent } from './chunk-2PYQ6BZJ.js';
22
- import { socketPathProblem, vsockLinks, linkVsockSockets, GUEST_CID, LEASE_LINK } from './chunk-RI37FFPL.js';
23
- import { describeCallFailure, shortClaim, retrySend, SELF_GOVERNED, resolveGoverningSource, prefetchGovernedContext, refusalCode, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, formatClock, stageGovernedContext, keptContextTree, removeStagedContext, keepStagedContext, pinnedSourceOf, WORKFLOW_PERMISSION_REASON, shimScript, controlPlaneTokenFailure } from './chunk-JRM35NTG.js';
24
- import { postJson, WorkCommandError, isSignatureRefusal, getJson } from './chunk-ZNQRFRBW.js';
25
- import { isBuiltinContainer, isBuiltinVm, isBuiltinSandbox, keyRunnerFactory, runnerCredentialProblem, builtinKind, sandboxSizeOf, networkOf, sandboxTimeLimitMinutes, DEFAULT_SETUP_MINUTES, optionalEgress, egressSummary, DEFAULT_WORKSPACE_KEEP_DAYS, DEFAULT_CHECK_TIME_LIMIT_MINUTES, allowHostProblem, systemClock, emptyEgressCounts, hostAllowed, imageBuildOf, DEFAULT_SAVED_STATE_MAX_BYTES } from './chunk-4JKCTCCC.js';
26
- import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-MV5WJEIB.js';
27
- import { runnerLabelValue, said, RUNNER_LABEL, BUILD_LABEL, prepareRunArgs, containerRunArgs, judgeProgramRecord, firstLine, writeJobFile, readJobFile, spawnRun, NETWORK_HELPER_MEMORY, createDockerHost, builderSpec, mountableHome, homeCopyRunArgs, containerExecArgs, childEnvironment, readDu, identityRunArgs, parseIdentity, withAnswerProtocol, JUDGE_TREE, MAX_REPORT_BYTES, parseSize, readJobFileChecked, NETWORK_READY_FILE, networkHelperName, judgeCommand, JUDGE_DIR, WORKSPACE_LABEL, networkHelperRunArgs, JUDGE_MAX_TURNS, JUDGE_ITEMS, answerFileName, hashJudgeProgram, identityScript, identityArgs, homeCopyScript, homeCopyArgs, COMMAND_DIR, MAX_ITEMS_BYTES, JUDGE_REPORT, SAVED_STATE_LABEL } from './chunk-J3KW7GPE.js';
28
- import { universalImagePending, RecipeError, writeSanitizedRecipe, classifyProperties, DOCKER_FEATURE } from './chunk-IWJFARST.js';
21
+ import { vmExecArgs, runExec, execCapture, waitForAgent } from './chunk-7HYXJVU6.js';
22
+ import { socketPathProblem, vsockLinks, linkVsockSockets, GUEST_CID, LEASE_LINK } from './chunk-ZGCIFD5Y.js';
23
+ import { describeCallFailure, shortClaim, retrySend, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, formatClock, stageGovernedContext, keptContextTree, removeStagedContext, WORKFLOW_PERMISSION_REASON, refusalCode, SELF_GOVERNED, resolveGoverningSource, prefetchGovernedContext, controlPlaneTokenFailure, keepStagedContext, pinnedSourceOf, shimScript } from './chunk-R35KRBTS.js';
24
+ import { postJson, WorkCommandError, isSignatureRefusal, getJson } from './chunk-YFMVZBLO.js';
25
+ import { isBuiltinContainer, isBuiltinVm, isBuiltinSandbox, keyRunnerFactory, runnerCredentialProblem, builtinKind, optionalEgress, egressSummary, DEFAULT_WORKSPACE_KEEP_DAYS, DEFAULT_CHECK_TIME_LIMIT_MINUTES, DEFAULT_SETUP_MINUTES, allowHostProblem, systemClock, emptyEgressCounts, hostAllowed, sandboxSizeOf, networkOf, sandboxTimeLimitMinutes, imageBuildOf, DEFAULT_SAVED_STATE_MAX_BYTES } from './chunk-O53PVAUY.js';
26
+ import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-GGKXYHT5.js';
27
+ import { firstLine, writeJobFile, runnerLabelValue, readJobFile, spawnRun, RUNNER_LABEL, BUILD_LABEL, said, NETWORK_HELPER_MEMORY, createDockerHost, builderSpec, containerExecArgs, childEnvironment, readDu, withAnswerProtocol, JUDGE_TREE, MAX_REPORT_BYTES, parseSize, readJobFileChecked, NETWORK_READY_FILE, networkHelperName, prepareRunArgs, containerRunArgs, judgeCommand, JUDGE_DIR, WORKSPACE_LABEL, homeCopyRunArgs, networkHelperRunArgs, JUDGE_MAX_TURNS, JUDGE_ITEMS, answerFileName, hashJudgeProgram, identityScript, identityArgs, parseIdentity, mountableHome, homeCopyScript, homeCopyArgs, COMMAND_DIR, judgeProgramRecord, MAX_ITEMS_BYTES, JUDGE_REPORT, identityRunArgs, SAVED_STATE_LABEL } from './chunk-REZNW2YV.js';
28
+ import { universalImagePending, RecipeError, writeSanitizedRecipe, classifyProperties, DOCKER_FEATURE } from './chunk-2FUJ5PSV.js';
29
29
  import { sanitizeTerminal, planArtifactUpload } from './chunk-QHYCF36K.js';
30
30
  import { sharedSkillNames, skillConflictMessage } from './chunk-MRSN67FY.js';
31
31
  import { readCloudConfig } from './chunk-HWVMRETT.js';
32
- import { signerFor } from './chunk-OWBNADRQ.js';
33
- import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir, buildDir, ghConfigDir, workspacesDir, jobsDir, storeScopes, spoolDir, orgStoreIn, rootfsDir, imagesDir, sessionsDir, builderCacheDiskPath, buildsDir, runnerDirOfBuild, mirrorDir, orgHomeDir, deliveryDir, jobConsolePath, vsockPath, SHIM_DIR, builderConsolePath, rootfsPathFor, diskTemplatesDir } from './chunk-6Z5AC4ZK.js';
34
- import { BUILD_HOME, repositoriesOf, primaryOf, SandboxError, describeSandboxError, buildEnvironment, CHECKOUT_DIR, JOB_MOUNT, heldBuildAuthority, IMAGE_BUNDLES, writeTrustFiles, trustEnvironment, parseIsolation, containerLocalAddress, destinationEnvironment, withHooksPath, startBuildServices, headerVariables, authorEnvironment, sharedUserEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, parseProxied, toolProxyEnvironment, readBundleScript, applyTrust, STAND_IN, proxyEnvironment, dropBuildAuthority, sandboxSettingsFrom, createSandboxClient, hostKeyOf, buildAuthorityFor, checkEnvironment, withMiseVariables, checkoutPath, variablesFor, carried6to4, classifyAddress, EVAL_INPUT_DIR, isMiseVariableAllowed, variableForPath, droppedRequestHeaders, hasDotDotSegment, routeNameOf } from './chunk-4S6ML2YX.js';
35
- import { agentCredential, RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON, UNKNOWN_TRANSPORT_REASON, RELAY_USER, builtInNames, currentAgentCredential, removedAgentCredentialValues, managedAgentCredential, realFdStorePort, RELAY_ID, MAX_MALFORMED_SESSIONS, MALFORMED_REASON, JOIN_WAIT_MS, MAX_OPEN_STREAMS, STREAM_RESET_RATE, STREAM_RESET_BURST, STREAM_WINDOW, SESSION_WINDOW, targetOfPath, AGENT_CREDENTIAL_MISSING, fdName, DOCKER_DESKTOP_REASON } from './chunk-MX5XVAOI.js';
32
+ import { signerFor } from './chunk-KM2IMFAG.js';
33
+ import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir, workspacesDir, jobsDir, storeScopes, spoolDir, orgStoreIn, rootfsDir, imagesDir, sessionsDir, builderCacheDiskPath, buildsDir, buildDir, ghConfigDir, runnerDirOfBuild, mirrorDir, orgHomeDir, jobConsolePath, vsockPath, builderConsolePath, rootfsPathFor, diskTemplatesDir, deliveryDir, SHIM_DIR } from './chunk-T5CN56UE.js';
34
+ import { BUILD_HOME, SandboxError, JOB_MOUNT, heldBuildAuthority, IMAGE_BUNDLES, writeTrustFiles, buildEnvironment, trustEnvironment, CHECKOUT_DIR, parseIsolation, containerLocalAddress, destinationEnvironment, withHooksPath, startBuildServices, headerVariables, authorEnvironment, sharedUserEnvironment, withTunnelRecord, commitIdentityRecord, repositoriesOf, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, parseProxied, readBundleScript, applyTrust, STAND_IN, proxyEnvironment, describeSandboxError, dropBuildAuthority, sandboxSettingsFrom, createSandboxClient, hostKeyOf, buildAuthorityFor, checkEnvironment, withMiseVariables, checkoutPath, primaryOf, variablesFor, carried6to4, classifyAddress, EVAL_INPUT_DIR, isMiseVariableAllowed, variableForPath, droppedRequestHeaders, hasDotDotSegment, routeNameOf, toolProxyEnvironment } from './chunk-4YAFIOIV.js';
35
+ import { agentCredential, RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON, UNKNOWN_TRANSPORT_REASON, RELAY_USER, currentAgentCredential, removedAgentCredentialValues, managedAgentCredential, realFdStorePort, builtInNames, MAX_MALFORMED_SESSIONS, MALFORMED_REASON, JOIN_WAIT_MS, MAX_OPEN_STREAMS, STREAM_RESET_RATE, STREAM_RESET_BURST, STREAM_WINDOW, SESSION_WINDOW, targetOfPath, AGENT_CREDENTIAL_MISSING, fdName, RELAY_ID, DOCKER_DESKTOP_REASON } from './chunk-7U43B2UB.js';
36
36
  import { versionAtLeast } from './chunk-SF2TBJBP.js';
37
- import { jobAgent, firecrackerArch, MICROVM_ARCHES, FIRECRACKER_VERSION, hostHelperPinned, builderDiskPinned, guestAssetsPinned, runnerAgentNamed, credentialForms, orList, credentialShapes, agentCommands, defaultCommandLine, credentialAgent, agentVersionOf, GUEST_HELPER_SHA256, GUEST_ASSETS_BASE_URL, GUEST_KERNEL_SHA256, JAILER_BINARY_SHA256, FIRECRACKER_BINARY_SHA256, FIRECRACKER_SHA256, GUEST_ASSETS_VERSION, BUILDER_DISK_VERSION, credentialHandOffKey, strippedVariables, runnerAgents, judgeAgent, networkHelperPinned, NETWORK_HELPER_SHA256, universalImagePinned, UNIVERSAL_IMAGE, FFMPEG_STATIC, FFMPEG_BUILD_TAG, GH_SHA256, GH_VERSION, TINI_SHA256, TINI_VERSION, HEADLESS_SHELL, HEADLESS_SHELL_BASE_URL, PLAYWRIGHT_FFMPEG, PLAYWRIGHT_FFMPEG_BASE_URL, VOICE_PACK_FILES, BUILDER_DISK_SHA256, BUILDER_DISK_BASE_URL, DEFAULT_BASE_IMAGE } from './chunk-CKA4JQKR.js';
37
+ import { jobAgent, firecrackerArch, MICROVM_ARCHES, FIRECRACKER_VERSION, hostHelperPinned, builderDiskPinned, guestAssetsPinned, runnerAgentNamed, credentialForms, orList, credentialShapes, agentCommands, defaultCommandLine, credentialAgent, GUEST_HELPER_SHA256, GUEST_ASSETS_BASE_URL, GUEST_KERNEL_SHA256, JAILER_BINARY_SHA256, FIRECRACKER_BINARY_SHA256, FIRECRACKER_SHA256, GUEST_ASSETS_VERSION, BUILDER_DISK_VERSION, credentialHandOffKey, strippedVariables, runnerAgents, judgeAgent, networkHelperPinned, NETWORK_HELPER_SHA256, universalImagePinned, UNIVERSAL_IMAGE, FFMPEG_STATIC, FFMPEG_BUILD_TAG, GH_SHA256, GH_VERSION, TINI_SHA256, TINI_VERSION, HEADLESS_SHELL, HEADLESS_SHELL_BASE_URL, PLAYWRIGHT_FFMPEG, PLAYWRIGHT_FFMPEG_BASE_URL, VOICE_PACK_FILES, BUILDER_DISK_SHA256, BUILDER_DISK_BASE_URL, DEFAULT_BASE_IMAGE, agentVersionOf } from './chunk-IEX5FXEU.js';
38
38
  import { sessionFolderIn, legacyAgent, defaultAgents, parsePolicy, AGENTS_SKILLS_BASE, OUTERLAYER_SKILLS_BASE, orderSessions, extractFacts, sessionsToTimeline, resolvePackRoot } from './chunk-G2CRPYFO.js';
39
39
  import { MAINTAINED_SKILLS } from './chunk-ATEMEVFF.js';
40
40
  import { resolveVersion } from './chunk-RA3O54FC.js';
@@ -42,13 +42,13 @@ import { homeFromEnv } from './chunk-352DHUTP.js';
42
42
  import { scrubText, MIN_SCRUBBED_LITERAL, setCustomScrubbers, setLocalIdentity, setRepoRoot, forkFor, scrubDeep } from './chunk-E5G6K7C4.js';
43
43
  import { CheckItemsSchema, CODE_REVIEW_NAME, JUDGE_NOT_RUN_ITEM_ID, REVIEWED_COMMIT_ITEM_ID, CHECK_ITEM_REASON_MAX_LENGTH, UNREVIEWED_FILES_ITEM_ID, CHECK_ITEM_HREF_MAX_LENGTH, downconvertSession } from './chunk-NEXKR5RR.js';
44
44
  import { randomUUID, randomBytes, createHash } from 'crypto';
45
- import { rmSync, mkdirSync, existsSync, readFileSync, mkdtempSync, statSync, appendFileSync, writeFileSync, readdirSync, accessSync, constants, openSync, closeSync, renameSync, rmdirSync, chmodSync, createReadStream, lstatSync, unlinkSync, realpathSync, writeSync, truncateSync, cpSync, fstatSync, readSync, createWriteStream, copyFileSync, linkSync } from 'fs';
45
+ import { rmSync, mkdirSync, existsSync, readFileSync, mkdtempSync, statSync, appendFileSync, writeFileSync, readdirSync, accessSync, constants, openSync, closeSync, renameSync, rmdirSync, createReadStream, lstatSync, unlinkSync, chmodSync, realpathSync, writeSync, truncateSync, cpSync, fstatSync, readSync, createWriteStream, copyFileSync, linkSync } from 'fs';
46
46
  import { join, dirname, isAbsolute, delimiter, basename, sep, relative, resolve } from 'path';
47
- import { Readable, PassThrough, Writable, Duplex, Transform } from 'stream';
47
+ import { Readable, PassThrough, Duplex, Transform, Writable } from 'stream';
48
48
  import { spawn, spawnSync } from 'child_process';
49
49
  import { createServer, createConnection, connect, isIP } from 'net';
50
50
  import { availableParallelism, networkInterfaces, tmpdir } from 'os';
51
- import { gzipSync, createGunzip, gunzipSync, constants as constants$1 } from 'zlib';
51
+ import { createGunzip, gunzipSync, constants as constants$1, gzipSync } from 'zlib';
52
52
  import http2 from 'http2';
53
53
  import { once } from 'events';
54
54
  import { createServer as createServer$1, request, Agent } from 'http';
@@ -3492,1070 +3492,1255 @@ function recordingCause(log) {
3492
3492
  const cause = proof ?? lines.find((line2) => line2.startsWith("setup failed: "));
3493
3493
  return cause === void 0 ? void 0 : cause.slice(0, MAX_CAUSE_LENGTH);
3494
3494
  }
3495
-
3496
- // src/runner/sandbox/sweep.ts
3497
- var RUNNER_METADATA = "outerlayerRunner";
3498
- var BUILD_METADATA = "outerlayerBuild";
3499
- function orphanedSandboxes(sandboxes, runnerId, heldBuilds) {
3500
- return sandboxes.filter((sandbox) => {
3501
- if (sandbox.metadata[RUNNER_METADATA] !== runnerId) return false;
3502
- if (sandbox.state === "ended") return false;
3503
- const build = sandbox.metadata[BUILD_METADATA];
3504
- return build === void 0 || !heldBuilds.has(build);
3505
- });
3495
+ function savedStatesDir(runnerDirPath) {
3496
+ return join(runnerDirPath, "saved-states");
3506
3497
  }
3507
- async function sweepSandboxes(input) {
3508
- let listed;
3498
+ function writeRecord2(dir, record) {
3499
+ mkdirSync(dir, { recursive: true, mode: RUNNER_DIR_MODE });
3500
+ const target2 = join(dir, `${record.id}.json`);
3501
+ const partial = `${target2}.partial`;
3502
+ writeFileSync(partial, `${JSON.stringify(record, null, 2)}
3503
+ `, { mode: RUNNER_FILE_MODE });
3504
+ renameSync(partial, target2);
3505
+ }
3506
+ function isRecord(value) {
3507
+ if (typeof value !== "object" || value === null) return false;
3508
+ const r = value;
3509
+ return typeof r.id === "string" && (r.kind === "volume" || r.kind === "tar" || r.kind === "disk") && typeof r.savedAt === "string" && Number.isFinite(Date.parse(r.savedAt)) && typeof r.bytes === "number" && typeof r.baseKey === "string" && typeof r.key === "string" && typeof r.fileHashes === "object" && r.fileHashes !== null && typeof r.complete === "boolean";
3510
+ }
3511
+ function readRecords(dir) {
3512
+ let names;
3509
3513
  try {
3510
- listed = await input.client.list();
3511
- } catch (err) {
3512
- input.log?.(`could not list sandboxes: ${err instanceof Error ? err.message : String(err)}`);
3513
- return 0;
3514
+ names = readdirSync(dir);
3515
+ } catch {
3516
+ return [];
3514
3517
  }
3515
- let ended = 0;
3516
- for (const sandbox of orphanedSandboxes(listed, input.runnerId, input.heldBuilds())) {
3518
+ const records = [];
3519
+ for (const name of names) {
3520
+ if (!name.endsWith(".json")) continue;
3517
3521
  try {
3518
- await input.client.end(sandbox.id);
3519
- ended++;
3520
- input.log?.(`ended sandbox ${sandbox.id} (${sandbox.name}): no unfinished build holds it`);
3521
- } catch (err) {
3522
- input.log?.(`could not end sandbox ${sandbox.id}: ${err instanceof Error ? err.message : String(err)}`);
3522
+ const parsed = JSON.parse(readFileSync(join(dir, name), "utf8"));
3523
+ if (isRecord(parsed)) records.push(parsed);
3524
+ } catch {
3523
3525
  }
3524
3526
  }
3525
- return ended;
3527
+ return records;
3526
3528
  }
3527
- var BUFFER_HIGH_WATER = 1024 * 1024;
3528
- var DRAIN_POLL_MS = 5;
3529
- var OPEN_TIMEOUT_MS = 3e4;
3530
- var defaultWebSocket = (url) => new WebSocket(url);
3531
- function sandboxExecStream(opts) {
3532
- const create = opts.webSocket ?? defaultWebSocket;
3533
- return () => {
3534
- const stdout = new PassThrough();
3535
- let socket;
3536
- let open = false;
3537
- let finished2 = false;
3538
- let tail = "";
3539
- let finish2;
3540
- const exited = new Promise((resolve2) => finish2 = resolve2);
3541
- const end = (status, why) => {
3542
- if (finished2) return;
3543
- finished2 = true;
3544
- if (why !== void 0) tail = `${tail}${why}`.slice(-400);
3545
- stdout.end();
3546
- try {
3547
- socket?.close();
3548
- } catch {
3549
- }
3550
- finish2(status);
3551
- };
3552
- const opened = new Promise((resolve2, reject) => {
3553
- const timer = setTimeout(() => reject(new Error("the WebSocket did not open in time")), OPEN_TIMEOUT_MS);
3554
- timer.unref();
3555
- opts.url().then((url) => {
3556
- socket = create(url);
3557
- socket.binaryType = "arraybuffer";
3558
- socket.addEventListener("open", () => {
3559
- clearTimeout(timer);
3560
- open = true;
3561
- socket.send(opts.secret);
3562
- resolve2();
3563
- });
3564
- socket.addEventListener("message", (event) => {
3565
- if (event.data instanceof ArrayBuffer) stdout.write(Buffer.from(event.data));
3566
- else if (ArrayBuffer.isView(event.data)) stdout.write(Buffer.from(event.data.buffer, event.data.byteOffset, event.data.byteLength));
3567
- });
3568
- socket.addEventListener("error", (event) => {
3569
- clearTimeout(timer);
3570
- reject(new Error(event.message ?? "the WebSocket failed"));
3571
- });
3572
- socket.addEventListener("close", (event) => {
3573
- clearTimeout(timer);
3574
- reject(new Error("the WebSocket closed before it opened"));
3575
- end(event.code === 1e3 || event.code === void 0 ? 0 : 1, event.code === 1008 ? "the relay refused the join secret" : void 0);
3576
- });
3577
- }).catch((err) => {
3578
- clearTimeout(timer);
3579
- reject(err);
3580
- });
3581
- });
3582
- opened.catch((err) => end(1, err instanceof Error ? err.message : String(err)));
3583
- const stdin = new Writable({
3584
- write(chunk, _encoding, callback) {
3585
- opened.then(
3586
- async () => {
3587
- if (finished2) return callback(new Error("the connection is closed"));
3588
- socket.send(chunk);
3589
- while (open && !finished2 && socket.bufferedAmount > BUFFER_HIGH_WATER) await new Promise((resolve2) => setTimeout(resolve2, DRAIN_POLL_MS));
3590
- callback();
3591
- },
3592
- () => callback(new Error("the connection did not open"))
3593
- );
3594
- },
3595
- final(callback) {
3596
- end(0);
3597
- callback();
3598
- }
3599
- });
3600
- stdin.on("error", () => void 0);
3601
- stdout.on("error", () => void 0);
3602
- return { stdin, stdout, exited, kill: () => end(-1), stderrTail: () => tail };
3603
- };
3529
+ function deleteRecord(dir, id) {
3530
+ rmSync(join(dir, `${id}.json`), { force: true });
3604
3531
  }
3605
- var RUN_DIR = "/tmp/ol-run";
3606
- var CHUNK_BYTES = 9e4;
3607
- var READ_ATTEMPTS = 4;
3608
- var POLL_FIRST_MS = 50;
3609
- var POLL_LONGEST_MS = 1e3;
3610
- function quote(word) {
3611
- return /^[A-Za-z0-9_@%+=:,./-]+$/.test(word) ? word : `'${word.replace(/'/g, `'\\''`)}'`;
3532
+
3533
+ // src/runner/saved-state/container.ts
3534
+ var SAVED_FOLDERS = ["home", "work"];
3535
+ function stopProcessesScript() {
3536
+ return [
3537
+ "me=$$; uid=$(id -u); n=0; pass=0",
3538
+ "while [ $pass -lt 25 ]; do",
3539
+ ' left=""',
3540
+ " for p in /proc/[0-9]*; do",
3541
+ " pid=${p##*/}",
3542
+ ' [ "$pid" = "$me" ] && continue',
3543
+ ` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
3544
+ ' case "$info" in *"S=Z"*) continue ;; esac',
3545
+ // The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
3546
+ ' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
3547
+ " done",
3548
+ ' [ -z "$left" ] && break',
3549
+ ' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
3550
+ " pass=$((pass+1))",
3551
+ " sleep 0.2",
3552
+ "done",
3553
+ '[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
3554
+ 'echo "stopped $n"'
3555
+ ].join("\n");
3612
3556
  }
3613
- function sandboxLine(command, args, env) {
3614
- const values = Object.entries(env ?? {}).filter((entry) => entry[1] !== void 0).map(([name, value]) => `${name}=${quote(value)}`);
3615
- return ["sudo", "env", ...values, command, ...args.map(quote)].join(" ");
3557
+ var HELPER_FLAGS = [
3558
+ "--rm",
3559
+ "--network",
3560
+ "none",
3561
+ "--cap-drop",
3562
+ "ALL",
3563
+ "--cap-add",
3564
+ "CHOWN",
3565
+ "--cap-add",
3566
+ "DAC_OVERRIDE",
3567
+ "--cap-add",
3568
+ "DAC_READ_SEARCH",
3569
+ "--cap-add",
3570
+ "FOWNER",
3571
+ "--security-opt",
3572
+ "no-new-privileges",
3573
+ "--read-only",
3574
+ "--user",
3575
+ "0",
3576
+ "--pids-limit",
3577
+ "256"
3578
+ ];
3579
+ function helperArgs(input) {
3580
+ return [
3581
+ ...HELPER_FLAGS,
3582
+ "--label",
3583
+ `${RUNNER_LABEL}=${input.runnerId}`,
3584
+ "--mount",
3585
+ `type=volume,src=${input.from},dst=/from,readonly`,
3586
+ "--mount",
3587
+ `type=volume,src=${input.to},dst=/to`,
3588
+ "--entrypoint",
3589
+ "sh",
3590
+ input.image,
3591
+ "-c",
3592
+ input.script
3593
+ ];
3616
3594
  }
3617
- function sandboxRunner(client2, sandbox, options = {}) {
3618
- const sleep = options.sleep ?? ((ms) => new Promise((resolve2) => setTimeout(resolve2, ms)));
3619
- return async (command, args, opts = {}) => {
3620
- const id = randomBytes(6).toString("hex");
3621
- const file = (suffix) => `${RUN_DIR}/${id}.${suffix}`;
3622
- const line2 = sandboxLine(command, args, opts.env);
3623
- options.onCommand?.(line2);
3624
- if (opts.input !== void 0) {
3625
- await client2.command(sandbox, `mkdir -p ${RUN_DIR}`);
3626
- await client2.upload(sandbox, file("in"), Buffer.isBuffer(opts.input) ? opts.input : Buffer.from(opts.input));
3627
- }
3628
- const start = `mkdir -p ${RUN_DIR}; echo $$ > ${file("pid")}; ${line2} < ${opts.input === void 0 ? "/dev/null" : file("in")} > ${file("out")} 2> ${file("err")}; echo $? > ${file("code.tmp")}; mv -f ${file("code.tmp")} ${file("code")}`;
3629
- await client2.command(sandbox, start, { background: true });
3630
- const deadline = opts.timeoutMs === void 0 ? void 0 : Date.now() + opts.timeoutMs;
3631
- const collect = `if [ -f ${file("code")} ]; then cat ${file("code")}; head -c ${CHUNK_BYTES} ${file("out")} | base64 | tr -d '\\n'; echo; head -c ${CHUNK_BYTES} ${file("err")} | base64 | tr -d '\\n'; echo; wc -c < ${file("out")}; wc -c < ${file("err")}; fi`;
3632
- const ask2 = async (cmd) => {
3633
- for (let attempt = 1; ; attempt++) {
3634
- try {
3635
- return await client2.command(sandbox, cmd);
3636
- } catch (err) {
3637
- const transient = err instanceof SandboxError && (err.status === 0 || err.status >= 500);
3638
- if (!transient || attempt >= READ_ATTEMPTS) throw err;
3639
- await sleep(POLL_FIRST_MS * attempt);
3640
- }
3641
- }
3642
- };
3643
- let wait = POLL_FIRST_MS;
3644
- for (; ; ) {
3645
- const polled = await ask2(collect);
3646
- if (polled.stdout.trim() !== "") {
3647
- const [code = "", out = "", err = "", outSize = "0", errSize = "0"] = polled.stdout.split("\n").map((part) => part.trim());
3648
- const pieces = [Buffer.from(out, "base64")];
3649
- const errPieces = [Buffer.from(err, "base64")];
3650
- const more = async (name, size, into) => {
3651
- for (let read = CHUNK_BYTES; read < size; read += CHUNK_BYTES) {
3652
- const next = await ask2(`tail -c +${read + 1} ${file(name)} | head -c ${CHUNK_BYTES} | base64 | tr -d '\\n'`);
3653
- into.push(Buffer.from(next.stdout.trim(), "base64"));
3654
- }
3655
- };
3656
- await more("out", Number(outSize), pieces);
3657
- await more("err", Number(errSize), errPieces);
3658
- await client2.command(sandbox, `rm -f ${RUN_DIR}/${id}.*`);
3659
- const result = { status: Number(code) || 0, stdout: Buffer.concat(pieces).toString("utf8"), stderr: Buffer.concat(errPieces).toString("utf8") };
3660
- if (opts.logPath !== void 0) {
3661
- mkdirSync(dirname(opts.logPath), { recursive: true });
3662
- appendFileSync(opts.logPath, result.stdout + result.stderr);
3663
- }
3664
- return result;
3665
- }
3666
- if (deadline !== void 0 && Date.now() >= deadline) {
3667
- await client2.command(sandbox, `kill -9 $(cat ${file("pid")}) 2>/dev/null; rm -f ${RUN_DIR}/${id}.*`).catch(() => void 0);
3668
- const extra = `${command}: no answer within ${Math.round(opts.timeoutMs / 1e3)}s
3669
- `;
3670
- if (opts.logPath !== void 0) {
3671
- mkdirSync(dirname(opts.logPath), { recursive: true });
3672
- appendFileSync(opts.logPath, extra);
3673
- }
3674
- return { status: 124, stdout: "", stderr: extra };
3675
- }
3676
- await sleep(wait);
3677
- wait = Math.min(wait * 2, POLL_LONGEST_MS);
3595
+ var ONE_BUILD_FOLDERS = ["tmp", ".command"];
3596
+ function clearOneBuildArgs(input) {
3597
+ return [
3598
+ ...HELPER_FLAGS,
3599
+ "--label",
3600
+ `${RUNNER_LABEL}=${input.runnerId}`,
3601
+ "--mount",
3602
+ `type=volume,src=${input.volume},dst=/build`,
3603
+ "--entrypoint",
3604
+ "sh",
3605
+ input.image,
3606
+ "-c",
3607
+ `rm -rf ${ONE_BUILD_FOLDERS.map((name) => `/build/${name}`).join(" ")}`
3608
+ ];
3609
+ }
3610
+ var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
3611
+ var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
3612
+ function failureText(result) {
3613
+ const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
3614
+ return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
3615
+ }
3616
+ async function saveContainerState(input) {
3617
+ const { host } = input;
3618
+ const id = randomBytes(6).toString("hex");
3619
+ const volume = `outerlayer-saved-${id}`;
3620
+ const record = {
3621
+ id,
3622
+ kind: "volume",
3623
+ volume,
3624
+ savedAt: input.now.toISOString(),
3625
+ bytes: 0,
3626
+ baseKey: input.key.baseKey,
3627
+ key: input.key.key,
3628
+ fileHashes: input.key.fileHashes,
3629
+ ...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
3630
+ complete: false
3631
+ };
3632
+ const discard = async (reason) => {
3633
+ const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
3634
+ if (removed.status !== 0) {
3635
+ input.log(`the volume ${volume} could not be removed (${failureText({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
3636
+ return { ok: false, reason };
3678
3637
  }
3638
+ deleteRecord(input.dir, id);
3639
+ return { ok: false, reason };
3679
3640
  };
3680
- }
3681
- function sandboxHost(client2, sandbox, options = {}) {
3682
- return createContainerHost({ run: sandboxRunner(client2, sandbox, options) });
3683
- }
3684
-
3685
- // src/runner/sandbox/hooks.ts
3686
- var SANDBOX_DIR = "/tmp/ol";
3687
- var CLI_DIR = `${SANDBOX_DIR}/dist`;
3688
- var SKILL_PACK_DIR = `${SANDBOX_DIR}/skill-pack`;
3689
- var SECRET_FILE = `${SANDBOX_DIR}/join-secret`;
3690
- var CONTAINER_SECRET_FILE = "/run/outerlayer-relay/join-secret";
3691
- var JOIN_WS_PORT = 7077;
3692
- var SANDBOX_RESERVE_MIB = 512;
3693
- var RELAY_ATTEMPTS = 40;
3694
- var RELAY_WAIT_MS = 500;
3695
- var SANDBOX_PROBE_BYTES = 96 * 1024;
3696
- var MAX_TAIL_READS2 = 8;
3697
- var NO_ENV = { literals: {}, passed: {} };
3698
- function sandboxNameOf(runnerLabel, buildId) {
3699
- const short = runnerLabel.toLowerCase().replace(/[^a-z0-9]/g, "").slice(0, 8);
3700
- return `ol-${short === "" ? "runner" : short}-${buildId}`;
3701
- }
3702
- async function endUnansweredSandbox(sandboxClient, runnerId, buildId, warn) {
3703
3641
  try {
3704
- for (const found of await sandboxClient.list()) {
3705
- if (found.state !== "ended" && found.metadata[RUNNER_METADATA] === runnerId && found.metadata[BUILD_METADATA] === buildId) await sandboxClient.end(found.id);
3706
- }
3642
+ const noEnv = { literals: {}, passed: {} };
3643
+ const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
3644
+ if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText(stopped)})` };
3645
+ input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
3646
+ const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
3647
+ if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText(flushed)})` };
3648
+ writeRecord2(input.dir, record);
3649
+ const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
3650
+ if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText(created)})`);
3651
+ const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
3652
+ if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText(copied)})`);
3653
+ const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
3654
+ const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
3655
+ writeRecord2(input.dir, done);
3656
+ return { ok: true, record: done };
3707
3657
  } catch (err) {
3708
- warn(`could not end a sandbox that a lost create may have made: ${describeCallFailure(err)} \xB7 the next sweep ends it`);
3658
+ return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
3709
3659
  }
3710
3660
  }
3711
- function client(runtime) {
3712
- return runtime.sandbox.client();
3713
- }
3714
- async function sandboxOfBuild(runtime, state) {
3715
- const held3 = runtime.sandbox.sandboxes.get(state.buildId);
3716
- if (held3 !== void 0) return held3;
3661
+ async function restoreContainerState(input) {
3717
3662
  try {
3718
- const found = await client(runtime).get(state.sandboxId);
3719
- if (found.state === "ended" || found.secret === "") return void 0;
3720
- runtime.sandbox.sandboxes.set(state.buildId, found);
3721
- return found;
3663
+ const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
3664
+ if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText(copied)})` };
3665
+ return { ok: true };
3722
3666
  } catch (err) {
3723
- if (err instanceof SandboxError && err.status === 404) return void 0;
3724
- throw err;
3667
+ return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
3725
3668
  }
3726
3669
  }
3727
- function runtimeInSandbox(runtime, sandbox) {
3728
- return { ...runtime, host: sandboxHost(client(runtime), sandbox, { sleep: runtime.sandbox.sleep }) };
3729
- }
3730
- var SandboxGoneError = class extends Error {
3731
- constructor(sandboxId, endedReason) {
3732
- super(`The provider ended sandbox ${sandboxId} while the build ran${endedReason === void 0 || endedReason === "" ? "" : ` (${endedReason})`}`);
3733
- this.endedReason = endedReason;
3734
- this.name = "SandboxGoneError";
3670
+ async function removeSavedState(record, deps) {
3671
+ if (record.kind === "volume" && record.volume !== void 0) {
3672
+ const result = await deps.removeVolume(record.volume);
3673
+ if (result !== void 0 && result.status !== 0) {
3674
+ const line2 = (result.stderr ?? "").trim().split("\n").filter((text2) => text2.trim() !== "").at(-1);
3675
+ throw new Error(`Docker could not remove volume ${record.volume} (exit ${result.status}${line2 === void 0 ? "" : `: ${line2}`})`);
3676
+ }
3735
3677
  }
3736
- };
3737
- async function inSandbox(runtime, state) {
3738
- const sandbox = await sandboxOfBuild(runtime, state);
3739
- if (sandbox === void 0) throw new SandboxGoneError(state.sandboxId);
3740
- return runtimeInSandbox(runtime, sandbox);
3741
- }
3742
- async function goneReason(runtime, state) {
3743
- try {
3744
- const found = await client(runtime).get(state.sandboxId);
3745
- return found.state === "ended" ? new SandboxGoneError(state.sandboxId, found.endedReason ?? void 0) : void 0;
3746
- } catch (err) {
3747
- return err instanceof SandboxError && err.status === 404 ? new SandboxGoneError(state.sandboxId) : void 0;
3678
+ if ((record.kind === "tar" || record.kind === "disk") && record.path !== void 0) {
3679
+ rmSync(`${record.path}.partial`, { force: true });
3680
+ rmSync(record.path, { force: true });
3748
3681
  }
3682
+ deleteRecord(deps.dir, record.id);
3749
3683
  }
3750
- function sandboxRunArgs(args, opts = {}) {
3751
- const out = [...args];
3752
- const network = out.indexOf("--network");
3753
- if (network !== -1) out[network + 1] = "host";
3754
- const entrypoint = out.indexOf("--entrypoint");
3755
- out.splice(
3756
- entrypoint,
3757
- 0,
3758
- "--mount",
3759
- `type=bind,src=${CLI_DIR},dst=/opt/outerlayer/cli/dist,readonly`,
3760
- ...opts.skillPack === true ? ["--mount", `type=bind,src=${SKILL_PACK_DIR},dst=/opt/outerlayer/cli/skill-pack,readonly`] : [],
3761
- "--mount",
3762
- `type=bind,src=${SECRET_FILE},dst=${CONTAINER_SECRET_FILE},readonly`,
3763
- ...Object.entries(toolProxyEnvironment()).flatMap(([name, value]) => ["--env", `${name}=${value}`])
3764
- );
3765
- out.push("--join-ws", String(JOIN_WS_PORT), "--join-secret-file", CONTAINER_SECRET_FILE);
3766
- return out;
3767
- }
3768
- var RELAY_PROBE = `const s=require("net").connect(3128,"127.0.0.1");s.on("connect",()=>process.exit(0));s.on("error",()=>process.exit(1));setTimeout(()=>process.exit(1),2000)`;
3769
- async function waitForRelay(runtime, container) {
3770
- for (let attempt = 0; attempt < RELAY_ATTEMPTS; attempt++) {
3771
- const probe = await runtime.host.exec({ container, user: RELAY_USER, env: NO_ENV, argv: ["/opt/outerlayer/bin/node", "-e", RELAY_PROBE] });
3772
- if (probe.status === 0) return true;
3773
- await runtime.sandbox.sleep(RELAY_WAIT_MS);
3684
+ async function removeOldestUntil(input) {
3685
+ const removed = [];
3686
+ for (const record of oldestFirst(input.records)) {
3687
+ if (input.enough()) break;
3688
+ try {
3689
+ await removeSavedState(record, input.deps);
3690
+ removed.push(record);
3691
+ input.log(`removed saved state ${record.id} from ${record.savedAt}`);
3692
+ } catch (err) {
3693
+ input.log(`could not remove saved state ${record.id}: ${describeCallFailure(err)}`);
3694
+ }
3774
3695
  }
3775
- return false;
3776
- }
3777
- function joinRelay(runtime, sandbox, build, services, joinSecret, log) {
3778
- const exec = sandboxExecStream({
3779
- url: () => client(runtime).portUrl(sandbox, JOIN_WS_PORT),
3780
- secret: joinSecret,
3781
- ...runtime.sandbox.webSocket === void 0 ? {} : { webSocket: runtime.sandbox.webSocket }
3782
- });
3783
- runtime.relays.set(
3784
- build.buildId,
3785
- superviseRelay({
3786
- container: build.container,
3787
- services,
3788
- execStream: () => exec(),
3789
- ...log === void 0 ? {} : { log },
3790
- onFatal: (reason) => runtime.relayFatal.set(build.buildId, reason),
3791
- builtIns: builtInNames(runtime.agents)
3792
- })
3793
- );
3696
+ return removed;
3794
3697
  }
3795
- async function cleanupSandbox(runtime, state) {
3796
- const failures = [];
3797
- const relay = runtime.relays.get(state.buildId);
3798
- if (relay !== void 0) {
3799
- runtime.relays.delete(state.buildId);
3800
- await relay.close();
3801
- }
3802
- runtime.relayFatal.delete(state.buildId);
3803
- runtime.leases.get(state.buildId)?.close();
3804
- runtime.leases.delete(state.buildId);
3805
- if (state.sandboxId !== "") {
3698
+ async function removeSuperseded(input) {
3699
+ for (const old of input.records.filter((record) => record.complete && record.key === input.made.key && record.id !== input.made.id && Date.parse(record.savedAt) < Date.parse(input.made.savedAt))) {
3806
3700
  try {
3807
- await client(runtime).end(state.sandboxId);
3701
+ await removeSavedState(old, input.deps);
3702
+ input.log(`removed saved state ${old.id}, saved state ${input.made.id} replaces it`);
3808
3703
  } catch (err) {
3809
- failures.push(`sandbox ${state.sandboxId} was not ended: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
3704
+ input.log(`could not remove saved state ${old.id}: ${describeCallFailure(err)}`);
3810
3705
  }
3811
3706
  }
3812
- runtime.sandbox.sandboxes.delete(state.buildId);
3813
- runtime.sandbox.joinSecrets.delete(state.buildId);
3814
- const tunnel = runtime.tunnels.get(state.buildId);
3815
- if (tunnel !== void 0) {
3816
- runtime.tunnels.delete(state.buildId);
3817
- await tunnel.close();
3707
+ }
3708
+ var RECORD_FILE = "record.json";
3709
+ var DISK_FILE = "disk.ext4";
3710
+ var WORK_ITEM_ID2 = /^[A-Za-z0-9_-]{1,64}$/;
3711
+ function itemDiskDir(configPath, organizationId, workItemId) {
3712
+ if (!WORK_ITEM_ID2.test(workItemId)) throw new Error(`cannot keep a disk for the work item ${JSON.stringify(workItemId)}`);
3713
+ return join(workspacesDir(configPath, organizationId), workItemId);
3714
+ }
3715
+ function isRecord2(value) {
3716
+ if (typeof value !== "object" || value === null) return false;
3717
+ const r = value;
3718
+ return typeof r.workItemId === "string" && typeof r.claimId === "string" && typeof r.queue === "string" && typeof r.imageId === "string" && typeof r.diskBytes === "number" && (r.kind === "disk" || r.kind === "volume" && typeof r.volume === "string" || r.kind === "sandbox" && typeof r.volume === "string" && typeof r.snapshot === "string") && typeof r.bytes === "number" && typeof r.keptAt === "string" && Number.isFinite(Date.parse(r.keptAt));
3719
+ }
3720
+ function readItemDisk(dir) {
3721
+ let parsed;
3722
+ try {
3723
+ parsed = JSON.parse(readFileSync(join(dir, RECORD_FILE), "utf8"));
3724
+ } catch {
3725
+ return void 0;
3818
3726
  }
3819
- if (state.container !== void 0) rmSync(state.container.buildDir, { recursive: true, force: true });
3820
- if (failures.length > 0) throw new Error(failures.join("; "));
3727
+ if (!isRecord2(parsed) || parsed.workItemId !== basename(dir)) return void 0;
3728
+ if (parsed.kind !== "disk") return { dir, record: parsed };
3729
+ const path = join(dir, DISK_FILE);
3730
+ try {
3731
+ if (!statSync(path).isFile()) return void 0;
3732
+ } catch {
3733
+ return void 0;
3734
+ }
3735
+ return { dir, record: parsed, path };
3821
3736
  }
3822
- async function provisionSandbox(input) {
3823
- const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
3824
- const tag = itemTag({ item: item.number, organizationId: input.organizationId, claimId: input.claimId });
3825
- const logPath = join(jobDir, "provision.log");
3826
- const fail = (reason) => ({ ok: false, reason });
3827
- const own = config.sandbox;
3828
- if (own === void 0) return fail("runner.sandbox is not set in the config file");
3829
- const repositories = repositoriesOf({ repositories: input.repositories, repository: item.specRepository, branch: input.branch });
3830
- const primary = primaryOf(repositories);
3831
- if (primary === void 0) return fail("the work item names no repository to clone");
3832
- const repository = primary.repository;
3833
- const sized = sandboxSizeOf(config.build);
3834
- if (!sized.ok) return fail(`${sized.field} ${sized.problem}`);
3835
- let sandboxClient;
3737
+ function swapInItemDisk(dir, incoming) {
3738
+ const older = readItemDisk(dir);
3739
+ const outgoing = join(dirname(dir), `.${basename(dir)}.${randomBytes(4).toString("hex")}.old`);
3740
+ let hadOlder = true;
3836
3741
  try {
3837
- sandboxClient = client(runtime);
3742
+ renameSync(dir, outgoing);
3838
3743
  } catch (err) {
3839
- return fail(err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err));
3744
+ if (err.code !== "ENOENT") throw err;
3745
+ hadOlder = false;
3840
3746
  }
3841
- const governing = input.queue === "run" ? SELF_GOVERNED : await resolveGoverningSource({ config, repository, fetchImpl: input.fetchImpl });
3842
- if (!governing.ok) return fail(governing.reason);
3843
- const governingSource = governing.source;
3844
- const controlPlaneUrl = (controlPlane) => runtime.remoteFor?.(controlPlane) ?? `${runtime.github.gitUrl}/${controlPlane}.git`;
3845
- if (governingSource.kind === "governed") {
3846
- const unfetched = await prefetchGovernedContext({
3847
- configPath: config.configPath,
3848
- repository,
3849
- source: governingSource,
3850
- controlPlaneUrl,
3851
- controlPlaneToken: () => tokens.controlPlane(),
3852
- log,
3853
- tag
3854
- });
3855
- if (unfetched !== void 0) return fail(unfetched);
3747
+ try {
3748
+ renameSync(incoming, dir);
3749
+ } catch (err) {
3750
+ if (hadOlder) renameSync(outgoing, dir);
3751
+ throw err;
3856
3752
  }
3857
- const buildId = randomBytes(6).toString("hex");
3858
- const runnerId = runnerLabelValue(config.configPath);
3859
- const name = sandboxNameOf(runnerId, buildId);
3860
- const container = `outerlayer-build-${buildId}`;
3861
- const buildDir2 = buildDir(config.configPath, buildId);
3862
- const image = own.image;
3863
- const containerState = {
3864
- buildId,
3865
- container,
3866
- volume: container,
3867
- buildDir: buildDir2,
3868
- repository,
3869
- imageId: image,
3870
- builtIns: true,
3871
- ...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository, ref: governingSource.ref, commit: governingSource.sha } } : {},
3872
- transport: "exec",
3873
- jobFiles: "copied"
3874
- };
3875
- const state = { sandboxId: "", name, dataPlaneUrl: "", buildId, container: containerState };
3876
- const secrets = secretsOf(runtime, jobDir);
3877
- const env = buildEnvironment({
3878
- config,
3879
- item: item.number,
3880
- ...input.queue === "run" ? { run: true } : {},
3881
- itemKey,
3882
- repository,
3883
- repositories,
3884
- hostEnv: runtime.hostEnv,
3885
- agents: runtime.agents,
3886
- ...input.job === void 0 ? {} : { job: input.job },
3887
- containerEnv: {},
3888
- builtIns: true
3889
- });
3890
- if (env.skippedTokenVariables.length > 0) {
3891
- log.line(`${tag} not passing ${env.skippedTokenVariables.join(", ")} from runner.build.variables into the build: its repository token comes from the runner`);
3753
+ rmSync(outgoing, { recursive: true, force: true });
3754
+ return older;
3755
+ }
3756
+ function stagingItemDiskDir(dir) {
3757
+ const parent = dirname(dir);
3758
+ mkdirSync(parent, { recursive: true, mode: 448 });
3759
+ const staging = join(parent, `.${basename(dir)}.${randomBytes(4).toString("hex")}.partial`);
3760
+ mkdirSync(staging, { mode: 448 });
3761
+ return staging;
3762
+ }
3763
+ function listItemDisks(configPath, scope) {
3764
+ const where = workspacesDir(configPath, scope);
3765
+ let names;
3766
+ try {
3767
+ names = readdirSync(where);
3768
+ } catch {
3769
+ return [];
3892
3770
  }
3893
- const environment = {
3894
- isolation: "vm",
3895
- imageId: image,
3896
- variables: env.variables,
3897
- cliVersion: input.cliVersion
3898
- };
3899
- const bail = async (reason, aborted2 = false) => {
3900
- try {
3901
- await cleanupSandbox(runtime, { buildId, sandboxId: state.sandboxId, container: containerState });
3902
- } catch (err) {
3903
- log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
3904
- }
3905
- return aborted2 ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
3906
- };
3907
- if (input.aborted()) return bail("", true);
3908
- mkdirSync(buildDir2, { recursive: true });
3909
- const ghDir = ghConfigDir(buildDir2);
3910
- let readToken;
3771
+ const kept = [];
3772
+ for (const name of names) {
3773
+ if (!WORK_ITEM_ID2.test(name)) continue;
3774
+ const found = readItemDisk(join(where, name));
3775
+ if (found !== void 0) kept.push(found);
3776
+ }
3777
+ return kept;
3778
+ }
3779
+ function listLeftovers(configPath, scope) {
3780
+ const where = workspacesDir(configPath, scope);
3781
+ let names;
3911
3782
  try {
3912
- readToken = await tokens.token("read");
3913
- writeGhConfig(ghDir, readToken);
3914
- } catch (err) {
3915
- return bail(err instanceof WorkCommandError ? `the build's read token could not be had: ${refusalCode(err)}` : `the build's gh configuration could not be written: ${describeCallFailure(err)}`);
3783
+ names = readdirSync(where);
3784
+ } catch {
3785
+ return [];
3916
3786
  }
3917
- for (const entry of repositories) {
3918
- if (entry.repository === primary.repository) continue;
3787
+ const found = [];
3788
+ for (const name of names) {
3789
+ if (!name.startsWith(".") && (!WORK_ITEM_ID2.test(name) || readItemDisk(join(where, name)) !== void 0)) continue;
3919
3790
  try {
3920
- await tokens.token("read", entry.repository);
3921
- } catch (err) {
3922
- return bail(`the build's read token for ${entry.repository} could not be had: ${refusalCode(err)}`);
3791
+ found.push({ path: join(where, name), mtimeMs: lstatSync(join(where, name)).mtimeMs });
3792
+ } catch {
3923
3793
  }
3924
3794
  }
3925
- keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`));
3926
- tokens.markUsed("read");
3927
- environment.credentials = tokens.used();
3928
- let services;
3929
- try {
3930
- const gitRemote = gitRemoteFor(runtime, { repositories, tokens, buildDir: buildDir2, log: (line2) => log.line(`${tag} ${line2}`), ...input.queue === "run" ? { readOnly: true } : {} });
3931
- services = await startServicesFor(runtime, config, buildDir2, jobDir, itemKey, true, gitRemote, "stream", input.leaseExpiryMs === void 0 ? void 0 : leaseFor(runtime, buildId, input.leaseExpiryMs), void 0, (line2) => log.line(`${tag} ${line2}`));
3932
- runtime.tunnels.set(buildId, services);
3933
- } catch (err) {
3934
- return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
3935
- }
3936
- if (input.aborted()) return bail("", true);
3937
- let sandbox;
3938
- runtime.sandbox.creating.add(buildId);
3795
+ return found;
3796
+ }
3797
+
3798
+ // src/runner/workspace/restore.ts
3799
+ var DISK_REASON_TEXT = {
3800
+ no_kept_disk: "no disk is kept for the work item on this host",
3801
+ other_image: "the kept disk was made on another image",
3802
+ other_disk_size: "the kept disk was made with another runner.build.disk size",
3803
+ disk_not_restored: "the kept disk could not be copied",
3804
+ checkout_not_switched: "the kept checkout could not be switched to the branch",
3805
+ build_type_cannot_restore: "this build type cannot restore a disk",
3806
+ fresh_requested: "a member asked for a fresh disk",
3807
+ holder_busy: "the machine that holds its disk was busy"
3808
+ };
3809
+ function itemDiskDecision(input) {
3810
+ if (input.fresh === true) return { state: { restored: false, reason: "fresh_requested" } };
3811
+ if (!input.builtin) return { state: { restored: false, reason: "build_type_cannot_restore" } };
3812
+ let kept;
3939
3813
  try {
3940
- sandbox = await sandboxClient.create({
3941
- name,
3942
- from: own.snapshot,
3943
- ...sized.size,
3944
- idleMinutes: own.idleMinutes,
3945
- timeLimitMinutes: sandboxTimeLimitMinutes(config.timeLimitMinutes),
3946
- network: networkOf(own.allowHosts),
3947
- metadata: { [RUNNER_METADATA]: runnerId, [BUILD_METADATA]: buildId }
3948
- });
3949
- } catch (err) {
3950
- if (err instanceof SandboxError && err.status === 0) await endUnansweredSandbox(sandboxClient, runnerId, buildId, (line2) => log.error(`${tag} ${line2}`));
3951
- runtime.sandbox.creating.delete(buildId);
3952
- return bail(`The provider refused to create the sandbox: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
3814
+ kept = readItemDisk(itemDiskDir(input.configPath, input.organizationId, input.workItemId));
3815
+ } catch {
3816
+ kept = void 0;
3953
3817
  }
3954
- runtime.sandbox.sandboxes.set(buildId, sandbox);
3955
- state.sandboxId = sandbox.id;
3956
- state.dataPlaneUrl = sandbox.dataPlaneUrl;
3957
- input.save(state, environment);
3958
- runtime.sandbox.creating.delete(buildId);
3959
- log.line(`${tag} sandbox ${sandbox.id} created \xB7 ${name}`);
3960
- if (input.aborted()) return bail("", true);
3961
- const sandboxed = runtimeInSandbox(runtime, sandbox);
3962
- const { host } = sandboxed;
3963
- const dockerFailed = (what, result) => bail(`${what} failed (exit ${result.status})${said(result.stderr)}`);
3964
- const joinSecret = randomBytes(32);
3965
- runtime.sandbox.joinSecrets.set(buildId, joinSecret);
3966
- const packDir = join(runtime.cliBuild().distDir, "..", "skill-pack");
3967
- const skillPack = existsSync(packDir) ? packDir : void 0;
3818
+ if (kept === void 0) return { state: { restored: false, reason: input.heldElsewhere === true ? "holder_busy" : "no_kept_disk" } };
3819
+ if (kept.record.repository !== input.repository) return { state: { restored: false, reason: "no_kept_disk" } };
3820
+ if (kept.record.imageId !== input.imageId) return { state: { restored: false, reason: "other_image" } };
3821
+ if (kept.record.diskBytes !== input.diskBytes) return { state: { restored: false, reason: "other_disk_size" } };
3822
+ if (kept.record.kind !== input.backend) return { state: { restored: false, reason: "disk_not_restored" } };
3823
+ return { kept, state: { restored: true, fromClaimId: kept.record.claimId } };
3824
+ }
3825
+ async function discardItemDisk(kept, removeVolume, onError, removeSnapshot) {
3968
3826
  try {
3969
- await sandboxClient.command(sandbox, `mkdir -p ${CLI_DIR}`);
3970
- await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/dist.tgz`, gzipSync(packFolder(runtime.cliBuild().distDir)));
3971
- if (skillPack !== void 0) {
3972
- await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/skill-pack.tgz`, gzipSync(packFolder(skillPack)));
3973
- const placed = await sandboxClient.command(sandbox, `mkdir -p ${SKILL_PACK_DIR} && tar xzf ${SANDBOX_DIR}/skill-pack.tgz -C ${SKILL_PACK_DIR} && rm -f ${SANDBOX_DIR}/skill-pack.tgz`);
3974
- if (placed.exitCode !== 0) return bail(`putting the skill pack in the sandbox failed (exit ${placed.exitCode ?? "none"})${said(placed.stderr)}`);
3827
+ if (kept.record.snapshot !== void 0) {
3828
+ if (removeSnapshot === void 0) return onError(`could not remove the snapshot ${kept.record.snapshot} of the kept disk: no sandbox provider is set`);
3829
+ await removeSnapshot(kept.record.snapshot);
3975
3830
  }
3976
- await sandboxClient.upload(sandbox, SECRET_FILE, joinSecret);
3977
- const unpacked = await sandboxClient.command(sandbox, `tar xzf ${SANDBOX_DIR}/dist.tgz -C ${CLI_DIR} && rm -f ${SANDBOX_DIR}/dist.tgz && sudo chown 60100:60100 ${SECRET_FILE} && sudo chmod 400 ${SECRET_FILE}`);
3978
- if (unpacked.exitCode !== 0) return bail(`putting the CLI in the sandbox failed (exit ${unpacked.exitCode ?? "none"})${said(unpacked.stderr)}`);
3831
+ if (kept.record.kind === "volume" && kept.record.volume !== void 0) {
3832
+ const removed = await removeVolume(kept.record.volume);
3833
+ if (removed !== void 0 && removed.status !== 0 && !/no such volume/i.test(removed.stderr ?? "")) {
3834
+ onError(`could not remove the volume ${kept.record.volume} of the kept disk: ${removed.stderr?.trim() ?? `exit ${removed.status}`}`);
3835
+ return;
3836
+ }
3837
+ }
3838
+ rmSync(kept.dir, { recursive: true, force: true });
3979
3839
  } catch (err) {
3980
- return bail(`putting the CLI in the sandbox failed: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
3840
+ onError(`could not remove the kept disk of work item ${kept.record.workItemId}: ${describeCallFailure(err)}`);
3981
3841
  }
3982
- const engine = await host.version();
3983
- if (engine === null) return bail("Docker is not available in the sandbox: `docker version` failed");
3984
- const limits2 = { memory: `${Math.max(512, sized.size.memoryMib - SANDBOX_RESERVE_MIB)}m`, cpus: sized.size.vcpu, pids: config.build?.pids ?? 4096 };
3985
- const volume = await host.createVolume(container, { [BUILD_LABEL]: buildId, [RUNNER_LABEL]: runnerId });
3986
- if (volume.status !== 0) return dockerFailed("creating the build's volume", volume);
3987
- const runInput = {
3988
- buildId,
3989
- runnerId,
3990
- container,
3991
- volume: container,
3992
- image,
3993
- user: ADDED_USER,
3994
- limits: limits2,
3995
- destinations: config.build?.destinations?.length ?? 0,
3996
- builtIns: true,
3997
- transport: "exec",
3998
- jobFiles: "copied"
3999
- };
4000
- const prepared = await host.run(prepareRunArgs(runInput), { logPath });
4001
- if (prepared.status !== 0) return dockerFailed("preparing the build's volume", prepared);
4002
- const imageConfig = await host.imageConfig(image);
4003
- const resolved = await resolveBuildUser(sandboxed, runInput, { imageUser: imageConfig.user }, logPath);
4004
- if (!resolved.ok) return bail(resolved.reason);
4005
- containerState.agentUser = resolved.user;
4006
- if (resolved.home !== void 0) containerState.home = resolved.home;
4007
- const agentUser = resolved.user;
4008
- input.save(state, environment);
4009
- if (input.aborted()) return bail("", true);
4010
- const started = await host.run(sandboxRunArgs(containerRunArgs({ ...runInput, home: containerState.home }, runtime.agents), { skillPack: skillPack !== void 0 }), { logPath });
4011
- if (started.status !== 0) return dockerFailed("starting the build's container", started);
4012
- joinRelay(runtime, sandbox, { buildId, container }, services.streams, joinSecret, (line2) => log.line(`${tag} ${line2}`));
4013
- if (input.aborted()) return bail("", true);
4014
- if (!await waitForRelay(sandboxed, container)) return bail("the relay in the build's container did not start");
4015
- const ghDelivered = await deliverJobFiles(sandboxed, containerState, ["gh"], secrets);
4016
- if (!ghDelivered.ok) return bail(ghDelivered.reason);
4017
- keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`), {
4018
- runtime: sandboxed,
4019
- state: containerState,
4020
- secrets,
4021
- delivered: readToken,
4022
- onFailure: (reason) => log.error(`${tag} ${reason}`)
4023
- });
4024
- const tools = await host.exec({ container, user: ADDED_USER, env: NO_ENV, argv: ["cat", "/opt/outerlayer/tools.json"] });
4025
- const agentVersion = agentVersionOf(tools.status === 0 ? tools.stdout : "", runtime.agents, input.job);
4026
- environment.agentVersion = agentVersion;
4027
- const judgeProgram = await judgeProgramRecord((argv) => host.exec({ container, user: ADDED_USER, env: NO_ENV, argv }), runtime.agents);
4028
- if (judgeProgram !== void 0) containerState.judgeProgram = judgeProgram;
4029
- input.save(state, environment);
4030
- const stepExec = async (argv, workdir, opts) => {
4031
- const ran = await host.exec({ container, user: agentUser, workdir, env: withMiseVariables(env, opts?.variables), argv }, { ...opts?.quiet === true ? {} : { logPath }, ...opts?.timeoutMs === void 0 ? {} : { timeoutMs: opts.timeoutMs } });
4032
- return endedAtLimit(ran) ? { ...ran, timedOut: true } : ran;
4033
- };
4034
- const setupMinutes = config.build?.setupMinutes ?? DEFAULT_SETUP_MINUTES;
4035
- const steps = await runBuildSteps({
4036
- repository,
4037
- branch: primary.branch,
4038
- repositories,
4039
- ...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository } } : {},
4040
- agentVersion,
4041
- agents: runtime.agents,
4042
- ...input.job === void 0 ? {} : { job: input.job },
4043
- lifecycle: {},
4044
- exec: stepExec,
4045
- log,
4046
- tag,
4047
- aborted: input.aborted,
4048
- stageContext: () => stageGovernedContext({
4049
- config,
4050
- repository,
4051
- source: pinnedSourceOf(governingSource),
4052
- buildId,
4053
- buildDir: buildDir2,
4054
- fetchImpl: input.fetchImpl,
4055
- controlPlaneUrl,
4056
- controlPlaneToken: () => tokens.controlPlane(),
4057
- log,
4058
- tag
4059
- }),
4060
- deliverContext: async () => {
4061
- const delivered = await deliverJobFiles(sandboxed, containerState, ["context"], secrets);
4062
- return delivered.ok ? delivered : { ok: false, reason: `context_unavailable: ${delivered.reason}` };
4063
- },
4064
- removeStagedContext: () => removeStagedContext(buildDir2),
4065
- keepIn: jobDir,
4066
- ...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
4067
- // The image in the snapshot is the universal image, so the build chooses and installs its runtimes.
4068
- ...runtimeStepInput({ base: {} }, services),
4069
- setup: { timeoutMs: setupMinutes * 6e4, limitMinutes: setupMinutes, restored: false }
4070
- });
4071
- if (!steps.ok) return steps.aborted === true ? bail("", true) : bail(steps.reason);
4072
- containerState.startCommit = steps.startCommit;
4073
- keepRuntimeStep(containerState, steps);
4074
- keepMissingSetupScripts(containerState, steps);
4075
- if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
4076
- input.save(state, environment);
4077
- return { ok: true, workdir: CHECKOUT_DIR, state, environment };
4078
3842
  }
4079
- function asContainerJob(job) {
4080
- return { ...job, container: job.sandbox.container };
3843
+
3844
+ // src/runner/saved-state/provision.ts
3845
+ var isRetryFresh = (value) => typeof value === "object" && value !== null && value.retryFresh === true;
3846
+ async function provisionWithFreshFallback(input) {
3847
+ const first = await input.attempt(true);
3848
+ if (!isRetryFresh(first)) return first;
3849
+ input.log(`${first.reason}; provisioning the build again from a fresh disk`);
3850
+ if (first.itemDisk === void 0) return await input.attempt(false);
3851
+ const second = await input.attempt(true, first.itemDisk);
3852
+ if (!isRetryFresh(second)) return second;
3853
+ input.log(`${second.reason}; provisioning the build again from a fresh disk`);
3854
+ return await input.attempt(false, first.itemDisk);
4081
3855
  }
4082
- function sandboxJudgeEnvironment(input) {
4083
- const own = input.job.sandbox;
4084
- const state = own.container;
4085
- const inner = async () => containerJudgeEnvironment({ runtime: await inSandbox(input.runtime, own), job: asContainerJob(input.job), jobDir: input.jobDir, ...input.log === void 0 ? {} : { log: input.log } });
3856
+ function keptDiskPlan() {
3857
+ return { keyCommits: {}, noSave: "it starts from the work item's kept disk" };
3858
+ }
3859
+ function clockOf(now) {
3860
+ return now ?? (() => /* @__PURE__ */ new Date());
3861
+ }
3862
+ function taggedLog(log, tag) {
3863
+ return (line2) => log.line(`${tag} ${line2}`);
3864
+ }
3865
+ function uidFromProbe(probe) {
3866
+ const uid = Number(probe.stdout);
3867
+ return probe.status === 0 && Number.isInteger(uid) ? uid : null;
3868
+ }
3869
+ function agentUidOf(probe) {
3870
+ return uidFromProbe(probe) ?? void 0;
3871
+ }
3872
+ function userIdChangedReason(savedUid, agentUid) {
3873
+ if (savedUid === void 0 || agentUid === savedUid) return void 0;
3874
+ return `the build user's id is ${agentUid ?? "unknown"} and it was ${savedUid} when the state was saved`;
3875
+ }
3876
+ function setupInputFor(input) {
3877
+ const minutes = input.setupMinutes ?? DEFAULT_SETUP_MINUTES;
4086
3878
  return {
4087
- stop: async () => {
4088
- const { runtime } = input;
4089
- const services = runtime.tunnels.get(state.buildId);
4090
- const joinSecret = runtime.sandbox.joinSecrets.get(state.buildId);
4091
- if (services?.streams === void 0 || joinSecret === void 0) throw new Error("the runner no longer holds the build's tunnel and join secret, so it cannot join the build's relay again");
4092
- const sandbox = await sandboxOfBuild(runtime, own);
4093
- if (sandbox === void 0) throw new SandboxGoneError(own.sandboxId);
4094
- const sandboxed = runtimeInSandbox(runtime, sandbox);
4095
- runtime.leases.get(state.buildId)?.stop();
4096
- const relay = runtime.relays.get(state.buildId);
4097
- if (relay !== void 0) {
4098
- runtime.relays.delete(state.buildId);
4099
- await relay.close();
4100
- }
4101
- await sandboxed.host.stop(state.container);
4102
- const started = await sandboxed.host.start(state.container);
4103
- if (started.status !== 0) throw new Error(`the build's container could not be started again (exit ${started.status})`);
4104
- if (!await waitForRelay(sandboxed, state.container)) throw new Error("the relay in the build's container did not start again");
4105
- joinRelay(runtime, sandbox, { buildId: state.buildId, container: state.container }, services.streams, joinSecret, input.log);
4106
- const delivered = await deliverJobFiles(sandboxed, state, ["gh", "git-hooks", "context"], secretsOf(runtime, input.jobDir));
4107
- if (!delivered.ok) throw new Error(delivered.reason);
4108
- },
4109
- hashProgram: async () => (await inner()).hashProgram(),
4110
- prepare: async (files, tree) => (await inner()).prepare(files, tree),
4111
- run: async (command, opts) => (await inner()).run(command, opts),
4112
- readReport: async () => (await inner()).readReport(),
4113
- readItems: async (name) => (await inner()).readItems(name),
4114
- clear: async () => (await inner()).clear?.()
3879
+ timeoutMs: minutes * 6e4,
3880
+ limitMinutes: minutes,
3881
+ restored: input.plan.restore !== void 0 || input.fromItemDisk === true,
3882
+ ...input.fromItemDisk === true ? { fromItemDisk: true } : {},
3883
+ keyCommits: input.plan.keyCommits,
3884
+ ...input.withoutBuildVariables === true ? { withoutBuildVariables: true } : {},
3885
+ // A plan has a key exactly when the build may save a state. A build that starts from one is not asked to save.
3886
+ ...input.plan.key === void 0 ? {} : { saveState: input.saveState }
4115
3887
  };
4116
3888
  }
4117
- async function deliverSandboxHooks(runtime, state, jobDir) {
3889
+ async function saveAndPrune(input) {
3890
+ const made = await input.save();
3891
+ if (!made.ok) return made;
3892
+ input.log(`saved state ${made.record.id}`);
3893
+ await removeSuperseded({ made: made.record, records: readRecords(input.dir), deps: { dir: input.dir, removeVolume: input.removeVolume }, log: input.log });
3894
+ return { ok: true };
3895
+ }
3896
+ async function discardUnusableState(input) {
4118
3897
  try {
4119
- return await deliverJobFiles(await inSandbox(runtime, state), state.container, ["git-hooks"], secretsOf(runtime, jobDir));
3898
+ await removeSavedState(input.record, { dir: input.dir, removeVolume: input.removeVolume });
4120
3899
  } catch (err) {
4121
- return { ok: false, reason: `the commit hooks could not be delivered: ${describeCallFailure(err)}` };
3900
+ input.onError(`could not remove saved state ${input.record.id}: ${describeCallFailure(err)}`);
4122
3901
  }
3902
+ return { retryFresh: true, reason: `saved state ${input.record.id} could not be used: ${input.reason}` };
4123
3903
  }
4124
- async function startSandboxCommand(input) {
4125
- const own = input.job.sandbox;
4126
- const runtime = await inSandbox(input.runtime, own);
4127
- const exec = containerCommandExec({ job: asContainerJob(input.job), cmdline: input.cmdline, runtime, config: input.config, ...input.extraEnv === void 0 ? {} : { extraEnv: input.extraEnv } });
4128
- const started = await runtime.host.execDetached(exec);
4129
- if (started.status !== 0) {
4130
- const first = started.stderr.trim().split("\n")[0] ?? "";
4131
- throw new Error(`docker exec exited ${started.status}${first === "" ? "" : `: ${first}`}`);
3904
+ function settleFailedSteps(steps, on) {
3905
+ return steps.restoreFailed === true ? on.giveUp(steps.reason, steps.fromItemDisk === true) : on.bail(steps.reason, steps.aborted === true);
3906
+ }
3907
+ async function discardUnusableItemDisk(input) {
3908
+ await discardItemDisk(input.kept, input.removeVolume, input.onError, input.removeSnapshot);
3909
+ return { retryFresh: true, reason: `the kept disk of claim ${shortClaim(input.kept.record.claimId)} could not be used: ${input.reason}`, itemDisk: input.why };
3910
+ }
3911
+
3912
+ // src/runner/sandbox/sweep.ts
3913
+ var RUNNER_METADATA = "outerlayerRunner";
3914
+ var BUILD_METADATA = "outerlayerBuild";
3915
+ function orphanedSandboxes(sandboxes, runnerId, heldBuilds) {
3916
+ return sandboxes.filter((sandbox) => {
3917
+ if (sandbox.metadata[RUNNER_METADATA] !== runnerId) return false;
3918
+ if (sandbox.state === "ended") return false;
3919
+ const build = sandbox.metadata[BUILD_METADATA];
3920
+ return build === void 0 || !heldBuilds.has(build);
3921
+ });
3922
+ }
3923
+ async function sweepSandboxes(input) {
3924
+ let listed;
3925
+ try {
3926
+ listed = await input.client.list();
3927
+ } catch (err) {
3928
+ input.log?.(`could not list sandboxes: ${err instanceof Error ? err.message : String(err)}`);
3929
+ return 0;
3930
+ }
3931
+ let ended = 0;
3932
+ for (const sandbox of orphanedSandboxes(listed, input.runnerId, input.heldBuilds())) {
3933
+ try {
3934
+ await input.client.end(sandbox.id);
3935
+ ended++;
3936
+ input.log?.(`ended sandbox ${sandbox.id} (${sandbox.name}): no unfinished build holds it`);
3937
+ } catch (err) {
3938
+ input.log?.(`could not end sandbox ${sandbox.id}: ${err instanceof Error ? err.message : String(err)}`);
3939
+ }
4132
3940
  }
3941
+ return ended;
3942
+ }
3943
+ var BUFFER_HIGH_WATER = 1024 * 1024;
3944
+ var DRAIN_POLL_MS = 5;
3945
+ var OPEN_TIMEOUT_MS = 3e4;
3946
+ var defaultWebSocket = (url) => new WebSocket(url);
3947
+ function sandboxExecStream(opts) {
3948
+ const create = opts.webSocket ?? defaultWebSocket;
3949
+ return () => {
3950
+ const stdout = new PassThrough();
3951
+ let socket;
3952
+ let open = false;
3953
+ let finished2 = false;
3954
+ let tail = "";
3955
+ let finish2;
3956
+ const exited = new Promise((resolve2) => finish2 = resolve2);
3957
+ const end = (status, why) => {
3958
+ if (finished2) return;
3959
+ finished2 = true;
3960
+ if (why !== void 0) tail = `${tail}${why}`.slice(-400);
3961
+ stdout.end();
3962
+ try {
3963
+ socket?.close();
3964
+ } catch {
3965
+ }
3966
+ finish2(status);
3967
+ };
3968
+ const opened = new Promise((resolve2, reject) => {
3969
+ const timer = setTimeout(() => reject(new Error("the WebSocket did not open in time")), OPEN_TIMEOUT_MS);
3970
+ timer.unref();
3971
+ opts.url().then((url) => {
3972
+ socket = create(url);
3973
+ socket.binaryType = "arraybuffer";
3974
+ socket.addEventListener("open", () => {
3975
+ clearTimeout(timer);
3976
+ open = true;
3977
+ socket.send(opts.secret);
3978
+ resolve2();
3979
+ });
3980
+ socket.addEventListener("message", (event) => {
3981
+ if (event.data instanceof ArrayBuffer) stdout.write(Buffer.from(event.data));
3982
+ else if (ArrayBuffer.isView(event.data)) stdout.write(Buffer.from(event.data.buffer, event.data.byteOffset, event.data.byteLength));
3983
+ });
3984
+ socket.addEventListener("error", (event) => {
3985
+ clearTimeout(timer);
3986
+ reject(new Error(event.message ?? "the WebSocket failed"));
3987
+ });
3988
+ socket.addEventListener("close", (event) => {
3989
+ clearTimeout(timer);
3990
+ reject(new Error("the WebSocket closed before it opened"));
3991
+ end(event.code === 1e3 || event.code === void 0 ? 0 : 1, event.code === 1008 ? "the relay refused the join secret" : void 0);
3992
+ });
3993
+ }).catch((err) => {
3994
+ clearTimeout(timer);
3995
+ reject(err);
3996
+ });
3997
+ });
3998
+ opened.catch((err) => end(1, err instanceof Error ? err.message : String(err)));
3999
+ const stdin = new Writable({
4000
+ write(chunk, _encoding, callback) {
4001
+ opened.then(
4002
+ async () => {
4003
+ if (finished2) return callback(new Error("the connection is closed"));
4004
+ socket.send(chunk);
4005
+ while (open && !finished2 && socket.bufferedAmount > BUFFER_HIGH_WATER) await new Promise((resolve2) => setTimeout(resolve2, DRAIN_POLL_MS));
4006
+ callback();
4007
+ },
4008
+ () => callback(new Error("the connection did not open"))
4009
+ );
4010
+ },
4011
+ final(callback) {
4012
+ end(0);
4013
+ callback();
4014
+ }
4015
+ });
4016
+ stdin.on("error", () => void 0);
4017
+ stdout.on("error", () => void 0);
4018
+ return { stdin, stdout, exited, kill: () => end(-1), stderrTail: () => tail };
4019
+ };
4020
+ }
4021
+ var RUN_DIR = "/tmp/ol-run";
4022
+ var CHUNK_BYTES = 9e4;
4023
+ var READ_ATTEMPTS = 4;
4024
+ var POLL_FIRST_MS = 50;
4025
+ var POLL_LONGEST_MS = 1e3;
4026
+ function quote(word) {
4027
+ return /^[A-Za-z0-9_@%+=:,./-]+$/.test(word) ? word : `'${word.replace(/'/g, `'\\''`)}'`;
4028
+ }
4029
+ function sandboxLine(command, args, env) {
4030
+ const values = Object.entries(env ?? {}).filter((entry) => entry[1] !== void 0).map(([name, value]) => `${name}=${quote(value)}`);
4031
+ return ["sudo", "env", ...values, command, ...args.map(quote)].join(" ");
4032
+ }
4033
+ function sandboxRunner(client2, sandbox, options = {}) {
4034
+ const sleep = options.sleep ?? ((ms) => new Promise((resolve2) => setTimeout(resolve2, ms)));
4035
+ return async (command, args, opts = {}) => {
4036
+ const id = randomBytes(6).toString("hex");
4037
+ const file = (suffix) => `${RUN_DIR}/${id}.${suffix}`;
4038
+ const line2 = sandboxLine(command, args, opts.env);
4039
+ options.onCommand?.(line2);
4040
+ if (opts.input !== void 0) {
4041
+ await client2.command(sandbox, `mkdir -p ${RUN_DIR}`);
4042
+ await client2.upload(sandbox, file("in"), Buffer.isBuffer(opts.input) ? opts.input : Buffer.from(opts.input));
4043
+ }
4044
+ const start = `mkdir -p ${RUN_DIR}; echo $$ > ${file("pid")}; ${line2} < ${opts.input === void 0 ? "/dev/null" : file("in")} > ${file("out")} 2> ${file("err")}; echo $? > ${file("code.tmp")}; mv -f ${file("code.tmp")} ${file("code")}`;
4045
+ await client2.command(sandbox, start, { background: true });
4046
+ const deadline = opts.timeoutMs === void 0 ? void 0 : Date.now() + opts.timeoutMs;
4047
+ const collect = `if [ -f ${file("code")} ]; then cat ${file("code")}; head -c ${CHUNK_BYTES} ${file("out")} | base64 | tr -d '\\n'; echo; head -c ${CHUNK_BYTES} ${file("err")} | base64 | tr -d '\\n'; echo; wc -c < ${file("out")}; wc -c < ${file("err")}; fi`;
4048
+ const ask2 = async (cmd) => {
4049
+ for (let attempt = 1; ; attempt++) {
4050
+ try {
4051
+ return await client2.command(sandbox, cmd);
4052
+ } catch (err) {
4053
+ const transient = err instanceof SandboxError && (err.status === 0 || err.status >= 500);
4054
+ if (!transient || attempt >= READ_ATTEMPTS) throw err;
4055
+ await sleep(POLL_FIRST_MS * attempt);
4056
+ }
4057
+ }
4058
+ };
4059
+ let wait = POLL_FIRST_MS;
4060
+ for (; ; ) {
4061
+ const polled = await ask2(collect);
4062
+ if (polled.stdout.trim() !== "") {
4063
+ const [code = "", out = "", err = "", outSize = "0", errSize = "0"] = polled.stdout.split("\n").map((part) => part.trim());
4064
+ const pieces = [Buffer.from(out, "base64")];
4065
+ const errPieces = [Buffer.from(err, "base64")];
4066
+ const more = async (name, size, into) => {
4067
+ for (let read = CHUNK_BYTES; read < size; read += CHUNK_BYTES) {
4068
+ const next = await ask2(`tail -c +${read + 1} ${file(name)} | head -c ${CHUNK_BYTES} | base64 | tr -d '\\n'`);
4069
+ into.push(Buffer.from(next.stdout.trim(), "base64"));
4070
+ }
4071
+ };
4072
+ await more("out", Number(outSize), pieces);
4073
+ await more("err", Number(errSize), errPieces);
4074
+ await client2.command(sandbox, `rm -f ${RUN_DIR}/${id}.*`);
4075
+ const result = { status: Number(code) || 0, stdout: Buffer.concat(pieces).toString("utf8"), stderr: Buffer.concat(errPieces).toString("utf8") };
4076
+ if (opts.logPath !== void 0) {
4077
+ mkdirSync(dirname(opts.logPath), { recursive: true });
4078
+ appendFileSync(opts.logPath, result.stdout + result.stderr);
4079
+ }
4080
+ return result;
4081
+ }
4082
+ if (deadline !== void 0 && Date.now() >= deadline) {
4083
+ await client2.command(sandbox, `kill -9 $(cat ${file("pid")}) 2>/dev/null; rm -f ${RUN_DIR}/${id}.*`).catch(() => void 0);
4084
+ const extra = `${command}: no answer within ${Math.round(opts.timeoutMs / 1e3)}s
4085
+ `;
4086
+ if (opts.logPath !== void 0) {
4087
+ mkdirSync(dirname(opts.logPath), { recursive: true });
4088
+ appendFileSync(opts.logPath, extra);
4089
+ }
4090
+ return { status: 124, stdout: "", stderr: extra };
4091
+ }
4092
+ await sleep(wait);
4093
+ wait = Math.min(wait * 2, POLL_LONGEST_MS);
4094
+ }
4095
+ };
4096
+ }
4097
+ function sandboxHost(client2, sandbox, options = {}) {
4098
+ return createContainerHost({ run: sandboxRunner(client2, sandbox, options) });
4099
+ }
4100
+
4101
+ // src/runner/sandbox/hooks.ts
4102
+ var SANDBOX_DIR = "/tmp/ol";
4103
+ var CLI_DIR = `${SANDBOX_DIR}/dist`;
4104
+ var SKILL_PACK_DIR = `${SANDBOX_DIR}/skill-pack`;
4105
+ var SECRET_FILE = `${SANDBOX_DIR}/join-secret`;
4106
+ var CONTAINER_SECRET_FILE = "/run/outerlayer-relay/join-secret";
4107
+ var JOIN_WS_PORT = 7077;
4108
+ var SANDBOX_RESERVE_MIB = 512;
4109
+ var RELAY_ATTEMPTS = 40;
4110
+ var RELAY_WAIT_MS = 500;
4111
+ var SANDBOX_PROBE_BYTES = 96 * 1024;
4112
+ var MAX_TAIL_READS2 = 8;
4113
+ var NO_ENV = { literals: {}, passed: {} };
4114
+ function sandboxNameOf(runnerLabel, buildId) {
4115
+ const short = runnerLabel.toLowerCase().replace(/[^a-z0-9]/g, "").slice(0, 8);
4116
+ return `ol-${short === "" ? "runner" : short}-${buildId}`;
4117
+ }
4118
+ function itemSnapshotName(runnerLabel, part = randomBytes(6).toString("hex")) {
4119
+ return sandboxNameOf(runnerLabel, `disk-${part}`);
4120
+ }
4121
+ function diskImageOf(own) {
4122
+ return `${own.snapshot}/${own.image}`;
4133
4123
  }
4134
- async function probeSandboxCommand(runtime, state, offset) {
4135
- let sandboxed;
4124
+ var GIB2 = 1024 ** 3;
4125
+ async function endUnansweredSandbox(sandboxClient, runnerId, buildId, warn) {
4136
4126
  try {
4137
- sandboxed = await inSandbox(runtime, state);
4127
+ for (const found of await sandboxClient.list()) {
4128
+ if (found.state !== "ended" && found.metadata[RUNNER_METADATA] === runnerId && found.metadata[BUILD_METADATA] === buildId) await sandboxClient.end(found.id);
4129
+ }
4138
4130
  } catch (err) {
4139
- if (err instanceof SandboxGoneError) throw await goneReason(runtime, state) ?? err;
4140
- return void 0;
4131
+ warn(`could not end a sandbox that a lost create may have made: ${describeCallFailure(err)} \xB7 the next sweep ends it`);
4141
4132
  }
4133
+ }
4134
+ function client(runtime) {
4135
+ return runtime.sandbox.client();
4136
+ }
4137
+ async function sandboxOfBuild(runtime, state) {
4138
+ const held3 = runtime.sandbox.sandboxes.get(state.buildId);
4139
+ if (held3 !== void 0) return held3;
4142
4140
  try {
4143
- const result = await sandboxed.host.exec(
4144
- { container: state.container.container, user: state.container.agentUser ?? ADDED_USER, env: NO_ENV, argv: probeArgvOf(offset) },
4145
- { timeoutMs: 6e4 }
4146
- );
4147
- return result.status === 0 ? parseProbe(result.stdout) : void 0;
4141
+ const found = await client(runtime).get(state.sandboxId);
4142
+ if (found.state === "ended" || found.secret === "") return void 0;
4143
+ runtime.sandbox.sandboxes.set(state.buildId, found);
4144
+ return found;
4148
4145
  } catch (err) {
4149
- if (err instanceof SandboxError && err.status === 404) throw await goneReason(runtime, state) ?? new SandboxGoneError(state.sandboxId);
4150
- if (err instanceof SandboxError) return void 0;
4146
+ if (err instanceof SandboxError && err.status === 404) return void 0;
4151
4147
  throw err;
4152
4148
  }
4153
4149
  }
4154
- function probeArgvOf(offset) {
4155
- const argv = probeArgv(offset);
4156
- argv[argv.length - 1] = String(SANDBOX_PROBE_BYTES);
4157
- return argv;
4150
+ function runtimeInSandbox(runtime, sandbox) {
4151
+ return { ...runtime, host: sandboxHost(client(runtime), sandbox, { sleep: runtime.sandbox.sleep }) };
4158
4152
  }
4159
- async function endSandboxCommand(runtime, state, jobDir, warn) {
4160
- try {
4161
- const sandboxed = await inSandbox(runtime, state);
4162
- for (let reads = 0; reads < MAX_TAIL_READS2; reads++) {
4163
- const offset = commandLogOffset(jobDir, state.commandLogBase);
4164
- const probe = await probeSandboxCommand(runtime, state, offset);
4165
- if (probe === void 0) break;
4166
- appendCommandLog(jobDir, probe.bytes);
4167
- if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) break;
4168
- }
4169
- await sandboxed.host.stop(state.container.container);
4170
- } catch (err) {
4171
- warn(`ending the build's command in its sandbox failed: ${describeCallFailure(err)} \xB7 continuing`);
4153
+ var SandboxGoneError = class extends Error {
4154
+ constructor(sandboxId, endedReason) {
4155
+ super(`The provider ended sandbox ${sandboxId} while the build ran${endedReason === void 0 || endedReason === "" ? "" : ` (${endedReason})`}`);
4156
+ this.endedReason = endedReason;
4157
+ this.name = "SandboxGoneError";
4172
4158
  }
4159
+ };
4160
+ async function inSandbox(runtime, state) {
4161
+ const sandbox = await sandboxOfBuild(runtime, state);
4162
+ if (sandbox === void 0) throw new SandboxGoneError(state.sandboxId);
4163
+ return runtimeInSandbox(runtime, sandbox);
4173
4164
  }
4174
- async function runCliInSandbox(input, argv, logName) {
4175
- const { job } = input;
4176
- const own = job.sandbox;
4165
+ async function goneReason(runtime, state) {
4177
4166
  try {
4178
- const sandboxed = await inSandbox(input.runtime, own);
4179
- const state = own.container;
4180
- const env = buildEnvironment({
4181
- config: claimCalls(input.config, job),
4182
- item: job.item,
4183
- ...job.queue === "run" ? { run: true } : {},
4184
- itemKey: job.itemKey,
4185
- branch: job.branch,
4186
- repository: state.repository,
4187
- repositories: job.repositories,
4188
- hostEnv: input.runtime.hostEnv,
4189
- agents: input.runtime.agents,
4190
- job,
4191
- containerEnv: state.containerEnv ?? {},
4192
- unpinnedRuntimes: state.unpinnedRuntimes,
4193
- missingSetupScripts: state.missingSetupScripts,
4194
- miseVariables: state.miseVariables,
4195
- builtIns: true,
4196
- identity: job.commitIdentity
4197
- });
4198
- const result = await sandboxed.host.exec(
4199
- { container: state.container, user: state.agentUser ?? ADDED_USER, workdir: job.workdir ?? CHECKOUT_DIR, env, argv },
4200
- { logPath: join(input.jobDir, logName) }
4201
- );
4202
- return result.status === 0 ? { ok: true } : { ok: false, reason: `exit ${result.status}` };
4167
+ const found = await client(runtime).get(state.sandboxId);
4168
+ return found.state === "ended" ? new SandboxGoneError(state.sandboxId, found.endedReason ?? void 0) : void 0;
4203
4169
  } catch (err) {
4204
- return { ok: false, reason: describeCallFailure(err) };
4170
+ return err instanceof SandboxError && err.status === 404 ? new SandboxGoneError(state.sandboxId) : void 0;
4205
4171
  }
4206
4172
  }
4207
- function syncInSandbox(input) {
4208
- return runCliInSandbox(input, ["outerlayer", "sync"], "sync.log");
4209
- }
4210
- function recordProofsInSandbox(input) {
4211
- return runCliInSandbox(input, [...RECORD_PROOFS_ARGV], PROOFS_LOG);
4212
- }
4213
- function savedStatesDir(runnerDirPath) {
4214
- return join(runnerDirPath, "saved-states");
4173
+ function sandboxRunArgs(args, opts = {}) {
4174
+ const out = [...args];
4175
+ const network = out.indexOf("--network");
4176
+ if (network !== -1) out[network + 1] = "host";
4177
+ const entrypoint = out.indexOf("--entrypoint");
4178
+ out.splice(
4179
+ entrypoint,
4180
+ 0,
4181
+ "--mount",
4182
+ `type=bind,src=${CLI_DIR},dst=/opt/outerlayer/cli/dist,readonly`,
4183
+ ...opts.skillPack === true ? ["--mount", `type=bind,src=${SKILL_PACK_DIR},dst=/opt/outerlayer/cli/skill-pack,readonly`] : [],
4184
+ "--mount",
4185
+ `type=bind,src=${SECRET_FILE},dst=${CONTAINER_SECRET_FILE},readonly`,
4186
+ ...Object.entries(toolProxyEnvironment()).flatMap(([name, value]) => ["--env", `${name}=${value}`])
4187
+ );
4188
+ out.push("--join-ws", String(JOIN_WS_PORT), "--join-secret-file", CONTAINER_SECRET_FILE);
4189
+ return out;
4215
4190
  }
4216
- function writeRecord2(dir, record) {
4217
- mkdirSync(dir, { recursive: true, mode: RUNNER_DIR_MODE });
4218
- const target2 = join(dir, `${record.id}.json`);
4219
- const partial = `${target2}.partial`;
4220
- writeFileSync(partial, `${JSON.stringify(record, null, 2)}
4221
- `, { mode: RUNNER_FILE_MODE });
4222
- renameSync(partial, target2);
4191
+ var RELAY_PROBE = `const s=require("net").connect(3128,"127.0.0.1");s.on("connect",()=>process.exit(0));s.on("error",()=>process.exit(1));setTimeout(()=>process.exit(1),2000)`;
4192
+ async function waitForRelay(runtime, container) {
4193
+ for (let attempt = 0; attempt < RELAY_ATTEMPTS; attempt++) {
4194
+ const probe = await runtime.host.exec({ container, user: RELAY_USER, env: NO_ENV, argv: ["/opt/outerlayer/bin/node", "-e", RELAY_PROBE] });
4195
+ if (probe.status === 0) return true;
4196
+ await runtime.sandbox.sleep(RELAY_WAIT_MS);
4197
+ }
4198
+ return false;
4223
4199
  }
4224
- function isRecord(value) {
4225
- if (typeof value !== "object" || value === null) return false;
4226
- const r = value;
4227
- return typeof r.id === "string" && (r.kind === "volume" || r.kind === "tar" || r.kind === "disk") && typeof r.savedAt === "string" && Number.isFinite(Date.parse(r.savedAt)) && typeof r.bytes === "number" && typeof r.baseKey === "string" && typeof r.key === "string" && typeof r.fileHashes === "object" && r.fileHashes !== null && typeof r.complete === "boolean";
4200
+ function joinRelay(runtime, sandbox, build, services, joinSecret, log) {
4201
+ const exec = sandboxExecStream({
4202
+ url: () => client(runtime).portUrl(sandbox, JOIN_WS_PORT),
4203
+ secret: joinSecret,
4204
+ ...runtime.sandbox.webSocket === void 0 ? {} : { webSocket: runtime.sandbox.webSocket }
4205
+ });
4206
+ runtime.relays.set(
4207
+ build.buildId,
4208
+ superviseRelay({
4209
+ container: build.container,
4210
+ services,
4211
+ execStream: () => exec(),
4212
+ ...log === void 0 ? {} : { log },
4213
+ onFatal: (reason) => runtime.relayFatal.set(build.buildId, reason),
4214
+ builtIns: builtInNames(runtime.agents)
4215
+ })
4216
+ );
4228
4217
  }
4229
- function readRecords(dir) {
4230
- let names;
4231
- try {
4232
- names = readdirSync(dir);
4233
- } catch {
4234
- return [];
4218
+ var SANDBOX_RUN_DIR = "/tmp/ol-run";
4219
+ async function keepSandboxDisk(runtime, state, kept) {
4220
+ const sandbox = await sandboxOfBuild(runtime, state);
4221
+ if (sandbox === void 0) throw new Error(`sandbox ${state.sandboxId} is gone`);
4222
+ const { host } = runtimeInSandbox(runtime, sandbox);
4223
+ const removed = await host.remove(kept.container);
4224
+ if (removed.status !== 0 && !/no such container/i.test(removed.stderr)) throw new Error(`container ${kept.container} was not removed${said(removed.stderr)}`);
4225
+ const emptied = await host.run(clearOneBuildArgs({ image: kept.imageId, volume: kept.volume, runnerId: kept.runnerId }));
4226
+ if (emptied.status !== 0) throw new Error(`the build's command state was not removed from the volume (exit ${emptied.status})${said(emptied.stderr)}`);
4227
+ const cleared = await client(runtime).command(sandbox, `sudo rm -rf ${SANDBOX_DIR} ${SANDBOX_RUN_DIR}`);
4228
+ if (cleared.exitCode !== 0) throw new Error(`the runner's files were not removed from the sandbox (exit ${cleared.exitCode ?? "none"})${said(cleared.stderr)}`);
4229
+ await kept.keep();
4230
+ }
4231
+ async function cleanupSandbox(runtime, state, keepDisk) {
4232
+ const failures = [];
4233
+ const relay = runtime.relays.get(state.buildId);
4234
+ if (relay !== void 0) {
4235
+ runtime.relays.delete(state.buildId);
4236
+ await relay.close();
4235
4237
  }
4236
- const records = [];
4237
- for (const name of names) {
4238
- if (!name.endsWith(".json")) continue;
4238
+ runtime.relayFatal.delete(state.buildId);
4239
+ runtime.leases.get(state.buildId)?.close();
4240
+ runtime.leases.delete(state.buildId);
4241
+ if (keepDisk !== void 0) {
4239
4242
  try {
4240
- const parsed = JSON.parse(readFileSync(join(dir, name), "utf8"));
4241
- if (isRecord(parsed)) records.push(parsed);
4242
- } catch {
4243
+ await keepSandboxDisk(runtime, state, keepDisk);
4244
+ } catch (err) {
4245
+ keepDisk.warn(`the build's disk was not kept: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
4243
4246
  }
4244
4247
  }
4245
- return records;
4248
+ if (state.sandboxId !== "") {
4249
+ try {
4250
+ await client(runtime).end(state.sandboxId);
4251
+ } catch (err) {
4252
+ failures.push(`sandbox ${state.sandboxId} was not ended: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
4253
+ }
4254
+ }
4255
+ runtime.sandbox.sandboxes.delete(state.buildId);
4256
+ runtime.sandbox.joinSecrets.delete(state.buildId);
4257
+ const tunnel = runtime.tunnels.get(state.buildId);
4258
+ if (tunnel !== void 0) {
4259
+ runtime.tunnels.delete(state.buildId);
4260
+ await tunnel.close();
4261
+ }
4262
+ if (state.container !== void 0) rmSync(state.container.buildDir, { recursive: true, force: true });
4263
+ if (failures.length > 0) throw new Error(failures.join("; "));
4246
4264
  }
4247
- function deleteRecord(dir, id) {
4248
- rmSync(join(dir, `${id}.json`), { force: true });
4265
+ async function provisionSandbox(input) {
4266
+ const tag = itemTag({ item: input.item.number, organizationId: input.organizationId, claimId: input.claimId });
4267
+ return provisionWithFreshFallback({
4268
+ attempt: (_allowRestore, itemDiskFailed) => provisionSandboxOnce(input, itemDiskFailed),
4269
+ log: (line2) => input.log.line(`${tag} ${line2}`)
4270
+ });
4249
4271
  }
4250
-
4251
- // src/runner/saved-state/remove.ts
4252
- async function removeSavedState(record, deps) {
4253
- if (record.kind === "volume" && record.volume !== void 0) {
4254
- const result = await deps.removeVolume(record.volume);
4255
- if (result !== void 0 && result.status !== 0) {
4256
- const line2 = (result.stderr ?? "").trim().split("\n").filter((text2) => text2.trim() !== "").at(-1);
4257
- throw new Error(`Docker could not remove volume ${record.volume} (exit ${result.status}${line2 === void 0 ? "" : `: ${line2}`})`);
4258
- }
4272
+ async function provisionSandboxOnce(input, itemDiskFailed) {
4273
+ const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
4274
+ const tag = itemTag({ item: item.number, organizationId: input.organizationId, claimId: input.claimId });
4275
+ const logPath = join(jobDir, "provision.log");
4276
+ const fail = (reason) => ({ ok: false, reason });
4277
+ const own = config.sandbox;
4278
+ if (own === void 0) return fail("runner.sandbox is not set in the config file");
4279
+ const repositories = repositoriesOf({ repositories: input.repositories, repository: item.specRepository, branch: input.branch });
4280
+ const primary = primaryOf(repositories);
4281
+ if (primary === void 0) return fail("the work item names no repository to clone");
4282
+ const repository = primary.repository;
4283
+ const sized = sandboxSizeOf(config.build);
4284
+ if (!sized.ok) return fail(`${sized.field} ${sized.problem}`);
4285
+ let sandboxClient;
4286
+ try {
4287
+ sandboxClient = client(runtime);
4288
+ } catch (err) {
4289
+ return fail(err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err));
4259
4290
  }
4260
- if ((record.kind === "tar" || record.kind === "disk") && record.path !== void 0) {
4261
- rmSync(`${record.path}.partial`, { force: true });
4262
- rmSync(record.path, { force: true });
4291
+ const governing = input.queue === "run" ? SELF_GOVERNED : await resolveGoverningSource({ config, repository, fetchImpl: input.fetchImpl });
4292
+ if (!governing.ok) return fail(governing.reason);
4293
+ const governingSource = governing.source;
4294
+ const controlPlaneUrl = (controlPlane) => runtime.remoteFor?.(controlPlane) ?? `${runtime.github.gitUrl}/${controlPlane}.git`;
4295
+ if (governingSource.kind === "governed") {
4296
+ const unfetched = await prefetchGovernedContext({
4297
+ configPath: config.configPath,
4298
+ repository,
4299
+ source: governingSource,
4300
+ controlPlaneUrl,
4301
+ controlPlaneToken: () => tokens.controlPlane(),
4302
+ log,
4303
+ tag
4304
+ });
4305
+ if (unfetched !== void 0) return fail(unfetched);
4306
+ }
4307
+ const diskKey = { imageId: diskImageOf(own), diskBytes: sized.size.diskGib * GIB2 };
4308
+ const itemDisk = input.workItemId === void 0 ? { state: void 0 } : itemDiskFailed !== void 0 ? { state: { restored: false, reason: itemDiskFailed } } : itemDiskDecision({ fresh: input.fresh, heldElsewhere: input.diskHeldElsewhere, builtin: true, backend: "sandbox", configPath: config.configPath, organizationId: input.organizationId, workItemId: input.workItemId, repository, ...diskKey });
4309
+ const keptDisk = itemDisk.kept;
4310
+ if (itemDisk.state?.restored === false) log.line(`${tag} this job starts without a kept disk: ${DISK_REASON_TEXT[itemDisk.state.reason]}`);
4311
+ const buildId = randomBytes(6).toString("hex");
4312
+ const runnerId = runnerLabelValue(config.configPath);
4313
+ const name = sandboxNameOf(runnerId, buildId);
4314
+ const container = `outerlayer-build-${buildId}`;
4315
+ const volumeName = keptDisk?.record.volume ?? container;
4316
+ const buildDir2 = buildDir(config.configPath, buildId);
4317
+ const image = own.image;
4318
+ const containerState = {
4319
+ buildId,
4320
+ container,
4321
+ volume: volumeName,
4322
+ buildDir: buildDir2,
4323
+ repository,
4324
+ imageId: image,
4325
+ builtIns: true,
4326
+ ...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository, ref: governingSource.ref, commit: governingSource.sha } } : {},
4327
+ transport: "exec",
4328
+ jobFiles: "copied"
4329
+ };
4330
+ const state = { sandboxId: "", name, dataPlaneUrl: "", buildId, diskKey, container: containerState };
4331
+ const secrets = secretsOf(runtime, jobDir);
4332
+ const env = buildEnvironment({
4333
+ config,
4334
+ item: item.number,
4335
+ ...input.queue === "run" ? { run: true } : {},
4336
+ itemKey,
4337
+ repository,
4338
+ repositories,
4339
+ hostEnv: runtime.hostEnv,
4340
+ agents: runtime.agents,
4341
+ ...input.job === void 0 ? {} : { job: input.job },
4342
+ containerEnv: {},
4343
+ builtIns: true
4344
+ });
4345
+ if (env.skippedTokenVariables.length > 0) {
4346
+ log.line(`${tag} not passing ${env.skippedTokenVariables.join(", ")} from runner.build.variables into the build: its repository token comes from the runner`);
4263
4347
  }
4264
- deleteRecord(deps.dir, record.id);
4265
- }
4266
- async function removeOldestUntil(input) {
4267
- const removed = [];
4268
- for (const record of oldestFirst(input.records)) {
4269
- if (input.enough()) break;
4348
+ const environment = {
4349
+ isolation: "vm",
4350
+ imageId: image,
4351
+ variables: env.variables,
4352
+ cliVersion: input.cliVersion
4353
+ };
4354
+ const bail = async (reason, aborted2 = false) => {
4270
4355
  try {
4271
- await removeSavedState(record, input.deps);
4272
- removed.push(record);
4273
- input.log(`removed saved state ${record.id} from ${record.savedAt}`);
4356
+ await cleanupSandbox(runtime, { buildId, sandboxId: state.sandboxId, container: containerState });
4274
4357
  } catch (err) {
4275
- input.log(`could not remove saved state ${record.id}: ${describeCallFailure(err)}`);
4358
+ log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
4276
4359
  }
4360
+ return aborted2 ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
4361
+ };
4362
+ const giveUpItemDisk = async (reason, why) => {
4363
+ await bail(reason);
4364
+ return discardUnusableItemDisk({ kept: keptDisk, reason, why, removeVolume: async () => void 0, removeSnapshot: (snapshot) => sandboxClient.removeSnapshot(snapshot), onError: (line2) => log.error(`${tag} ${line2}`) });
4365
+ };
4366
+ if (input.aborted()) return bail("", true);
4367
+ mkdirSync(buildDir2, { recursive: true });
4368
+ const ghDir = ghConfigDir(buildDir2);
4369
+ let readToken;
4370
+ try {
4371
+ readToken = await tokens.token("read");
4372
+ writeGhConfig(ghDir, readToken);
4373
+ } catch (err) {
4374
+ return bail(err instanceof WorkCommandError ? `the build's read token could not be had: ${refusalCode(err)}` : `the build's gh configuration could not be written: ${describeCallFailure(err)}`);
4277
4375
  }
4278
- return removed;
4279
- }
4280
- async function removeSuperseded(input) {
4281
- for (const old of input.records.filter((record) => record.complete && record.key === input.made.key && record.id !== input.made.id && Date.parse(record.savedAt) < Date.parse(input.made.savedAt))) {
4376
+ for (const entry of repositories) {
4377
+ if (entry.repository === primary.repository) continue;
4282
4378
  try {
4283
- await removeSavedState(old, input.deps);
4284
- input.log(`removed saved state ${old.id}, saved state ${input.made.id} replaces it`);
4379
+ await tokens.token("read", entry.repository);
4285
4380
  } catch (err) {
4286
- input.log(`could not remove saved state ${old.id}: ${describeCallFailure(err)}`);
4381
+ return bail(`the build's read token for ${entry.repository} could not be had: ${refusalCode(err)}`);
4287
4382
  }
4288
4383
  }
4289
- }
4290
-
4291
- // src/runner/saved-state/plan.ts
4292
- function savedStateBase(report) {
4293
- const digest = report.recipe === "default" ? report.base?.digest : void 0;
4294
- return digest === void 0 ? void 0 : { digest };
4295
- }
4296
- function savedStateMaxOf(config) {
4297
- return config.housekeeping?.savedStateMax ?? DEFAULT_SAVED_STATE_MAX_BYTES;
4298
- }
4299
- async function planSavedState(input) {
4300
- const { config, runtime, log } = input;
4301
- const refused2 = savingRefusedBecause({ universalImage: input.base !== void 0, fleet: config.fleet === true, savedStateMax: savedStateMaxOf(config) });
4302
- if (refused2 !== void 0 || input.base === void 0) {
4303
- const why = refused2 ?? "its image comes from a devcontainer file";
4304
- log(`this build saves no state: ${why}`);
4305
- return { keyCommits: {}, noSave: why };
4384
+ keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`));
4385
+ tokens.markUsed("read");
4386
+ environment.credentials = tokens.used();
4387
+ let services;
4388
+ try {
4389
+ const gitRemote = gitRemoteFor(runtime, { repositories, tokens, buildDir: buildDir2, log: (line2) => log.line(`${tag} ${line2}`), ...input.queue === "run" ? { readOnly: true } : {} });
4390
+ services = await startServicesFor(runtime, config, buildDir2, jobDir, itemKey, true, gitRemote, "stream", input.leaseExpiryMs === void 0 ? void 0 : leaseFor(runtime, buildId, input.leaseExpiryMs), void 0, (line2) => log.line(`${tag} ${line2}`));
4391
+ runtime.tunnels.set(buildId, services);
4392
+ } catch (err) {
4393
+ return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
4306
4394
  }
4307
- const root = runnerDir(config.configPath);
4308
- const repositories = [];
4309
- const keyCommits = {};
4310
- for (const entry of input.repositories) {
4311
- let token;
4312
- try {
4313
- token = entry.primary ? await input.tokens.token("read") : await input.tokens.token("read", entry.repository);
4314
- } catch (err) {
4315
- return unread(`the read token for ${entry.repository} could not be had: ${describeCallFailure(err)}`, log);
4316
- }
4317
- const [owner = "", name = ""] = entry.repository.split("/");
4318
- const read = await runtime.readKeyFiles({
4319
- remote: runtime.remoteFor?.(entry.repository) ?? `${runtime.github.gitUrl}/${entry.repository}.git`,
4320
- dir: join(root, "key-files", owner, name),
4321
- paths: keyFilePaths(),
4322
- token
4395
+ if (input.aborted()) return bail("", true);
4396
+ let sandbox;
4397
+ runtime.sandbox.creating.add(buildId);
4398
+ try {
4399
+ sandbox = await sandboxClient.create({
4400
+ name,
4401
+ from: keptDisk?.record.snapshot ?? own.snapshot,
4402
+ ...sized.size,
4403
+ idleMinutes: own.idleMinutes,
4404
+ timeLimitMinutes: sandboxTimeLimitMinutes(config.timeLimitMinutes),
4405
+ network: networkOf(own.allowHosts),
4406
+ metadata: { [RUNNER_METADATA]: runnerId, [BUILD_METADATA]: buildId }
4323
4407
  });
4324
- if (!read.ok) return unread(read.reason, log);
4325
- keyCommits[entry.repository] = read.commit;
4326
- repositories.push({ repository: entry.repository, commit: read.commit, files: read.files });
4408
+ } catch (err) {
4409
+ if (err instanceof SandboxError && err.status === 0) await endUnansweredSandbox(sandboxClient, runnerId, buildId, (line2) => log.error(`${tag} ${line2}`));
4410
+ runtime.sandbox.creating.delete(buildId);
4411
+ const refused2 = `The provider refused to create the sandbox: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`;
4412
+ if (keptDisk !== void 0 && err instanceof SandboxError && err.status === 404) return giveUpItemDisk(refused2, "disk_not_restored");
4413
+ return bail(refused2);
4327
4414
  }
4328
- const key = stateKey({
4329
- factory: keyRunnerFactory(config),
4330
- repositories,
4331
- imageDigest: input.base.digest,
4332
- diskBytes: input.diskBytes
4333
- });
4334
- const found = lookup(readRecords(savedStatesDir(root)), key, input.now.getTime());
4335
- if (found.kind !== "none" && found.record.kind === "tar") {
4336
- try {
4337
- await removeSavedState(found.record, { dir: savedStatesDir(root), removeVolume: async () => void 0 });
4338
- log(`saved state ${found.record.id} is a tar file, which this runner no longer restores; removed it; starting from a fresh disk`);
4339
- } catch (err) {
4340
- log(`saved state ${found.record.id} is a tar file, which this runner no longer restores, and it could not be removed: ${describeCallFailure(err)}; starting from a fresh disk`);
4415
+ runtime.sandbox.sandboxes.set(buildId, sandbox);
4416
+ state.sandboxId = sandbox.id;
4417
+ state.dataPlaneUrl = sandbox.dataPlaneUrl;
4418
+ input.save(state, environment);
4419
+ runtime.sandbox.creating.delete(buildId);
4420
+ log.line(`${tag} sandbox ${sandbox.id} created \xB7 ${name}`);
4421
+ if (keptDisk !== void 0) log.line(`${tag} restored the disk that claim ${shortClaim(keptDisk.record.claimId)} kept: snapshot ${keptDisk.record.snapshot}`);
4422
+ if (input.aborted()) return bail("", true);
4423
+ const sandboxed = runtimeInSandbox(runtime, sandbox);
4424
+ const { host } = sandboxed;
4425
+ const dockerFailed = (what, result) => bail(`${what} failed (exit ${result.status})${said(result.stderr)}`);
4426
+ const joinSecret = randomBytes(32);
4427
+ runtime.sandbox.joinSecrets.set(buildId, joinSecret);
4428
+ const packDir = join(runtime.cliBuild().distDir, "..", "skill-pack");
4429
+ const skillPack = existsSync(packDir) ? packDir : void 0;
4430
+ try {
4431
+ await sandboxClient.command(sandbox, `mkdir -p ${CLI_DIR}`);
4432
+ await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/dist.tgz`, gzipSync(packFolder(runtime.cliBuild().distDir)));
4433
+ if (skillPack !== void 0) {
4434
+ await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/skill-pack.tgz`, gzipSync(packFolder(skillPack)));
4435
+ const placed = await sandboxClient.command(sandbox, `mkdir -p ${SKILL_PACK_DIR} && tar xzf ${SANDBOX_DIR}/skill-pack.tgz -C ${SKILL_PACK_DIR} && rm -f ${SANDBOX_DIR}/skill-pack.tgz`);
4436
+ if (placed.exitCode !== 0) return bail(`putting the skill pack in the sandbox failed (exit ${placed.exitCode ?? "none"})${said(placed.stderr)}`);
4341
4437
  }
4342
- return { key, keyCommits };
4343
- }
4344
- if (found.kind === "usable" && input.allowRestore) {
4345
- log(`starting from saved state ${found.record.id}, saved ${formatDuration(found.ageMs)} ago`);
4346
- return { key, keyCommits, restore: { record: found.record, ageMs: found.ageMs } };
4347
- }
4348
- if (found.kind === "expired") {
4349
- log(`saved state ${found.record.id} is ${formatDuration(found.ageMs)} old, past the limit of ${SAVED_STATE_MAX_AGE_MS / 864e5} days; starting from a fresh disk`);
4350
- } else if (found.kind === "changed") {
4351
- log(`the setup script or version files changed on the default branch since saved state ${found.record.id}: ${found.files.join(", ")}; starting from a fresh disk`);
4352
- } else if (found.kind === "none") {
4353
- log("no saved state matches this build's key; starting from a fresh disk");
4438
+ await sandboxClient.upload(sandbox, SECRET_FILE, joinSecret);
4439
+ const unpacked = await sandboxClient.command(sandbox, `tar xzf ${SANDBOX_DIR}/dist.tgz -C ${CLI_DIR} && rm -f ${SANDBOX_DIR}/dist.tgz && sudo chown 60100:60100 ${SECRET_FILE} && sudo chmod 400 ${SECRET_FILE}`);
4440
+ if (unpacked.exitCode !== 0) return bail(`putting the CLI in the sandbox failed (exit ${unpacked.exitCode ?? "none"})${said(unpacked.stderr)}`);
4441
+ } catch (err) {
4442
+ return bail(`putting the CLI in the sandbox failed: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
4354
4443
  }
4355
- return { key, keyCommits };
4356
- }
4357
- function unread(reason, log) {
4358
- log(`this build starts from no saved state and saves none: ${reason}`);
4359
- return { keyCommits: {}, noSave: `its key files could not be read: ${reason}` };
4444
+ const engine = await host.version();
4445
+ if (engine === null) return bail("Docker is not available in the sandbox: `docker version` failed");
4446
+ const limits2 = { memory: `${Math.max(512, sized.size.memoryMib - SANDBOX_RESERVE_MIB)}m`, cpus: sized.size.vcpu, pids: config.build?.pids ?? 4096 };
4447
+ const volume = await host.createVolume(volumeName, { [BUILD_LABEL]: buildId, [RUNNER_LABEL]: runnerId });
4448
+ if (volume.status !== 0) return dockerFailed("creating the build's volume", volume);
4449
+ const runInput = {
4450
+ buildId,
4451
+ runnerId,
4452
+ container,
4453
+ volume: volumeName,
4454
+ image,
4455
+ user: ADDED_USER,
4456
+ limits: limits2,
4457
+ destinations: config.build?.destinations?.length ?? 0,
4458
+ builtIns: true,
4459
+ transport: "exec",
4460
+ jobFiles: "copied"
4461
+ };
4462
+ const prepared = await host.run(prepareRunArgs(runInput), { logPath });
4463
+ if (prepared.status !== 0) return dockerFailed("preparing the build's volume", prepared);
4464
+ const imageConfig = await host.imageConfig(image);
4465
+ const resolved = await resolveBuildUser(sandboxed, runInput, { imageUser: imageConfig.user }, logPath);
4466
+ if (!resolved.ok) return bail(resolved.reason);
4467
+ containerState.agentUser = resolved.user;
4468
+ if (resolved.home !== void 0) containerState.home = resolved.home;
4469
+ const agentUser = resolved.user;
4470
+ input.save(state, environment);
4471
+ if (input.aborted()) return bail("", true);
4472
+ const started = await host.run(sandboxRunArgs(containerRunArgs({ ...runInput, home: containerState.home }, runtime.agents), { skillPack: skillPack !== void 0 }), { logPath });
4473
+ if (started.status !== 0) return dockerFailed("starting the build's container", started);
4474
+ joinRelay(runtime, sandbox, { buildId, container }, services.streams, joinSecret, (line2) => log.line(`${tag} ${line2}`));
4475
+ if (input.aborted()) return bail("", true);
4476
+ if (!await waitForRelay(sandboxed, container)) return bail("the relay in the build's container did not start");
4477
+ const ghDelivered = await deliverJobFiles(sandboxed, containerState, ["gh"], secrets);
4478
+ if (!ghDelivered.ok) return bail(ghDelivered.reason);
4479
+ keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`), {
4480
+ runtime: sandboxed,
4481
+ state: containerState,
4482
+ secrets,
4483
+ delivered: readToken,
4484
+ onFailure: (reason) => log.error(`${tag} ${reason}`)
4485
+ });
4486
+ const tools = await host.exec({ container, user: ADDED_USER, env: NO_ENV, argv: ["cat", "/opt/outerlayer/tools.json"] });
4487
+ const agentVersion = agentVersionOf(tools.status === 0 ? tools.stdout : "", runtime.agents, input.job);
4488
+ environment.agentVersion = agentVersion;
4489
+ const judgeProgram = await judgeProgramRecord((argv) => host.exec({ container, user: ADDED_USER, env: NO_ENV, argv }), runtime.agents);
4490
+ if (judgeProgram !== void 0) containerState.judgeProgram = judgeProgram;
4491
+ input.save(state, environment);
4492
+ const stepExec = async (argv, workdir, opts) => {
4493
+ const ran = await host.exec({ container, user: agentUser, workdir, env: withMiseVariables(env, opts?.variables), argv }, { ...opts?.quiet === true ? {} : { logPath }, ...opts?.timeoutMs === void 0 ? {} : { timeoutMs: opts.timeoutMs } });
4494
+ return endedAtLimit(ran) ? { ...ran, timedOut: true } : ran;
4495
+ };
4496
+ const setupMinutes = config.build?.setupMinutes ?? DEFAULT_SETUP_MINUTES;
4497
+ const steps = await runBuildSteps({
4498
+ repository,
4499
+ branch: primary.branch,
4500
+ repositories,
4501
+ ...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository } } : {},
4502
+ agentVersion,
4503
+ agents: runtime.agents,
4504
+ ...input.job === void 0 ? {} : { job: input.job },
4505
+ lifecycle: {},
4506
+ exec: stepExec,
4507
+ log,
4508
+ tag,
4509
+ aborted: input.aborted,
4510
+ stageContext: () => stageGovernedContext({
4511
+ config,
4512
+ repository,
4513
+ source: pinnedSourceOf(governingSource),
4514
+ buildId,
4515
+ buildDir: buildDir2,
4516
+ fetchImpl: input.fetchImpl,
4517
+ controlPlaneUrl,
4518
+ controlPlaneToken: () => tokens.controlPlane(),
4519
+ log,
4520
+ tag
4521
+ }),
4522
+ deliverContext: async () => {
4523
+ const delivered = await deliverJobFiles(sandboxed, containerState, ["context"], secrets);
4524
+ return delivered.ok ? delivered : { ok: false, reason: `context_unavailable: ${delivered.reason}` };
4525
+ },
4526
+ removeStagedContext: () => removeStagedContext(buildDir2),
4527
+ keepIn: jobDir,
4528
+ ...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
4529
+ // The image in the snapshot is the universal image, so the build chooses and installs its runtimes.
4530
+ ...runtimeStepInput({ base: {} }, services),
4531
+ // A kept disk holds the checkouts already: each is switched to the branch, and keeps its untracked files.
4532
+ setup: { timeoutMs: setupMinutes * 6e4, limitMinutes: setupMinutes, restored: false, ...keptDisk === void 0 ? {} : { restored: true, fromItemDisk: true } }
4533
+ });
4534
+ if (!steps.ok) return settleFailedSteps(steps, { giveUp: (reason) => giveUpItemDisk(reason, "checkout_not_switched"), bail });
4535
+ containerState.startCommit = steps.startCommit;
4536
+ keepRuntimeStep(containerState, steps);
4537
+ keepMissingSetupScripts(containerState, steps);
4538
+ if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
4539
+ input.save(state, environment);
4540
+ return { ok: true, workdir: CHECKOUT_DIR, state, environment, ...itemDisk.state === void 0 ? {} : { disk: itemDisk.state } };
4360
4541
  }
4361
- var RECORD_FILE = "record.json";
4362
- var DISK_FILE = "disk.ext4";
4363
- var WORK_ITEM_ID2 = /^[A-Za-z0-9_-]{1,64}$/;
4364
- function itemDiskDir(configPath, organizationId, workItemId) {
4365
- if (!WORK_ITEM_ID2.test(workItemId)) throw new Error(`cannot keep a disk for the work item ${JSON.stringify(workItemId)}`);
4366
- return join(workspacesDir(configPath, organizationId), workItemId);
4542
+ function asContainerJob(job) {
4543
+ return { ...job, container: job.sandbox.container };
4367
4544
  }
4368
- function isRecord2(value) {
4369
- if (typeof value !== "object" || value === null) return false;
4370
- const r = value;
4371
- return typeof r.workItemId === "string" && typeof r.claimId === "string" && typeof r.queue === "string" && typeof r.imageId === "string" && typeof r.diskBytes === "number" && (r.kind === "disk" || r.kind === "volume" && typeof r.volume === "string") && typeof r.bytes === "number" && typeof r.keptAt === "string" && Number.isFinite(Date.parse(r.keptAt));
4545
+ function sandboxJudgeEnvironment(input) {
4546
+ const own = input.job.sandbox;
4547
+ const state = own.container;
4548
+ const inner = async () => containerJudgeEnvironment({ runtime: await inSandbox(input.runtime, own), job: asContainerJob(input.job), jobDir: input.jobDir, ...input.log === void 0 ? {} : { log: input.log } });
4549
+ return {
4550
+ stop: async () => {
4551
+ const { runtime } = input;
4552
+ const services = runtime.tunnels.get(state.buildId);
4553
+ const joinSecret = runtime.sandbox.joinSecrets.get(state.buildId);
4554
+ if (services?.streams === void 0 || joinSecret === void 0) throw new Error("the runner no longer holds the build's tunnel and join secret, so it cannot join the build's relay again");
4555
+ const sandbox = await sandboxOfBuild(runtime, own);
4556
+ if (sandbox === void 0) throw new SandboxGoneError(own.sandboxId);
4557
+ const sandboxed = runtimeInSandbox(runtime, sandbox);
4558
+ runtime.leases.get(state.buildId)?.stop();
4559
+ const relay = runtime.relays.get(state.buildId);
4560
+ if (relay !== void 0) {
4561
+ runtime.relays.delete(state.buildId);
4562
+ await relay.close();
4563
+ }
4564
+ await sandboxed.host.stop(state.container);
4565
+ const started = await sandboxed.host.start(state.container);
4566
+ if (started.status !== 0) throw new Error(`the build's container could not be started again (exit ${started.status})`);
4567
+ if (!await waitForRelay(sandboxed, state.container)) throw new Error("the relay in the build's container did not start again");
4568
+ joinRelay(runtime, sandbox, { buildId: state.buildId, container: state.container }, services.streams, joinSecret, input.log);
4569
+ const delivered = await deliverJobFiles(sandboxed, state, ["gh", "git-hooks", "context"], secretsOf(runtime, input.jobDir));
4570
+ if (!delivered.ok) throw new Error(delivered.reason);
4571
+ },
4572
+ hashProgram: async () => (await inner()).hashProgram(),
4573
+ prepare: async (files, tree) => (await inner()).prepare(files, tree),
4574
+ run: async (command, opts) => (await inner()).run(command, opts),
4575
+ readReport: async () => (await inner()).readReport(),
4576
+ readItems: async (name) => (await inner()).readItems(name),
4577
+ clear: async () => (await inner()).clear?.()
4578
+ };
4372
4579
  }
4373
- function readItemDisk(dir) {
4374
- let parsed;
4580
+ async function deliverSandboxHooks(runtime, state, jobDir) {
4375
4581
  try {
4376
- parsed = JSON.parse(readFileSync(join(dir, RECORD_FILE), "utf8"));
4377
- } catch {
4378
- return void 0;
4582
+ return await deliverJobFiles(await inSandbox(runtime, state), state.container, ["git-hooks"], secretsOf(runtime, jobDir));
4583
+ } catch (err) {
4584
+ return { ok: false, reason: `the commit hooks could not be delivered: ${describeCallFailure(err)}` };
4379
4585
  }
4380
- if (!isRecord2(parsed) || parsed.workItemId !== basename(dir)) return void 0;
4381
- if (parsed.kind === "volume") return { dir, record: parsed };
4382
- const path = join(dir, DISK_FILE);
4383
- try {
4384
- if (!statSync(path).isFile()) return void 0;
4385
- } catch {
4386
- return void 0;
4586
+ }
4587
+ async function startSandboxCommand(input) {
4588
+ const own = input.job.sandbox;
4589
+ const runtime = await inSandbox(input.runtime, own);
4590
+ const exec = containerCommandExec({ job: asContainerJob(input.job), cmdline: input.cmdline, runtime, config: input.config, ...input.extraEnv === void 0 ? {} : { extraEnv: input.extraEnv } });
4591
+ const started = await runtime.host.execDetached(exec);
4592
+ if (started.status !== 0) {
4593
+ const first = started.stderr.trim().split("\n")[0] ?? "";
4594
+ throw new Error(`docker exec exited ${started.status}${first === "" ? "" : `: ${first}`}`);
4387
4595
  }
4388
- return { dir, record: parsed, path };
4389
4596
  }
4390
- function swapInItemDisk(dir, incoming) {
4391
- const older = readItemDisk(dir);
4392
- const outgoing = join(dirname(dir), `.${basename(dir)}.${randomBytes(4).toString("hex")}.old`);
4393
- let hadOlder = true;
4597
+ async function probeSandboxCommand(runtime, state, offset) {
4598
+ let sandboxed;
4394
4599
  try {
4395
- renameSync(dir, outgoing);
4600
+ sandboxed = await inSandbox(runtime, state);
4396
4601
  } catch (err) {
4397
- if (err.code !== "ENOENT") throw err;
4398
- hadOlder = false;
4602
+ if (err instanceof SandboxGoneError) throw await goneReason(runtime, state) ?? err;
4603
+ return void 0;
4399
4604
  }
4400
4605
  try {
4401
- renameSync(incoming, dir);
4606
+ const result = await sandboxed.host.exec(
4607
+ { container: state.container.container, user: state.container.agentUser ?? ADDED_USER, env: NO_ENV, argv: probeArgvOf(offset) },
4608
+ { timeoutMs: 6e4 }
4609
+ );
4610
+ return result.status === 0 ? parseProbe(result.stdout) : void 0;
4402
4611
  } catch (err) {
4403
- if (hadOlder) renameSync(outgoing, dir);
4612
+ if (err instanceof SandboxError && err.status === 404) throw await goneReason(runtime, state) ?? new SandboxGoneError(state.sandboxId);
4613
+ if (err instanceof SandboxError) return void 0;
4404
4614
  throw err;
4405
4615
  }
4406
- rmSync(outgoing, { recursive: true, force: true });
4407
- return older;
4408
- }
4409
- function stagingItemDiskDir(dir) {
4410
- const parent = dirname(dir);
4411
- mkdirSync(parent, { recursive: true, mode: 448 });
4412
- const staging = join(parent, `.${basename(dir)}.${randomBytes(4).toString("hex")}.partial`);
4413
- mkdirSync(staging, { mode: 448 });
4414
- return staging;
4415
4616
  }
4416
- function listItemDisks(configPath, scope) {
4417
- const where = workspacesDir(configPath, scope);
4418
- let names;
4419
- try {
4420
- names = readdirSync(where);
4421
- } catch {
4422
- return [];
4423
- }
4424
- const kept = [];
4425
- for (const name of names) {
4426
- if (!WORK_ITEM_ID2.test(name)) continue;
4427
- const found = readItemDisk(join(where, name));
4428
- if (found !== void 0) kept.push(found);
4429
- }
4430
- return kept;
4617
+ function probeArgvOf(offset) {
4618
+ const argv = probeArgv(offset);
4619
+ argv[argv.length - 1] = String(SANDBOX_PROBE_BYTES);
4620
+ return argv;
4431
4621
  }
4432
- function listLeftovers(configPath, scope) {
4433
- const where = workspacesDir(configPath, scope);
4434
- let names;
4622
+ async function endSandboxCommand(runtime, state, jobDir, warn) {
4435
4623
  try {
4436
- names = readdirSync(where);
4437
- } catch {
4438
- return [];
4439
- }
4440
- const found = [];
4441
- for (const name of names) {
4442
- if (!name.startsWith(".") && (!WORK_ITEM_ID2.test(name) || readItemDisk(join(where, name)) !== void 0)) continue;
4443
- try {
4444
- found.push({ path: join(where, name), mtimeMs: lstatSync(join(where, name)).mtimeMs });
4445
- } catch {
4624
+ const sandboxed = await inSandbox(runtime, state);
4625
+ for (let reads = 0; reads < MAX_TAIL_READS2; reads++) {
4626
+ const offset = commandLogOffset(jobDir, state.commandLogBase);
4627
+ const probe = await probeSandboxCommand(runtime, state, offset);
4628
+ if (probe === void 0) break;
4629
+ appendCommandLog(jobDir, probe.bytes);
4630
+ if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) break;
4446
4631
  }
4632
+ await sandboxed.host.stop(state.container.container);
4633
+ } catch (err) {
4634
+ warn(`ending the build's command in its sandbox failed: ${describeCallFailure(err)} \xB7 continuing`);
4447
4635
  }
4448
- return found;
4449
- }
4450
-
4451
- // src/runner/workspace/restore.ts
4452
- var DISK_REASON_TEXT = {
4453
- no_kept_disk: "no disk is kept for the work item on this host",
4454
- other_image: "the kept disk was made on another image",
4455
- other_disk_size: "the kept disk was made with another runner.build.disk size",
4456
- disk_not_restored: "the kept disk could not be copied",
4457
- checkout_not_switched: "the kept checkout could not be switched to the branch",
4458
- build_type_cannot_restore: "this build type cannot restore a disk",
4459
- fresh_requested: "a member asked for a fresh disk",
4460
- holder_busy: "the machine that holds its disk was busy"
4461
- };
4462
- function itemDiskDecision(input) {
4463
- if (input.fresh === true) return { state: { restored: false, reason: "fresh_requested" } };
4464
- if (!input.builtin) return { state: { restored: false, reason: "build_type_cannot_restore" } };
4465
- let kept;
4466
- try {
4467
- kept = readItemDisk(itemDiskDir(input.configPath, input.organizationId, input.workItemId));
4468
- } catch {
4469
- kept = void 0;
4470
- }
4471
- if (kept === void 0) return { state: { restored: false, reason: input.heldElsewhere === true ? "holder_busy" : "no_kept_disk" } };
4472
- if (kept.record.repository !== input.repository) return { state: { restored: false, reason: "no_kept_disk" } };
4473
- if (kept.record.imageId !== input.imageId) return { state: { restored: false, reason: "other_image" } };
4474
- if (kept.record.diskBytes !== input.diskBytes) return { state: { restored: false, reason: "other_disk_size" } };
4475
- if (kept.record.kind !== input.backend) return { state: { restored: false, reason: "disk_not_restored" } };
4476
- return { kept, state: { restored: true, fromClaimId: kept.record.claimId } };
4477
4636
  }
4478
- async function discardItemDisk(kept, removeVolume, onError) {
4637
+ async function runCliInSandbox(input, argv, logName) {
4638
+ const { job } = input;
4639
+ const own = job.sandbox;
4479
4640
  try {
4480
- if (kept.record.kind === "volume" && kept.record.volume !== void 0) {
4481
- const removed = await removeVolume(kept.record.volume);
4482
- if (removed !== void 0 && removed.status !== 0 && !/no such volume/i.test(removed.stderr ?? "")) {
4483
- onError(`could not remove the volume ${kept.record.volume} of the kept disk: ${removed.stderr?.trim() ?? `exit ${removed.status}`}`);
4484
- return;
4485
- }
4486
- }
4487
- rmSync(kept.dir, { recursive: true, force: true });
4641
+ const sandboxed = await inSandbox(input.runtime, own);
4642
+ const state = own.container;
4643
+ const env = buildEnvironment({
4644
+ config: claimCalls(input.config, job),
4645
+ item: job.item,
4646
+ ...job.queue === "run" ? { run: true } : {},
4647
+ itemKey: job.itemKey,
4648
+ branch: job.branch,
4649
+ repository: state.repository,
4650
+ repositories: job.repositories,
4651
+ hostEnv: input.runtime.hostEnv,
4652
+ agents: input.runtime.agents,
4653
+ job,
4654
+ containerEnv: state.containerEnv ?? {},
4655
+ unpinnedRuntimes: state.unpinnedRuntimes,
4656
+ missingSetupScripts: state.missingSetupScripts,
4657
+ miseVariables: state.miseVariables,
4658
+ builtIns: true,
4659
+ identity: job.commitIdentity
4660
+ });
4661
+ const result = await sandboxed.host.exec(
4662
+ { container: state.container, user: state.agentUser ?? ADDED_USER, workdir: job.workdir ?? CHECKOUT_DIR, env, argv },
4663
+ { logPath: join(input.jobDir, logName) }
4664
+ );
4665
+ return result.status === 0 ? { ok: true } : { ok: false, reason: `exit ${result.status}` };
4488
4666
  } catch (err) {
4489
- onError(`could not remove the kept disk of work item ${kept.record.workItemId}: ${describeCallFailure(err)}`);
4667
+ return { ok: false, reason: describeCallFailure(err) };
4490
4668
  }
4491
4669
  }
4492
-
4493
- // src/runner/saved-state/provision.ts
4494
- var isRetryFresh = (value) => typeof value === "object" && value !== null && value.retryFresh === true;
4495
- async function provisionWithFreshFallback(input) {
4496
- const first = await input.attempt(true);
4497
- if (!isRetryFresh(first)) return first;
4498
- input.log(`${first.reason}; provisioning the build again from a fresh disk`);
4499
- if (first.itemDisk === void 0) return await input.attempt(false);
4500
- const second = await input.attempt(true, first.itemDisk);
4501
- if (!isRetryFresh(second)) return second;
4502
- input.log(`${second.reason}; provisioning the build again from a fresh disk`);
4503
- return await input.attempt(false, first.itemDisk);
4504
- }
4505
- function keptDiskPlan() {
4506
- return { keyCommits: {}, noSave: "it starts from the work item's kept disk" };
4507
- }
4508
- function clockOf(now) {
4509
- return now ?? (() => /* @__PURE__ */ new Date());
4510
- }
4511
- function taggedLog(log, tag) {
4512
- return (line2) => log.line(`${tag} ${line2}`);
4513
- }
4514
- function uidFromProbe(probe) {
4515
- const uid = Number(probe.stdout);
4516
- return probe.status === 0 && Number.isInteger(uid) ? uid : null;
4517
- }
4518
- function agentUidOf(probe) {
4519
- return uidFromProbe(probe) ?? void 0;
4670
+ function syncInSandbox(input) {
4671
+ return runCliInSandbox(input, ["outerlayer", "sync"], "sync.log");
4520
4672
  }
4521
- function userIdChangedReason(savedUid, agentUid) {
4522
- if (savedUid === void 0 || agentUid === savedUid) return void 0;
4523
- return `the build user's id is ${agentUid ?? "unknown"} and it was ${savedUid} when the state was saved`;
4673
+ function recordProofsInSandbox(input) {
4674
+ return runCliInSandbox(input, [...RECORD_PROOFS_ARGV], PROOFS_LOG);
4524
4675
  }
4525
- function setupInputFor(input) {
4526
- const minutes = input.setupMinutes ?? DEFAULT_SETUP_MINUTES;
4527
- return {
4528
- timeoutMs: minutes * 6e4,
4529
- limitMinutes: minutes,
4530
- restored: input.plan.restore !== void 0 || input.fromItemDisk === true,
4531
- ...input.fromItemDisk === true ? { fromItemDisk: true } : {},
4532
- keyCommits: input.plan.keyCommits,
4533
- ...input.withoutBuildVariables === true ? { withoutBuildVariables: true } : {},
4534
- // A plan has a key exactly when the build may save a state. A build that starts from one is not asked to save.
4535
- ...input.plan.key === void 0 ? {} : { saveState: input.saveState }
4536
- };
4676
+ function savedStateBase(report) {
4677
+ const digest = report.recipe === "default" ? report.base?.digest : void 0;
4678
+ return digest === void 0 ? void 0 : { digest };
4537
4679
  }
4538
- async function saveAndPrune(input) {
4539
- const made = await input.save();
4540
- if (!made.ok) return made;
4541
- input.log(`saved state ${made.record.id}`);
4542
- await removeSuperseded({ made: made.record, records: readRecords(input.dir), deps: { dir: input.dir, removeVolume: input.removeVolume }, log: input.log });
4543
- return { ok: true };
4680
+ function savedStateMaxOf(config) {
4681
+ return config.housekeeping?.savedStateMax ?? DEFAULT_SAVED_STATE_MAX_BYTES;
4544
4682
  }
4545
- async function discardUnusableState(input) {
4546
- try {
4547
- await removeSavedState(input.record, { dir: input.dir, removeVolume: input.removeVolume });
4548
- } catch (err) {
4549
- input.onError(`could not remove saved state ${input.record.id}: ${describeCallFailure(err)}`);
4683
+ async function planSavedState(input) {
4684
+ const { config, runtime, log } = input;
4685
+ const refused2 = savingRefusedBecause({ universalImage: input.base !== void 0, fleet: config.fleet === true, savedStateMax: savedStateMaxOf(config) });
4686
+ if (refused2 !== void 0 || input.base === void 0) {
4687
+ const why = refused2 ?? "its image comes from a devcontainer file";
4688
+ log(`this build saves no state: ${why}`);
4689
+ return { keyCommits: {}, noSave: why };
4550
4690
  }
4551
- return { retryFresh: true, reason: `saved state ${input.record.id} could not be used: ${input.reason}` };
4552
- }
4553
- function settleFailedSteps(steps, on) {
4554
- return steps.restoreFailed === true ? on.giveUp(steps.reason, steps.fromItemDisk === true) : on.bail(steps.reason, steps.aborted === true);
4691
+ const root = runnerDir(config.configPath);
4692
+ const repositories = [];
4693
+ const keyCommits = {};
4694
+ for (const entry of input.repositories) {
4695
+ let token;
4696
+ try {
4697
+ token = entry.primary ? await input.tokens.token("read") : await input.tokens.token("read", entry.repository);
4698
+ } catch (err) {
4699
+ return unread(`the read token for ${entry.repository} could not be had: ${describeCallFailure(err)}`, log);
4700
+ }
4701
+ const [owner = "", name = ""] = entry.repository.split("/");
4702
+ const read = await runtime.readKeyFiles({
4703
+ remote: runtime.remoteFor?.(entry.repository) ?? `${runtime.github.gitUrl}/${entry.repository}.git`,
4704
+ dir: join(root, "key-files", owner, name),
4705
+ paths: keyFilePaths(),
4706
+ token
4707
+ });
4708
+ if (!read.ok) return unread(read.reason, log);
4709
+ keyCommits[entry.repository] = read.commit;
4710
+ repositories.push({ repository: entry.repository, commit: read.commit, files: read.files });
4711
+ }
4712
+ const key = stateKey({
4713
+ factory: keyRunnerFactory(config),
4714
+ repositories,
4715
+ imageDigest: input.base.digest,
4716
+ diskBytes: input.diskBytes
4717
+ });
4718
+ const found = lookup(readRecords(savedStatesDir(root)), key, input.now.getTime());
4719
+ if (found.kind !== "none" && found.record.kind === "tar") {
4720
+ try {
4721
+ await removeSavedState(found.record, { dir: savedStatesDir(root), removeVolume: async () => void 0 });
4722
+ log(`saved state ${found.record.id} is a tar file, which this runner no longer restores; removed it; starting from a fresh disk`);
4723
+ } catch (err) {
4724
+ log(`saved state ${found.record.id} is a tar file, which this runner no longer restores, and it could not be removed: ${describeCallFailure(err)}; starting from a fresh disk`);
4725
+ }
4726
+ return { key, keyCommits };
4727
+ }
4728
+ if (found.kind === "usable" && input.allowRestore) {
4729
+ log(`starting from saved state ${found.record.id}, saved ${formatDuration(found.ageMs)} ago`);
4730
+ return { key, keyCommits, restore: { record: found.record, ageMs: found.ageMs } };
4731
+ }
4732
+ if (found.kind === "expired") {
4733
+ log(`saved state ${found.record.id} is ${formatDuration(found.ageMs)} old, past the limit of ${SAVED_STATE_MAX_AGE_MS / 864e5} days; starting from a fresh disk`);
4734
+ } else if (found.kind === "changed") {
4735
+ log(`the setup script or version files changed on the default branch since saved state ${found.record.id}: ${found.files.join(", ")}; starting from a fresh disk`);
4736
+ } else if (found.kind === "none") {
4737
+ log("no saved state matches this build's key; starting from a fresh disk");
4738
+ }
4739
+ return { key, keyCommits };
4555
4740
  }
4556
- async function discardUnusableItemDisk(input) {
4557
- await discardItemDisk(input.kept, input.removeVolume, input.onError);
4558
- return { retryFresh: true, reason: `the kept disk of claim ${shortClaim(input.kept.record.claimId)} could not be used: ${input.reason}`, itemDisk: input.why };
4741
+ function unread(reason, log) {
4742
+ log(`this build starts from no saved state and saves none: ${reason}`);
4743
+ return { keyCommits: {}, noSave: `its key files could not be read: ${reason}` };
4559
4744
  }
4560
4745
  var FREEZE_DEADLINE_SECONDS = 12 * 60;
4561
4746
  function freezeScript() {
@@ -4578,7 +4763,7 @@ function freezeScript() {
4578
4763
  "exit 5"
4579
4764
  ].join("\n");
4580
4765
  }
4581
- var failureText = (status, stderr) => {
4766
+ var failureText2 = (status, stderr) => {
4582
4767
  const line2 = stderr.trim().split("\n").at(-1) ?? "";
4583
4768
  return `exit ${status}${line2 === "" ? "" : `: ${line2}`}`;
4584
4769
  };
@@ -4621,7 +4806,7 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
4621
4806
  const first = await Promise.race([frozen, ended]);
4622
4807
  if (first !== "frozen") {
4623
4808
  stdin.end();
4624
- return { ok: false, reason: "error" in first ? `the guest could not freeze /build: ${first.error}` : `the guest could not freeze /build (${failureText(first.status, stderr)})` };
4809
+ return { ok: false, reason: "error" in first ? `the guest could not freeze /build: ${first.error}` : `the guest could not freeze /build (${failureText2(first.status, stderr)})` };
4625
4810
  }
4626
4811
  return {
4627
4812
  ok: true,
@@ -4632,8 +4817,8 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
4632
4817
  const result = await ended;
4633
4818
  if ("error" in result) return { ok: false, reason: `the freeze command's connection failed: ${result.error}` };
4634
4819
  if (thawedBeforeAsked) return { ok: false, reason: "the freeze command thawed /build before the runner asked" };
4635
- if (!thawedAfterAsked) return { ok: false, reason: `the freeze command did not say it thawed /build (${failureText(result.status, stderr)})` };
4636
- if (result.status !== 0) return { ok: false, reason: `the freeze command ended badly after it thawed /build (${failureText(result.status, stderr)})` };
4820
+ if (!thawedAfterAsked) return { ok: false, reason: `the freeze command did not say it thawed /build (${failureText2(result.status, stderr)})` };
4821
+ if (result.status !== 0) return { ok: false, reason: `the freeze command ended badly after it thawed /build (${failureText2(result.status, stderr)})` };
4637
4822
  return { ok: true };
4638
4823
  }
4639
4824
  }
@@ -4642,7 +4827,7 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
4642
4827
 
4643
4828
  // src/runner/saved-state/microvm.ts
4644
4829
  var HELPER_COPY_NAME = "saved-state.ext4";
4645
- var failureText2 = (status, stderr) => {
4830
+ var failureText3 = (status, stderr) => {
4646
4831
  const line2 = stderr.trim().split("\n").at(-1) ?? "";
4647
4832
  return `exit ${status}${line2 === "" ? "" : `: ${line2}`}`;
4648
4833
  };
@@ -4681,9 +4866,9 @@ async function saveMicroVmDiskState(input) {
4681
4866
  const { savedStateMax } = input.limits;
4682
4867
  if (savedStateMax === "off") return { ok: false, reason: "runner.housekeeping.savedStateMax is off" };
4683
4868
  const stopped = await input.guest.run(["/bin/sh", "-c", guestStopScript()]);
4684
- if (stopped.status !== 0) return { ok: false, reason: `the guest's processes could not be stopped (${failureText2(stopped.status, stopped.stderr)})` };
4869
+ if (stopped.status !== 0) return { ok: false, reason: `the guest's processes could not be stopped (${failureText3(stopped.status, stopped.stderr)})` };
4685
4870
  const flushed = await input.guest.run(["sync", "-f", "/build"]);
4686
- if (flushed.status !== 0) return { ok: false, reason: `the build disk could not be flushed (${failureText2(flushed.status, flushed.stderr)})` };
4871
+ if (flushed.status !== 0) return { ok: false, reason: `the build disk could not be flushed (${failureText3(flushed.status, flushed.stderr)})` };
4687
4872
  const allocated = allocatedBytes(input.buildDisk);
4688
4873
  if (allocated > savedStateMax) {
4689
4874
  return { ok: false, reason: `the build disk holds ${formatBytes(allocated)}, more than the ${formatBytes(savedStateMax)} of runner.housekeeping.savedStateMax` };
@@ -7449,129 +7634,6 @@ function buildReleaseLog(jobDirPath, endStage, scrub) {
7449
7634
  if (parts.length === 0) return void 0;
7450
7635
  return capRunnerLog(scrub(parts.join("\n")) + "\n");
7451
7636
  }
7452
- var SAVED_FOLDERS = ["home", "work"];
7453
- function stopProcessesScript() {
7454
- return [
7455
- "me=$$; uid=$(id -u); n=0; pass=0",
7456
- "while [ $pass -lt 25 ]; do",
7457
- ' left=""',
7458
- " for p in /proc/[0-9]*; do",
7459
- " pid=${p##*/}",
7460
- ' [ "$pid" = "$me" ] && continue',
7461
- ` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
7462
- ' case "$info" in *"S=Z"*) continue ;; esac',
7463
- // The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
7464
- ' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
7465
- " done",
7466
- ' [ -z "$left" ] && break',
7467
- ' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
7468
- " pass=$((pass+1))",
7469
- " sleep 0.2",
7470
- "done",
7471
- '[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
7472
- 'echo "stopped $n"'
7473
- ].join("\n");
7474
- }
7475
- var HELPER_FLAGS = [
7476
- "--rm",
7477
- "--network",
7478
- "none",
7479
- "--cap-drop",
7480
- "ALL",
7481
- "--cap-add",
7482
- "CHOWN",
7483
- "--cap-add",
7484
- "DAC_OVERRIDE",
7485
- "--cap-add",
7486
- "DAC_READ_SEARCH",
7487
- "--cap-add",
7488
- "FOWNER",
7489
- "--security-opt",
7490
- "no-new-privileges",
7491
- "--read-only",
7492
- "--user",
7493
- "0",
7494
- "--pids-limit",
7495
- "256"
7496
- ];
7497
- function helperArgs(input) {
7498
- return [
7499
- ...HELPER_FLAGS,
7500
- "--label",
7501
- `${RUNNER_LABEL}=${input.runnerId}`,
7502
- "--mount",
7503
- `type=volume,src=${input.from},dst=/from,readonly`,
7504
- "--mount",
7505
- `type=volume,src=${input.to},dst=/to`,
7506
- "--entrypoint",
7507
- "sh",
7508
- input.image,
7509
- "-c",
7510
- input.script
7511
- ];
7512
- }
7513
- var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
7514
- var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
7515
- function failureText3(result) {
7516
- const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
7517
- return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
7518
- }
7519
- async function saveContainerState(input) {
7520
- const { host } = input;
7521
- const id = randomBytes(6).toString("hex");
7522
- const volume = `outerlayer-saved-${id}`;
7523
- const record = {
7524
- id,
7525
- kind: "volume",
7526
- volume,
7527
- savedAt: input.now.toISOString(),
7528
- bytes: 0,
7529
- baseKey: input.key.baseKey,
7530
- key: input.key.key,
7531
- fileHashes: input.key.fileHashes,
7532
- ...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
7533
- complete: false
7534
- };
7535
- const discard = async (reason) => {
7536
- const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
7537
- if (removed.status !== 0) {
7538
- input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
7539
- return { ok: false, reason };
7540
- }
7541
- deleteRecord(input.dir, id);
7542
- return { ok: false, reason };
7543
- };
7544
- try {
7545
- const noEnv = { literals: {}, passed: {} };
7546
- const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
7547
- if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText3(stopped)})` };
7548
- input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
7549
- const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
7550
- if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText3(flushed)})` };
7551
- writeRecord2(input.dir, record);
7552
- const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
7553
- if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText3(created)})`);
7554
- const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
7555
- if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText3(copied)})`);
7556
- const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
7557
- const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
7558
- writeRecord2(input.dir, done);
7559
- return { ok: true, record: done };
7560
- } catch (err) {
7561
- return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
7562
- }
7563
- }
7564
- async function restoreContainerState(input) {
7565
- try {
7566
- const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
7567
- if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText3(copied)})` };
7568
- return { ok: true };
7569
- } catch (err) {
7570
- return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
7571
- }
7572
- }
7573
-
7574
- // src/runner/workspace/keep.ts
7575
7637
  function recordOf(input, kind, bytes, volume) {
7576
7638
  return {
7577
7639
  workItemId: input.workItemId,
@@ -7639,16 +7701,16 @@ async function keepContainerDisk(input) {
7639
7701
  const dropNew = async () => {
7640
7702
  if (!created) return;
7641
7703
  const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
7642
- if (removed.status !== 0) input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })})`);
7704
+ if (removed.status !== 0) input.log(`the volume ${volume} could not be removed (${failureText({ status: removed.status, stderr: removed.stderr })})`);
7643
7705
  };
7644
7706
  try {
7645
7707
  const made = await host.createVolume(volume, { [WORKSPACE_LABEL]: input.workItemId, [RUNNER_LABEL]: input.runnerId });
7646
- if (made.status !== 0) return { ok: false, reason: `the volume for the kept disk could not be made (${failureText3(made)})` };
7708
+ if (made.status !== 0) return { ok: false, reason: `the volume for the kept disk could not be made (${failureText(made)})` };
7647
7709
  created = true;
7648
7710
  const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
7649
7711
  if (copied.status !== 0) {
7650
7712
  await dropNew();
7651
- return { ok: false, reason: `the copy into the kept disk failed (${failureText3(copied)})` };
7713
+ return { ok: false, reason: `the copy into the kept disk failed (${failureText(copied)})` };
7652
7714
  }
7653
7715
  const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
7654
7716
  const record = recordOf(input, "volume", Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, volume);
@@ -7657,7 +7719,7 @@ async function keepContainerDisk(input) {
7657
7719
  const removed = await host.removeVolume(older.record.volume);
7658
7720
  if (removed.status !== 0 && !/no such volume/i.test(removed.stderr)) {
7659
7721
  await dropNew();
7660
- return { ok: false, reason: `the work item's older disk could not be replaced: its volume ${older.record.volume} was not removed (${failureText3(removed)})` };
7722
+ return { ok: false, reason: `the work item's older disk could not be replaced: its volume ${older.record.volume} was not removed (${failureText(removed)})` };
7661
7723
  }
7662
7724
  }
7663
7725
  staging = stagingItemDiskDir(dir);
@@ -7671,6 +7733,48 @@ async function keepContainerDisk(input) {
7671
7733
  if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
7672
7734
  }
7673
7735
  }
7736
+ function saidBy(err) {
7737
+ return err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err);
7738
+ }
7739
+ async function keepSandboxDisk2(input) {
7740
+ let dir;
7741
+ try {
7742
+ dir = itemDiskDir(input.configPath, input.organizationId, input.workItemId);
7743
+ } catch (err) {
7744
+ return { ok: false, reason: describeCallFailure(err) };
7745
+ }
7746
+ const { client: client2, snapshotName } = input;
7747
+ const dropNew = async () => {
7748
+ try {
7749
+ await client2.removeSnapshot(snapshotName);
7750
+ } catch (err) {
7751
+ input.log(`the snapshot ${snapshotName} could not be removed (${saidBy(err)}); remove it at the sandbox provider`);
7752
+ }
7753
+ };
7754
+ let staging;
7755
+ try {
7756
+ const saved = await client2.saveSnapshot(input.sandboxId, snapshotName);
7757
+ const record = { ...recordOf(input, "sandbox", 0, input.buildVolume), snapshot: snapshotName, storedBytes: saved.storedBytes };
7758
+ const older = readItemDisk(dir);
7759
+ if (older?.record.snapshot !== void 0) {
7760
+ try {
7761
+ await client2.removeSnapshot(older.record.snapshot);
7762
+ } catch (err) {
7763
+ await dropNew();
7764
+ return { ok: false, reason: `the work item's older disk could not be replaced: its snapshot ${older.record.snapshot} was not removed (${saidBy(err)})` };
7765
+ }
7766
+ }
7767
+ staging = stagingItemDiskDir(dir);
7768
+ finish(input, dir, staging, record);
7769
+ staging = void 0;
7770
+ return { ok: true, record };
7771
+ } catch (err) {
7772
+ await dropNew();
7773
+ return { ok: false, reason: `the sandbox could not be saved: ${saidBy(err)}` };
7774
+ } finally {
7775
+ if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
7776
+ }
7777
+ }
7674
7778
 
7675
7779
  // src/runner/workspace/finish.ts
7676
7780
  function noDiskReason(job, config) {
@@ -7735,6 +7839,31 @@ function containerDiskKeeper(input) {
7735
7839
  input.log.line(`${input.tag} kept the build disk for the work item: ${formatBytes(kept.record.bytes)} allocated`);
7736
7840
  };
7737
7841
  }
7842
+ function sandboxDiskKeeper(input) {
7843
+ const state = input.job.sandbox;
7844
+ if (state === void 0) return void 0;
7845
+ const refused2 = noDiskReason(input.job, input.config);
7846
+ if (refused2 !== void 0) {
7847
+ if (input.job.workItemId !== void 0) input.log.line(`${input.tag} the build disk is not kept: ${refused2}`);
7848
+ return void 0;
7849
+ }
7850
+ if (state.sandboxId === "" || state.diskKey === void 0 || state.container.imageId === void 0) return void 0;
7851
+ const { imageId, diskBytes } = state.diskKey;
7852
+ const { sandboxId } = state;
7853
+ const keep = async () => {
7854
+ const kept = await keepSandboxDisk2({ ...target(input, imageId, diskBytes), client: input.client(), sandboxId, snapshotName: input.snapshotName, buildVolume: state.container.volume });
7855
+ if (!kept.ok) throw new Error(kept.reason);
7856
+ input.log.line(`${input.tag} kept the build disk for the work item: snapshot ${kept.record.snapshot} of sandbox ${sandboxId}, ${formatBytes(kept.record.storedBytes)} stored`);
7857
+ };
7858
+ return {
7859
+ container: state.container.container,
7860
+ volume: state.container.volume,
7861
+ imageId: state.container.imageId,
7862
+ runnerId: runnerLabelValue(input.config.configPath),
7863
+ keep,
7864
+ warn: (text2) => input.log.error(`${input.tag} ${text2}`)
7865
+ };
7866
+ }
7738
7867
  var SESSION_TAR = "session.tar";
7739
7868
  async function guestOf(job, runtime) {
7740
7869
  if (job.microvm !== void 0) {
@@ -9370,7 +9499,12 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
9370
9499
  let cleanedUp = true;
9371
9500
  tunnelRecord = sandboxBuild.runtime.tunnels.get(sandboxBuild.state.buildId)?.record();
9372
9501
  try {
9373
- await cleanupSandbox(sandboxBuild.runtime, sandboxBuild.state);
9502
+ const sandboxRuntime = sandboxBuild.runtime;
9503
+ await cleanupSandbox(
9504
+ sandboxRuntime,
9505
+ sandboxBuild.state,
9506
+ sandboxDiskKeeper({ job, config, runtime: sandboxRuntime, now, log, tag, jobDirPath, client: () => sandboxRuntime.sandbox.client(), snapshotName: itemSnapshotName(runnerLabelValue(config.configPath)) })
9507
+ );
9374
9508
  log.line(`${tag} ended sandbox ${sandboxBuild.state.sandboxId}`);
9375
9509
  } catch (err) {
9376
9510
  cleanedUp = false;
@@ -9871,8 +10005,12 @@ var bytesOf = (entries) => entries.reduce((sum, entry) => sum + entry.kept.recor
9871
10005
  function allEntries(configPath) {
9872
10006
  return storeScopes(configPath).flatMap((scope) => listItemDisks(configPath, scope).map((kept) => ({ scope, kept })));
9873
10007
  }
9874
- async function removeEntry(entry, removeVolume) {
10008
+ async function removeEntry(entry, removeVolume, removeSnapshot) {
9875
10009
  const { record, dir } = entry.kept;
10010
+ if (record.snapshot !== void 0) {
10011
+ if (removeSnapshot === void 0) throw new Error("no sandbox provider is set");
10012
+ await removeSnapshot(record.snapshot);
10013
+ }
9876
10014
  if (record.kind === "volume" && record.volume !== void 0) {
9877
10015
  if (removeVolume === void 0) throw new Error("Docker is not available");
9878
10016
  await removeVolume(record.volume);
@@ -9909,7 +10047,7 @@ async function removeItemDisks(input) {
9909
10047
  const gone = /* @__PURE__ */ new Set();
9910
10048
  const remove = async (entry, why) => {
9911
10049
  try {
9912
- await removeEntry(entry, input.removeVolume);
10050
+ await removeEntry(entry, input.removeVolume, input.removeSnapshot);
9913
10051
  gone.add(entry);
9914
10052
  freed += entry.kept.record.bytes;
9915
10053
  input.log(`housekeeping: removed the kept disk of work item ${entry.kept.record.workItemId} (${formatBytes(entry.kept.record.bytes)}), ${why}`);
@@ -9960,7 +10098,7 @@ async function removeItemDisks(input) {
9960
10098
  }
9961
10099
  async function freeItemDisksUntil(input) {
9962
10100
  let freed = 0;
9963
- for (const entry of oldestFirst2(allEntries(input.configPath))) {
10101
+ for (const entry of oldestFirst2(allEntries(input.configPath)).filter((candidate) => candidate.kept.record.kind !== "sandbox")) {
9964
10102
  if (input.enough()) break;
9965
10103
  try {
9966
10104
  await removeEntry(entry, input.removeVolume);
@@ -10221,6 +10359,7 @@ async function runHousekeeping(options) {
10221
10359
  limits: { max: options.workspaceMax ?? Number.POSITIVE_INFINITY, orgMax: options.workspaceOrgMax ?? Number.POSITIVE_INFINITY, keepDays: options.workspaceKeepDays ?? DEFAULT_WORKSPACE_KEEP_DAYS },
10222
10360
  nowMs,
10223
10361
  removeVolume: options.docker === void 0 ? void 0 : (name) => options.docker.removeVolume(name),
10362
+ removeSnapshot: options.removeSnapshot,
10224
10363
  ...options.itemStates === void 0 ? {} : { itemStates: options.itemStates },
10225
10364
  log: (line2) => options.log.line(line2),
10226
10365
  error: (line2) => options.log.error(line2)