@outerlayer/cli 0.38.0-next.20261111004457 → 0.38.0-next.20261111024231
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/{agent-setup-OU7CGYSN.js → agent-setup-T2VITZUY.js} +9 -9
- package/dist/{agent-subcommand-GG4PSHYT.js → agent-subcommand-KV3JO5YD.js} +1 -1
- package/dist/{bridge-MVW5RNQD.js → bridge-7MCKZXCR.js} +4 -4
- package/dist/{build-D4Y3LBPO.js → build-N2MRLYFW.js} +10 -10
- package/dist/{build-disk-VDMWH3C3.js → build-disk-IJZYZ2K5.js} +8 -8
- package/dist/build-info.json +1 -1
- package/dist/{builder-6L5ZMXBE.js → builder-ATKIRCDJ.js} +7 -7
- package/dist/{caller-FFSXVMLB.js → caller-OWXJDYPI.js} +10 -10
- package/dist/{check-U5DOQU64.js → check-F7RN65GG.js} +40 -40
- package/dist/{chunk-IWJFARST.js → chunk-2FUJ5PSV.js} +1 -1
- package/dist/{chunk-EKV7QATX.js → chunk-2PRDLE4L.js} +1 -1
- package/dist/{chunk-IZHI643L.js → chunk-3VEPVZV4.js} +5 -5
- package/dist/{chunk-3IL6H7A4.js → chunk-4HUYH65P.js} +1 -1
- package/dist/{chunk-4S6ML2YX.js → chunk-4YAFIOIV.js} +17 -2
- package/dist/{chunk-GELUV567.js → chunk-5NJ367OD.js} +2 -2
- package/dist/{chunk-R6UNNKV3.js → chunk-64ROJ5LD.js} +1 -1
- package/dist/{chunk-5U4IKVF5.js → chunk-6T2X2JI6.js} +3 -3
- package/dist/{chunk-7ILAGC3U.js → chunk-6U3UTAWQ.js} +1 -1
- package/dist/{chunk-WVJDXOIG.js → chunk-7G2HRTSG.js} +3 -3
- package/dist/{chunk-2PYQ6BZJ.js → chunk-7HYXJVU6.js} +1 -1
- package/dist/{chunk-X7BVDOS2.js → chunk-7LWASV77.js} +3 -3
- package/dist/{chunk-MX5XVAOI.js → chunk-7U43B2UB.js} +1 -1
- package/dist/{chunk-JASHMKDN.js → chunk-AB4GMYIH.js} +1 -1
- package/dist/{chunk-NBJPY3OC.js → chunk-ASCXEXYJ.js} +5 -5
- package/dist/{chunk-UF3X5ECJ.js → chunk-BY2YRJSC.js} +2 -2
- package/dist/{chunk-U43K2JS5.js → chunk-ECMQNH2H.js} +2 -2
- package/dist/{chunk-YWQ3S3F5.js → chunk-EWGHEMEO.js} +2 -2
- package/dist/{chunk-F5ONFHHU.js → chunk-FB2UT5XA.js} +31 -27
- package/dist/{chunk-BZSGU2WC.js → chunk-FT5NDSSG.js} +4 -4
- package/dist/{chunk-MV5WJEIB.js → chunk-GGKXYHT5.js} +2 -2
- package/dist/{chunk-7NRULS6C.js → chunk-GPP5EZV7.js} +3 -3
- package/dist/{chunk-2ZPXDOCU.js → chunk-I5C667TJ.js} +9 -9
- package/dist/{chunk-QB5MU2WS.js → chunk-IB2UHXZ7.js} +2 -2
- package/dist/{chunk-EN3OWMED.js → chunk-ICW75TS5.js} +4 -4
- package/dist/{chunk-CKA4JQKR.js → chunk-IEX5FXEU.js} +2 -2
- package/dist/{chunk-OZDUVL5L.js → chunk-JPAO6HRN.js} +2 -2
- package/dist/{chunk-DDEJP7YT.js → chunk-K6KUWUUT.js} +1 -1
- package/dist/{chunk-JNEK36QK.js → chunk-KGJAYD7F.js} +1239 -1100
- package/dist/{chunk-OWBNADRQ.js → chunk-KM2IMFAG.js} +1 -1
- package/dist/{chunk-3AHABZMA.js → chunk-LXREVZX7.js} +2 -2
- package/dist/{chunk-A6CCWLUP.js → chunk-M5DYOU5G.js} +3 -3
- package/dist/{chunk-YE3KPAT5.js → chunk-MPF7CVQM.js} +1 -1
- package/dist/{chunk-W5OAKC2W.js → chunk-NC6HUEMI.js} +9 -9
- package/dist/{chunk-P5XX7VZ6.js → chunk-NWSKP5VI.js} +2 -2
- package/dist/{chunk-4JKCTCCC.js → chunk-O53PVAUY.js} +4 -4
- package/dist/{chunk-D7OVS2LL.js → chunk-OBSWWJJ3.js} +1 -1
- package/dist/{chunk-ONZNFKCH.js → chunk-OBXRHBOM.js} +2 -2
- package/dist/{chunk-JP5D37RZ.js → chunk-PP7MPUX7.js} +3 -3
- package/dist/{chunk-JPO6RJEW.js → chunk-QM3M7QN3.js} +1 -1
- package/dist/{chunk-JRM35NTG.js → chunk-R35KRBTS.js} +5 -5
- package/dist/{chunk-NIQXLC6A.js → chunk-R7BS36HP.js} +1 -1
- package/dist/{chunk-J3KW7GPE.js → chunk-REZNW2YV.js} +6 -6
- package/dist/{chunk-LNTNFSRI.js → chunk-RPIYENP2.js} +2 -2
- package/dist/{chunk-6Z5AC4ZK.js → chunk-T5CN56UE.js} +1 -1
- package/dist/{chunk-WWGXWKLT.js → chunk-U5YHNSSI.js} +5 -5
- package/dist/{chunk-DNCF3XOC.js → chunk-W2H6JURI.js} +3 -3
- package/dist/{chunk-BYWVHMBR.js → chunk-XMIL73Y2.js} +1 -1
- package/dist/{chunk-JDXYBQBZ.js → chunk-XSVZFWZ4.js} +7 -7
- package/dist/{chunk-ZNQRFRBW.js → chunk-YFMVZBLO.js} +5 -5
- package/dist/{chunk-NCPZP6PQ.js → chunk-YFQIK6LZ.js} +7 -7
- package/dist/{chunk-5TODASD5.js → chunk-YI2XPK7A.js} +1 -1
- package/dist/{chunk-RI37FFPL.js → chunk-ZGCIFD5Y.js} +2 -2
- package/dist/{claude-launcher-GNRTFQMI.js → claude-launcher-ZC664P33.js} +1 -1
- package/dist/{cli-SNHD2D6X.js → cli-XWXX6PJD.js} +130 -130
- package/dist/{command-NTKAT7KB.js → command-D656K4AE.js} +35 -35
- package/dist/{config-LUIWZESD.js → config-U2OR7MQW.js} +9 -9
- package/dist/{connect-cmd-4M4DN6GO.js → connect-cmd-73BBTTU5.js} +7 -7
- package/dist/{container-start-BVEDUA7N.js → container-start-JFJYC4GQ.js} +33 -33
- package/dist/{context-show-cmd-DAMQJOAL.js → context-show-cmd-ETWKBHXZ.js} +8 -8
- package/dist/{control-plane-launch-SD45JYKF.js → control-plane-launch-JOH7FS5U.js} +7 -7
- package/dist/{demo-record-cmd-CFOXCR42.js → demo-record-cmd-GBHPIYJA.js} +5 -5
- package/dist/{demo-voice-cmd-4YZJPUIP.js → demo-voice-cmd-GUYFA2NG.js} +13 -13
- package/dist/{docker-2DLKQOK3.js → docker-RC4CFNKI.js} +7 -7
- package/dist/{doctor-VFCDAKM4.js → doctor-TW6VREAT.js} +35 -35
- package/dist/{emit-artifact-cmd-IWZDXJBY.js → emit-artifact-cmd-CQLCPM4R.js} +12 -12
- package/dist/{emit-criteria-cmd-UVCLLYEZ.js → emit-criteria-cmd-VM7AO3E3.js} +9 -9
- package/dist/{emit-demo-cmd-BHKNT36Y.js → emit-demo-cmd-3NSODQTO.js} +13 -13
- package/dist/{emit-finding-cmd-Y3XX26B4.js → emit-finding-cmd-4CRLSMOK.js} +9 -9
- package/dist/{emit-result-cmd-TTQ3MLY4.js → emit-result-cmd-GM43QMCM.js} +11 -11
- package/dist/{emit-step-cmd-EASDIM5Q.js → emit-step-cmd-J7KS2QXL.js} +9 -9
- package/dist/{exec-client-MEIRWXLR.js → exec-client-DY7673SD.js} +5 -5
- package/dist/find-unit-ZVHHFTL2.js +24 -0
- package/dist/{findings-cmd-KQOL6NJG.js → findings-cmd-25F44AB4.js} +9 -9
- package/dist/{fleet-readiness-R2BMWLXQ.js → fleet-readiness-Z4C6W7XN.js} +7 -7
- package/dist/{governing-source-ARSS7R7L.js → governing-source-IURJZXUR.js} +11 -11
- package/dist/{guest-init-UGBFOWNC.js → guest-init-4B5MMXIQ.js} +6 -6
- package/dist/{hook-fast-DC556GE2.js → hook-fast-LB3VTEIW.js} +1 -1
- package/dist/{host-checks-UTZBDSWV.js → host-checks-AQTZ7IKK.js} +31 -31
- package/dist/{pending-COVCIH4Q.js → host-key-ZWMN7VIH.js} +5 -5
- package/dist/{image-tag-RU4RHPIT.js → image-tag-4DKLSTO6.js} +5 -5
- package/dist/index.js +4 -4
- package/dist/{init-I5HVGOQC.js → init-P2PDGLTU.js} +31 -31
- package/dist/{init-QFTRSBBT.js → init-XYFDIEOA.js} +31 -31
- package/dist/{init-agent-TKQ67BS4.js → init-agent-EGNFBPM5.js} +31 -31
- package/dist/{init-cmd-I36LJ6YC.js → init-cmd-C2I22ZJA.js} +12 -12
- package/dist/{init-update-KG7V3VJF.js → init-update-H4LSR7H4.js} +10 -10
- package/dist/{install-cmd-66Z7TY52.js → install-cmd-BI4C3MGF.js} +7 -7
- package/dist/{logs-XWMVEAUP.js → logs-7DD6J3M6.js} +4 -4
- package/dist/{loop-YCOSJRLW.js → loop-KEGBBAV4.js} +39 -39
- package/dist/{machine-BJEAYCHT.js → machine-JK27TGT6.js} +31 -31
- package/dist/{mcp-serve-cmd-M7CX4RO3.js → mcp-serve-cmd-HK2ODMHU.js} +6 -6
- package/dist/{paths-GQ66KHNT.js → paths-EHWU5QWB.js} +4 -4
- package/dist/{host-key-CTRFLP2W.js → pending-K4A23JIC.js} +5 -5
- package/dist/{pidfile-RBG6FQ7K.js → pidfile-PG3L2CVV.js} +5 -5
- package/dist/{record-6IAZ26H3.js → record-PXCRTDQU.js} +5 -5
- package/dist/{relay-VEBGGPYV.js → relay-2MJY25BV.js} +2 -2
- package/dist/{relay-protocol-2AJCUAMJ.js → relay-protocol-VBCRA347.js} +2 -2
- package/dist/{run-cmd-EJPHCENL.js → run-cmd-F7ANZPSX.js} +9 -9
- package/dist/{runner-agents-P562WZJB.js → runner-agents-HZ5FIFPU.js} +1 -1
- package/dist/{runner-env-XPZ3FPQL.js → runner-env-5AM2XYEN.js} +11 -11
- package/dist/{runner-identity-TAJAY5M5.js → runner-identity-CL2KMFGC.js} +7 -7
- package/dist/{runner-key-I3M7GEJE.js → runner-key-CJ66VZE3.js} +31 -31
- package/dist/{session-sender-OIOE4QR2.js → session-sender-RSAHCLL6.js} +12 -12
- package/dist/{spec-cmd-2J43XP3L.js → spec-cmd-ED6DKB54.js} +10 -10
- package/dist/{state-NAZKB77R.js → state-MV53BATE.js} +5 -5
- package/dist/{status-BFRDAZQP.js → status-CZTCYZLU.js} +31 -31
- package/dist/{sync-cmd-KWTFVQCR.js → sync-cmd-PQZD77PE.js} +11 -11
- package/dist/{update-steps-LINCNPGV.js → update-steps-DCFEKNW2.js} +32 -32
- package/dist/{watch-3IBFF2SQ.js → watch-FAQXAAUA.js} +11 -11
- package/dist/{work-claim-cmd-C2WFSOKE.js → work-claim-cmd-XPXHRV74.js} +15 -15
- package/dist/{work-cmd-DO2QNBKA.js → work-cmd-ME7DWBI6.js} +9 -9
- package/dist/{work-comment-cmd-P5VJHCX7.js → work-comment-cmd-ZT4O56QR.js} +11 -11
- package/dist/{work-demo-cmd-UJBZLOMQ.js → work-demo-cmd-YV22MZXG.js} +9 -9
- package/dist/{work-logs-cmd-OVE5JUMH.js → work-logs-cmd-LUK7THB2.js} +9 -9
- package/dist/{work-needs-attention-cmd-QNCOUXMT.js → work-needs-attention-cmd-KTLWMBJV.js} +9 -9
- package/dist/{work-no-change-cmd-KQSONMJX.js → work-no-change-cmd-TWGW5ZOO.js} +9 -9
- package/dist/{work-open-pr-cmd-HBE74CZU.js → work-open-pr-cmd-VEM5MEUS.js} +10 -10
- package/dist/{work-pr-cmd-62BGKUJM.js → work-pr-cmd-M4G35ZIV.js} +9 -9
- package/dist/{work-proofs-cmd-SLUFPOEN.js → work-proofs-cmd-7EZ4QZUR.js} +10 -10
- package/dist/{work-proofs-record-6KGPMUHD.js → work-proofs-record-AY72YKNC.js} +16 -16
- package/dist/{work-stop-cmd-7WEHA365.js → work-stop-cmd-FX5QTUYJ.js} +9 -9
- package/package.json +1 -1
- package/dist/find-unit-2RZ4NCXQ.js +0 -24
|
@@ -4,37 +4,37 @@ import { itemsSentence, criteriaOf, hostItemsOf, loadCitationFinder, testFileMat
|
|
|
4
4
|
import { RUNNER_PROTOCOL } from './chunk-6XMC6R4W.js';
|
|
5
5
|
import { sha256OfFile, downloadPinned, VoiceDownloadError } from './chunk-2PGI6RDE.js';
|
|
6
6
|
import { zodProblem } from './chunk-I3LTIAOR.js';
|
|
7
|
-
import { hasStoredCredential, storedCredentialVariables, storeAgentCredential, isAgentName, credentialFromStdin } from './chunk-
|
|
7
|
+
import { hasStoredCredential, storedCredentialVariables, storeAgentCredential, isAgentName, credentialFromStdin } from './chunk-ICW75TS5.js';
|
|
8
8
|
import { upgradedStarterText } from './chunk-OUKSF3LK.js';
|
|
9
|
-
import { HELPER_PATH, HELPER_RECORD, DOCKERD_LOG } from './chunk-
|
|
10
|
-
import { writeDockerClientConfig } from './chunk-
|
|
11
|
-
import { senderPidfile } from './chunk-
|
|
12
|
-
import { installScrubContext } from './chunk-
|
|
9
|
+
import { HELPER_PATH, HELPER_RECORD, DOCKERD_LOG } from './chunk-7LWASV77.js';
|
|
10
|
+
import { writeDockerClientConfig } from './chunk-2PRDLE4L.js';
|
|
11
|
+
import { senderPidfile } from './chunk-NWSKP5VI.js';
|
|
12
|
+
import { installScrubContext } from './chunk-U5YHNSSI.js';
|
|
13
13
|
import { readsJobState, READ_JOB_STATE_FORMATS, jobStateFormatOf } from './chunk-ZVB57WI6.js';
|
|
14
|
-
import { runnerImageVersion } from './chunk-
|
|
15
|
-
import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey, configForJob } from './chunk-
|
|
14
|
+
import { runnerImageVersion } from './chunk-64ROJ5LD.js';
|
|
15
|
+
import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey, configForJob } from './chunk-JPAO6HRN.js';
|
|
16
16
|
import { receiveFiles } from './chunk-NN33FYXG.js';
|
|
17
|
-
import { readDefaultBranchFiles, readDefaultBranchHead, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-
|
|
18
|
-
import { FETCHED_DIR, toolsDockerfile, FETCHED_FFMPEG, FETCHED_GH, FETCHED_TINI, FETCHED_HEADLESS_SHELL, FETCHED_PLAYWRIGHT_FFMPEG, FETCHED_VOICE_DIR } from './chunk-
|
|
17
|
+
import { readDefaultBranchFiles, readDefaultBranchHead, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-YFQIK6LZ.js';
|
|
18
|
+
import { FETCHED_DIR, toolsDockerfile, FETCHED_FFMPEG, FETCHED_GH, FETCHED_TINI, FETCHED_HEADLESS_SHELL, FETCHED_PLAYWRIGHT_FFMPEG, FETCHED_VOICE_DIR } from './chunk-W2H6JURI.js';
|
|
19
19
|
import { isSameProcess, startTimeOf } from './chunk-JBJTC3UU.js';
|
|
20
20
|
import { currentCliBuild } from './chunk-OGFGZQA3.js';
|
|
21
|
-
import { vmExecArgs, runExec, execCapture, waitForAgent } from './chunk-
|
|
22
|
-
import { socketPathProblem, vsockLinks, linkVsockSockets, GUEST_CID, LEASE_LINK } from './chunk-
|
|
23
|
-
import { describeCallFailure, shortClaim, retrySend,
|
|
24
|
-
import { postJson, WorkCommandError, isSignatureRefusal, getJson } from './chunk-
|
|
25
|
-
import { isBuiltinContainer, isBuiltinVm, isBuiltinSandbox, keyRunnerFactory, runnerCredentialProblem, builtinKind,
|
|
26
|
-
import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-
|
|
27
|
-
import {
|
|
28
|
-
import { universalImagePending, RecipeError, writeSanitizedRecipe, classifyProperties, DOCKER_FEATURE } from './chunk-
|
|
21
|
+
import { vmExecArgs, runExec, execCapture, waitForAgent } from './chunk-7HYXJVU6.js';
|
|
22
|
+
import { socketPathProblem, vsockLinks, linkVsockSockets, GUEST_CID, LEASE_LINK } from './chunk-ZGCIFD5Y.js';
|
|
23
|
+
import { describeCallFailure, shortClaim, retrySend, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, formatClock, stageGovernedContext, keptContextTree, removeStagedContext, WORKFLOW_PERMISSION_REASON, refusalCode, SELF_GOVERNED, resolveGoverningSource, prefetchGovernedContext, controlPlaneTokenFailure, keepStagedContext, pinnedSourceOf, shimScript } from './chunk-R35KRBTS.js';
|
|
24
|
+
import { postJson, WorkCommandError, isSignatureRefusal, getJson } from './chunk-YFMVZBLO.js';
|
|
25
|
+
import { isBuiltinContainer, isBuiltinVm, isBuiltinSandbox, keyRunnerFactory, runnerCredentialProblem, builtinKind, optionalEgress, egressSummary, DEFAULT_WORKSPACE_KEEP_DAYS, DEFAULT_CHECK_TIME_LIMIT_MINUTES, DEFAULT_SETUP_MINUTES, allowHostProblem, systemClock, emptyEgressCounts, hostAllowed, sandboxSizeOf, networkOf, sandboxTimeLimitMinutes, imageBuildOf, DEFAULT_SAVED_STATE_MAX_BYTES } from './chunk-O53PVAUY.js';
|
|
26
|
+
import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-GGKXYHT5.js';
|
|
27
|
+
import { firstLine, writeJobFile, runnerLabelValue, readJobFile, spawnRun, RUNNER_LABEL, BUILD_LABEL, said, NETWORK_HELPER_MEMORY, createDockerHost, builderSpec, containerExecArgs, childEnvironment, readDu, withAnswerProtocol, JUDGE_TREE, MAX_REPORT_BYTES, parseSize, readJobFileChecked, NETWORK_READY_FILE, networkHelperName, prepareRunArgs, containerRunArgs, judgeCommand, JUDGE_DIR, WORKSPACE_LABEL, homeCopyRunArgs, networkHelperRunArgs, JUDGE_MAX_TURNS, JUDGE_ITEMS, answerFileName, hashJudgeProgram, identityScript, identityArgs, parseIdentity, mountableHome, homeCopyScript, homeCopyArgs, COMMAND_DIR, judgeProgramRecord, MAX_ITEMS_BYTES, JUDGE_REPORT, identityRunArgs, SAVED_STATE_LABEL } from './chunk-REZNW2YV.js';
|
|
28
|
+
import { universalImagePending, RecipeError, writeSanitizedRecipe, classifyProperties, DOCKER_FEATURE } from './chunk-2FUJ5PSV.js';
|
|
29
29
|
import { sanitizeTerminal, planArtifactUpload } from './chunk-QHYCF36K.js';
|
|
30
30
|
import { sharedSkillNames, skillConflictMessage } from './chunk-MRSN67FY.js';
|
|
31
31
|
import { readCloudConfig } from './chunk-HWVMRETT.js';
|
|
32
|
-
import { signerFor } from './chunk-
|
|
33
|
-
import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir,
|
|
34
|
-
import { BUILD_HOME,
|
|
35
|
-
import { agentCredential, RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON, UNKNOWN_TRANSPORT_REASON, RELAY_USER,
|
|
32
|
+
import { signerFor } from './chunk-KM2IMFAG.js';
|
|
33
|
+
import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir, workspacesDir, jobsDir, storeScopes, spoolDir, orgStoreIn, rootfsDir, imagesDir, sessionsDir, builderCacheDiskPath, buildsDir, buildDir, ghConfigDir, runnerDirOfBuild, mirrorDir, orgHomeDir, jobConsolePath, vsockPath, builderConsolePath, rootfsPathFor, diskTemplatesDir, deliveryDir, SHIM_DIR } from './chunk-T5CN56UE.js';
|
|
34
|
+
import { BUILD_HOME, SandboxError, JOB_MOUNT, heldBuildAuthority, IMAGE_BUNDLES, writeTrustFiles, buildEnvironment, trustEnvironment, CHECKOUT_DIR, parseIsolation, containerLocalAddress, destinationEnvironment, withHooksPath, startBuildServices, headerVariables, authorEnvironment, sharedUserEnvironment, withTunnelRecord, commitIdentityRecord, repositoriesOf, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, parseProxied, readBundleScript, applyTrust, STAND_IN, proxyEnvironment, describeSandboxError, dropBuildAuthority, sandboxSettingsFrom, createSandboxClient, hostKeyOf, buildAuthorityFor, checkEnvironment, withMiseVariables, checkoutPath, primaryOf, variablesFor, carried6to4, classifyAddress, EVAL_INPUT_DIR, isMiseVariableAllowed, variableForPath, droppedRequestHeaders, hasDotDotSegment, routeNameOf, toolProxyEnvironment } from './chunk-4YAFIOIV.js';
|
|
35
|
+
import { agentCredential, RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON, UNKNOWN_TRANSPORT_REASON, RELAY_USER, currentAgentCredential, removedAgentCredentialValues, managedAgentCredential, realFdStorePort, builtInNames, MAX_MALFORMED_SESSIONS, MALFORMED_REASON, JOIN_WAIT_MS, MAX_OPEN_STREAMS, STREAM_RESET_RATE, STREAM_RESET_BURST, STREAM_WINDOW, SESSION_WINDOW, targetOfPath, AGENT_CREDENTIAL_MISSING, fdName, RELAY_ID, DOCKER_DESKTOP_REASON } from './chunk-7U43B2UB.js';
|
|
36
36
|
import { versionAtLeast } from './chunk-SF2TBJBP.js';
|
|
37
|
-
import { jobAgent, firecrackerArch, MICROVM_ARCHES, FIRECRACKER_VERSION, hostHelperPinned, builderDiskPinned, guestAssetsPinned, runnerAgentNamed, credentialForms, orList, credentialShapes, agentCommands, defaultCommandLine, credentialAgent,
|
|
37
|
+
import { jobAgent, firecrackerArch, MICROVM_ARCHES, FIRECRACKER_VERSION, hostHelperPinned, builderDiskPinned, guestAssetsPinned, runnerAgentNamed, credentialForms, orList, credentialShapes, agentCommands, defaultCommandLine, credentialAgent, GUEST_HELPER_SHA256, GUEST_ASSETS_BASE_URL, GUEST_KERNEL_SHA256, JAILER_BINARY_SHA256, FIRECRACKER_BINARY_SHA256, FIRECRACKER_SHA256, GUEST_ASSETS_VERSION, BUILDER_DISK_VERSION, credentialHandOffKey, strippedVariables, runnerAgents, judgeAgent, networkHelperPinned, NETWORK_HELPER_SHA256, universalImagePinned, UNIVERSAL_IMAGE, FFMPEG_STATIC, FFMPEG_BUILD_TAG, GH_SHA256, GH_VERSION, TINI_SHA256, TINI_VERSION, HEADLESS_SHELL, HEADLESS_SHELL_BASE_URL, PLAYWRIGHT_FFMPEG, PLAYWRIGHT_FFMPEG_BASE_URL, VOICE_PACK_FILES, BUILDER_DISK_SHA256, BUILDER_DISK_BASE_URL, DEFAULT_BASE_IMAGE, agentVersionOf } from './chunk-IEX5FXEU.js';
|
|
38
38
|
import { sessionFolderIn, legacyAgent, defaultAgents, parsePolicy, AGENTS_SKILLS_BASE, OUTERLAYER_SKILLS_BASE, orderSessions, extractFacts, sessionsToTimeline, resolvePackRoot } from './chunk-G2CRPYFO.js';
|
|
39
39
|
import { MAINTAINED_SKILLS } from './chunk-ATEMEVFF.js';
|
|
40
40
|
import { resolveVersion } from './chunk-RA3O54FC.js';
|
|
@@ -42,13 +42,13 @@ import { homeFromEnv } from './chunk-352DHUTP.js';
|
|
|
42
42
|
import { scrubText, MIN_SCRUBBED_LITERAL, setCustomScrubbers, setLocalIdentity, setRepoRoot, forkFor, scrubDeep } from './chunk-E5G6K7C4.js';
|
|
43
43
|
import { CheckItemsSchema, CODE_REVIEW_NAME, JUDGE_NOT_RUN_ITEM_ID, REVIEWED_COMMIT_ITEM_ID, CHECK_ITEM_REASON_MAX_LENGTH, UNREVIEWED_FILES_ITEM_ID, CHECK_ITEM_HREF_MAX_LENGTH, downconvertSession } from './chunk-NEXKR5RR.js';
|
|
44
44
|
import { randomUUID, randomBytes, createHash } from 'crypto';
|
|
45
|
-
import { rmSync, mkdirSync, existsSync, readFileSync, mkdtempSync, statSync, appendFileSync, writeFileSync, readdirSync, accessSync, constants, openSync, closeSync, renameSync, rmdirSync,
|
|
45
|
+
import { rmSync, mkdirSync, existsSync, readFileSync, mkdtempSync, statSync, appendFileSync, writeFileSync, readdirSync, accessSync, constants, openSync, closeSync, renameSync, rmdirSync, createReadStream, lstatSync, unlinkSync, chmodSync, realpathSync, writeSync, truncateSync, cpSync, fstatSync, readSync, createWriteStream, copyFileSync, linkSync } from 'fs';
|
|
46
46
|
import { join, dirname, isAbsolute, delimiter, basename, sep, relative, resolve } from 'path';
|
|
47
|
-
import { Readable, PassThrough,
|
|
47
|
+
import { Readable, PassThrough, Duplex, Transform, Writable } from 'stream';
|
|
48
48
|
import { spawn, spawnSync } from 'child_process';
|
|
49
49
|
import { createServer, createConnection, connect, isIP } from 'net';
|
|
50
50
|
import { availableParallelism, networkInterfaces, tmpdir } from 'os';
|
|
51
|
-
import {
|
|
51
|
+
import { createGunzip, gunzipSync, constants as constants$1, gzipSync } from 'zlib';
|
|
52
52
|
import http2 from 'http2';
|
|
53
53
|
import { once } from 'events';
|
|
54
54
|
import { createServer as createServer$1, request, Agent } from 'http';
|
|
@@ -3492,1070 +3492,1255 @@ function recordingCause(log) {
|
|
|
3492
3492
|
const cause = proof ?? lines.find((line2) => line2.startsWith("setup failed: "));
|
|
3493
3493
|
return cause === void 0 ? void 0 : cause.slice(0, MAX_CAUSE_LENGTH);
|
|
3494
3494
|
}
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
var RUNNER_METADATA = "outerlayerRunner";
|
|
3498
|
-
var BUILD_METADATA = "outerlayerBuild";
|
|
3499
|
-
function orphanedSandboxes(sandboxes, runnerId, heldBuilds) {
|
|
3500
|
-
return sandboxes.filter((sandbox) => {
|
|
3501
|
-
if (sandbox.metadata[RUNNER_METADATA] !== runnerId) return false;
|
|
3502
|
-
if (sandbox.state === "ended") return false;
|
|
3503
|
-
const build = sandbox.metadata[BUILD_METADATA];
|
|
3504
|
-
return build === void 0 || !heldBuilds.has(build);
|
|
3505
|
-
});
|
|
3495
|
+
function savedStatesDir(runnerDirPath) {
|
|
3496
|
+
return join(runnerDirPath, "saved-states");
|
|
3506
3497
|
}
|
|
3507
|
-
|
|
3508
|
-
|
|
3498
|
+
function writeRecord2(dir, record) {
|
|
3499
|
+
mkdirSync(dir, { recursive: true, mode: RUNNER_DIR_MODE });
|
|
3500
|
+
const target2 = join(dir, `${record.id}.json`);
|
|
3501
|
+
const partial = `${target2}.partial`;
|
|
3502
|
+
writeFileSync(partial, `${JSON.stringify(record, null, 2)}
|
|
3503
|
+
`, { mode: RUNNER_FILE_MODE });
|
|
3504
|
+
renameSync(partial, target2);
|
|
3505
|
+
}
|
|
3506
|
+
function isRecord(value) {
|
|
3507
|
+
if (typeof value !== "object" || value === null) return false;
|
|
3508
|
+
const r = value;
|
|
3509
|
+
return typeof r.id === "string" && (r.kind === "volume" || r.kind === "tar" || r.kind === "disk") && typeof r.savedAt === "string" && Number.isFinite(Date.parse(r.savedAt)) && typeof r.bytes === "number" && typeof r.baseKey === "string" && typeof r.key === "string" && typeof r.fileHashes === "object" && r.fileHashes !== null && typeof r.complete === "boolean";
|
|
3510
|
+
}
|
|
3511
|
+
function readRecords(dir) {
|
|
3512
|
+
let names;
|
|
3509
3513
|
try {
|
|
3510
|
-
|
|
3511
|
-
} catch
|
|
3512
|
-
|
|
3513
|
-
return 0;
|
|
3514
|
+
names = readdirSync(dir);
|
|
3515
|
+
} catch {
|
|
3516
|
+
return [];
|
|
3514
3517
|
}
|
|
3515
|
-
|
|
3516
|
-
for (const
|
|
3518
|
+
const records = [];
|
|
3519
|
+
for (const name of names) {
|
|
3520
|
+
if (!name.endsWith(".json")) continue;
|
|
3517
3521
|
try {
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
} catch (err) {
|
|
3522
|
-
input.log?.(`could not end sandbox ${sandbox.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3522
|
+
const parsed = JSON.parse(readFileSync(join(dir, name), "utf8"));
|
|
3523
|
+
if (isRecord(parsed)) records.push(parsed);
|
|
3524
|
+
} catch {
|
|
3523
3525
|
}
|
|
3524
3526
|
}
|
|
3525
|
-
return
|
|
3527
|
+
return records;
|
|
3526
3528
|
}
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
var OPEN_TIMEOUT_MS = 3e4;
|
|
3530
|
-
var defaultWebSocket = (url) => new WebSocket(url);
|
|
3531
|
-
function sandboxExecStream(opts) {
|
|
3532
|
-
const create = opts.webSocket ?? defaultWebSocket;
|
|
3533
|
-
return () => {
|
|
3534
|
-
const stdout = new PassThrough();
|
|
3535
|
-
let socket;
|
|
3536
|
-
let open = false;
|
|
3537
|
-
let finished2 = false;
|
|
3538
|
-
let tail = "";
|
|
3539
|
-
let finish2;
|
|
3540
|
-
const exited = new Promise((resolve2) => finish2 = resolve2);
|
|
3541
|
-
const end = (status, why) => {
|
|
3542
|
-
if (finished2) return;
|
|
3543
|
-
finished2 = true;
|
|
3544
|
-
if (why !== void 0) tail = `${tail}${why}`.slice(-400);
|
|
3545
|
-
stdout.end();
|
|
3546
|
-
try {
|
|
3547
|
-
socket?.close();
|
|
3548
|
-
} catch {
|
|
3549
|
-
}
|
|
3550
|
-
finish2(status);
|
|
3551
|
-
};
|
|
3552
|
-
const opened = new Promise((resolve2, reject) => {
|
|
3553
|
-
const timer = setTimeout(() => reject(new Error("the WebSocket did not open in time")), OPEN_TIMEOUT_MS);
|
|
3554
|
-
timer.unref();
|
|
3555
|
-
opts.url().then((url) => {
|
|
3556
|
-
socket = create(url);
|
|
3557
|
-
socket.binaryType = "arraybuffer";
|
|
3558
|
-
socket.addEventListener("open", () => {
|
|
3559
|
-
clearTimeout(timer);
|
|
3560
|
-
open = true;
|
|
3561
|
-
socket.send(opts.secret);
|
|
3562
|
-
resolve2();
|
|
3563
|
-
});
|
|
3564
|
-
socket.addEventListener("message", (event) => {
|
|
3565
|
-
if (event.data instanceof ArrayBuffer) stdout.write(Buffer.from(event.data));
|
|
3566
|
-
else if (ArrayBuffer.isView(event.data)) stdout.write(Buffer.from(event.data.buffer, event.data.byteOffset, event.data.byteLength));
|
|
3567
|
-
});
|
|
3568
|
-
socket.addEventListener("error", (event) => {
|
|
3569
|
-
clearTimeout(timer);
|
|
3570
|
-
reject(new Error(event.message ?? "the WebSocket failed"));
|
|
3571
|
-
});
|
|
3572
|
-
socket.addEventListener("close", (event) => {
|
|
3573
|
-
clearTimeout(timer);
|
|
3574
|
-
reject(new Error("the WebSocket closed before it opened"));
|
|
3575
|
-
end(event.code === 1e3 || event.code === void 0 ? 0 : 1, event.code === 1008 ? "the relay refused the join secret" : void 0);
|
|
3576
|
-
});
|
|
3577
|
-
}).catch((err) => {
|
|
3578
|
-
clearTimeout(timer);
|
|
3579
|
-
reject(err);
|
|
3580
|
-
});
|
|
3581
|
-
});
|
|
3582
|
-
opened.catch((err) => end(1, err instanceof Error ? err.message : String(err)));
|
|
3583
|
-
const stdin = new Writable({
|
|
3584
|
-
write(chunk, _encoding, callback) {
|
|
3585
|
-
opened.then(
|
|
3586
|
-
async () => {
|
|
3587
|
-
if (finished2) return callback(new Error("the connection is closed"));
|
|
3588
|
-
socket.send(chunk);
|
|
3589
|
-
while (open && !finished2 && socket.bufferedAmount > BUFFER_HIGH_WATER) await new Promise((resolve2) => setTimeout(resolve2, DRAIN_POLL_MS));
|
|
3590
|
-
callback();
|
|
3591
|
-
},
|
|
3592
|
-
() => callback(new Error("the connection did not open"))
|
|
3593
|
-
);
|
|
3594
|
-
},
|
|
3595
|
-
final(callback) {
|
|
3596
|
-
end(0);
|
|
3597
|
-
callback();
|
|
3598
|
-
}
|
|
3599
|
-
});
|
|
3600
|
-
stdin.on("error", () => void 0);
|
|
3601
|
-
stdout.on("error", () => void 0);
|
|
3602
|
-
return { stdin, stdout, exited, kill: () => end(-1), stderrTail: () => tail };
|
|
3603
|
-
};
|
|
3529
|
+
function deleteRecord(dir, id) {
|
|
3530
|
+
rmSync(join(dir, `${id}.json`), { force: true });
|
|
3604
3531
|
}
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
var
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3532
|
+
|
|
3533
|
+
// src/runner/saved-state/container.ts
|
|
3534
|
+
var SAVED_FOLDERS = ["home", "work"];
|
|
3535
|
+
function stopProcessesScript() {
|
|
3536
|
+
return [
|
|
3537
|
+
"me=$$; uid=$(id -u); n=0; pass=0",
|
|
3538
|
+
"while [ $pass -lt 25 ]; do",
|
|
3539
|
+
' left=""',
|
|
3540
|
+
" for p in /proc/[0-9]*; do",
|
|
3541
|
+
" pid=${p##*/}",
|
|
3542
|
+
' [ "$pid" = "$me" ] && continue',
|
|
3543
|
+
` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
|
|
3544
|
+
' case "$info" in *"S=Z"*) continue ;; esac',
|
|
3545
|
+
// The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
|
|
3546
|
+
' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
|
|
3547
|
+
" done",
|
|
3548
|
+
' [ -z "$left" ] && break',
|
|
3549
|
+
' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
|
|
3550
|
+
" pass=$((pass+1))",
|
|
3551
|
+
" sleep 0.2",
|
|
3552
|
+
"done",
|
|
3553
|
+
'[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
|
|
3554
|
+
'echo "stopped $n"'
|
|
3555
|
+
].join("\n");
|
|
3612
3556
|
}
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3557
|
+
var HELPER_FLAGS = [
|
|
3558
|
+
"--rm",
|
|
3559
|
+
"--network",
|
|
3560
|
+
"none",
|
|
3561
|
+
"--cap-drop",
|
|
3562
|
+
"ALL",
|
|
3563
|
+
"--cap-add",
|
|
3564
|
+
"CHOWN",
|
|
3565
|
+
"--cap-add",
|
|
3566
|
+
"DAC_OVERRIDE",
|
|
3567
|
+
"--cap-add",
|
|
3568
|
+
"DAC_READ_SEARCH",
|
|
3569
|
+
"--cap-add",
|
|
3570
|
+
"FOWNER",
|
|
3571
|
+
"--security-opt",
|
|
3572
|
+
"no-new-privileges",
|
|
3573
|
+
"--read-only",
|
|
3574
|
+
"--user",
|
|
3575
|
+
"0",
|
|
3576
|
+
"--pids-limit",
|
|
3577
|
+
"256"
|
|
3578
|
+
];
|
|
3579
|
+
function helperArgs(input) {
|
|
3580
|
+
return [
|
|
3581
|
+
...HELPER_FLAGS,
|
|
3582
|
+
"--label",
|
|
3583
|
+
`${RUNNER_LABEL}=${input.runnerId}`,
|
|
3584
|
+
"--mount",
|
|
3585
|
+
`type=volume,src=${input.from},dst=/from,readonly`,
|
|
3586
|
+
"--mount",
|
|
3587
|
+
`type=volume,src=${input.to},dst=/to`,
|
|
3588
|
+
"--entrypoint",
|
|
3589
|
+
"sh",
|
|
3590
|
+
input.image,
|
|
3591
|
+
"-c",
|
|
3592
|
+
input.script
|
|
3593
|
+
];
|
|
3616
3594
|
}
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
return
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
const result = { status: Number(code) || 0, stdout: Buffer.concat(pieces).toString("utf8"), stderr: Buffer.concat(errPieces).toString("utf8") };
|
|
3660
|
-
if (opts.logPath !== void 0) {
|
|
3661
|
-
mkdirSync(dirname(opts.logPath), { recursive: true });
|
|
3662
|
-
appendFileSync(opts.logPath, result.stdout + result.stderr);
|
|
3663
|
-
}
|
|
3664
|
-
return result;
|
|
3665
|
-
}
|
|
3666
|
-
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
3667
|
-
await client2.command(sandbox, `kill -9 $(cat ${file("pid")}) 2>/dev/null; rm -f ${RUN_DIR}/${id}.*`).catch(() => void 0);
|
|
3668
|
-
const extra = `${command}: no answer within ${Math.round(opts.timeoutMs / 1e3)}s
|
|
3669
|
-
`;
|
|
3670
|
-
if (opts.logPath !== void 0) {
|
|
3671
|
-
mkdirSync(dirname(opts.logPath), { recursive: true });
|
|
3672
|
-
appendFileSync(opts.logPath, extra);
|
|
3673
|
-
}
|
|
3674
|
-
return { status: 124, stdout: "", stderr: extra };
|
|
3675
|
-
}
|
|
3676
|
-
await sleep(wait);
|
|
3677
|
-
wait = Math.min(wait * 2, POLL_LONGEST_MS);
|
|
3595
|
+
var ONE_BUILD_FOLDERS = ["tmp", ".command"];
|
|
3596
|
+
function clearOneBuildArgs(input) {
|
|
3597
|
+
return [
|
|
3598
|
+
...HELPER_FLAGS,
|
|
3599
|
+
"--label",
|
|
3600
|
+
`${RUNNER_LABEL}=${input.runnerId}`,
|
|
3601
|
+
"--mount",
|
|
3602
|
+
`type=volume,src=${input.volume},dst=/build`,
|
|
3603
|
+
"--entrypoint",
|
|
3604
|
+
"sh",
|
|
3605
|
+
input.image,
|
|
3606
|
+
"-c",
|
|
3607
|
+
`rm -rf ${ONE_BUILD_FOLDERS.map((name) => `/build/${name}`).join(" ")}`
|
|
3608
|
+
];
|
|
3609
|
+
}
|
|
3610
|
+
var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
|
|
3611
|
+
var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
|
|
3612
|
+
function failureText(result) {
|
|
3613
|
+
const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
|
|
3614
|
+
return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
3615
|
+
}
|
|
3616
|
+
async function saveContainerState(input) {
|
|
3617
|
+
const { host } = input;
|
|
3618
|
+
const id = randomBytes(6).toString("hex");
|
|
3619
|
+
const volume = `outerlayer-saved-${id}`;
|
|
3620
|
+
const record = {
|
|
3621
|
+
id,
|
|
3622
|
+
kind: "volume",
|
|
3623
|
+
volume,
|
|
3624
|
+
savedAt: input.now.toISOString(),
|
|
3625
|
+
bytes: 0,
|
|
3626
|
+
baseKey: input.key.baseKey,
|
|
3627
|
+
key: input.key.key,
|
|
3628
|
+
fileHashes: input.key.fileHashes,
|
|
3629
|
+
...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
|
|
3630
|
+
complete: false
|
|
3631
|
+
};
|
|
3632
|
+
const discard = async (reason) => {
|
|
3633
|
+
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
3634
|
+
if (removed.status !== 0) {
|
|
3635
|
+
input.log(`the volume ${volume} could not be removed (${failureText({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
|
|
3636
|
+
return { ok: false, reason };
|
|
3678
3637
|
}
|
|
3638
|
+
deleteRecord(input.dir, id);
|
|
3639
|
+
return { ok: false, reason };
|
|
3679
3640
|
};
|
|
3680
|
-
}
|
|
3681
|
-
function sandboxHost(client2, sandbox, options = {}) {
|
|
3682
|
-
return createContainerHost({ run: sandboxRunner(client2, sandbox, options) });
|
|
3683
|
-
}
|
|
3684
|
-
|
|
3685
|
-
// src/runner/sandbox/hooks.ts
|
|
3686
|
-
var SANDBOX_DIR = "/tmp/ol";
|
|
3687
|
-
var CLI_DIR = `${SANDBOX_DIR}/dist`;
|
|
3688
|
-
var SKILL_PACK_DIR = `${SANDBOX_DIR}/skill-pack`;
|
|
3689
|
-
var SECRET_FILE = `${SANDBOX_DIR}/join-secret`;
|
|
3690
|
-
var CONTAINER_SECRET_FILE = "/run/outerlayer-relay/join-secret";
|
|
3691
|
-
var JOIN_WS_PORT = 7077;
|
|
3692
|
-
var SANDBOX_RESERVE_MIB = 512;
|
|
3693
|
-
var RELAY_ATTEMPTS = 40;
|
|
3694
|
-
var RELAY_WAIT_MS = 500;
|
|
3695
|
-
var SANDBOX_PROBE_BYTES = 96 * 1024;
|
|
3696
|
-
var MAX_TAIL_READS2 = 8;
|
|
3697
|
-
var NO_ENV = { literals: {}, passed: {} };
|
|
3698
|
-
function sandboxNameOf(runnerLabel, buildId) {
|
|
3699
|
-
const short = runnerLabel.toLowerCase().replace(/[^a-z0-9]/g, "").slice(0, 8);
|
|
3700
|
-
return `ol-${short === "" ? "runner" : short}-${buildId}`;
|
|
3701
|
-
}
|
|
3702
|
-
async function endUnansweredSandbox(sandboxClient, runnerId, buildId, warn) {
|
|
3703
3641
|
try {
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
}
|
|
3642
|
+
const noEnv = { literals: {}, passed: {} };
|
|
3643
|
+
const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
|
|
3644
|
+
if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText(stopped)})` };
|
|
3645
|
+
input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
|
|
3646
|
+
const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
|
|
3647
|
+
if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText(flushed)})` };
|
|
3648
|
+
writeRecord2(input.dir, record);
|
|
3649
|
+
const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
|
|
3650
|
+
if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText(created)})`);
|
|
3651
|
+
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
3652
|
+
if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText(copied)})`);
|
|
3653
|
+
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
3654
|
+
const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
|
|
3655
|
+
writeRecord2(input.dir, done);
|
|
3656
|
+
return { ok: true, record: done };
|
|
3707
3657
|
} catch (err) {
|
|
3708
|
-
|
|
3658
|
+
return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
|
|
3709
3659
|
}
|
|
3710
3660
|
}
|
|
3711
|
-
function
|
|
3712
|
-
return runtime.sandbox.client();
|
|
3713
|
-
}
|
|
3714
|
-
async function sandboxOfBuild(runtime, state) {
|
|
3715
|
-
const held3 = runtime.sandbox.sandboxes.get(state.buildId);
|
|
3716
|
-
if (held3 !== void 0) return held3;
|
|
3661
|
+
async function restoreContainerState(input) {
|
|
3717
3662
|
try {
|
|
3718
|
-
const
|
|
3719
|
-
if (
|
|
3720
|
-
|
|
3721
|
-
return found;
|
|
3663
|
+
const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
3664
|
+
if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText(copied)})` };
|
|
3665
|
+
return { ok: true };
|
|
3722
3666
|
} catch (err) {
|
|
3723
|
-
|
|
3724
|
-
throw err;
|
|
3667
|
+
return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
|
|
3725
3668
|
}
|
|
3726
3669
|
}
|
|
3727
|
-
function
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
this.name = "SandboxGoneError";
|
|
3670
|
+
async function removeSavedState(record, deps) {
|
|
3671
|
+
if (record.kind === "volume" && record.volume !== void 0) {
|
|
3672
|
+
const result = await deps.removeVolume(record.volume);
|
|
3673
|
+
if (result !== void 0 && result.status !== 0) {
|
|
3674
|
+
const line2 = (result.stderr ?? "").trim().split("\n").filter((text2) => text2.trim() !== "").at(-1);
|
|
3675
|
+
throw new Error(`Docker could not remove volume ${record.volume} (exit ${result.status}${line2 === void 0 ? "" : `: ${line2}`})`);
|
|
3676
|
+
}
|
|
3735
3677
|
}
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
if (sandbox === void 0) throw new SandboxGoneError(state.sandboxId);
|
|
3740
|
-
return runtimeInSandbox(runtime, sandbox);
|
|
3741
|
-
}
|
|
3742
|
-
async function goneReason(runtime, state) {
|
|
3743
|
-
try {
|
|
3744
|
-
const found = await client(runtime).get(state.sandboxId);
|
|
3745
|
-
return found.state === "ended" ? new SandboxGoneError(state.sandboxId, found.endedReason ?? void 0) : void 0;
|
|
3746
|
-
} catch (err) {
|
|
3747
|
-
return err instanceof SandboxError && err.status === 404 ? new SandboxGoneError(state.sandboxId) : void 0;
|
|
3678
|
+
if ((record.kind === "tar" || record.kind === "disk") && record.path !== void 0) {
|
|
3679
|
+
rmSync(`${record.path}.partial`, { force: true });
|
|
3680
|
+
rmSync(record.path, { force: true });
|
|
3748
3681
|
}
|
|
3682
|
+
deleteRecord(deps.dir, record.id);
|
|
3749
3683
|
}
|
|
3750
|
-
function
|
|
3751
|
-
const
|
|
3752
|
-
const
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
"--mount",
|
|
3762
|
-
`type=bind,src=${SECRET_FILE},dst=${CONTAINER_SECRET_FILE},readonly`,
|
|
3763
|
-
...Object.entries(toolProxyEnvironment()).flatMap(([name, value]) => ["--env", `${name}=${value}`])
|
|
3764
|
-
);
|
|
3765
|
-
out.push("--join-ws", String(JOIN_WS_PORT), "--join-secret-file", CONTAINER_SECRET_FILE);
|
|
3766
|
-
return out;
|
|
3767
|
-
}
|
|
3768
|
-
var RELAY_PROBE = `const s=require("net").connect(3128,"127.0.0.1");s.on("connect",()=>process.exit(0));s.on("error",()=>process.exit(1));setTimeout(()=>process.exit(1),2000)`;
|
|
3769
|
-
async function waitForRelay(runtime, container) {
|
|
3770
|
-
for (let attempt = 0; attempt < RELAY_ATTEMPTS; attempt++) {
|
|
3771
|
-
const probe = await runtime.host.exec({ container, user: RELAY_USER, env: NO_ENV, argv: ["/opt/outerlayer/bin/node", "-e", RELAY_PROBE] });
|
|
3772
|
-
if (probe.status === 0) return true;
|
|
3773
|
-
await runtime.sandbox.sleep(RELAY_WAIT_MS);
|
|
3684
|
+
async function removeOldestUntil(input) {
|
|
3685
|
+
const removed = [];
|
|
3686
|
+
for (const record of oldestFirst(input.records)) {
|
|
3687
|
+
if (input.enough()) break;
|
|
3688
|
+
try {
|
|
3689
|
+
await removeSavedState(record, input.deps);
|
|
3690
|
+
removed.push(record);
|
|
3691
|
+
input.log(`removed saved state ${record.id} from ${record.savedAt}`);
|
|
3692
|
+
} catch (err) {
|
|
3693
|
+
input.log(`could not remove saved state ${record.id}: ${describeCallFailure(err)}`);
|
|
3694
|
+
}
|
|
3774
3695
|
}
|
|
3775
|
-
return
|
|
3776
|
-
}
|
|
3777
|
-
function joinRelay(runtime, sandbox, build, services, joinSecret, log) {
|
|
3778
|
-
const exec = sandboxExecStream({
|
|
3779
|
-
url: () => client(runtime).portUrl(sandbox, JOIN_WS_PORT),
|
|
3780
|
-
secret: joinSecret,
|
|
3781
|
-
...runtime.sandbox.webSocket === void 0 ? {} : { webSocket: runtime.sandbox.webSocket }
|
|
3782
|
-
});
|
|
3783
|
-
runtime.relays.set(
|
|
3784
|
-
build.buildId,
|
|
3785
|
-
superviseRelay({
|
|
3786
|
-
container: build.container,
|
|
3787
|
-
services,
|
|
3788
|
-
execStream: () => exec(),
|
|
3789
|
-
...log === void 0 ? {} : { log },
|
|
3790
|
-
onFatal: (reason) => runtime.relayFatal.set(build.buildId, reason),
|
|
3791
|
-
builtIns: builtInNames(runtime.agents)
|
|
3792
|
-
})
|
|
3793
|
-
);
|
|
3696
|
+
return removed;
|
|
3794
3697
|
}
|
|
3795
|
-
async function
|
|
3796
|
-
const
|
|
3797
|
-
const relay = runtime.relays.get(state.buildId);
|
|
3798
|
-
if (relay !== void 0) {
|
|
3799
|
-
runtime.relays.delete(state.buildId);
|
|
3800
|
-
await relay.close();
|
|
3801
|
-
}
|
|
3802
|
-
runtime.relayFatal.delete(state.buildId);
|
|
3803
|
-
runtime.leases.get(state.buildId)?.close();
|
|
3804
|
-
runtime.leases.delete(state.buildId);
|
|
3805
|
-
if (state.sandboxId !== "") {
|
|
3698
|
+
async function removeSuperseded(input) {
|
|
3699
|
+
for (const old of input.records.filter((record) => record.complete && record.key === input.made.key && record.id !== input.made.id && Date.parse(record.savedAt) < Date.parse(input.made.savedAt))) {
|
|
3806
3700
|
try {
|
|
3807
|
-
await
|
|
3701
|
+
await removeSavedState(old, input.deps);
|
|
3702
|
+
input.log(`removed saved state ${old.id}, saved state ${input.made.id} replaces it`);
|
|
3808
3703
|
} catch (err) {
|
|
3809
|
-
|
|
3704
|
+
input.log(`could not remove saved state ${old.id}: ${describeCallFailure(err)}`);
|
|
3810
3705
|
}
|
|
3811
3706
|
}
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3707
|
+
}
|
|
3708
|
+
var RECORD_FILE = "record.json";
|
|
3709
|
+
var DISK_FILE = "disk.ext4";
|
|
3710
|
+
var WORK_ITEM_ID2 = /^[A-Za-z0-9_-]{1,64}$/;
|
|
3711
|
+
function itemDiskDir(configPath, organizationId, workItemId) {
|
|
3712
|
+
if (!WORK_ITEM_ID2.test(workItemId)) throw new Error(`cannot keep a disk for the work item ${JSON.stringify(workItemId)}`);
|
|
3713
|
+
return join(workspacesDir(configPath, organizationId), workItemId);
|
|
3714
|
+
}
|
|
3715
|
+
function isRecord2(value) {
|
|
3716
|
+
if (typeof value !== "object" || value === null) return false;
|
|
3717
|
+
const r = value;
|
|
3718
|
+
return typeof r.workItemId === "string" && typeof r.claimId === "string" && typeof r.queue === "string" && typeof r.imageId === "string" && typeof r.diskBytes === "number" && (r.kind === "disk" || r.kind === "volume" && typeof r.volume === "string" || r.kind === "sandbox" && typeof r.volume === "string" && typeof r.snapshot === "string") && typeof r.bytes === "number" && typeof r.keptAt === "string" && Number.isFinite(Date.parse(r.keptAt));
|
|
3719
|
+
}
|
|
3720
|
+
function readItemDisk(dir) {
|
|
3721
|
+
let parsed;
|
|
3722
|
+
try {
|
|
3723
|
+
parsed = JSON.parse(readFileSync(join(dir, RECORD_FILE), "utf8"));
|
|
3724
|
+
} catch {
|
|
3725
|
+
return void 0;
|
|
3818
3726
|
}
|
|
3819
|
-
if (
|
|
3820
|
-
if (
|
|
3727
|
+
if (!isRecord2(parsed) || parsed.workItemId !== basename(dir)) return void 0;
|
|
3728
|
+
if (parsed.kind !== "disk") return { dir, record: parsed };
|
|
3729
|
+
const path = join(dir, DISK_FILE);
|
|
3730
|
+
try {
|
|
3731
|
+
if (!statSync(path).isFile()) return void 0;
|
|
3732
|
+
} catch {
|
|
3733
|
+
return void 0;
|
|
3734
|
+
}
|
|
3735
|
+
return { dir, record: parsed, path };
|
|
3821
3736
|
}
|
|
3822
|
-
|
|
3823
|
-
const
|
|
3824
|
-
const
|
|
3825
|
-
|
|
3826
|
-
const fail = (reason) => ({ ok: false, reason });
|
|
3827
|
-
const own = config.sandbox;
|
|
3828
|
-
if (own === void 0) return fail("runner.sandbox is not set in the config file");
|
|
3829
|
-
const repositories = repositoriesOf({ repositories: input.repositories, repository: item.specRepository, branch: input.branch });
|
|
3830
|
-
const primary = primaryOf(repositories);
|
|
3831
|
-
if (primary === void 0) return fail("the work item names no repository to clone");
|
|
3832
|
-
const repository = primary.repository;
|
|
3833
|
-
const sized = sandboxSizeOf(config.build);
|
|
3834
|
-
if (!sized.ok) return fail(`${sized.field} ${sized.problem}`);
|
|
3835
|
-
let sandboxClient;
|
|
3737
|
+
function swapInItemDisk(dir, incoming) {
|
|
3738
|
+
const older = readItemDisk(dir);
|
|
3739
|
+
const outgoing = join(dirname(dir), `.${basename(dir)}.${randomBytes(4).toString("hex")}.old`);
|
|
3740
|
+
let hadOlder = true;
|
|
3836
3741
|
try {
|
|
3837
|
-
|
|
3742
|
+
renameSync(dir, outgoing);
|
|
3838
3743
|
} catch (err) {
|
|
3839
|
-
|
|
3744
|
+
if (err.code !== "ENOENT") throw err;
|
|
3745
|
+
hadOlder = false;
|
|
3840
3746
|
}
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
const unfetched = await prefetchGovernedContext({
|
|
3847
|
-
configPath: config.configPath,
|
|
3848
|
-
repository,
|
|
3849
|
-
source: governingSource,
|
|
3850
|
-
controlPlaneUrl,
|
|
3851
|
-
controlPlaneToken: () => tokens.controlPlane(),
|
|
3852
|
-
log,
|
|
3853
|
-
tag
|
|
3854
|
-
});
|
|
3855
|
-
if (unfetched !== void 0) return fail(unfetched);
|
|
3747
|
+
try {
|
|
3748
|
+
renameSync(incoming, dir);
|
|
3749
|
+
} catch (err) {
|
|
3750
|
+
if (hadOlder) renameSync(outgoing, dir);
|
|
3751
|
+
throw err;
|
|
3856
3752
|
}
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
const
|
|
3862
|
-
|
|
3863
|
-
const
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
};
|
|
3875
|
-
const state = { sandboxId: "", name, dataPlaneUrl: "", buildId, container: containerState };
|
|
3876
|
-
const secrets = secretsOf(runtime, jobDir);
|
|
3877
|
-
const env = buildEnvironment({
|
|
3878
|
-
config,
|
|
3879
|
-
item: item.number,
|
|
3880
|
-
...input.queue === "run" ? { run: true } : {},
|
|
3881
|
-
itemKey,
|
|
3882
|
-
repository,
|
|
3883
|
-
repositories,
|
|
3884
|
-
hostEnv: runtime.hostEnv,
|
|
3885
|
-
agents: runtime.agents,
|
|
3886
|
-
...input.job === void 0 ? {} : { job: input.job },
|
|
3887
|
-
containerEnv: {},
|
|
3888
|
-
builtIns: true
|
|
3889
|
-
});
|
|
3890
|
-
if (env.skippedTokenVariables.length > 0) {
|
|
3891
|
-
log.line(`${tag} not passing ${env.skippedTokenVariables.join(", ")} from runner.build.variables into the build: its repository token comes from the runner`);
|
|
3753
|
+
rmSync(outgoing, { recursive: true, force: true });
|
|
3754
|
+
return older;
|
|
3755
|
+
}
|
|
3756
|
+
function stagingItemDiskDir(dir) {
|
|
3757
|
+
const parent = dirname(dir);
|
|
3758
|
+
mkdirSync(parent, { recursive: true, mode: 448 });
|
|
3759
|
+
const staging = join(parent, `.${basename(dir)}.${randomBytes(4).toString("hex")}.partial`);
|
|
3760
|
+
mkdirSync(staging, { mode: 448 });
|
|
3761
|
+
return staging;
|
|
3762
|
+
}
|
|
3763
|
+
function listItemDisks(configPath, scope) {
|
|
3764
|
+
const where = workspacesDir(configPath, scope);
|
|
3765
|
+
let names;
|
|
3766
|
+
try {
|
|
3767
|
+
names = readdirSync(where);
|
|
3768
|
+
} catch {
|
|
3769
|
+
return [];
|
|
3892
3770
|
}
|
|
3893
|
-
const
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
}
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
}
|
|
3905
|
-
return aborted2 ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
|
|
3906
|
-
};
|
|
3907
|
-
if (input.aborted()) return bail("", true);
|
|
3908
|
-
mkdirSync(buildDir2, { recursive: true });
|
|
3909
|
-
const ghDir = ghConfigDir(buildDir2);
|
|
3910
|
-
let readToken;
|
|
3771
|
+
const kept = [];
|
|
3772
|
+
for (const name of names) {
|
|
3773
|
+
if (!WORK_ITEM_ID2.test(name)) continue;
|
|
3774
|
+
const found = readItemDisk(join(where, name));
|
|
3775
|
+
if (found !== void 0) kept.push(found);
|
|
3776
|
+
}
|
|
3777
|
+
return kept;
|
|
3778
|
+
}
|
|
3779
|
+
function listLeftovers(configPath, scope) {
|
|
3780
|
+
const where = workspacesDir(configPath, scope);
|
|
3781
|
+
let names;
|
|
3911
3782
|
try {
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
return bail(err instanceof WorkCommandError ? `the build's read token could not be had: ${refusalCode(err)}` : `the build's gh configuration could not be written: ${describeCallFailure(err)}`);
|
|
3783
|
+
names = readdirSync(where);
|
|
3784
|
+
} catch {
|
|
3785
|
+
return [];
|
|
3916
3786
|
}
|
|
3917
|
-
|
|
3918
|
-
|
|
3787
|
+
const found = [];
|
|
3788
|
+
for (const name of names) {
|
|
3789
|
+
if (!name.startsWith(".") && (!WORK_ITEM_ID2.test(name) || readItemDisk(join(where, name)) !== void 0)) continue;
|
|
3919
3790
|
try {
|
|
3920
|
-
|
|
3921
|
-
} catch
|
|
3922
|
-
return bail(`the build's read token for ${entry.repository} could not be had: ${refusalCode(err)}`);
|
|
3791
|
+
found.push({ path: join(where, name), mtimeMs: lstatSync(join(where, name)).mtimeMs });
|
|
3792
|
+
} catch {
|
|
3923
3793
|
}
|
|
3924
3794
|
}
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
|
|
3934
|
-
|
|
3935
|
-
|
|
3936
|
-
|
|
3937
|
-
|
|
3938
|
-
|
|
3795
|
+
return found;
|
|
3796
|
+
}
|
|
3797
|
+
|
|
3798
|
+
// src/runner/workspace/restore.ts
|
|
3799
|
+
var DISK_REASON_TEXT = {
|
|
3800
|
+
no_kept_disk: "no disk is kept for the work item on this host",
|
|
3801
|
+
other_image: "the kept disk was made on another image",
|
|
3802
|
+
other_disk_size: "the kept disk was made with another runner.build.disk size",
|
|
3803
|
+
disk_not_restored: "the kept disk could not be copied",
|
|
3804
|
+
checkout_not_switched: "the kept checkout could not be switched to the branch",
|
|
3805
|
+
build_type_cannot_restore: "this build type cannot restore a disk",
|
|
3806
|
+
fresh_requested: "a member asked for a fresh disk",
|
|
3807
|
+
holder_busy: "the machine that holds its disk was busy"
|
|
3808
|
+
};
|
|
3809
|
+
function itemDiskDecision(input) {
|
|
3810
|
+
if (input.fresh === true) return { state: { restored: false, reason: "fresh_requested" } };
|
|
3811
|
+
if (!input.builtin) return { state: { restored: false, reason: "build_type_cannot_restore" } };
|
|
3812
|
+
let kept;
|
|
3939
3813
|
try {
|
|
3940
|
-
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
...sized.size,
|
|
3944
|
-
idleMinutes: own.idleMinutes,
|
|
3945
|
-
timeLimitMinutes: sandboxTimeLimitMinutes(config.timeLimitMinutes),
|
|
3946
|
-
network: networkOf(own.allowHosts),
|
|
3947
|
-
metadata: { [RUNNER_METADATA]: runnerId, [BUILD_METADATA]: buildId }
|
|
3948
|
-
});
|
|
3949
|
-
} catch (err) {
|
|
3950
|
-
if (err instanceof SandboxError && err.status === 0) await endUnansweredSandbox(sandboxClient, runnerId, buildId, (line2) => log.error(`${tag} ${line2}`));
|
|
3951
|
-
runtime.sandbox.creating.delete(buildId);
|
|
3952
|
-
return bail(`The provider refused to create the sandbox: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
|
|
3814
|
+
kept = readItemDisk(itemDiskDir(input.configPath, input.organizationId, input.workItemId));
|
|
3815
|
+
} catch {
|
|
3816
|
+
kept = void 0;
|
|
3953
3817
|
}
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
input.
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
const { host } = sandboxed;
|
|
3963
|
-
const dockerFailed = (what, result) => bail(`${what} failed (exit ${result.status})${said(result.stderr)}`);
|
|
3964
|
-
const joinSecret = randomBytes(32);
|
|
3965
|
-
runtime.sandbox.joinSecrets.set(buildId, joinSecret);
|
|
3966
|
-
const packDir = join(runtime.cliBuild().distDir, "..", "skill-pack");
|
|
3967
|
-
const skillPack = existsSync(packDir) ? packDir : void 0;
|
|
3818
|
+
if (kept === void 0) return { state: { restored: false, reason: input.heldElsewhere === true ? "holder_busy" : "no_kept_disk" } };
|
|
3819
|
+
if (kept.record.repository !== input.repository) return { state: { restored: false, reason: "no_kept_disk" } };
|
|
3820
|
+
if (kept.record.imageId !== input.imageId) return { state: { restored: false, reason: "other_image" } };
|
|
3821
|
+
if (kept.record.diskBytes !== input.diskBytes) return { state: { restored: false, reason: "other_disk_size" } };
|
|
3822
|
+
if (kept.record.kind !== input.backend) return { state: { restored: false, reason: "disk_not_restored" } };
|
|
3823
|
+
return { kept, state: { restored: true, fromClaimId: kept.record.claimId } };
|
|
3824
|
+
}
|
|
3825
|
+
async function discardItemDisk(kept, removeVolume, onError, removeSnapshot) {
|
|
3968
3826
|
try {
|
|
3969
|
-
|
|
3970
|
-
|
|
3971
|
-
|
|
3972
|
-
await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/skill-pack.tgz`, gzipSync(packFolder(skillPack)));
|
|
3973
|
-
const placed = await sandboxClient.command(sandbox, `mkdir -p ${SKILL_PACK_DIR} && tar xzf ${SANDBOX_DIR}/skill-pack.tgz -C ${SKILL_PACK_DIR} && rm -f ${SANDBOX_DIR}/skill-pack.tgz`);
|
|
3974
|
-
if (placed.exitCode !== 0) return bail(`putting the skill pack in the sandbox failed (exit ${placed.exitCode ?? "none"})${said(placed.stderr)}`);
|
|
3827
|
+
if (kept.record.snapshot !== void 0) {
|
|
3828
|
+
if (removeSnapshot === void 0) return onError(`could not remove the snapshot ${kept.record.snapshot} of the kept disk: no sandbox provider is set`);
|
|
3829
|
+
await removeSnapshot(kept.record.snapshot);
|
|
3975
3830
|
}
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3831
|
+
if (kept.record.kind === "volume" && kept.record.volume !== void 0) {
|
|
3832
|
+
const removed = await removeVolume(kept.record.volume);
|
|
3833
|
+
if (removed !== void 0 && removed.status !== 0 && !/no such volume/i.test(removed.stderr ?? "")) {
|
|
3834
|
+
onError(`could not remove the volume ${kept.record.volume} of the kept disk: ${removed.stderr?.trim() ?? `exit ${removed.status}`}`);
|
|
3835
|
+
return;
|
|
3836
|
+
}
|
|
3837
|
+
}
|
|
3838
|
+
rmSync(kept.dir, { recursive: true, force: true });
|
|
3979
3839
|
} catch (err) {
|
|
3980
|
-
|
|
3840
|
+
onError(`could not remove the kept disk of work item ${kept.record.workItemId}: ${describeCallFailure(err)}`);
|
|
3981
3841
|
}
|
|
3982
|
-
const engine = await host.version();
|
|
3983
|
-
if (engine === null) return bail("Docker is not available in the sandbox: `docker version` failed");
|
|
3984
|
-
const limits2 = { memory: `${Math.max(512, sized.size.memoryMib - SANDBOX_RESERVE_MIB)}m`, cpus: sized.size.vcpu, pids: config.build?.pids ?? 4096 };
|
|
3985
|
-
const volume = await host.createVolume(container, { [BUILD_LABEL]: buildId, [RUNNER_LABEL]: runnerId });
|
|
3986
|
-
if (volume.status !== 0) return dockerFailed("creating the build's volume", volume);
|
|
3987
|
-
const runInput = {
|
|
3988
|
-
buildId,
|
|
3989
|
-
runnerId,
|
|
3990
|
-
container,
|
|
3991
|
-
volume: container,
|
|
3992
|
-
image,
|
|
3993
|
-
user: ADDED_USER,
|
|
3994
|
-
limits: limits2,
|
|
3995
|
-
destinations: config.build?.destinations?.length ?? 0,
|
|
3996
|
-
builtIns: true,
|
|
3997
|
-
transport: "exec",
|
|
3998
|
-
jobFiles: "copied"
|
|
3999
|
-
};
|
|
4000
|
-
const prepared = await host.run(prepareRunArgs(runInput), { logPath });
|
|
4001
|
-
if (prepared.status !== 0) return dockerFailed("preparing the build's volume", prepared);
|
|
4002
|
-
const imageConfig = await host.imageConfig(image);
|
|
4003
|
-
const resolved = await resolveBuildUser(sandboxed, runInput, { imageUser: imageConfig.user }, logPath);
|
|
4004
|
-
if (!resolved.ok) return bail(resolved.reason);
|
|
4005
|
-
containerState.agentUser = resolved.user;
|
|
4006
|
-
if (resolved.home !== void 0) containerState.home = resolved.home;
|
|
4007
|
-
const agentUser = resolved.user;
|
|
4008
|
-
input.save(state, environment);
|
|
4009
|
-
if (input.aborted()) return bail("", true);
|
|
4010
|
-
const started = await host.run(sandboxRunArgs(containerRunArgs({ ...runInput, home: containerState.home }, runtime.agents), { skillPack: skillPack !== void 0 }), { logPath });
|
|
4011
|
-
if (started.status !== 0) return dockerFailed("starting the build's container", started);
|
|
4012
|
-
joinRelay(runtime, sandbox, { buildId, container }, services.streams, joinSecret, (line2) => log.line(`${tag} ${line2}`));
|
|
4013
|
-
if (input.aborted()) return bail("", true);
|
|
4014
|
-
if (!await waitForRelay(sandboxed, container)) return bail("the relay in the build's container did not start");
|
|
4015
|
-
const ghDelivered = await deliverJobFiles(sandboxed, containerState, ["gh"], secrets);
|
|
4016
|
-
if (!ghDelivered.ok) return bail(ghDelivered.reason);
|
|
4017
|
-
keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`), {
|
|
4018
|
-
runtime: sandboxed,
|
|
4019
|
-
state: containerState,
|
|
4020
|
-
secrets,
|
|
4021
|
-
delivered: readToken,
|
|
4022
|
-
onFailure: (reason) => log.error(`${tag} ${reason}`)
|
|
4023
|
-
});
|
|
4024
|
-
const tools = await host.exec({ container, user: ADDED_USER, env: NO_ENV, argv: ["cat", "/opt/outerlayer/tools.json"] });
|
|
4025
|
-
const agentVersion = agentVersionOf(tools.status === 0 ? tools.stdout : "", runtime.agents, input.job);
|
|
4026
|
-
environment.agentVersion = agentVersion;
|
|
4027
|
-
const judgeProgram = await judgeProgramRecord((argv) => host.exec({ container, user: ADDED_USER, env: NO_ENV, argv }), runtime.agents);
|
|
4028
|
-
if (judgeProgram !== void 0) containerState.judgeProgram = judgeProgram;
|
|
4029
|
-
input.save(state, environment);
|
|
4030
|
-
const stepExec = async (argv, workdir, opts) => {
|
|
4031
|
-
const ran = await host.exec({ container, user: agentUser, workdir, env: withMiseVariables(env, opts?.variables), argv }, { ...opts?.quiet === true ? {} : { logPath }, ...opts?.timeoutMs === void 0 ? {} : { timeoutMs: opts.timeoutMs } });
|
|
4032
|
-
return endedAtLimit(ran) ? { ...ran, timedOut: true } : ran;
|
|
4033
|
-
};
|
|
4034
|
-
const setupMinutes = config.build?.setupMinutes ?? DEFAULT_SETUP_MINUTES;
|
|
4035
|
-
const steps = await runBuildSteps({
|
|
4036
|
-
repository,
|
|
4037
|
-
branch: primary.branch,
|
|
4038
|
-
repositories,
|
|
4039
|
-
...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository } } : {},
|
|
4040
|
-
agentVersion,
|
|
4041
|
-
agents: runtime.agents,
|
|
4042
|
-
...input.job === void 0 ? {} : { job: input.job },
|
|
4043
|
-
lifecycle: {},
|
|
4044
|
-
exec: stepExec,
|
|
4045
|
-
log,
|
|
4046
|
-
tag,
|
|
4047
|
-
aborted: input.aborted,
|
|
4048
|
-
stageContext: () => stageGovernedContext({
|
|
4049
|
-
config,
|
|
4050
|
-
repository,
|
|
4051
|
-
source: pinnedSourceOf(governingSource),
|
|
4052
|
-
buildId,
|
|
4053
|
-
buildDir: buildDir2,
|
|
4054
|
-
fetchImpl: input.fetchImpl,
|
|
4055
|
-
controlPlaneUrl,
|
|
4056
|
-
controlPlaneToken: () => tokens.controlPlane(),
|
|
4057
|
-
log,
|
|
4058
|
-
tag
|
|
4059
|
-
}),
|
|
4060
|
-
deliverContext: async () => {
|
|
4061
|
-
const delivered = await deliverJobFiles(sandboxed, containerState, ["context"], secrets);
|
|
4062
|
-
return delivered.ok ? delivered : { ok: false, reason: `context_unavailable: ${delivered.reason}` };
|
|
4063
|
-
},
|
|
4064
|
-
removeStagedContext: () => removeStagedContext(buildDir2),
|
|
4065
|
-
keepIn: jobDir,
|
|
4066
|
-
...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
|
|
4067
|
-
// The image in the snapshot is the universal image, so the build chooses and installs its runtimes.
|
|
4068
|
-
...runtimeStepInput({ base: {} }, services),
|
|
4069
|
-
setup: { timeoutMs: setupMinutes * 6e4, limitMinutes: setupMinutes, restored: false }
|
|
4070
|
-
});
|
|
4071
|
-
if (!steps.ok) return steps.aborted === true ? bail("", true) : bail(steps.reason);
|
|
4072
|
-
containerState.startCommit = steps.startCommit;
|
|
4073
|
-
keepRuntimeStep(containerState, steps);
|
|
4074
|
-
keepMissingSetupScripts(containerState, steps);
|
|
4075
|
-
if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
|
|
4076
|
-
input.save(state, environment);
|
|
4077
|
-
return { ok: true, workdir: CHECKOUT_DIR, state, environment };
|
|
4078
3842
|
}
|
|
4079
|
-
|
|
4080
|
-
|
|
3843
|
+
|
|
3844
|
+
// src/runner/saved-state/provision.ts
|
|
3845
|
+
var isRetryFresh = (value) => typeof value === "object" && value !== null && value.retryFresh === true;
|
|
3846
|
+
async function provisionWithFreshFallback(input) {
|
|
3847
|
+
const first = await input.attempt(true);
|
|
3848
|
+
if (!isRetryFresh(first)) return first;
|
|
3849
|
+
input.log(`${first.reason}; provisioning the build again from a fresh disk`);
|
|
3850
|
+
if (first.itemDisk === void 0) return await input.attempt(false);
|
|
3851
|
+
const second = await input.attempt(true, first.itemDisk);
|
|
3852
|
+
if (!isRetryFresh(second)) return second;
|
|
3853
|
+
input.log(`${second.reason}; provisioning the build again from a fresh disk`);
|
|
3854
|
+
return await input.attempt(false, first.itemDisk);
|
|
4081
3855
|
}
|
|
4082
|
-
function
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
3856
|
+
function keptDiskPlan() {
|
|
3857
|
+
return { keyCommits: {}, noSave: "it starts from the work item's kept disk" };
|
|
3858
|
+
}
|
|
3859
|
+
function clockOf(now) {
|
|
3860
|
+
return now ?? (() => /* @__PURE__ */ new Date());
|
|
3861
|
+
}
|
|
3862
|
+
function taggedLog(log, tag) {
|
|
3863
|
+
return (line2) => log.line(`${tag} ${line2}`);
|
|
3864
|
+
}
|
|
3865
|
+
function uidFromProbe(probe) {
|
|
3866
|
+
const uid = Number(probe.stdout);
|
|
3867
|
+
return probe.status === 0 && Number.isInteger(uid) ? uid : null;
|
|
3868
|
+
}
|
|
3869
|
+
function agentUidOf(probe) {
|
|
3870
|
+
return uidFromProbe(probe) ?? void 0;
|
|
3871
|
+
}
|
|
3872
|
+
function userIdChangedReason(savedUid, agentUid) {
|
|
3873
|
+
if (savedUid === void 0 || agentUid === savedUid) return void 0;
|
|
3874
|
+
return `the build user's id is ${agentUid ?? "unknown"} and it was ${savedUid} when the state was saved`;
|
|
3875
|
+
}
|
|
3876
|
+
function setupInputFor(input) {
|
|
3877
|
+
const minutes = input.setupMinutes ?? DEFAULT_SETUP_MINUTES;
|
|
4086
3878
|
return {
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
runtime.leases.get(state.buildId)?.stop();
|
|
4096
|
-
const relay = runtime.relays.get(state.buildId);
|
|
4097
|
-
if (relay !== void 0) {
|
|
4098
|
-
runtime.relays.delete(state.buildId);
|
|
4099
|
-
await relay.close();
|
|
4100
|
-
}
|
|
4101
|
-
await sandboxed.host.stop(state.container);
|
|
4102
|
-
const started = await sandboxed.host.start(state.container);
|
|
4103
|
-
if (started.status !== 0) throw new Error(`the build's container could not be started again (exit ${started.status})`);
|
|
4104
|
-
if (!await waitForRelay(sandboxed, state.container)) throw new Error("the relay in the build's container did not start again");
|
|
4105
|
-
joinRelay(runtime, sandbox, { buildId: state.buildId, container: state.container }, services.streams, joinSecret, input.log);
|
|
4106
|
-
const delivered = await deliverJobFiles(sandboxed, state, ["gh", "git-hooks", "context"], secretsOf(runtime, input.jobDir));
|
|
4107
|
-
if (!delivered.ok) throw new Error(delivered.reason);
|
|
4108
|
-
},
|
|
4109
|
-
hashProgram: async () => (await inner()).hashProgram(),
|
|
4110
|
-
prepare: async (files, tree) => (await inner()).prepare(files, tree),
|
|
4111
|
-
run: async (command, opts) => (await inner()).run(command, opts),
|
|
4112
|
-
readReport: async () => (await inner()).readReport(),
|
|
4113
|
-
readItems: async (name) => (await inner()).readItems(name),
|
|
4114
|
-
clear: async () => (await inner()).clear?.()
|
|
3879
|
+
timeoutMs: minutes * 6e4,
|
|
3880
|
+
limitMinutes: minutes,
|
|
3881
|
+
restored: input.plan.restore !== void 0 || input.fromItemDisk === true,
|
|
3882
|
+
...input.fromItemDisk === true ? { fromItemDisk: true } : {},
|
|
3883
|
+
keyCommits: input.plan.keyCommits,
|
|
3884
|
+
...input.withoutBuildVariables === true ? { withoutBuildVariables: true } : {},
|
|
3885
|
+
// A plan has a key exactly when the build may save a state. A build that starts from one is not asked to save.
|
|
3886
|
+
...input.plan.key === void 0 ? {} : { saveState: input.saveState }
|
|
4115
3887
|
};
|
|
4116
3888
|
}
|
|
4117
|
-
async function
|
|
3889
|
+
async function saveAndPrune(input) {
|
|
3890
|
+
const made = await input.save();
|
|
3891
|
+
if (!made.ok) return made;
|
|
3892
|
+
input.log(`saved state ${made.record.id}`);
|
|
3893
|
+
await removeSuperseded({ made: made.record, records: readRecords(input.dir), deps: { dir: input.dir, removeVolume: input.removeVolume }, log: input.log });
|
|
3894
|
+
return { ok: true };
|
|
3895
|
+
}
|
|
3896
|
+
async function discardUnusableState(input) {
|
|
4118
3897
|
try {
|
|
4119
|
-
|
|
3898
|
+
await removeSavedState(input.record, { dir: input.dir, removeVolume: input.removeVolume });
|
|
4120
3899
|
} catch (err) {
|
|
4121
|
-
|
|
3900
|
+
input.onError(`could not remove saved state ${input.record.id}: ${describeCallFailure(err)}`);
|
|
4122
3901
|
}
|
|
3902
|
+
return { retryFresh: true, reason: `saved state ${input.record.id} could not be used: ${input.reason}` };
|
|
4123
3903
|
}
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
3904
|
+
function settleFailedSteps(steps, on) {
|
|
3905
|
+
return steps.restoreFailed === true ? on.giveUp(steps.reason, steps.fromItemDisk === true) : on.bail(steps.reason, steps.aborted === true);
|
|
3906
|
+
}
|
|
3907
|
+
async function discardUnusableItemDisk(input) {
|
|
3908
|
+
await discardItemDisk(input.kept, input.removeVolume, input.onError, input.removeSnapshot);
|
|
3909
|
+
return { retryFresh: true, reason: `the kept disk of claim ${shortClaim(input.kept.record.claimId)} could not be used: ${input.reason}`, itemDisk: input.why };
|
|
3910
|
+
}
|
|
3911
|
+
|
|
3912
|
+
// src/runner/sandbox/sweep.ts
|
|
3913
|
+
var RUNNER_METADATA = "outerlayerRunner";
|
|
3914
|
+
var BUILD_METADATA = "outerlayerBuild";
|
|
3915
|
+
function orphanedSandboxes(sandboxes, runnerId, heldBuilds) {
|
|
3916
|
+
return sandboxes.filter((sandbox) => {
|
|
3917
|
+
if (sandbox.metadata[RUNNER_METADATA] !== runnerId) return false;
|
|
3918
|
+
if (sandbox.state === "ended") return false;
|
|
3919
|
+
const build = sandbox.metadata[BUILD_METADATA];
|
|
3920
|
+
return build === void 0 || !heldBuilds.has(build);
|
|
3921
|
+
});
|
|
3922
|
+
}
|
|
3923
|
+
async function sweepSandboxes(input) {
|
|
3924
|
+
let listed;
|
|
3925
|
+
try {
|
|
3926
|
+
listed = await input.client.list();
|
|
3927
|
+
} catch (err) {
|
|
3928
|
+
input.log?.(`could not list sandboxes: ${err instanceof Error ? err.message : String(err)}`);
|
|
3929
|
+
return 0;
|
|
3930
|
+
}
|
|
3931
|
+
let ended = 0;
|
|
3932
|
+
for (const sandbox of orphanedSandboxes(listed, input.runnerId, input.heldBuilds())) {
|
|
3933
|
+
try {
|
|
3934
|
+
await input.client.end(sandbox.id);
|
|
3935
|
+
ended++;
|
|
3936
|
+
input.log?.(`ended sandbox ${sandbox.id} (${sandbox.name}): no unfinished build holds it`);
|
|
3937
|
+
} catch (err) {
|
|
3938
|
+
input.log?.(`could not end sandbox ${sandbox.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
3939
|
+
}
|
|
4132
3940
|
}
|
|
3941
|
+
return ended;
|
|
3942
|
+
}
|
|
3943
|
+
var BUFFER_HIGH_WATER = 1024 * 1024;
|
|
3944
|
+
var DRAIN_POLL_MS = 5;
|
|
3945
|
+
var OPEN_TIMEOUT_MS = 3e4;
|
|
3946
|
+
var defaultWebSocket = (url) => new WebSocket(url);
|
|
3947
|
+
function sandboxExecStream(opts) {
|
|
3948
|
+
const create = opts.webSocket ?? defaultWebSocket;
|
|
3949
|
+
return () => {
|
|
3950
|
+
const stdout = new PassThrough();
|
|
3951
|
+
let socket;
|
|
3952
|
+
let open = false;
|
|
3953
|
+
let finished2 = false;
|
|
3954
|
+
let tail = "";
|
|
3955
|
+
let finish2;
|
|
3956
|
+
const exited = new Promise((resolve2) => finish2 = resolve2);
|
|
3957
|
+
const end = (status, why) => {
|
|
3958
|
+
if (finished2) return;
|
|
3959
|
+
finished2 = true;
|
|
3960
|
+
if (why !== void 0) tail = `${tail}${why}`.slice(-400);
|
|
3961
|
+
stdout.end();
|
|
3962
|
+
try {
|
|
3963
|
+
socket?.close();
|
|
3964
|
+
} catch {
|
|
3965
|
+
}
|
|
3966
|
+
finish2(status);
|
|
3967
|
+
};
|
|
3968
|
+
const opened = new Promise((resolve2, reject) => {
|
|
3969
|
+
const timer = setTimeout(() => reject(new Error("the WebSocket did not open in time")), OPEN_TIMEOUT_MS);
|
|
3970
|
+
timer.unref();
|
|
3971
|
+
opts.url().then((url) => {
|
|
3972
|
+
socket = create(url);
|
|
3973
|
+
socket.binaryType = "arraybuffer";
|
|
3974
|
+
socket.addEventListener("open", () => {
|
|
3975
|
+
clearTimeout(timer);
|
|
3976
|
+
open = true;
|
|
3977
|
+
socket.send(opts.secret);
|
|
3978
|
+
resolve2();
|
|
3979
|
+
});
|
|
3980
|
+
socket.addEventListener("message", (event) => {
|
|
3981
|
+
if (event.data instanceof ArrayBuffer) stdout.write(Buffer.from(event.data));
|
|
3982
|
+
else if (ArrayBuffer.isView(event.data)) stdout.write(Buffer.from(event.data.buffer, event.data.byteOffset, event.data.byteLength));
|
|
3983
|
+
});
|
|
3984
|
+
socket.addEventListener("error", (event) => {
|
|
3985
|
+
clearTimeout(timer);
|
|
3986
|
+
reject(new Error(event.message ?? "the WebSocket failed"));
|
|
3987
|
+
});
|
|
3988
|
+
socket.addEventListener("close", (event) => {
|
|
3989
|
+
clearTimeout(timer);
|
|
3990
|
+
reject(new Error("the WebSocket closed before it opened"));
|
|
3991
|
+
end(event.code === 1e3 || event.code === void 0 ? 0 : 1, event.code === 1008 ? "the relay refused the join secret" : void 0);
|
|
3992
|
+
});
|
|
3993
|
+
}).catch((err) => {
|
|
3994
|
+
clearTimeout(timer);
|
|
3995
|
+
reject(err);
|
|
3996
|
+
});
|
|
3997
|
+
});
|
|
3998
|
+
opened.catch((err) => end(1, err instanceof Error ? err.message : String(err)));
|
|
3999
|
+
const stdin = new Writable({
|
|
4000
|
+
write(chunk, _encoding, callback) {
|
|
4001
|
+
opened.then(
|
|
4002
|
+
async () => {
|
|
4003
|
+
if (finished2) return callback(new Error("the connection is closed"));
|
|
4004
|
+
socket.send(chunk);
|
|
4005
|
+
while (open && !finished2 && socket.bufferedAmount > BUFFER_HIGH_WATER) await new Promise((resolve2) => setTimeout(resolve2, DRAIN_POLL_MS));
|
|
4006
|
+
callback();
|
|
4007
|
+
},
|
|
4008
|
+
() => callback(new Error("the connection did not open"))
|
|
4009
|
+
);
|
|
4010
|
+
},
|
|
4011
|
+
final(callback) {
|
|
4012
|
+
end(0);
|
|
4013
|
+
callback();
|
|
4014
|
+
}
|
|
4015
|
+
});
|
|
4016
|
+
stdin.on("error", () => void 0);
|
|
4017
|
+
stdout.on("error", () => void 0);
|
|
4018
|
+
return { stdin, stdout, exited, kill: () => end(-1), stderrTail: () => tail };
|
|
4019
|
+
};
|
|
4020
|
+
}
|
|
4021
|
+
var RUN_DIR = "/tmp/ol-run";
|
|
4022
|
+
var CHUNK_BYTES = 9e4;
|
|
4023
|
+
var READ_ATTEMPTS = 4;
|
|
4024
|
+
var POLL_FIRST_MS = 50;
|
|
4025
|
+
var POLL_LONGEST_MS = 1e3;
|
|
4026
|
+
function quote(word) {
|
|
4027
|
+
return /^[A-Za-z0-9_@%+=:,./-]+$/.test(word) ? word : `'${word.replace(/'/g, `'\\''`)}'`;
|
|
4028
|
+
}
|
|
4029
|
+
function sandboxLine(command, args, env) {
|
|
4030
|
+
const values = Object.entries(env ?? {}).filter((entry) => entry[1] !== void 0).map(([name, value]) => `${name}=${quote(value)}`);
|
|
4031
|
+
return ["sudo", "env", ...values, command, ...args.map(quote)].join(" ");
|
|
4032
|
+
}
|
|
4033
|
+
function sandboxRunner(client2, sandbox, options = {}) {
|
|
4034
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolve2) => setTimeout(resolve2, ms)));
|
|
4035
|
+
return async (command, args, opts = {}) => {
|
|
4036
|
+
const id = randomBytes(6).toString("hex");
|
|
4037
|
+
const file = (suffix) => `${RUN_DIR}/${id}.${suffix}`;
|
|
4038
|
+
const line2 = sandboxLine(command, args, opts.env);
|
|
4039
|
+
options.onCommand?.(line2);
|
|
4040
|
+
if (opts.input !== void 0) {
|
|
4041
|
+
await client2.command(sandbox, `mkdir -p ${RUN_DIR}`);
|
|
4042
|
+
await client2.upload(sandbox, file("in"), Buffer.isBuffer(opts.input) ? opts.input : Buffer.from(opts.input));
|
|
4043
|
+
}
|
|
4044
|
+
const start = `mkdir -p ${RUN_DIR}; echo $$ > ${file("pid")}; ${line2} < ${opts.input === void 0 ? "/dev/null" : file("in")} > ${file("out")} 2> ${file("err")}; echo $? > ${file("code.tmp")}; mv -f ${file("code.tmp")} ${file("code")}`;
|
|
4045
|
+
await client2.command(sandbox, start, { background: true });
|
|
4046
|
+
const deadline = opts.timeoutMs === void 0 ? void 0 : Date.now() + opts.timeoutMs;
|
|
4047
|
+
const collect = `if [ -f ${file("code")} ]; then cat ${file("code")}; head -c ${CHUNK_BYTES} ${file("out")} | base64 | tr -d '\\n'; echo; head -c ${CHUNK_BYTES} ${file("err")} | base64 | tr -d '\\n'; echo; wc -c < ${file("out")}; wc -c < ${file("err")}; fi`;
|
|
4048
|
+
const ask2 = async (cmd) => {
|
|
4049
|
+
for (let attempt = 1; ; attempt++) {
|
|
4050
|
+
try {
|
|
4051
|
+
return await client2.command(sandbox, cmd);
|
|
4052
|
+
} catch (err) {
|
|
4053
|
+
const transient = err instanceof SandboxError && (err.status === 0 || err.status >= 500);
|
|
4054
|
+
if (!transient || attempt >= READ_ATTEMPTS) throw err;
|
|
4055
|
+
await sleep(POLL_FIRST_MS * attempt);
|
|
4056
|
+
}
|
|
4057
|
+
}
|
|
4058
|
+
};
|
|
4059
|
+
let wait = POLL_FIRST_MS;
|
|
4060
|
+
for (; ; ) {
|
|
4061
|
+
const polled = await ask2(collect);
|
|
4062
|
+
if (polled.stdout.trim() !== "") {
|
|
4063
|
+
const [code = "", out = "", err = "", outSize = "0", errSize = "0"] = polled.stdout.split("\n").map((part) => part.trim());
|
|
4064
|
+
const pieces = [Buffer.from(out, "base64")];
|
|
4065
|
+
const errPieces = [Buffer.from(err, "base64")];
|
|
4066
|
+
const more = async (name, size, into) => {
|
|
4067
|
+
for (let read = CHUNK_BYTES; read < size; read += CHUNK_BYTES) {
|
|
4068
|
+
const next = await ask2(`tail -c +${read + 1} ${file(name)} | head -c ${CHUNK_BYTES} | base64 | tr -d '\\n'`);
|
|
4069
|
+
into.push(Buffer.from(next.stdout.trim(), "base64"));
|
|
4070
|
+
}
|
|
4071
|
+
};
|
|
4072
|
+
await more("out", Number(outSize), pieces);
|
|
4073
|
+
await more("err", Number(errSize), errPieces);
|
|
4074
|
+
await client2.command(sandbox, `rm -f ${RUN_DIR}/${id}.*`);
|
|
4075
|
+
const result = { status: Number(code) || 0, stdout: Buffer.concat(pieces).toString("utf8"), stderr: Buffer.concat(errPieces).toString("utf8") };
|
|
4076
|
+
if (opts.logPath !== void 0) {
|
|
4077
|
+
mkdirSync(dirname(opts.logPath), { recursive: true });
|
|
4078
|
+
appendFileSync(opts.logPath, result.stdout + result.stderr);
|
|
4079
|
+
}
|
|
4080
|
+
return result;
|
|
4081
|
+
}
|
|
4082
|
+
if (deadline !== void 0 && Date.now() >= deadline) {
|
|
4083
|
+
await client2.command(sandbox, `kill -9 $(cat ${file("pid")}) 2>/dev/null; rm -f ${RUN_DIR}/${id}.*`).catch(() => void 0);
|
|
4084
|
+
const extra = `${command}: no answer within ${Math.round(opts.timeoutMs / 1e3)}s
|
|
4085
|
+
`;
|
|
4086
|
+
if (opts.logPath !== void 0) {
|
|
4087
|
+
mkdirSync(dirname(opts.logPath), { recursive: true });
|
|
4088
|
+
appendFileSync(opts.logPath, extra);
|
|
4089
|
+
}
|
|
4090
|
+
return { status: 124, stdout: "", stderr: extra };
|
|
4091
|
+
}
|
|
4092
|
+
await sleep(wait);
|
|
4093
|
+
wait = Math.min(wait * 2, POLL_LONGEST_MS);
|
|
4094
|
+
}
|
|
4095
|
+
};
|
|
4096
|
+
}
|
|
4097
|
+
function sandboxHost(client2, sandbox, options = {}) {
|
|
4098
|
+
return createContainerHost({ run: sandboxRunner(client2, sandbox, options) });
|
|
4099
|
+
}
|
|
4100
|
+
|
|
4101
|
+
// src/runner/sandbox/hooks.ts
|
|
4102
|
+
var SANDBOX_DIR = "/tmp/ol";
|
|
4103
|
+
var CLI_DIR = `${SANDBOX_DIR}/dist`;
|
|
4104
|
+
var SKILL_PACK_DIR = `${SANDBOX_DIR}/skill-pack`;
|
|
4105
|
+
var SECRET_FILE = `${SANDBOX_DIR}/join-secret`;
|
|
4106
|
+
var CONTAINER_SECRET_FILE = "/run/outerlayer-relay/join-secret";
|
|
4107
|
+
var JOIN_WS_PORT = 7077;
|
|
4108
|
+
var SANDBOX_RESERVE_MIB = 512;
|
|
4109
|
+
var RELAY_ATTEMPTS = 40;
|
|
4110
|
+
var RELAY_WAIT_MS = 500;
|
|
4111
|
+
var SANDBOX_PROBE_BYTES = 96 * 1024;
|
|
4112
|
+
var MAX_TAIL_READS2 = 8;
|
|
4113
|
+
var NO_ENV = { literals: {}, passed: {} };
|
|
4114
|
+
function sandboxNameOf(runnerLabel, buildId) {
|
|
4115
|
+
const short = runnerLabel.toLowerCase().replace(/[^a-z0-9]/g, "").slice(0, 8);
|
|
4116
|
+
return `ol-${short === "" ? "runner" : short}-${buildId}`;
|
|
4117
|
+
}
|
|
4118
|
+
function itemSnapshotName(runnerLabel, part = randomBytes(6).toString("hex")) {
|
|
4119
|
+
return sandboxNameOf(runnerLabel, `disk-${part}`);
|
|
4120
|
+
}
|
|
4121
|
+
function diskImageOf(own) {
|
|
4122
|
+
return `${own.snapshot}/${own.image}`;
|
|
4133
4123
|
}
|
|
4134
|
-
|
|
4135
|
-
|
|
4124
|
+
var GIB2 = 1024 ** 3;
|
|
4125
|
+
async function endUnansweredSandbox(sandboxClient, runnerId, buildId, warn) {
|
|
4136
4126
|
try {
|
|
4137
|
-
|
|
4127
|
+
for (const found of await sandboxClient.list()) {
|
|
4128
|
+
if (found.state !== "ended" && found.metadata[RUNNER_METADATA] === runnerId && found.metadata[BUILD_METADATA] === buildId) await sandboxClient.end(found.id);
|
|
4129
|
+
}
|
|
4138
4130
|
} catch (err) {
|
|
4139
|
-
|
|
4140
|
-
return void 0;
|
|
4131
|
+
warn(`could not end a sandbox that a lost create may have made: ${describeCallFailure(err)} \xB7 the next sweep ends it`);
|
|
4141
4132
|
}
|
|
4133
|
+
}
|
|
4134
|
+
function client(runtime) {
|
|
4135
|
+
return runtime.sandbox.client();
|
|
4136
|
+
}
|
|
4137
|
+
async function sandboxOfBuild(runtime, state) {
|
|
4138
|
+
const held3 = runtime.sandbox.sandboxes.get(state.buildId);
|
|
4139
|
+
if (held3 !== void 0) return held3;
|
|
4142
4140
|
try {
|
|
4143
|
-
const
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
return result.status === 0 ? parseProbe(result.stdout) : void 0;
|
|
4141
|
+
const found = await client(runtime).get(state.sandboxId);
|
|
4142
|
+
if (found.state === "ended" || found.secret === "") return void 0;
|
|
4143
|
+
runtime.sandbox.sandboxes.set(state.buildId, found);
|
|
4144
|
+
return found;
|
|
4148
4145
|
} catch (err) {
|
|
4149
|
-
if (err instanceof SandboxError && err.status === 404)
|
|
4150
|
-
if (err instanceof SandboxError) return void 0;
|
|
4146
|
+
if (err instanceof SandboxError && err.status === 404) return void 0;
|
|
4151
4147
|
throw err;
|
|
4152
4148
|
}
|
|
4153
4149
|
}
|
|
4154
|
-
function
|
|
4155
|
-
|
|
4156
|
-
argv[argv.length - 1] = String(SANDBOX_PROBE_BYTES);
|
|
4157
|
-
return argv;
|
|
4150
|
+
function runtimeInSandbox(runtime, sandbox) {
|
|
4151
|
+
return { ...runtime, host: sandboxHost(client(runtime), sandbox, { sleep: runtime.sandbox.sleep }) };
|
|
4158
4152
|
}
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
const probe = await probeSandboxCommand(runtime, state, offset);
|
|
4165
|
-
if (probe === void 0) break;
|
|
4166
|
-
appendCommandLog(jobDir, probe.bytes);
|
|
4167
|
-
if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) break;
|
|
4168
|
-
}
|
|
4169
|
-
await sandboxed.host.stop(state.container.container);
|
|
4170
|
-
} catch (err) {
|
|
4171
|
-
warn(`ending the build's command in its sandbox failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
4153
|
+
var SandboxGoneError = class extends Error {
|
|
4154
|
+
constructor(sandboxId, endedReason) {
|
|
4155
|
+
super(`The provider ended sandbox ${sandboxId} while the build ran${endedReason === void 0 || endedReason === "" ? "" : ` (${endedReason})`}`);
|
|
4156
|
+
this.endedReason = endedReason;
|
|
4157
|
+
this.name = "SandboxGoneError";
|
|
4172
4158
|
}
|
|
4159
|
+
};
|
|
4160
|
+
async function inSandbox(runtime, state) {
|
|
4161
|
+
const sandbox = await sandboxOfBuild(runtime, state);
|
|
4162
|
+
if (sandbox === void 0) throw new SandboxGoneError(state.sandboxId);
|
|
4163
|
+
return runtimeInSandbox(runtime, sandbox);
|
|
4173
4164
|
}
|
|
4174
|
-
async function
|
|
4175
|
-
const { job } = input;
|
|
4176
|
-
const own = job.sandbox;
|
|
4165
|
+
async function goneReason(runtime, state) {
|
|
4177
4166
|
try {
|
|
4178
|
-
const
|
|
4179
|
-
|
|
4180
|
-
const env = buildEnvironment({
|
|
4181
|
-
config: claimCalls(input.config, job),
|
|
4182
|
-
item: job.item,
|
|
4183
|
-
...job.queue === "run" ? { run: true } : {},
|
|
4184
|
-
itemKey: job.itemKey,
|
|
4185
|
-
branch: job.branch,
|
|
4186
|
-
repository: state.repository,
|
|
4187
|
-
repositories: job.repositories,
|
|
4188
|
-
hostEnv: input.runtime.hostEnv,
|
|
4189
|
-
agents: input.runtime.agents,
|
|
4190
|
-
job,
|
|
4191
|
-
containerEnv: state.containerEnv ?? {},
|
|
4192
|
-
unpinnedRuntimes: state.unpinnedRuntimes,
|
|
4193
|
-
missingSetupScripts: state.missingSetupScripts,
|
|
4194
|
-
miseVariables: state.miseVariables,
|
|
4195
|
-
builtIns: true,
|
|
4196
|
-
identity: job.commitIdentity
|
|
4197
|
-
});
|
|
4198
|
-
const result = await sandboxed.host.exec(
|
|
4199
|
-
{ container: state.container, user: state.agentUser ?? ADDED_USER, workdir: job.workdir ?? CHECKOUT_DIR, env, argv },
|
|
4200
|
-
{ logPath: join(input.jobDir, logName) }
|
|
4201
|
-
);
|
|
4202
|
-
return result.status === 0 ? { ok: true } : { ok: false, reason: `exit ${result.status}` };
|
|
4167
|
+
const found = await client(runtime).get(state.sandboxId);
|
|
4168
|
+
return found.state === "ended" ? new SandboxGoneError(state.sandboxId, found.endedReason ?? void 0) : void 0;
|
|
4203
4169
|
} catch (err) {
|
|
4204
|
-
return
|
|
4170
|
+
return err instanceof SandboxError && err.status === 404 ? new SandboxGoneError(state.sandboxId) : void 0;
|
|
4205
4171
|
}
|
|
4206
4172
|
}
|
|
4207
|
-
function
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4173
|
+
function sandboxRunArgs(args, opts = {}) {
|
|
4174
|
+
const out = [...args];
|
|
4175
|
+
const network = out.indexOf("--network");
|
|
4176
|
+
if (network !== -1) out[network + 1] = "host";
|
|
4177
|
+
const entrypoint = out.indexOf("--entrypoint");
|
|
4178
|
+
out.splice(
|
|
4179
|
+
entrypoint,
|
|
4180
|
+
0,
|
|
4181
|
+
"--mount",
|
|
4182
|
+
`type=bind,src=${CLI_DIR},dst=/opt/outerlayer/cli/dist,readonly`,
|
|
4183
|
+
...opts.skillPack === true ? ["--mount", `type=bind,src=${SKILL_PACK_DIR},dst=/opt/outerlayer/cli/skill-pack,readonly`] : [],
|
|
4184
|
+
"--mount",
|
|
4185
|
+
`type=bind,src=${SECRET_FILE},dst=${CONTAINER_SECRET_FILE},readonly`,
|
|
4186
|
+
...Object.entries(toolProxyEnvironment()).flatMap(([name, value]) => ["--env", `${name}=${value}`])
|
|
4187
|
+
);
|
|
4188
|
+
out.push("--join-ws", String(JOIN_WS_PORT), "--join-secret-file", CONTAINER_SECRET_FILE);
|
|
4189
|
+
return out;
|
|
4215
4190
|
}
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4191
|
+
var RELAY_PROBE = `const s=require("net").connect(3128,"127.0.0.1");s.on("connect",()=>process.exit(0));s.on("error",()=>process.exit(1));setTimeout(()=>process.exit(1),2000)`;
|
|
4192
|
+
async function waitForRelay(runtime, container) {
|
|
4193
|
+
for (let attempt = 0; attempt < RELAY_ATTEMPTS; attempt++) {
|
|
4194
|
+
const probe = await runtime.host.exec({ container, user: RELAY_USER, env: NO_ENV, argv: ["/opt/outerlayer/bin/node", "-e", RELAY_PROBE] });
|
|
4195
|
+
if (probe.status === 0) return true;
|
|
4196
|
+
await runtime.sandbox.sleep(RELAY_WAIT_MS);
|
|
4197
|
+
}
|
|
4198
|
+
return false;
|
|
4223
4199
|
}
|
|
4224
|
-
function
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4200
|
+
function joinRelay(runtime, sandbox, build, services, joinSecret, log) {
|
|
4201
|
+
const exec = sandboxExecStream({
|
|
4202
|
+
url: () => client(runtime).portUrl(sandbox, JOIN_WS_PORT),
|
|
4203
|
+
secret: joinSecret,
|
|
4204
|
+
...runtime.sandbox.webSocket === void 0 ? {} : { webSocket: runtime.sandbox.webSocket }
|
|
4205
|
+
});
|
|
4206
|
+
runtime.relays.set(
|
|
4207
|
+
build.buildId,
|
|
4208
|
+
superviseRelay({
|
|
4209
|
+
container: build.container,
|
|
4210
|
+
services,
|
|
4211
|
+
execStream: () => exec(),
|
|
4212
|
+
...log === void 0 ? {} : { log },
|
|
4213
|
+
onFatal: (reason) => runtime.relayFatal.set(build.buildId, reason),
|
|
4214
|
+
builtIns: builtInNames(runtime.agents)
|
|
4215
|
+
})
|
|
4216
|
+
);
|
|
4228
4217
|
}
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4233
|
-
}
|
|
4234
|
-
|
|
4218
|
+
var SANDBOX_RUN_DIR = "/tmp/ol-run";
|
|
4219
|
+
async function keepSandboxDisk(runtime, state, kept) {
|
|
4220
|
+
const sandbox = await sandboxOfBuild(runtime, state);
|
|
4221
|
+
if (sandbox === void 0) throw new Error(`sandbox ${state.sandboxId} is gone`);
|
|
4222
|
+
const { host } = runtimeInSandbox(runtime, sandbox);
|
|
4223
|
+
const removed = await host.remove(kept.container);
|
|
4224
|
+
if (removed.status !== 0 && !/no such container/i.test(removed.stderr)) throw new Error(`container ${kept.container} was not removed${said(removed.stderr)}`);
|
|
4225
|
+
const emptied = await host.run(clearOneBuildArgs({ image: kept.imageId, volume: kept.volume, runnerId: kept.runnerId }));
|
|
4226
|
+
if (emptied.status !== 0) throw new Error(`the build's command state was not removed from the volume (exit ${emptied.status})${said(emptied.stderr)}`);
|
|
4227
|
+
const cleared = await client(runtime).command(sandbox, `sudo rm -rf ${SANDBOX_DIR} ${SANDBOX_RUN_DIR}`);
|
|
4228
|
+
if (cleared.exitCode !== 0) throw new Error(`the runner's files were not removed from the sandbox (exit ${cleared.exitCode ?? "none"})${said(cleared.stderr)}`);
|
|
4229
|
+
await kept.keep();
|
|
4230
|
+
}
|
|
4231
|
+
async function cleanupSandbox(runtime, state, keepDisk) {
|
|
4232
|
+
const failures = [];
|
|
4233
|
+
const relay = runtime.relays.get(state.buildId);
|
|
4234
|
+
if (relay !== void 0) {
|
|
4235
|
+
runtime.relays.delete(state.buildId);
|
|
4236
|
+
await relay.close();
|
|
4235
4237
|
}
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4238
|
+
runtime.relayFatal.delete(state.buildId);
|
|
4239
|
+
runtime.leases.get(state.buildId)?.close();
|
|
4240
|
+
runtime.leases.delete(state.buildId);
|
|
4241
|
+
if (keepDisk !== void 0) {
|
|
4239
4242
|
try {
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
+
await keepSandboxDisk(runtime, state, keepDisk);
|
|
4244
|
+
} catch (err) {
|
|
4245
|
+
keepDisk.warn(`the build's disk was not kept: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
|
|
4243
4246
|
}
|
|
4244
4247
|
}
|
|
4245
|
-
|
|
4248
|
+
if (state.sandboxId !== "") {
|
|
4249
|
+
try {
|
|
4250
|
+
await client(runtime).end(state.sandboxId);
|
|
4251
|
+
} catch (err) {
|
|
4252
|
+
failures.push(`sandbox ${state.sandboxId} was not ended: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
|
|
4253
|
+
}
|
|
4254
|
+
}
|
|
4255
|
+
runtime.sandbox.sandboxes.delete(state.buildId);
|
|
4256
|
+
runtime.sandbox.joinSecrets.delete(state.buildId);
|
|
4257
|
+
const tunnel = runtime.tunnels.get(state.buildId);
|
|
4258
|
+
if (tunnel !== void 0) {
|
|
4259
|
+
runtime.tunnels.delete(state.buildId);
|
|
4260
|
+
await tunnel.close();
|
|
4261
|
+
}
|
|
4262
|
+
if (state.container !== void 0) rmSync(state.container.buildDir, { recursive: true, force: true });
|
|
4263
|
+
if (failures.length > 0) throw new Error(failures.join("; "));
|
|
4246
4264
|
}
|
|
4247
|
-
function
|
|
4248
|
-
|
|
4265
|
+
async function provisionSandbox(input) {
|
|
4266
|
+
const tag = itemTag({ item: input.item.number, organizationId: input.organizationId, claimId: input.claimId });
|
|
4267
|
+
return provisionWithFreshFallback({
|
|
4268
|
+
attempt: (_allowRestore, itemDiskFailed) => provisionSandboxOnce(input, itemDiskFailed),
|
|
4269
|
+
log: (line2) => input.log.line(`${tag} ${line2}`)
|
|
4270
|
+
});
|
|
4249
4271
|
}
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4272
|
+
async function provisionSandboxOnce(input, itemDiskFailed) {
|
|
4273
|
+
const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
|
|
4274
|
+
const tag = itemTag({ item: item.number, organizationId: input.organizationId, claimId: input.claimId });
|
|
4275
|
+
const logPath = join(jobDir, "provision.log");
|
|
4276
|
+
const fail = (reason) => ({ ok: false, reason });
|
|
4277
|
+
const own = config.sandbox;
|
|
4278
|
+
if (own === void 0) return fail("runner.sandbox is not set in the config file");
|
|
4279
|
+
const repositories = repositoriesOf({ repositories: input.repositories, repository: item.specRepository, branch: input.branch });
|
|
4280
|
+
const primary = primaryOf(repositories);
|
|
4281
|
+
if (primary === void 0) return fail("the work item names no repository to clone");
|
|
4282
|
+
const repository = primary.repository;
|
|
4283
|
+
const sized = sandboxSizeOf(config.build);
|
|
4284
|
+
if (!sized.ok) return fail(`${sized.field} ${sized.problem}`);
|
|
4285
|
+
let sandboxClient;
|
|
4286
|
+
try {
|
|
4287
|
+
sandboxClient = client(runtime);
|
|
4288
|
+
} catch (err) {
|
|
4289
|
+
return fail(err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err));
|
|
4259
4290
|
}
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4291
|
+
const governing = input.queue === "run" ? SELF_GOVERNED : await resolveGoverningSource({ config, repository, fetchImpl: input.fetchImpl });
|
|
4292
|
+
if (!governing.ok) return fail(governing.reason);
|
|
4293
|
+
const governingSource = governing.source;
|
|
4294
|
+
const controlPlaneUrl = (controlPlane) => runtime.remoteFor?.(controlPlane) ?? `${runtime.github.gitUrl}/${controlPlane}.git`;
|
|
4295
|
+
if (governingSource.kind === "governed") {
|
|
4296
|
+
const unfetched = await prefetchGovernedContext({
|
|
4297
|
+
configPath: config.configPath,
|
|
4298
|
+
repository,
|
|
4299
|
+
source: governingSource,
|
|
4300
|
+
controlPlaneUrl,
|
|
4301
|
+
controlPlaneToken: () => tokens.controlPlane(),
|
|
4302
|
+
log,
|
|
4303
|
+
tag
|
|
4304
|
+
});
|
|
4305
|
+
if (unfetched !== void 0) return fail(unfetched);
|
|
4306
|
+
}
|
|
4307
|
+
const diskKey = { imageId: diskImageOf(own), diskBytes: sized.size.diskGib * GIB2 };
|
|
4308
|
+
const itemDisk = input.workItemId === void 0 ? { state: void 0 } : itemDiskFailed !== void 0 ? { state: { restored: false, reason: itemDiskFailed } } : itemDiskDecision({ fresh: input.fresh, heldElsewhere: input.diskHeldElsewhere, builtin: true, backend: "sandbox", configPath: config.configPath, organizationId: input.organizationId, workItemId: input.workItemId, repository, ...diskKey });
|
|
4309
|
+
const keptDisk = itemDisk.kept;
|
|
4310
|
+
if (itemDisk.state?.restored === false) log.line(`${tag} this job starts without a kept disk: ${DISK_REASON_TEXT[itemDisk.state.reason]}`);
|
|
4311
|
+
const buildId = randomBytes(6).toString("hex");
|
|
4312
|
+
const runnerId = runnerLabelValue(config.configPath);
|
|
4313
|
+
const name = sandboxNameOf(runnerId, buildId);
|
|
4314
|
+
const container = `outerlayer-build-${buildId}`;
|
|
4315
|
+
const volumeName = keptDisk?.record.volume ?? container;
|
|
4316
|
+
const buildDir2 = buildDir(config.configPath, buildId);
|
|
4317
|
+
const image = own.image;
|
|
4318
|
+
const containerState = {
|
|
4319
|
+
buildId,
|
|
4320
|
+
container,
|
|
4321
|
+
volume: volumeName,
|
|
4322
|
+
buildDir: buildDir2,
|
|
4323
|
+
repository,
|
|
4324
|
+
imageId: image,
|
|
4325
|
+
builtIns: true,
|
|
4326
|
+
...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository, ref: governingSource.ref, commit: governingSource.sha } } : {},
|
|
4327
|
+
transport: "exec",
|
|
4328
|
+
jobFiles: "copied"
|
|
4329
|
+
};
|
|
4330
|
+
const state = { sandboxId: "", name, dataPlaneUrl: "", buildId, diskKey, container: containerState };
|
|
4331
|
+
const secrets = secretsOf(runtime, jobDir);
|
|
4332
|
+
const env = buildEnvironment({
|
|
4333
|
+
config,
|
|
4334
|
+
item: item.number,
|
|
4335
|
+
...input.queue === "run" ? { run: true } : {},
|
|
4336
|
+
itemKey,
|
|
4337
|
+
repository,
|
|
4338
|
+
repositories,
|
|
4339
|
+
hostEnv: runtime.hostEnv,
|
|
4340
|
+
agents: runtime.agents,
|
|
4341
|
+
...input.job === void 0 ? {} : { job: input.job },
|
|
4342
|
+
containerEnv: {},
|
|
4343
|
+
builtIns: true
|
|
4344
|
+
});
|
|
4345
|
+
if (env.skippedTokenVariables.length > 0) {
|
|
4346
|
+
log.line(`${tag} not passing ${env.skippedTokenVariables.join(", ")} from runner.build.variables into the build: its repository token comes from the runner`);
|
|
4263
4347
|
}
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4348
|
+
const environment = {
|
|
4349
|
+
isolation: "vm",
|
|
4350
|
+
imageId: image,
|
|
4351
|
+
variables: env.variables,
|
|
4352
|
+
cliVersion: input.cliVersion
|
|
4353
|
+
};
|
|
4354
|
+
const bail = async (reason, aborted2 = false) => {
|
|
4270
4355
|
try {
|
|
4271
|
-
await
|
|
4272
|
-
removed.push(record);
|
|
4273
|
-
input.log(`removed saved state ${record.id} from ${record.savedAt}`);
|
|
4356
|
+
await cleanupSandbox(runtime, { buildId, sandboxId: state.sandboxId, container: containerState });
|
|
4274
4357
|
} catch (err) {
|
|
4275
|
-
|
|
4358
|
+
log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
|
|
4276
4359
|
}
|
|
4360
|
+
return aborted2 ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(reason);
|
|
4361
|
+
};
|
|
4362
|
+
const giveUpItemDisk = async (reason, why) => {
|
|
4363
|
+
await bail(reason);
|
|
4364
|
+
return discardUnusableItemDisk({ kept: keptDisk, reason, why, removeVolume: async () => void 0, removeSnapshot: (snapshot) => sandboxClient.removeSnapshot(snapshot), onError: (line2) => log.error(`${tag} ${line2}`) });
|
|
4365
|
+
};
|
|
4366
|
+
if (input.aborted()) return bail("", true);
|
|
4367
|
+
mkdirSync(buildDir2, { recursive: true });
|
|
4368
|
+
const ghDir = ghConfigDir(buildDir2);
|
|
4369
|
+
let readToken;
|
|
4370
|
+
try {
|
|
4371
|
+
readToken = await tokens.token("read");
|
|
4372
|
+
writeGhConfig(ghDir, readToken);
|
|
4373
|
+
} catch (err) {
|
|
4374
|
+
return bail(err instanceof WorkCommandError ? `the build's read token could not be had: ${refusalCode(err)}` : `the build's gh configuration could not be written: ${describeCallFailure(err)}`);
|
|
4277
4375
|
}
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
async function removeSuperseded(input) {
|
|
4281
|
-
for (const old of input.records.filter((record) => record.complete && record.key === input.made.key && record.id !== input.made.id && Date.parse(record.savedAt) < Date.parse(input.made.savedAt))) {
|
|
4376
|
+
for (const entry of repositories) {
|
|
4377
|
+
if (entry.repository === primary.repository) continue;
|
|
4282
4378
|
try {
|
|
4283
|
-
await
|
|
4284
|
-
input.log(`removed saved state ${old.id}, saved state ${input.made.id} replaces it`);
|
|
4379
|
+
await tokens.token("read", entry.repository);
|
|
4285
4380
|
} catch (err) {
|
|
4286
|
-
|
|
4381
|
+
return bail(`the build's read token for ${entry.repository} could not be had: ${refusalCode(err)}`);
|
|
4287
4382
|
}
|
|
4288
4383
|
}
|
|
4289
|
-
}
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4295
|
-
}
|
|
4296
|
-
|
|
4297
|
-
|
|
4298
|
-
}
|
|
4299
|
-
async function planSavedState(input) {
|
|
4300
|
-
const { config, runtime, log } = input;
|
|
4301
|
-
const refused2 = savingRefusedBecause({ universalImage: input.base !== void 0, fleet: config.fleet === true, savedStateMax: savedStateMaxOf(config) });
|
|
4302
|
-
if (refused2 !== void 0 || input.base === void 0) {
|
|
4303
|
-
const why = refused2 ?? "its image comes from a devcontainer file";
|
|
4304
|
-
log(`this build saves no state: ${why}`);
|
|
4305
|
-
return { keyCommits: {}, noSave: why };
|
|
4384
|
+
keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`));
|
|
4385
|
+
tokens.markUsed("read");
|
|
4386
|
+
environment.credentials = tokens.used();
|
|
4387
|
+
let services;
|
|
4388
|
+
try {
|
|
4389
|
+
const gitRemote = gitRemoteFor(runtime, { repositories, tokens, buildDir: buildDir2, log: (line2) => log.line(`${tag} ${line2}`), ...input.queue === "run" ? { readOnly: true } : {} });
|
|
4390
|
+
services = await startServicesFor(runtime, config, buildDir2, jobDir, itemKey, true, gitRemote, "stream", input.leaseExpiryMs === void 0 ? void 0 : leaseFor(runtime, buildId, input.leaseExpiryMs), void 0, (line2) => log.line(`${tag} ${line2}`));
|
|
4391
|
+
runtime.tunnels.set(buildId, services);
|
|
4392
|
+
} catch (err) {
|
|
4393
|
+
return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
|
|
4306
4394
|
}
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
remote: runtime.remoteFor?.(entry.repository) ?? `${runtime.github.gitUrl}/${entry.repository}.git`,
|
|
4320
|
-
dir: join(root, "key-files", owner, name),
|
|
4321
|
-
paths: keyFilePaths(),
|
|
4322
|
-
token
|
|
4395
|
+
if (input.aborted()) return bail("", true);
|
|
4396
|
+
let sandbox;
|
|
4397
|
+
runtime.sandbox.creating.add(buildId);
|
|
4398
|
+
try {
|
|
4399
|
+
sandbox = await sandboxClient.create({
|
|
4400
|
+
name,
|
|
4401
|
+
from: keptDisk?.record.snapshot ?? own.snapshot,
|
|
4402
|
+
...sized.size,
|
|
4403
|
+
idleMinutes: own.idleMinutes,
|
|
4404
|
+
timeLimitMinutes: sandboxTimeLimitMinutes(config.timeLimitMinutes),
|
|
4405
|
+
network: networkOf(own.allowHosts),
|
|
4406
|
+
metadata: { [RUNNER_METADATA]: runnerId, [BUILD_METADATA]: buildId }
|
|
4323
4407
|
});
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4408
|
+
} catch (err) {
|
|
4409
|
+
if (err instanceof SandboxError && err.status === 0) await endUnansweredSandbox(sandboxClient, runnerId, buildId, (line2) => log.error(`${tag} ${line2}`));
|
|
4410
|
+
runtime.sandbox.creating.delete(buildId);
|
|
4411
|
+
const refused2 = `The provider refused to create the sandbox: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`;
|
|
4412
|
+
if (keptDisk !== void 0 && err instanceof SandboxError && err.status === 404) return giveUpItemDisk(refused2, "disk_not_restored");
|
|
4413
|
+
return bail(refused2);
|
|
4327
4414
|
}
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
});
|
|
4334
|
-
|
|
4335
|
-
if (
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4415
|
+
runtime.sandbox.sandboxes.set(buildId, sandbox);
|
|
4416
|
+
state.sandboxId = sandbox.id;
|
|
4417
|
+
state.dataPlaneUrl = sandbox.dataPlaneUrl;
|
|
4418
|
+
input.save(state, environment);
|
|
4419
|
+
runtime.sandbox.creating.delete(buildId);
|
|
4420
|
+
log.line(`${tag} sandbox ${sandbox.id} created \xB7 ${name}`);
|
|
4421
|
+
if (keptDisk !== void 0) log.line(`${tag} restored the disk that claim ${shortClaim(keptDisk.record.claimId)} kept: snapshot ${keptDisk.record.snapshot}`);
|
|
4422
|
+
if (input.aborted()) return bail("", true);
|
|
4423
|
+
const sandboxed = runtimeInSandbox(runtime, sandbox);
|
|
4424
|
+
const { host } = sandboxed;
|
|
4425
|
+
const dockerFailed = (what, result) => bail(`${what} failed (exit ${result.status})${said(result.stderr)}`);
|
|
4426
|
+
const joinSecret = randomBytes(32);
|
|
4427
|
+
runtime.sandbox.joinSecrets.set(buildId, joinSecret);
|
|
4428
|
+
const packDir = join(runtime.cliBuild().distDir, "..", "skill-pack");
|
|
4429
|
+
const skillPack = existsSync(packDir) ? packDir : void 0;
|
|
4430
|
+
try {
|
|
4431
|
+
await sandboxClient.command(sandbox, `mkdir -p ${CLI_DIR}`);
|
|
4432
|
+
await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/dist.tgz`, gzipSync(packFolder(runtime.cliBuild().distDir)));
|
|
4433
|
+
if (skillPack !== void 0) {
|
|
4434
|
+
await sandboxClient.upload(sandbox, `${SANDBOX_DIR}/skill-pack.tgz`, gzipSync(packFolder(skillPack)));
|
|
4435
|
+
const placed = await sandboxClient.command(sandbox, `mkdir -p ${SKILL_PACK_DIR} && tar xzf ${SANDBOX_DIR}/skill-pack.tgz -C ${SKILL_PACK_DIR} && rm -f ${SANDBOX_DIR}/skill-pack.tgz`);
|
|
4436
|
+
if (placed.exitCode !== 0) return bail(`putting the skill pack in the sandbox failed (exit ${placed.exitCode ?? "none"})${said(placed.stderr)}`);
|
|
4341
4437
|
}
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
return
|
|
4347
|
-
}
|
|
4348
|
-
if (found.kind === "expired") {
|
|
4349
|
-
log(`saved state ${found.record.id} is ${formatDuration(found.ageMs)} old, past the limit of ${SAVED_STATE_MAX_AGE_MS / 864e5} days; starting from a fresh disk`);
|
|
4350
|
-
} else if (found.kind === "changed") {
|
|
4351
|
-
log(`the setup script or version files changed on the default branch since saved state ${found.record.id}: ${found.files.join(", ")}; starting from a fresh disk`);
|
|
4352
|
-
} else if (found.kind === "none") {
|
|
4353
|
-
log("no saved state matches this build's key; starting from a fresh disk");
|
|
4438
|
+
await sandboxClient.upload(sandbox, SECRET_FILE, joinSecret);
|
|
4439
|
+
const unpacked = await sandboxClient.command(sandbox, `tar xzf ${SANDBOX_DIR}/dist.tgz -C ${CLI_DIR} && rm -f ${SANDBOX_DIR}/dist.tgz && sudo chown 60100:60100 ${SECRET_FILE} && sudo chmod 400 ${SECRET_FILE}`);
|
|
4440
|
+
if (unpacked.exitCode !== 0) return bail(`putting the CLI in the sandbox failed (exit ${unpacked.exitCode ?? "none"})${said(unpacked.stderr)}`);
|
|
4441
|
+
} catch (err) {
|
|
4442
|
+
return bail(`putting the CLI in the sandbox failed: ${err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err)}`);
|
|
4354
4443
|
}
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4444
|
+
const engine = await host.version();
|
|
4445
|
+
if (engine === null) return bail("Docker is not available in the sandbox: `docker version` failed");
|
|
4446
|
+
const limits2 = { memory: `${Math.max(512, sized.size.memoryMib - SANDBOX_RESERVE_MIB)}m`, cpus: sized.size.vcpu, pids: config.build?.pids ?? 4096 };
|
|
4447
|
+
const volume = await host.createVolume(volumeName, { [BUILD_LABEL]: buildId, [RUNNER_LABEL]: runnerId });
|
|
4448
|
+
if (volume.status !== 0) return dockerFailed("creating the build's volume", volume);
|
|
4449
|
+
const runInput = {
|
|
4450
|
+
buildId,
|
|
4451
|
+
runnerId,
|
|
4452
|
+
container,
|
|
4453
|
+
volume: volumeName,
|
|
4454
|
+
image,
|
|
4455
|
+
user: ADDED_USER,
|
|
4456
|
+
limits: limits2,
|
|
4457
|
+
destinations: config.build?.destinations?.length ?? 0,
|
|
4458
|
+
builtIns: true,
|
|
4459
|
+
transport: "exec",
|
|
4460
|
+
jobFiles: "copied"
|
|
4461
|
+
};
|
|
4462
|
+
const prepared = await host.run(prepareRunArgs(runInput), { logPath });
|
|
4463
|
+
if (prepared.status !== 0) return dockerFailed("preparing the build's volume", prepared);
|
|
4464
|
+
const imageConfig = await host.imageConfig(image);
|
|
4465
|
+
const resolved = await resolveBuildUser(sandboxed, runInput, { imageUser: imageConfig.user }, logPath);
|
|
4466
|
+
if (!resolved.ok) return bail(resolved.reason);
|
|
4467
|
+
containerState.agentUser = resolved.user;
|
|
4468
|
+
if (resolved.home !== void 0) containerState.home = resolved.home;
|
|
4469
|
+
const agentUser = resolved.user;
|
|
4470
|
+
input.save(state, environment);
|
|
4471
|
+
if (input.aborted()) return bail("", true);
|
|
4472
|
+
const started = await host.run(sandboxRunArgs(containerRunArgs({ ...runInput, home: containerState.home }, runtime.agents), { skillPack: skillPack !== void 0 }), { logPath });
|
|
4473
|
+
if (started.status !== 0) return dockerFailed("starting the build's container", started);
|
|
4474
|
+
joinRelay(runtime, sandbox, { buildId, container }, services.streams, joinSecret, (line2) => log.line(`${tag} ${line2}`));
|
|
4475
|
+
if (input.aborted()) return bail("", true);
|
|
4476
|
+
if (!await waitForRelay(sandboxed, container)) return bail("the relay in the build's container did not start");
|
|
4477
|
+
const ghDelivered = await deliverJobFiles(sandboxed, containerState, ["gh"], secrets);
|
|
4478
|
+
if (!ghDelivered.ok) return bail(ghDelivered.reason);
|
|
4479
|
+
keepGhConfigCurrent(tokens, buildDir2, (err) => log.error(`${tag} the read token could not be written for gh: ${describeCallFailure(err)}`), {
|
|
4480
|
+
runtime: sandboxed,
|
|
4481
|
+
state: containerState,
|
|
4482
|
+
secrets,
|
|
4483
|
+
delivered: readToken,
|
|
4484
|
+
onFailure: (reason) => log.error(`${tag} ${reason}`)
|
|
4485
|
+
});
|
|
4486
|
+
const tools = await host.exec({ container, user: ADDED_USER, env: NO_ENV, argv: ["cat", "/opt/outerlayer/tools.json"] });
|
|
4487
|
+
const agentVersion = agentVersionOf(tools.status === 0 ? tools.stdout : "", runtime.agents, input.job);
|
|
4488
|
+
environment.agentVersion = agentVersion;
|
|
4489
|
+
const judgeProgram = await judgeProgramRecord((argv) => host.exec({ container, user: ADDED_USER, env: NO_ENV, argv }), runtime.agents);
|
|
4490
|
+
if (judgeProgram !== void 0) containerState.judgeProgram = judgeProgram;
|
|
4491
|
+
input.save(state, environment);
|
|
4492
|
+
const stepExec = async (argv, workdir, opts) => {
|
|
4493
|
+
const ran = await host.exec({ container, user: agentUser, workdir, env: withMiseVariables(env, opts?.variables), argv }, { ...opts?.quiet === true ? {} : { logPath }, ...opts?.timeoutMs === void 0 ? {} : { timeoutMs: opts.timeoutMs } });
|
|
4494
|
+
return endedAtLimit(ran) ? { ...ran, timedOut: true } : ran;
|
|
4495
|
+
};
|
|
4496
|
+
const setupMinutes = config.build?.setupMinutes ?? DEFAULT_SETUP_MINUTES;
|
|
4497
|
+
const steps = await runBuildSteps({
|
|
4498
|
+
repository,
|
|
4499
|
+
branch: primary.branch,
|
|
4500
|
+
repositories,
|
|
4501
|
+
...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository } } : {},
|
|
4502
|
+
agentVersion,
|
|
4503
|
+
agents: runtime.agents,
|
|
4504
|
+
...input.job === void 0 ? {} : { job: input.job },
|
|
4505
|
+
lifecycle: {},
|
|
4506
|
+
exec: stepExec,
|
|
4507
|
+
log,
|
|
4508
|
+
tag,
|
|
4509
|
+
aborted: input.aborted,
|
|
4510
|
+
stageContext: () => stageGovernedContext({
|
|
4511
|
+
config,
|
|
4512
|
+
repository,
|
|
4513
|
+
source: pinnedSourceOf(governingSource),
|
|
4514
|
+
buildId,
|
|
4515
|
+
buildDir: buildDir2,
|
|
4516
|
+
fetchImpl: input.fetchImpl,
|
|
4517
|
+
controlPlaneUrl,
|
|
4518
|
+
controlPlaneToken: () => tokens.controlPlane(),
|
|
4519
|
+
log,
|
|
4520
|
+
tag
|
|
4521
|
+
}),
|
|
4522
|
+
deliverContext: async () => {
|
|
4523
|
+
const delivered = await deliverJobFiles(sandboxed, containerState, ["context"], secrets);
|
|
4524
|
+
return delivered.ok ? delivered : { ok: false, reason: `context_unavailable: ${delivered.reason}` };
|
|
4525
|
+
},
|
|
4526
|
+
removeStagedContext: () => removeStagedContext(buildDir2),
|
|
4527
|
+
keepIn: jobDir,
|
|
4528
|
+
...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
|
|
4529
|
+
// The image in the snapshot is the universal image, so the build chooses and installs its runtimes.
|
|
4530
|
+
...runtimeStepInput({ base: {} }, services),
|
|
4531
|
+
// A kept disk holds the checkouts already: each is switched to the branch, and keeps its untracked files.
|
|
4532
|
+
setup: { timeoutMs: setupMinutes * 6e4, limitMinutes: setupMinutes, restored: false, ...keptDisk === void 0 ? {} : { restored: true, fromItemDisk: true } }
|
|
4533
|
+
});
|
|
4534
|
+
if (!steps.ok) return settleFailedSteps(steps, { giveUp: (reason) => giveUpItemDisk(reason, "checkout_not_switched"), bail });
|
|
4535
|
+
containerState.startCommit = steps.startCommit;
|
|
4536
|
+
keepRuntimeStep(containerState, steps);
|
|
4537
|
+
keepMissingSetupScripts(containerState, steps);
|
|
4538
|
+
if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
|
|
4539
|
+
input.save(state, environment);
|
|
4540
|
+
return { ok: true, workdir: CHECKOUT_DIR, state, environment, ...itemDisk.state === void 0 ? {} : { disk: itemDisk.state } };
|
|
4360
4541
|
}
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
var WORK_ITEM_ID2 = /^[A-Za-z0-9_-]{1,64}$/;
|
|
4364
|
-
function itemDiskDir(configPath, organizationId, workItemId) {
|
|
4365
|
-
if (!WORK_ITEM_ID2.test(workItemId)) throw new Error(`cannot keep a disk for the work item ${JSON.stringify(workItemId)}`);
|
|
4366
|
-
return join(workspacesDir(configPath, organizationId), workItemId);
|
|
4542
|
+
function asContainerJob(job) {
|
|
4543
|
+
return { ...job, container: job.sandbox.container };
|
|
4367
4544
|
}
|
|
4368
|
-
function
|
|
4369
|
-
|
|
4370
|
-
const
|
|
4371
|
-
|
|
4545
|
+
function sandboxJudgeEnvironment(input) {
|
|
4546
|
+
const own = input.job.sandbox;
|
|
4547
|
+
const state = own.container;
|
|
4548
|
+
const inner = async () => containerJudgeEnvironment({ runtime: await inSandbox(input.runtime, own), job: asContainerJob(input.job), jobDir: input.jobDir, ...input.log === void 0 ? {} : { log: input.log } });
|
|
4549
|
+
return {
|
|
4550
|
+
stop: async () => {
|
|
4551
|
+
const { runtime } = input;
|
|
4552
|
+
const services = runtime.tunnels.get(state.buildId);
|
|
4553
|
+
const joinSecret = runtime.sandbox.joinSecrets.get(state.buildId);
|
|
4554
|
+
if (services?.streams === void 0 || joinSecret === void 0) throw new Error("the runner no longer holds the build's tunnel and join secret, so it cannot join the build's relay again");
|
|
4555
|
+
const sandbox = await sandboxOfBuild(runtime, own);
|
|
4556
|
+
if (sandbox === void 0) throw new SandboxGoneError(own.sandboxId);
|
|
4557
|
+
const sandboxed = runtimeInSandbox(runtime, sandbox);
|
|
4558
|
+
runtime.leases.get(state.buildId)?.stop();
|
|
4559
|
+
const relay = runtime.relays.get(state.buildId);
|
|
4560
|
+
if (relay !== void 0) {
|
|
4561
|
+
runtime.relays.delete(state.buildId);
|
|
4562
|
+
await relay.close();
|
|
4563
|
+
}
|
|
4564
|
+
await sandboxed.host.stop(state.container);
|
|
4565
|
+
const started = await sandboxed.host.start(state.container);
|
|
4566
|
+
if (started.status !== 0) throw new Error(`the build's container could not be started again (exit ${started.status})`);
|
|
4567
|
+
if (!await waitForRelay(sandboxed, state.container)) throw new Error("the relay in the build's container did not start again");
|
|
4568
|
+
joinRelay(runtime, sandbox, { buildId: state.buildId, container: state.container }, services.streams, joinSecret, input.log);
|
|
4569
|
+
const delivered = await deliverJobFiles(sandboxed, state, ["gh", "git-hooks", "context"], secretsOf(runtime, input.jobDir));
|
|
4570
|
+
if (!delivered.ok) throw new Error(delivered.reason);
|
|
4571
|
+
},
|
|
4572
|
+
hashProgram: async () => (await inner()).hashProgram(),
|
|
4573
|
+
prepare: async (files, tree) => (await inner()).prepare(files, tree),
|
|
4574
|
+
run: async (command, opts) => (await inner()).run(command, opts),
|
|
4575
|
+
readReport: async () => (await inner()).readReport(),
|
|
4576
|
+
readItems: async (name) => (await inner()).readItems(name),
|
|
4577
|
+
clear: async () => (await inner()).clear?.()
|
|
4578
|
+
};
|
|
4372
4579
|
}
|
|
4373
|
-
function
|
|
4374
|
-
let parsed;
|
|
4580
|
+
async function deliverSandboxHooks(runtime, state, jobDir) {
|
|
4375
4581
|
try {
|
|
4376
|
-
|
|
4377
|
-
} catch {
|
|
4378
|
-
return
|
|
4582
|
+
return await deliverJobFiles(await inSandbox(runtime, state), state.container, ["git-hooks"], secretsOf(runtime, jobDir));
|
|
4583
|
+
} catch (err) {
|
|
4584
|
+
return { ok: false, reason: `the commit hooks could not be delivered: ${describeCallFailure(err)}` };
|
|
4379
4585
|
}
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
const
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4586
|
+
}
|
|
4587
|
+
async function startSandboxCommand(input) {
|
|
4588
|
+
const own = input.job.sandbox;
|
|
4589
|
+
const runtime = await inSandbox(input.runtime, own);
|
|
4590
|
+
const exec = containerCommandExec({ job: asContainerJob(input.job), cmdline: input.cmdline, runtime, config: input.config, ...input.extraEnv === void 0 ? {} : { extraEnv: input.extraEnv } });
|
|
4591
|
+
const started = await runtime.host.execDetached(exec);
|
|
4592
|
+
if (started.status !== 0) {
|
|
4593
|
+
const first = started.stderr.trim().split("\n")[0] ?? "";
|
|
4594
|
+
throw new Error(`docker exec exited ${started.status}${first === "" ? "" : `: ${first}`}`);
|
|
4387
4595
|
}
|
|
4388
|
-
return { dir, record: parsed, path };
|
|
4389
4596
|
}
|
|
4390
|
-
function
|
|
4391
|
-
|
|
4392
|
-
const outgoing = join(dirname(dir), `.${basename(dir)}.${randomBytes(4).toString("hex")}.old`);
|
|
4393
|
-
let hadOlder = true;
|
|
4597
|
+
async function probeSandboxCommand(runtime, state, offset) {
|
|
4598
|
+
let sandboxed;
|
|
4394
4599
|
try {
|
|
4395
|
-
|
|
4600
|
+
sandboxed = await inSandbox(runtime, state);
|
|
4396
4601
|
} catch (err) {
|
|
4397
|
-
if (err
|
|
4398
|
-
|
|
4602
|
+
if (err instanceof SandboxGoneError) throw await goneReason(runtime, state) ?? err;
|
|
4603
|
+
return void 0;
|
|
4399
4604
|
}
|
|
4400
4605
|
try {
|
|
4401
|
-
|
|
4606
|
+
const result = await sandboxed.host.exec(
|
|
4607
|
+
{ container: state.container.container, user: state.container.agentUser ?? ADDED_USER, env: NO_ENV, argv: probeArgvOf(offset) },
|
|
4608
|
+
{ timeoutMs: 6e4 }
|
|
4609
|
+
);
|
|
4610
|
+
return result.status === 0 ? parseProbe(result.stdout) : void 0;
|
|
4402
4611
|
} catch (err) {
|
|
4403
|
-
if (
|
|
4612
|
+
if (err instanceof SandboxError && err.status === 404) throw await goneReason(runtime, state) ?? new SandboxGoneError(state.sandboxId);
|
|
4613
|
+
if (err instanceof SandboxError) return void 0;
|
|
4404
4614
|
throw err;
|
|
4405
4615
|
}
|
|
4406
|
-
rmSync(outgoing, { recursive: true, force: true });
|
|
4407
|
-
return older;
|
|
4408
|
-
}
|
|
4409
|
-
function stagingItemDiskDir(dir) {
|
|
4410
|
-
const parent = dirname(dir);
|
|
4411
|
-
mkdirSync(parent, { recursive: true, mode: 448 });
|
|
4412
|
-
const staging = join(parent, `.${basename(dir)}.${randomBytes(4).toString("hex")}.partial`);
|
|
4413
|
-
mkdirSync(staging, { mode: 448 });
|
|
4414
|
-
return staging;
|
|
4415
4616
|
}
|
|
4416
|
-
function
|
|
4417
|
-
const
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
names = readdirSync(where);
|
|
4421
|
-
} catch {
|
|
4422
|
-
return [];
|
|
4423
|
-
}
|
|
4424
|
-
const kept = [];
|
|
4425
|
-
for (const name of names) {
|
|
4426
|
-
if (!WORK_ITEM_ID2.test(name)) continue;
|
|
4427
|
-
const found = readItemDisk(join(where, name));
|
|
4428
|
-
if (found !== void 0) kept.push(found);
|
|
4429
|
-
}
|
|
4430
|
-
return kept;
|
|
4617
|
+
function probeArgvOf(offset) {
|
|
4618
|
+
const argv = probeArgv(offset);
|
|
4619
|
+
argv[argv.length - 1] = String(SANDBOX_PROBE_BYTES);
|
|
4620
|
+
return argv;
|
|
4431
4621
|
}
|
|
4432
|
-
function
|
|
4433
|
-
const where = workspacesDir(configPath, scope);
|
|
4434
|
-
let names;
|
|
4622
|
+
async function endSandboxCommand(runtime, state, jobDir, warn) {
|
|
4435
4623
|
try {
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
try {
|
|
4444
|
-
found.push({ path: join(where, name), mtimeMs: lstatSync(join(where, name)).mtimeMs });
|
|
4445
|
-
} catch {
|
|
4624
|
+
const sandboxed = await inSandbox(runtime, state);
|
|
4625
|
+
for (let reads = 0; reads < MAX_TAIL_READS2; reads++) {
|
|
4626
|
+
const offset = commandLogOffset(jobDir, state.commandLogBase);
|
|
4627
|
+
const probe = await probeSandboxCommand(runtime, state, offset);
|
|
4628
|
+
if (probe === void 0) break;
|
|
4629
|
+
appendCommandLog(jobDir, probe.bytes);
|
|
4630
|
+
if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) break;
|
|
4446
4631
|
}
|
|
4632
|
+
await sandboxed.host.stop(state.container.container);
|
|
4633
|
+
} catch (err) {
|
|
4634
|
+
warn(`ending the build's command in its sandbox failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
4447
4635
|
}
|
|
4448
|
-
return found;
|
|
4449
|
-
}
|
|
4450
|
-
|
|
4451
|
-
// src/runner/workspace/restore.ts
|
|
4452
|
-
var DISK_REASON_TEXT = {
|
|
4453
|
-
no_kept_disk: "no disk is kept for the work item on this host",
|
|
4454
|
-
other_image: "the kept disk was made on another image",
|
|
4455
|
-
other_disk_size: "the kept disk was made with another runner.build.disk size",
|
|
4456
|
-
disk_not_restored: "the kept disk could not be copied",
|
|
4457
|
-
checkout_not_switched: "the kept checkout could not be switched to the branch",
|
|
4458
|
-
build_type_cannot_restore: "this build type cannot restore a disk",
|
|
4459
|
-
fresh_requested: "a member asked for a fresh disk",
|
|
4460
|
-
holder_busy: "the machine that holds its disk was busy"
|
|
4461
|
-
};
|
|
4462
|
-
function itemDiskDecision(input) {
|
|
4463
|
-
if (input.fresh === true) return { state: { restored: false, reason: "fresh_requested" } };
|
|
4464
|
-
if (!input.builtin) return { state: { restored: false, reason: "build_type_cannot_restore" } };
|
|
4465
|
-
let kept;
|
|
4466
|
-
try {
|
|
4467
|
-
kept = readItemDisk(itemDiskDir(input.configPath, input.organizationId, input.workItemId));
|
|
4468
|
-
} catch {
|
|
4469
|
-
kept = void 0;
|
|
4470
|
-
}
|
|
4471
|
-
if (kept === void 0) return { state: { restored: false, reason: input.heldElsewhere === true ? "holder_busy" : "no_kept_disk" } };
|
|
4472
|
-
if (kept.record.repository !== input.repository) return { state: { restored: false, reason: "no_kept_disk" } };
|
|
4473
|
-
if (kept.record.imageId !== input.imageId) return { state: { restored: false, reason: "other_image" } };
|
|
4474
|
-
if (kept.record.diskBytes !== input.diskBytes) return { state: { restored: false, reason: "other_disk_size" } };
|
|
4475
|
-
if (kept.record.kind !== input.backend) return { state: { restored: false, reason: "disk_not_restored" } };
|
|
4476
|
-
return { kept, state: { restored: true, fromClaimId: kept.record.claimId } };
|
|
4477
4636
|
}
|
|
4478
|
-
async function
|
|
4637
|
+
async function runCliInSandbox(input, argv, logName) {
|
|
4638
|
+
const { job } = input;
|
|
4639
|
+
const own = job.sandbox;
|
|
4479
4640
|
try {
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
}
|
|
4486
|
-
|
|
4487
|
-
|
|
4641
|
+
const sandboxed = await inSandbox(input.runtime, own);
|
|
4642
|
+
const state = own.container;
|
|
4643
|
+
const env = buildEnvironment({
|
|
4644
|
+
config: claimCalls(input.config, job),
|
|
4645
|
+
item: job.item,
|
|
4646
|
+
...job.queue === "run" ? { run: true } : {},
|
|
4647
|
+
itemKey: job.itemKey,
|
|
4648
|
+
branch: job.branch,
|
|
4649
|
+
repository: state.repository,
|
|
4650
|
+
repositories: job.repositories,
|
|
4651
|
+
hostEnv: input.runtime.hostEnv,
|
|
4652
|
+
agents: input.runtime.agents,
|
|
4653
|
+
job,
|
|
4654
|
+
containerEnv: state.containerEnv ?? {},
|
|
4655
|
+
unpinnedRuntimes: state.unpinnedRuntimes,
|
|
4656
|
+
missingSetupScripts: state.missingSetupScripts,
|
|
4657
|
+
miseVariables: state.miseVariables,
|
|
4658
|
+
builtIns: true,
|
|
4659
|
+
identity: job.commitIdentity
|
|
4660
|
+
});
|
|
4661
|
+
const result = await sandboxed.host.exec(
|
|
4662
|
+
{ container: state.container, user: state.agentUser ?? ADDED_USER, workdir: job.workdir ?? CHECKOUT_DIR, env, argv },
|
|
4663
|
+
{ logPath: join(input.jobDir, logName) }
|
|
4664
|
+
);
|
|
4665
|
+
return result.status === 0 ? { ok: true } : { ok: false, reason: `exit ${result.status}` };
|
|
4488
4666
|
} catch (err) {
|
|
4489
|
-
|
|
4667
|
+
return { ok: false, reason: describeCallFailure(err) };
|
|
4490
4668
|
}
|
|
4491
4669
|
}
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
var isRetryFresh = (value) => typeof value === "object" && value !== null && value.retryFresh === true;
|
|
4495
|
-
async function provisionWithFreshFallback(input) {
|
|
4496
|
-
const first = await input.attempt(true);
|
|
4497
|
-
if (!isRetryFresh(first)) return first;
|
|
4498
|
-
input.log(`${first.reason}; provisioning the build again from a fresh disk`);
|
|
4499
|
-
if (first.itemDisk === void 0) return await input.attempt(false);
|
|
4500
|
-
const second = await input.attempt(true, first.itemDisk);
|
|
4501
|
-
if (!isRetryFresh(second)) return second;
|
|
4502
|
-
input.log(`${second.reason}; provisioning the build again from a fresh disk`);
|
|
4503
|
-
return await input.attempt(false, first.itemDisk);
|
|
4504
|
-
}
|
|
4505
|
-
function keptDiskPlan() {
|
|
4506
|
-
return { keyCommits: {}, noSave: "it starts from the work item's kept disk" };
|
|
4507
|
-
}
|
|
4508
|
-
function clockOf(now) {
|
|
4509
|
-
return now ?? (() => /* @__PURE__ */ new Date());
|
|
4510
|
-
}
|
|
4511
|
-
function taggedLog(log, tag) {
|
|
4512
|
-
return (line2) => log.line(`${tag} ${line2}`);
|
|
4513
|
-
}
|
|
4514
|
-
function uidFromProbe(probe) {
|
|
4515
|
-
const uid = Number(probe.stdout);
|
|
4516
|
-
return probe.status === 0 && Number.isInteger(uid) ? uid : null;
|
|
4517
|
-
}
|
|
4518
|
-
function agentUidOf(probe) {
|
|
4519
|
-
return uidFromProbe(probe) ?? void 0;
|
|
4670
|
+
function syncInSandbox(input) {
|
|
4671
|
+
return runCliInSandbox(input, ["outerlayer", "sync"], "sync.log");
|
|
4520
4672
|
}
|
|
4521
|
-
function
|
|
4522
|
-
|
|
4523
|
-
return `the build user's id is ${agentUid ?? "unknown"} and it was ${savedUid} when the state was saved`;
|
|
4673
|
+
function recordProofsInSandbox(input) {
|
|
4674
|
+
return runCliInSandbox(input, [...RECORD_PROOFS_ARGV], PROOFS_LOG);
|
|
4524
4675
|
}
|
|
4525
|
-
function
|
|
4526
|
-
const
|
|
4527
|
-
return {
|
|
4528
|
-
timeoutMs: minutes * 6e4,
|
|
4529
|
-
limitMinutes: minutes,
|
|
4530
|
-
restored: input.plan.restore !== void 0 || input.fromItemDisk === true,
|
|
4531
|
-
...input.fromItemDisk === true ? { fromItemDisk: true } : {},
|
|
4532
|
-
keyCommits: input.plan.keyCommits,
|
|
4533
|
-
...input.withoutBuildVariables === true ? { withoutBuildVariables: true } : {},
|
|
4534
|
-
// A plan has a key exactly when the build may save a state. A build that starts from one is not asked to save.
|
|
4535
|
-
...input.plan.key === void 0 ? {} : { saveState: input.saveState }
|
|
4536
|
-
};
|
|
4676
|
+
function savedStateBase(report) {
|
|
4677
|
+
const digest = report.recipe === "default" ? report.base?.digest : void 0;
|
|
4678
|
+
return digest === void 0 ? void 0 : { digest };
|
|
4537
4679
|
}
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
if (!made.ok) return made;
|
|
4541
|
-
input.log(`saved state ${made.record.id}`);
|
|
4542
|
-
await removeSuperseded({ made: made.record, records: readRecords(input.dir), deps: { dir: input.dir, removeVolume: input.removeVolume }, log: input.log });
|
|
4543
|
-
return { ok: true };
|
|
4680
|
+
function savedStateMaxOf(config) {
|
|
4681
|
+
return config.housekeeping?.savedStateMax ?? DEFAULT_SAVED_STATE_MAX_BYTES;
|
|
4544
4682
|
}
|
|
4545
|
-
async function
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4683
|
+
async function planSavedState(input) {
|
|
4684
|
+
const { config, runtime, log } = input;
|
|
4685
|
+
const refused2 = savingRefusedBecause({ universalImage: input.base !== void 0, fleet: config.fleet === true, savedStateMax: savedStateMaxOf(config) });
|
|
4686
|
+
if (refused2 !== void 0 || input.base === void 0) {
|
|
4687
|
+
const why = refused2 ?? "its image comes from a devcontainer file";
|
|
4688
|
+
log(`this build saves no state: ${why}`);
|
|
4689
|
+
return { keyCommits: {}, noSave: why };
|
|
4550
4690
|
}
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4691
|
+
const root = runnerDir(config.configPath);
|
|
4692
|
+
const repositories = [];
|
|
4693
|
+
const keyCommits = {};
|
|
4694
|
+
for (const entry of input.repositories) {
|
|
4695
|
+
let token;
|
|
4696
|
+
try {
|
|
4697
|
+
token = entry.primary ? await input.tokens.token("read") : await input.tokens.token("read", entry.repository);
|
|
4698
|
+
} catch (err) {
|
|
4699
|
+
return unread(`the read token for ${entry.repository} could not be had: ${describeCallFailure(err)}`, log);
|
|
4700
|
+
}
|
|
4701
|
+
const [owner = "", name = ""] = entry.repository.split("/");
|
|
4702
|
+
const read = await runtime.readKeyFiles({
|
|
4703
|
+
remote: runtime.remoteFor?.(entry.repository) ?? `${runtime.github.gitUrl}/${entry.repository}.git`,
|
|
4704
|
+
dir: join(root, "key-files", owner, name),
|
|
4705
|
+
paths: keyFilePaths(),
|
|
4706
|
+
token
|
|
4707
|
+
});
|
|
4708
|
+
if (!read.ok) return unread(read.reason, log);
|
|
4709
|
+
keyCommits[entry.repository] = read.commit;
|
|
4710
|
+
repositories.push({ repository: entry.repository, commit: read.commit, files: read.files });
|
|
4711
|
+
}
|
|
4712
|
+
const key = stateKey({
|
|
4713
|
+
factory: keyRunnerFactory(config),
|
|
4714
|
+
repositories,
|
|
4715
|
+
imageDigest: input.base.digest,
|
|
4716
|
+
diskBytes: input.diskBytes
|
|
4717
|
+
});
|
|
4718
|
+
const found = lookup(readRecords(savedStatesDir(root)), key, input.now.getTime());
|
|
4719
|
+
if (found.kind !== "none" && found.record.kind === "tar") {
|
|
4720
|
+
try {
|
|
4721
|
+
await removeSavedState(found.record, { dir: savedStatesDir(root), removeVolume: async () => void 0 });
|
|
4722
|
+
log(`saved state ${found.record.id} is a tar file, which this runner no longer restores; removed it; starting from a fresh disk`);
|
|
4723
|
+
} catch (err) {
|
|
4724
|
+
log(`saved state ${found.record.id} is a tar file, which this runner no longer restores, and it could not be removed: ${describeCallFailure(err)}; starting from a fresh disk`);
|
|
4725
|
+
}
|
|
4726
|
+
return { key, keyCommits };
|
|
4727
|
+
}
|
|
4728
|
+
if (found.kind === "usable" && input.allowRestore) {
|
|
4729
|
+
log(`starting from saved state ${found.record.id}, saved ${formatDuration(found.ageMs)} ago`);
|
|
4730
|
+
return { key, keyCommits, restore: { record: found.record, ageMs: found.ageMs } };
|
|
4731
|
+
}
|
|
4732
|
+
if (found.kind === "expired") {
|
|
4733
|
+
log(`saved state ${found.record.id} is ${formatDuration(found.ageMs)} old, past the limit of ${SAVED_STATE_MAX_AGE_MS / 864e5} days; starting from a fresh disk`);
|
|
4734
|
+
} else if (found.kind === "changed") {
|
|
4735
|
+
log(`the setup script or version files changed on the default branch since saved state ${found.record.id}: ${found.files.join(", ")}; starting from a fresh disk`);
|
|
4736
|
+
} else if (found.kind === "none") {
|
|
4737
|
+
log("no saved state matches this build's key; starting from a fresh disk");
|
|
4738
|
+
}
|
|
4739
|
+
return { key, keyCommits };
|
|
4555
4740
|
}
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
return {
|
|
4741
|
+
function unread(reason, log) {
|
|
4742
|
+
log(`this build starts from no saved state and saves none: ${reason}`);
|
|
4743
|
+
return { keyCommits: {}, noSave: `its key files could not be read: ${reason}` };
|
|
4559
4744
|
}
|
|
4560
4745
|
var FREEZE_DEADLINE_SECONDS = 12 * 60;
|
|
4561
4746
|
function freezeScript() {
|
|
@@ -4578,7 +4763,7 @@ function freezeScript() {
|
|
|
4578
4763
|
"exit 5"
|
|
4579
4764
|
].join("\n");
|
|
4580
4765
|
}
|
|
4581
|
-
var
|
|
4766
|
+
var failureText2 = (status, stderr) => {
|
|
4582
4767
|
const line2 = stderr.trim().split("\n").at(-1) ?? "";
|
|
4583
4768
|
return `exit ${status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
4584
4769
|
};
|
|
@@ -4621,7 +4806,7 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
|
|
|
4621
4806
|
const first = await Promise.race([frozen, ended]);
|
|
4622
4807
|
if (first !== "frozen") {
|
|
4623
4808
|
stdin.end();
|
|
4624
|
-
return { ok: false, reason: "error" in first ? `the guest could not freeze /build: ${first.error}` : `the guest could not freeze /build (${
|
|
4809
|
+
return { ok: false, reason: "error" in first ? `the guest could not freeze /build: ${first.error}` : `the guest could not freeze /build (${failureText2(first.status, stderr)})` };
|
|
4625
4810
|
}
|
|
4626
4811
|
return {
|
|
4627
4812
|
ok: true,
|
|
@@ -4632,8 +4817,8 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
|
|
|
4632
4817
|
const result = await ended;
|
|
4633
4818
|
if ("error" in result) return { ok: false, reason: `the freeze command's connection failed: ${result.error}` };
|
|
4634
4819
|
if (thawedBeforeAsked) return { ok: false, reason: "the freeze command thawed /build before the runner asked" };
|
|
4635
|
-
if (!thawedAfterAsked) return { ok: false, reason: `the freeze command did not say it thawed /build (${
|
|
4636
|
-
if (result.status !== 0) return { ok: false, reason: `the freeze command ended badly after it thawed /build (${
|
|
4820
|
+
if (!thawedAfterAsked) return { ok: false, reason: `the freeze command did not say it thawed /build (${failureText2(result.status, stderr)})` };
|
|
4821
|
+
if (result.status !== 0) return { ok: false, reason: `the freeze command ended badly after it thawed /build (${failureText2(result.status, stderr)})` };
|
|
4637
4822
|
return { ok: true };
|
|
4638
4823
|
}
|
|
4639
4824
|
}
|
|
@@ -4642,7 +4827,7 @@ async function freezeGuest(guest, deadlineSeconds = FREEZE_DEADLINE_SECONDS) {
|
|
|
4642
4827
|
|
|
4643
4828
|
// src/runner/saved-state/microvm.ts
|
|
4644
4829
|
var HELPER_COPY_NAME = "saved-state.ext4";
|
|
4645
|
-
var
|
|
4830
|
+
var failureText3 = (status, stderr) => {
|
|
4646
4831
|
const line2 = stderr.trim().split("\n").at(-1) ?? "";
|
|
4647
4832
|
return `exit ${status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
4648
4833
|
};
|
|
@@ -4681,9 +4866,9 @@ async function saveMicroVmDiskState(input) {
|
|
|
4681
4866
|
const { savedStateMax } = input.limits;
|
|
4682
4867
|
if (savedStateMax === "off") return { ok: false, reason: "runner.housekeeping.savedStateMax is off" };
|
|
4683
4868
|
const stopped = await input.guest.run(["/bin/sh", "-c", guestStopScript()]);
|
|
4684
|
-
if (stopped.status !== 0) return { ok: false, reason: `the guest's processes could not be stopped (${
|
|
4869
|
+
if (stopped.status !== 0) return { ok: false, reason: `the guest's processes could not be stopped (${failureText3(stopped.status, stopped.stderr)})` };
|
|
4685
4870
|
const flushed = await input.guest.run(["sync", "-f", "/build"]);
|
|
4686
|
-
if (flushed.status !== 0) return { ok: false, reason: `the build disk could not be flushed (${
|
|
4871
|
+
if (flushed.status !== 0) return { ok: false, reason: `the build disk could not be flushed (${failureText3(flushed.status, flushed.stderr)})` };
|
|
4687
4872
|
const allocated = allocatedBytes(input.buildDisk);
|
|
4688
4873
|
if (allocated > savedStateMax) {
|
|
4689
4874
|
return { ok: false, reason: `the build disk holds ${formatBytes(allocated)}, more than the ${formatBytes(savedStateMax)} of runner.housekeeping.savedStateMax` };
|
|
@@ -7449,129 +7634,6 @@ function buildReleaseLog(jobDirPath, endStage, scrub) {
|
|
|
7449
7634
|
if (parts.length === 0) return void 0;
|
|
7450
7635
|
return capRunnerLog(scrub(parts.join("\n")) + "\n");
|
|
7451
7636
|
}
|
|
7452
|
-
var SAVED_FOLDERS = ["home", "work"];
|
|
7453
|
-
function stopProcessesScript() {
|
|
7454
|
-
return [
|
|
7455
|
-
"me=$$; uid=$(id -u); n=0; pass=0",
|
|
7456
|
-
"while [ $pass -lt 25 ]; do",
|
|
7457
|
-
' left=""',
|
|
7458
|
-
" for p in /proc/[0-9]*; do",
|
|
7459
|
-
" pid=${p##*/}",
|
|
7460
|
-
' [ "$pid" = "$me" ] && continue',
|
|
7461
|
-
` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
|
|
7462
|
-
' case "$info" in *"S=Z"*) continue ;; esac',
|
|
7463
|
-
// The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
|
|
7464
|
-
' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
|
|
7465
|
-
" done",
|
|
7466
|
-
' [ -z "$left" ] && break',
|
|
7467
|
-
' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
|
|
7468
|
-
" pass=$((pass+1))",
|
|
7469
|
-
" sleep 0.2",
|
|
7470
|
-
"done",
|
|
7471
|
-
'[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
|
|
7472
|
-
'echo "stopped $n"'
|
|
7473
|
-
].join("\n");
|
|
7474
|
-
}
|
|
7475
|
-
var HELPER_FLAGS = [
|
|
7476
|
-
"--rm",
|
|
7477
|
-
"--network",
|
|
7478
|
-
"none",
|
|
7479
|
-
"--cap-drop",
|
|
7480
|
-
"ALL",
|
|
7481
|
-
"--cap-add",
|
|
7482
|
-
"CHOWN",
|
|
7483
|
-
"--cap-add",
|
|
7484
|
-
"DAC_OVERRIDE",
|
|
7485
|
-
"--cap-add",
|
|
7486
|
-
"DAC_READ_SEARCH",
|
|
7487
|
-
"--cap-add",
|
|
7488
|
-
"FOWNER",
|
|
7489
|
-
"--security-opt",
|
|
7490
|
-
"no-new-privileges",
|
|
7491
|
-
"--read-only",
|
|
7492
|
-
"--user",
|
|
7493
|
-
"0",
|
|
7494
|
-
"--pids-limit",
|
|
7495
|
-
"256"
|
|
7496
|
-
];
|
|
7497
|
-
function helperArgs(input) {
|
|
7498
|
-
return [
|
|
7499
|
-
...HELPER_FLAGS,
|
|
7500
|
-
"--label",
|
|
7501
|
-
`${RUNNER_LABEL}=${input.runnerId}`,
|
|
7502
|
-
"--mount",
|
|
7503
|
-
`type=volume,src=${input.from},dst=/from,readonly`,
|
|
7504
|
-
"--mount",
|
|
7505
|
-
`type=volume,src=${input.to},dst=/to`,
|
|
7506
|
-
"--entrypoint",
|
|
7507
|
-
"sh",
|
|
7508
|
-
input.image,
|
|
7509
|
-
"-c",
|
|
7510
|
-
input.script
|
|
7511
|
-
];
|
|
7512
|
-
}
|
|
7513
|
-
var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
|
|
7514
|
-
var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
|
|
7515
|
-
function failureText3(result) {
|
|
7516
|
-
const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
|
|
7517
|
-
return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
7518
|
-
}
|
|
7519
|
-
async function saveContainerState(input) {
|
|
7520
|
-
const { host } = input;
|
|
7521
|
-
const id = randomBytes(6).toString("hex");
|
|
7522
|
-
const volume = `outerlayer-saved-${id}`;
|
|
7523
|
-
const record = {
|
|
7524
|
-
id,
|
|
7525
|
-
kind: "volume",
|
|
7526
|
-
volume,
|
|
7527
|
-
savedAt: input.now.toISOString(),
|
|
7528
|
-
bytes: 0,
|
|
7529
|
-
baseKey: input.key.baseKey,
|
|
7530
|
-
key: input.key.key,
|
|
7531
|
-
fileHashes: input.key.fileHashes,
|
|
7532
|
-
...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
|
|
7533
|
-
complete: false
|
|
7534
|
-
};
|
|
7535
|
-
const discard = async (reason) => {
|
|
7536
|
-
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
7537
|
-
if (removed.status !== 0) {
|
|
7538
|
-
input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
|
|
7539
|
-
return { ok: false, reason };
|
|
7540
|
-
}
|
|
7541
|
-
deleteRecord(input.dir, id);
|
|
7542
|
-
return { ok: false, reason };
|
|
7543
|
-
};
|
|
7544
|
-
try {
|
|
7545
|
-
const noEnv = { literals: {}, passed: {} };
|
|
7546
|
-
const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
|
|
7547
|
-
if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText3(stopped)})` };
|
|
7548
|
-
input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
|
|
7549
|
-
const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
|
|
7550
|
-
if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText3(flushed)})` };
|
|
7551
|
-
writeRecord2(input.dir, record);
|
|
7552
|
-
const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
|
|
7553
|
-
if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText3(created)})`);
|
|
7554
|
-
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
7555
|
-
if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText3(copied)})`);
|
|
7556
|
-
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
7557
|
-
const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
|
|
7558
|
-
writeRecord2(input.dir, done);
|
|
7559
|
-
return { ok: true, record: done };
|
|
7560
|
-
} catch (err) {
|
|
7561
|
-
return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
|
|
7562
|
-
}
|
|
7563
|
-
}
|
|
7564
|
-
async function restoreContainerState(input) {
|
|
7565
|
-
try {
|
|
7566
|
-
const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
7567
|
-
if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText3(copied)})` };
|
|
7568
|
-
return { ok: true };
|
|
7569
|
-
} catch (err) {
|
|
7570
|
-
return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
|
|
7571
|
-
}
|
|
7572
|
-
}
|
|
7573
|
-
|
|
7574
|
-
// src/runner/workspace/keep.ts
|
|
7575
7637
|
function recordOf(input, kind, bytes, volume) {
|
|
7576
7638
|
return {
|
|
7577
7639
|
workItemId: input.workItemId,
|
|
@@ -7639,16 +7701,16 @@ async function keepContainerDisk(input) {
|
|
|
7639
7701
|
const dropNew = async () => {
|
|
7640
7702
|
if (!created) return;
|
|
7641
7703
|
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
7642
|
-
if (removed.status !== 0) input.log(`the volume ${volume} could not be removed (${
|
|
7704
|
+
if (removed.status !== 0) input.log(`the volume ${volume} could not be removed (${failureText({ status: removed.status, stderr: removed.stderr })})`);
|
|
7643
7705
|
};
|
|
7644
7706
|
try {
|
|
7645
7707
|
const made = await host.createVolume(volume, { [WORKSPACE_LABEL]: input.workItemId, [RUNNER_LABEL]: input.runnerId });
|
|
7646
|
-
if (made.status !== 0) return { ok: false, reason: `the volume for the kept disk could not be made (${
|
|
7708
|
+
if (made.status !== 0) return { ok: false, reason: `the volume for the kept disk could not be made (${failureText(made)})` };
|
|
7647
7709
|
created = true;
|
|
7648
7710
|
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
7649
7711
|
if (copied.status !== 0) {
|
|
7650
7712
|
await dropNew();
|
|
7651
|
-
return { ok: false, reason: `the copy into the kept disk failed (${
|
|
7713
|
+
return { ok: false, reason: `the copy into the kept disk failed (${failureText(copied)})` };
|
|
7652
7714
|
}
|
|
7653
7715
|
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
7654
7716
|
const record = recordOf(input, "volume", Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, volume);
|
|
@@ -7657,7 +7719,7 @@ async function keepContainerDisk(input) {
|
|
|
7657
7719
|
const removed = await host.removeVolume(older.record.volume);
|
|
7658
7720
|
if (removed.status !== 0 && !/no such volume/i.test(removed.stderr)) {
|
|
7659
7721
|
await dropNew();
|
|
7660
|
-
return { ok: false, reason: `the work item's older disk could not be replaced: its volume ${older.record.volume} was not removed (${
|
|
7722
|
+
return { ok: false, reason: `the work item's older disk could not be replaced: its volume ${older.record.volume} was not removed (${failureText(removed)})` };
|
|
7661
7723
|
}
|
|
7662
7724
|
}
|
|
7663
7725
|
staging = stagingItemDiskDir(dir);
|
|
@@ -7671,6 +7733,48 @@ async function keepContainerDisk(input) {
|
|
|
7671
7733
|
if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
|
|
7672
7734
|
}
|
|
7673
7735
|
}
|
|
7736
|
+
function saidBy(err) {
|
|
7737
|
+
return err instanceof SandboxError ? describeSandboxError(err) : describeCallFailure(err);
|
|
7738
|
+
}
|
|
7739
|
+
async function keepSandboxDisk2(input) {
|
|
7740
|
+
let dir;
|
|
7741
|
+
try {
|
|
7742
|
+
dir = itemDiskDir(input.configPath, input.organizationId, input.workItemId);
|
|
7743
|
+
} catch (err) {
|
|
7744
|
+
return { ok: false, reason: describeCallFailure(err) };
|
|
7745
|
+
}
|
|
7746
|
+
const { client: client2, snapshotName } = input;
|
|
7747
|
+
const dropNew = async () => {
|
|
7748
|
+
try {
|
|
7749
|
+
await client2.removeSnapshot(snapshotName);
|
|
7750
|
+
} catch (err) {
|
|
7751
|
+
input.log(`the snapshot ${snapshotName} could not be removed (${saidBy(err)}); remove it at the sandbox provider`);
|
|
7752
|
+
}
|
|
7753
|
+
};
|
|
7754
|
+
let staging;
|
|
7755
|
+
try {
|
|
7756
|
+
const saved = await client2.saveSnapshot(input.sandboxId, snapshotName);
|
|
7757
|
+
const record = { ...recordOf(input, "sandbox", 0, input.buildVolume), snapshot: snapshotName, storedBytes: saved.storedBytes };
|
|
7758
|
+
const older = readItemDisk(dir);
|
|
7759
|
+
if (older?.record.snapshot !== void 0) {
|
|
7760
|
+
try {
|
|
7761
|
+
await client2.removeSnapshot(older.record.snapshot);
|
|
7762
|
+
} catch (err) {
|
|
7763
|
+
await dropNew();
|
|
7764
|
+
return { ok: false, reason: `the work item's older disk could not be replaced: its snapshot ${older.record.snapshot} was not removed (${saidBy(err)})` };
|
|
7765
|
+
}
|
|
7766
|
+
}
|
|
7767
|
+
staging = stagingItemDiskDir(dir);
|
|
7768
|
+
finish(input, dir, staging, record);
|
|
7769
|
+
staging = void 0;
|
|
7770
|
+
return { ok: true, record };
|
|
7771
|
+
} catch (err) {
|
|
7772
|
+
await dropNew();
|
|
7773
|
+
return { ok: false, reason: `the sandbox could not be saved: ${saidBy(err)}` };
|
|
7774
|
+
} finally {
|
|
7775
|
+
if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
|
|
7776
|
+
}
|
|
7777
|
+
}
|
|
7674
7778
|
|
|
7675
7779
|
// src/runner/workspace/finish.ts
|
|
7676
7780
|
function noDiskReason(job, config) {
|
|
@@ -7735,6 +7839,31 @@ function containerDiskKeeper(input) {
|
|
|
7735
7839
|
input.log.line(`${input.tag} kept the build disk for the work item: ${formatBytes(kept.record.bytes)} allocated`);
|
|
7736
7840
|
};
|
|
7737
7841
|
}
|
|
7842
|
+
function sandboxDiskKeeper(input) {
|
|
7843
|
+
const state = input.job.sandbox;
|
|
7844
|
+
if (state === void 0) return void 0;
|
|
7845
|
+
const refused2 = noDiskReason(input.job, input.config);
|
|
7846
|
+
if (refused2 !== void 0) {
|
|
7847
|
+
if (input.job.workItemId !== void 0) input.log.line(`${input.tag} the build disk is not kept: ${refused2}`);
|
|
7848
|
+
return void 0;
|
|
7849
|
+
}
|
|
7850
|
+
if (state.sandboxId === "" || state.diskKey === void 0 || state.container.imageId === void 0) return void 0;
|
|
7851
|
+
const { imageId, diskBytes } = state.diskKey;
|
|
7852
|
+
const { sandboxId } = state;
|
|
7853
|
+
const keep = async () => {
|
|
7854
|
+
const kept = await keepSandboxDisk2({ ...target(input, imageId, diskBytes), client: input.client(), sandboxId, snapshotName: input.snapshotName, buildVolume: state.container.volume });
|
|
7855
|
+
if (!kept.ok) throw new Error(kept.reason);
|
|
7856
|
+
input.log.line(`${input.tag} kept the build disk for the work item: snapshot ${kept.record.snapshot} of sandbox ${sandboxId}, ${formatBytes(kept.record.storedBytes)} stored`);
|
|
7857
|
+
};
|
|
7858
|
+
return {
|
|
7859
|
+
container: state.container.container,
|
|
7860
|
+
volume: state.container.volume,
|
|
7861
|
+
imageId: state.container.imageId,
|
|
7862
|
+
runnerId: runnerLabelValue(input.config.configPath),
|
|
7863
|
+
keep,
|
|
7864
|
+
warn: (text2) => input.log.error(`${input.tag} ${text2}`)
|
|
7865
|
+
};
|
|
7866
|
+
}
|
|
7738
7867
|
var SESSION_TAR = "session.tar";
|
|
7739
7868
|
async function guestOf(job, runtime) {
|
|
7740
7869
|
if (job.microvm !== void 0) {
|
|
@@ -9370,7 +9499,12 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
|
|
|
9370
9499
|
let cleanedUp = true;
|
|
9371
9500
|
tunnelRecord = sandboxBuild.runtime.tunnels.get(sandboxBuild.state.buildId)?.record();
|
|
9372
9501
|
try {
|
|
9373
|
-
|
|
9502
|
+
const sandboxRuntime = sandboxBuild.runtime;
|
|
9503
|
+
await cleanupSandbox(
|
|
9504
|
+
sandboxRuntime,
|
|
9505
|
+
sandboxBuild.state,
|
|
9506
|
+
sandboxDiskKeeper({ job, config, runtime: sandboxRuntime, now, log, tag, jobDirPath, client: () => sandboxRuntime.sandbox.client(), snapshotName: itemSnapshotName(runnerLabelValue(config.configPath)) })
|
|
9507
|
+
);
|
|
9374
9508
|
log.line(`${tag} ended sandbox ${sandboxBuild.state.sandboxId}`);
|
|
9375
9509
|
} catch (err) {
|
|
9376
9510
|
cleanedUp = false;
|
|
@@ -9871,8 +10005,12 @@ var bytesOf = (entries) => entries.reduce((sum, entry) => sum + entry.kept.recor
|
|
|
9871
10005
|
function allEntries(configPath) {
|
|
9872
10006
|
return storeScopes(configPath).flatMap((scope) => listItemDisks(configPath, scope).map((kept) => ({ scope, kept })));
|
|
9873
10007
|
}
|
|
9874
|
-
async function removeEntry(entry, removeVolume) {
|
|
10008
|
+
async function removeEntry(entry, removeVolume, removeSnapshot) {
|
|
9875
10009
|
const { record, dir } = entry.kept;
|
|
10010
|
+
if (record.snapshot !== void 0) {
|
|
10011
|
+
if (removeSnapshot === void 0) throw new Error("no sandbox provider is set");
|
|
10012
|
+
await removeSnapshot(record.snapshot);
|
|
10013
|
+
}
|
|
9876
10014
|
if (record.kind === "volume" && record.volume !== void 0) {
|
|
9877
10015
|
if (removeVolume === void 0) throw new Error("Docker is not available");
|
|
9878
10016
|
await removeVolume(record.volume);
|
|
@@ -9909,7 +10047,7 @@ async function removeItemDisks(input) {
|
|
|
9909
10047
|
const gone = /* @__PURE__ */ new Set();
|
|
9910
10048
|
const remove = async (entry, why) => {
|
|
9911
10049
|
try {
|
|
9912
|
-
await removeEntry(entry, input.removeVolume);
|
|
10050
|
+
await removeEntry(entry, input.removeVolume, input.removeSnapshot);
|
|
9913
10051
|
gone.add(entry);
|
|
9914
10052
|
freed += entry.kept.record.bytes;
|
|
9915
10053
|
input.log(`housekeeping: removed the kept disk of work item ${entry.kept.record.workItemId} (${formatBytes(entry.kept.record.bytes)}), ${why}`);
|
|
@@ -9960,7 +10098,7 @@ async function removeItemDisks(input) {
|
|
|
9960
10098
|
}
|
|
9961
10099
|
async function freeItemDisksUntil(input) {
|
|
9962
10100
|
let freed = 0;
|
|
9963
|
-
for (const entry of oldestFirst2(allEntries(input.configPath))) {
|
|
10101
|
+
for (const entry of oldestFirst2(allEntries(input.configPath)).filter((candidate) => candidate.kept.record.kind !== "sandbox")) {
|
|
9964
10102
|
if (input.enough()) break;
|
|
9965
10103
|
try {
|
|
9966
10104
|
await removeEntry(entry, input.removeVolume);
|
|
@@ -10221,6 +10359,7 @@ async function runHousekeeping(options) {
|
|
|
10221
10359
|
limits: { max: options.workspaceMax ?? Number.POSITIVE_INFINITY, orgMax: options.workspaceOrgMax ?? Number.POSITIVE_INFINITY, keepDays: options.workspaceKeepDays ?? DEFAULT_WORKSPACE_KEEP_DAYS },
|
|
10222
10360
|
nowMs,
|
|
10223
10361
|
removeVolume: options.docker === void 0 ? void 0 : (name) => options.docker.removeVolume(name),
|
|
10362
|
+
removeSnapshot: options.removeSnapshot,
|
|
10224
10363
|
...options.itemStates === void 0 ? {} : { itemStates: options.itemStates },
|
|
10225
10364
|
log: (line2) => options.log.line(line2),
|
|
10226
10365
|
error: (line2) => options.log.error(line2)
|