@outerlayer/cli 0.36.0-next.20261102135508 → 0.36.0-next.20261102195652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-setup-Q3HDN23U.js → agent-setup-5UWZZMZM.js} +6 -6
- package/dist/{agent-subcommand-Z6SOLWZW.js → agent-subcommand-4TEUJJNL.js} +1 -1
- package/dist/{build-KYXFZQ33.js → build-3CEFKWGY.js} +5 -5
- package/dist/{build-disk-W6AL5WHI.js → build-disk-DLW3WKC5.js} +4 -4
- package/dist/build-info.json +1 -1
- package/dist/{builder-IMNOEYV7.js → builder-ALMQ5AWP.js} +3 -3
- package/dist/{caller-NLRE272L.js → caller-XNBGW4SP.js} +6 -6
- package/dist/{check-WWX32IGY.js → check-AEDFTYH6.js} +32 -32
- package/dist/{chunk-PMA67TIN.js → chunk-2XN34EKQ.js} +3 -3
- package/dist/{chunk-IKFAJTLZ.js → chunk-42DBDDHP.js} +1 -1
- package/dist/{chunk-FTP7SE6Y.js → chunk-4UUKUOS7.js} +3 -3
- package/dist/{chunk-KRCU66AM.js → chunk-5XX75FJ6.js} +5 -5
- package/dist/{chunk-DG3M5DZU.js → chunk-6QSIK5VA.js} +1 -1
- package/dist/{chunk-3YEPLVNW.js → chunk-6Z7R24XA.js} +5 -5
- package/dist/{chunk-URHS76GV.js → chunk-73E33ZA2.js} +2 -2
- package/dist/{chunk-3UFX6KH2.js → chunk-7UGMB5VS.js} +4 -4
- package/dist/{chunk-VAYVTUU4.js → chunk-7XTLS5PG.js} +1 -1
- package/dist/{chunk-GFZ7PELR.js → chunk-A24PDRNU.js} +2 -2
- package/dist/{chunk-QKOJ22OV.js → chunk-A2U7T4ZH.js} +9 -9
- package/dist/{chunk-WACPWYMH.js → chunk-BACWWK3E.js} +20 -5
- package/dist/{chunk-6B3V6EGQ.js → chunk-BN5U4RJU.js} +3 -3
- package/dist/{chunk-K7N6LAPC.js → chunk-CA4OE2QT.js} +2 -2
- package/dist/{chunk-Y4E7IS7H.js → chunk-CI5LFLKH.js} +2 -2
- package/dist/{chunk-B2P72DAE.js → chunk-D74RGORK.js} +1 -1
- package/dist/{chunk-ENBOAE2W.js → chunk-DL2RYNXQ.js} +64 -23
- package/dist/{chunk-XEWG4AB2.js → chunk-F36EKGMP.js} +697 -234
- package/dist/{chunk-YJQ7YA7J.js → chunk-FIUUESD4.js} +2 -2
- package/dist/{chunk-YMVRCSKD.js → chunk-FWKQHKCR.js} +2 -2
- package/dist/{chunk-DTCTXDJP.js → chunk-G35Y4LN5.js} +4 -1
- package/dist/{chunk-ENMX3VQ7.js → chunk-GBGJGYEE.js} +2 -2
- package/dist/{chunk-K2G3VB7W.js → chunk-H23JP73I.js} +1 -1
- package/dist/{chunk-FRMAQ3C4.js → chunk-HRBSEXYC.js} +1 -1
- package/dist/{chunk-RF4JHGGB.js → chunk-HZVS4MGL.js} +3 -3
- package/dist/{chunk-AQE6E54X.js → chunk-IX6CFXSW.js} +5 -5
- package/dist/{chunk-CDG475ZP.js → chunk-J6HWRVN2.js} +4 -3
- package/dist/{chunk-E3TP4QTM.js → chunk-J6VC3JCG.js} +1 -1
- package/dist/{chunk-BUYXEU7J.js → chunk-JBPVEXHQ.js} +6 -6
- package/dist/{chunk-O3MJAHLI.js → chunk-JVHWWJ2J.js} +1 -1
- package/dist/{chunk-TJIFYV2Z.js → chunk-KN3TDNF7.js} +3 -3
- package/dist/{chunk-NXQK2I3G.js → chunk-LZS5EYZ6.js} +1 -1
- package/dist/{chunk-4XXBV37R.js → chunk-M4HH2F64.js} +3 -3
- package/dist/{chunk-ULBYRPAP.js → chunk-ML5YKV3B.js} +1 -1
- package/dist/{chunk-YYQ5NOT5.js → chunk-NMBXA3AT.js} +3 -3
- package/dist/{chunk-L2BA2HXM.js → chunk-PN4LWT5F.js} +1 -1
- package/dist/{chunk-CSBW4LGL.js → chunk-PSQEGJPN.js} +3 -3
- package/dist/{chunk-CSVJGW3I.js → chunk-SBZ545DH.js} +2 -2
- package/dist/{chunk-VUH7KKVD.js → chunk-T2KBHW6P.js} +2 -2
- package/dist/{chunk-6HDXRDC5.js → chunk-UCMIS74X.js} +1 -1
- package/dist/{chunk-ATKCHMFD.js → chunk-VKKKVYA4.js} +1 -1
- package/dist/{chunk-TKY3SDXH.js → chunk-Y2P3HWUG.js} +2 -2
- package/dist/{chunk-D5I5SSPU.js → chunk-Y5S2G2EZ.js} +7 -7
- package/dist/{chunk-SJJXDU2J.js → chunk-Z3MB7SV7.js} +1 -1
- package/dist/{chunk-D4VGJQY7.js → chunk-Z66CKWOM.js} +1 -1
- package/dist/{claude-launcher-LGKJZH33.js → claude-launcher-NJGZUB3P.js} +1 -1
- package/dist/{cli-Q5L64EEG.js → cli-POIDBCZA.js} +108 -108
- package/dist/{command-CGEFJZNC.js → command-ZLBFX553.js} +26 -26
- package/dist/{config-T22KVOOT.js → config-GKSSGMXD.js} +5 -5
- package/dist/{connect-cmd-RUBP466E.js → connect-cmd-6YKNFJJR.js} +4 -4
- package/dist/{container-start-JRF3SBUA.js → container-start-55QELXPJ.js} +24 -24
- package/dist/{context-show-cmd-4UXO5WQJ.js → context-show-cmd-FGMRJ7IG.js} +5 -5
- package/dist/{control-plane-launch-AAH4YH56.js → control-plane-launch-DGPA3ONU.js} +4 -4
- package/dist/{demo-voice-cmd-GEPX7GIG.js → demo-voice-cmd-BPTSL7JA.js} +10 -10
- package/dist/{docker-5VWK335X.js → docker-GFEEV42Z.js} +3 -3
- package/dist/{doctor-EQM3JRZI.js → doctor-ZOQSI3HL.js} +26 -26
- package/dist/{emit-artifact-cmd-TA3MPHLU.js → emit-artifact-cmd-3LDJ4VRW.js} +9 -9
- package/dist/{emit-criteria-cmd-NMTUV2KM.js → emit-criteria-cmd-V3ERJXHB.js} +6 -6
- package/dist/{emit-demo-cmd-Y2Y6LEL7.js → emit-demo-cmd-5KL64L74.js} +10 -10
- package/dist/{emit-finding-cmd-VECM64UR.js → emit-finding-cmd-R4OEWQQR.js} +6 -6
- package/dist/{emit-result-cmd-2ERWFDUN.js → emit-result-cmd-5YNQDTYG.js} +8 -8
- package/dist/{emit-step-cmd-V2ET66N4.js → emit-step-cmd-Z5WC7UEQ.js} +6 -6
- package/dist/{find-unit-CNAPU7XQ.js → find-unit-OI56YEX5.js} +7 -7
- package/dist/{findings-cmd-QJPPF6BY.js → findings-cmd-J5ZCY2AT.js} +6 -6
- package/dist/{fleet-readiness-DKAPHAZN.js → fleet-readiness-DKCFOUYP.js} +4 -4
- package/dist/{governing-source-7PZ6DAJU.js → governing-source-7PD4ISRN.js} +8 -8
- package/dist/{hook-fast-VXROB4KF.js → hook-fast-ICT2PRQX.js} +1 -1
- package/dist/{host-checks-WAPO5X7X.js → host-checks-GADKYJFN.js} +22 -22
- package/dist/{pending-VOFQRXP4.js → host-key-FI6KWMJC.js} +2 -2
- package/dist/{image-tag-PBWQBATH.js → image-tag-RX4WTHJH.js} +2 -2
- package/dist/index.js +4 -4
- package/dist/{init-Y2URPVVY.js → init-BZXVQO22.js} +22 -22
- package/dist/{init-EK3PUU6W.js → init-EKF4ILHF.js} +22 -22
- package/dist/{init-cmd-XNANUDIW.js → init-cmd-3LWFEKHD.js} +9 -9
- package/dist/{init-update-EFCTKF6D.js → init-update-THEHC25C.js} +7 -7
- package/dist/{install-cmd-RHFUZFV4.js → install-cmd-WBVTTE5Z.js} +4 -4
- package/dist/{logs-E7MGQINH.js → logs-7YIXBO2K.js} +1 -1
- package/dist/{loop-L6TX6OR4.js → loop-RUQQ4L5A.js} +31 -31
- package/dist/{machine-VJKAY2GX.js → machine-IGHZW3IO.js} +22 -22
- package/dist/{mcp-serve-cmd-XXERT673.js → mcp-serve-cmd-HLS24XCG.js} +3 -3
- package/dist/{paths-F2ZR2EKC.js → paths-2YTU4QIO.js} +1 -1
- package/dist/{host-key-HLLV37NV.js → pending-S2SKIBCB.js} +2 -2
- package/dist/{pidfile-7M7O2M7T.js → pidfile-SPBTMTCJ.js} +2 -2
- package/dist/{record-YCJ6DZOY.js → record-3KXRFK5C.js} +2 -2
- package/dist/{runner-env-YW55JVU7.js → runner-env-HOWRS6I6.js} +7 -7
- package/dist/{runner-identity-Y5ZFCC7V.js → runner-identity-32MTGA2F.js} +3 -3
- package/dist/{runner-key-R47C6374.js → runner-key-FXYWHMMO.js} +22 -22
- package/dist/{session-sender-UPACLRGG.js → session-sender-B6INUY7G.js} +9 -9
- package/dist/{spec-cmd-7QRE2SZK.js → spec-cmd-ZEZFMC4A.js} +7 -7
- package/dist/{state-VAHBIMS5.js → state-DVKFUWB3.js} +2 -2
- package/dist/{status-BQEYEG2V.js → status-VPFIIHOZ.js} +22 -22
- package/dist/{sync-cmd-NBLYEPQG.js → sync-cmd-W756BQWX.js} +8 -8
- package/dist/{update-steps-VGKNGWTZ.js → update-steps-D6L6AG6E.js} +23 -23
- package/dist/{watch-IHV4V3HZ.js → watch-6EDRJZMG.js} +8 -8
- package/dist/{work-claim-cmd-WGTSU5IT.js → work-claim-cmd-7HR4AVCT.js} +12 -12
- package/dist/{work-cmd-HKOJU5FJ.js → work-cmd-ZP4UIHBT.js} +6 -6
- package/dist/{work-comment-cmd-H3JCWQ2J.js → work-comment-cmd-CYDOEMW3.js} +8 -8
- package/dist/{work-demo-cmd-J4AZAIVP.js → work-demo-cmd-3A7Z3VVS.js} +6 -6
- package/dist/{work-logs-cmd-UOO4JVLG.js → work-logs-cmd-NDPSRSCM.js} +6 -6
- package/dist/{work-open-pr-cmd-QFRDDBAM.js → work-open-pr-cmd-Z5B3ERAU.js} +7 -7
- package/dist/{work-pr-cmd-3Q2D5RQT.js → work-pr-cmd-ZVPGCDWD.js} +6 -6
- package/dist/{work-proofs-cmd-7Y3HGT7C.js → work-proofs-cmd-VAXJ5UBF.js} +7 -7
- package/dist/{work-proofs-record-QPUM6DY4.js → work-proofs-record-K467JH24.js} +13 -13
- package/dist/{work-stop-cmd-SD2ZRMSY.js → work-stop-cmd-J45Q2QT7.js} +6 -6
- package/package.json +3 -3
|
@@ -3,35 +3,35 @@ import { createRequire } from 'node:module';
|
|
|
3
3
|
import { itemsSentence, criteriaOf, hostItemsOf, loadCitationFinder, testFileMatcher, MAX_TEST_FILE_BYTES, MAX_TEST_FILES, criterionInputs, uncitedIds, planCriteria, parseJudgeAnswer, inCriteriaOrder, oneLineReason, judgedItems, IdScan, firstInPathOrder, isSearchCandidate } from './chunk-FOSYM7TQ.js';
|
|
4
4
|
import { zodProblem } from './chunk-I3LTIAOR.js';
|
|
5
5
|
import { sha256OfFile, downloadPinned, VoiceDownloadError } from './chunk-2PGI6RDE.js';
|
|
6
|
-
import { hasStoredCredential, AGENT_COMMANDS, storeAgentCredential, credentialFromStdin, isAgentName } from './chunk-
|
|
6
|
+
import { hasStoredCredential, AGENT_COMMANDS, storeAgentCredential, credentialFromStdin, isAgentName } from './chunk-CA4OE2QT.js';
|
|
7
7
|
import { upgradedStarterText } from './chunk-LHHDX7FB.js';
|
|
8
8
|
import { HELPER_PATH, HELPER_RECORD, DOCKERD_LOG } from './chunk-J2OMKFYB.js';
|
|
9
|
-
import { senderPidfile } from './chunk-
|
|
10
|
-
import { installScrubContext } from './chunk-
|
|
9
|
+
import { senderPidfile } from './chunk-FWKQHKCR.js';
|
|
10
|
+
import { installScrubContext } from './chunk-7UGMB5VS.js';
|
|
11
11
|
import { readsJobState, READ_JOB_STATE_FORMATS, jobStateFormatOf } from './chunk-ZVB57WI6.js';
|
|
12
|
-
import { runnerImageVersion } from './chunk-
|
|
13
|
-
import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey } from './chunk-
|
|
12
|
+
import { runnerImageVersion } from './chunk-ML5YKV3B.js';
|
|
13
|
+
import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey } from './chunk-SBZ545DH.js';
|
|
14
14
|
import { receiveFiles } from './chunk-NN33FYXG.js';
|
|
15
|
-
import { readDefaultBranchFiles, readDefaultBranchHead, cloneRepository, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-
|
|
15
|
+
import { readDefaultBranchFiles, readDefaultBranchHead, cloneRepository, buildRepositoryImage, fetchCommit, readBranchHead, diffBetween, prepareImageBuild, KEY_PREFIX, TAG_HOST, SHARED_OWNER, SHARED_NAME, universalBase, metadataFromLabels, writeRecord } from './chunk-KN3TDNF7.js';
|
|
16
16
|
import { FETCHED_DIR, toolsDockerfile, FETCHED_FFMPEG, FETCHED_GH, FETCHED_TINI, FETCHED_HEADLESS_SHELL, FETCHED_PLAYWRIGHT_FFMPEG, FETCHED_VOICE_DIR } from './chunk-ZSRMJOAK.js';
|
|
17
17
|
import { isSameProcess, startTimeOf } from './chunk-JBJTC3UU.js';
|
|
18
18
|
import { currentCliBuild } from './chunk-OGFGZQA3.js';
|
|
19
19
|
import { vmExecArgs, runExec, execCapture, waitForAgent } from './chunk-HRHWA6TZ.js';
|
|
20
20
|
import { socketPathProblem, vsockLinks, linkVsockSockets, GUEST_CID, LEASE_LINK } from './chunk-H2NJG3TE.js';
|
|
21
|
-
import { describeCallFailure, shortClaim, retrySend, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, formatClock, stageGovernedContext, keptContextTree, removeStagedContext, WORKFLOW_PERMISSION_REASON, refusalCode, resolveGoverningSource, prefetchGovernedContext, controlPlaneTokenFailure, keepStagedContext, pinnedSourceOf, shimScript } from './chunk-
|
|
21
|
+
import { describeCallFailure, shortClaim, retrySend, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, formatClock, stageGovernedContext, keptContextTree, removeStagedContext, WORKFLOW_PERMISSION_REASON, refusalCode, resolveGoverningSource, prefetchGovernedContext, controlPlaneTokenFailure, keepStagedContext, pinnedSourceOf, shimScript } from './chunk-6Z7R24XA.js';
|
|
22
22
|
import { writeDockerClientConfig } from './chunk-2YC3TXWX.js';
|
|
23
|
-
import { runnerCredentialProblem, builtinKind, keyRunnerFactory, optionalEgress, isBuiltinContainer, isBuiltinProcess, isBuiltinVm, egressSummary, DEFAULT_CHECK_TIME_LIMIT_MINUTES, DEFAULT_SETUP_MINUTES, allowHostProblem, systemClock, emptyEgressCounts, hostAllowed, imageBuildOf, DEFAULT_SAVED_STATE_MAX_BYTES } from './chunk-
|
|
24
|
-
import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-
|
|
25
|
-
import { writeJobFile, readJobFile, spawnRun, RUNNER_LABEL, BUILD_LABEL, NETWORK_HELPER_MEMORY, createDockerHost, builderSpec, containerExecArgs, childEnvironment, readDu, withAnswerProtocol, judgeCommand, MAX_REPORT_BYTES, readJobFileChecked, NETWORK_READY_FILE, networkHelperName,
|
|
23
|
+
import { runnerCredentialProblem, builtinKind, keyRunnerFactory, optionalEgress, isBuiltinContainer, isBuiltinProcess, isBuiltinVm, egressSummary, DEFAULT_WORKSPACE_KEEP_DAYS, DEFAULT_CHECK_TIME_LIMIT_MINUTES, DEFAULT_SETUP_MINUTES, allowHostProblem, systemClock, emptyEgressCounts, hostAllowed, imageBuildOf, DEFAULT_SAVED_STATE_MAX_BYTES } from './chunk-BACWWK3E.js';
|
|
24
|
+
import { formatBytes, volumeBytes, allocatedBytes, DEFAULT_BUILD_DISK, DEFAULT_CACHE_DISK, freeBytes } from './chunk-CI5LFLKH.js';
|
|
25
|
+
import { writeJobFile, readJobFile, spawnRun, RUNNER_LABEL, BUILD_LABEL, NETWORK_HELPER_MEMORY, createDockerHost, builderSpec, containerExecArgs, childEnvironment, readDu, withAnswerProtocol, judgeCommand, MAX_REPORT_BYTES, parseSize, runnerLabelValue, readJobFileChecked, NETWORK_READY_FILE, networkHelperName, prepareRunArgs, containerRunArgs, JUDGE_MAX_TURNS, WORKSPACE_LABEL, homeCopyRunArgs, networkHelperRunArgs, JUDGE_DIR, HASH_ARGV, parseProgramHash, identityScript, identityArgs, parseIdentity, mountableHome, homeCopyScript, homeCopyArgs, COMMAND_DIR, JUDGE_PROGRAM, MAX_ITEMS_BYTES, JUDGE_ITEMS, JUDGE_REPORT, identityRunArgs, SAVED_STATE_LABEL } from './chunk-J6HWRVN2.js';
|
|
26
26
|
import { universalImagePending, RecipeError, writeSanitizedRecipe, classifyProperties, DOCKER_FEATURE } from './chunk-BZSJ77MO.js';
|
|
27
|
-
import { postJson, WorkCommandError, getJson, isSignatureRefusal } from './chunk-
|
|
27
|
+
import { postJson, WorkCommandError, getJson, isSignatureRefusal } from './chunk-IX6CFXSW.js';
|
|
28
28
|
import { sanitizeTerminal, planArtifactUpload } from './chunk-P6PHY33P.js';
|
|
29
29
|
import { scrubText, MIN_SCRUBBED_LITERAL, setCustomScrubbers, setLocalIdentity, setRepoRoot, forkFor, parseTranscript, scrubDeep } from './chunk-6KH3NMIT.js';
|
|
30
30
|
import { readCloudConfig } from './chunk-HWVMRETT.js';
|
|
31
|
-
import { signerFor } from './chunk-
|
|
31
|
+
import { signerFor } from './chunk-42DBDDHP.js';
|
|
32
32
|
import { sharedSkillNames, skillConflictMessage } from './chunk-6JLYU322.js';
|
|
33
33
|
import { versionAtLeast, MINIMUM_CLAUDE_VERSION } from './chunk-BSPJI5T7.js';
|
|
34
|
-
import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir, buildDir, jobsDir, storeScopes, spoolDir, orgStoreIn, rootfsDir, imagesDir, sessionsDir, builderCacheDiskPath, buildsDir, ghConfigDir, runnerDirOfBuild, mirrorDir, orgHomeDir, jobConsolePath, vsockPath, builderConsolePath, rootfsPathFor, diskTemplatesDir, deliveryDir, SHIM_DIR } from './chunk-
|
|
34
|
+
import { vmAssetsDir, RUNNER_DIR_MODE, RUNNER_FILE_MODE, runnerDir, buildDir, jobsDir, storeScopes, spoolDir, orgStoreIn, rootfsDir, imagesDir, sessionsDir, builderCacheDiskPath, buildsDir, ghConfigDir, runnerDirOfBuild, mirrorDir, orgHomeDir, jobConsolePath, vsockPath, workspacesDir, builderConsolePath, rootfsPathFor, diskTemplatesDir, deliveryDir, SHIM_DIR } from './chunk-G35Y4LN5.js';
|
|
35
35
|
import { CHECKOUT_DIR, JOB_MOUNT, buildEnvironment, parseIsolation, proxyEnvironment, withProxyWarningHidden, containerLocalAddress, destinationEnvironment, withHooksPath, startBuildServices, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, repositoriesOf, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, checkEnvironment, withMiseVariables, checkoutPath, EVAL_INPUT_DIR, isMiseVariableAllowed, BUILD_HOME, primaryOf } from './chunk-CXJC3JLN.js';
|
|
36
36
|
import { firecrackerArch, MICROVM_ARCHES, FIRECRACKER_VERSION, hostHelperPinned, builderDiskPinned, guestAssetsPinned, runnerAgentNamed, credentialForms, orList, credentialShapes, credentialAgent, agentCredential, strippedVariables, builtInEnvironment, builtInNames, buildAgent, AGENT_CREDENTIAL_MISSING, RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON, UNKNOWN_TRANSPORT_REASON, GUEST_HELPER_SHA256, GUEST_ASSETS_BASE_URL, GUEST_KERNEL_SHA256, JAILER_BINARY_SHA256, FIRECRACKER_BINARY_SHA256, FIRECRACKER_SHA256, GUEST_ASSETS_VERSION, BUILDER_DISK_VERSION, credentialHandOffKey, runnerAgents, currentAgentCredential, removedAgentCredentialValues, managedAgentCredential, RELAY_USER, realFdStorePort, networkHelperPinned, NETWORK_HELPER_SHA256, universalImagePinned, UNIVERSAL_IMAGE, MAX_MALFORMED_SESSIONS, MALFORMED_REASON, JOIN_WAIT_MS, FFMPEG_STATIC, FFMPEG_BUILD_TAG, GH_SHA256, GH_VERSION, TINI_SHA256, TINI_VERSION, HEADLESS_SHELL, HEADLESS_SHELL_BASE_URL, PLAYWRIGHT_FFMPEG, PLAYWRIGHT_FFMPEG_BASE_URL, VOICE_PACK_FILES, BUILDER_DISK_SHA256, BUILDER_DISK_BASE_URL, DEFAULT_BASE_IMAGE, MAX_OPEN_STREAMS, STREAM_RESET_RATE, STREAM_RESET_BURST, STREAM_WINDOW, SESSION_WINDOW, targetOfPath, agentVersionOf, fdName, RELAY_ID, DOCKER_DESKTOP_REASON } from './chunk-DTQPDMHG.js';
|
|
37
37
|
import { defaultAgents, parsePolicy, AGENTS_SKILLS_BASE, OUTERLAYER_SKILLS_BASE, resolvePackRoot, orderSessions, extractFacts, sessionsToTimeline } from './chunk-I6U6QPN6.js';
|
|
@@ -1237,11 +1237,11 @@ function ask(socketPath, request, timeoutMs) {
|
|
|
1237
1237
|
const conn = createConnection(socketPath);
|
|
1238
1238
|
let text2 = "";
|
|
1239
1239
|
let settled = false;
|
|
1240
|
-
const settle = (
|
|
1240
|
+
const settle = (finish2) => {
|
|
1241
1241
|
if (settled) return;
|
|
1242
1242
|
settled = true;
|
|
1243
1243
|
conn.destroy();
|
|
1244
|
-
|
|
1244
|
+
finish2();
|
|
1245
1245
|
};
|
|
1246
1246
|
conn.setTimeout(timeoutMs, () => settle(() => reject(new Error(`the helper at ${socketPath} did not answer within ${Math.round(timeoutMs / 1e3)} seconds`))));
|
|
1247
1247
|
conn.on("connect", () => conn.write(`${JSON.stringify(request)}
|
|
@@ -2475,16 +2475,18 @@ async function deliver(input, staged) {
|
|
|
2475
2475
|
if (recorded.status !== 0) return { ok: false, reason: failure("recording the delivery failed", recorded) };
|
|
2476
2476
|
return { ok: true };
|
|
2477
2477
|
}
|
|
2478
|
-
async function checkOut(input, entry, restored) {
|
|
2478
|
+
async function checkOut(input, entry, restored, fromItemDisk) {
|
|
2479
2479
|
const { exec, log, tag } = input;
|
|
2480
2480
|
const path = checkoutPath(entry);
|
|
2481
|
-
const failed = (reason) => ({ ok: false, reason, ...restored ? { restoreFailed: true } : {} });
|
|
2481
|
+
const failed = (reason) => ({ ok: false, reason, ...restored ? { restoreFailed: true } : {}, ...restored && fromItemDisk ? { fromItemDisk: true } : {} });
|
|
2482
|
+
const what = fromItemDisk ? "kept" : "saved";
|
|
2482
2483
|
if (restored) {
|
|
2483
|
-
|
|
2484
|
+
const steps = fromItemDisk ? [["git", "reset", "--hard", "-q"]] : [["git", "reset", "--hard", "-q"], ["git", "clean", "-fdq"]];
|
|
2485
|
+
for (const argv of steps) {
|
|
2484
2486
|
const cleaned = await exec(argv, path, { quiet: true });
|
|
2485
2487
|
if (cleaned.status !== 0) {
|
|
2486
2488
|
const detail = gitErrorLine(cleaned.stderr);
|
|
2487
|
-
return failed(`the
|
|
2489
|
+
return failed(`the ${what} checkout of ${entry.repository} could not be cleaned (${argv[1]} exit ${cleaned.status})${detail === "" ? "" : `: ${detail}`}`);
|
|
2488
2490
|
}
|
|
2489
2491
|
}
|
|
2490
2492
|
} else {
|
|
@@ -2508,7 +2510,7 @@ async function checkOut(input, entry, restored) {
|
|
|
2508
2510
|
} catch (err) {
|
|
2509
2511
|
if (err instanceof BranchPrepareError) {
|
|
2510
2512
|
const reason = entry.primary ? err.message : `${entry.repository}: ${err.message}`;
|
|
2511
|
-
return failed(restored ? `the
|
|
2513
|
+
return failed(restored ? `the ${what} checkout of ${entry.repository} could not be switched to ${entry.branch}: ${err.message}` : reason);
|
|
2512
2514
|
}
|
|
2513
2515
|
throw err;
|
|
2514
2516
|
}
|
|
@@ -2533,9 +2535,10 @@ async function runBuildSteps(input) {
|
|
|
2533
2535
|
const rewrite = await exec(["git", "config", "--global", `url.http://127.0.0.1:${RELAY_PORT}/github.com/.insteadOf`, "https://github.com/"], "/build/work");
|
|
2534
2536
|
if (rewrite.status !== 0) return { ok: false, reason: `configuring the build's git remote failed (exit ${rewrite.status})` };
|
|
2535
2537
|
const restored = input.setup?.restored === true;
|
|
2538
|
+
const fromItemDisk = input.setup?.restored === true && input.setup.fromItemDisk === true;
|
|
2536
2539
|
const heads = [];
|
|
2537
2540
|
const keyCommits = input.setup?.keyCommits ?? {};
|
|
2538
|
-
const first = await checkOut(input, { ...primary, primary: true }, restored);
|
|
2541
|
+
const first = await checkOut(input, { ...primary, primary: true }, restored, fromItemDisk);
|
|
2539
2542
|
if (!first.ok) return first;
|
|
2540
2543
|
startCommit = first.startCommit;
|
|
2541
2544
|
heads.push({ repository: primary.repository, head: first.head, startCommit: first.startCommit, keyCommit: keyCommits[primary.repository] });
|
|
@@ -2546,7 +2549,7 @@ async function runBuildSteps(input) {
|
|
|
2546
2549
|
}
|
|
2547
2550
|
for (const entry of repositories.filter((candidate) => candidate.repository !== primary.repository)) {
|
|
2548
2551
|
if (input.aborted()) return aborted();
|
|
2549
|
-
const checked = await checkOut(input, { ...entry, primary: false }, restored);
|
|
2552
|
+
const checked = await checkOut(input, { ...entry, primary: false }, restored, fromItemDisk);
|
|
2550
2553
|
if (!checked.ok) return checked;
|
|
2551
2554
|
heads.push({ repository: entry.repository, head: checked.head, startCommit: checked.startCommit, keyCommit: keyCommits[entry.repository] });
|
|
2552
2555
|
}
|
|
@@ -2659,11 +2662,11 @@ function savedStatesDir(runnerDirPath) {
|
|
|
2659
2662
|
}
|
|
2660
2663
|
function writeRecord2(dir, record) {
|
|
2661
2664
|
mkdirSync(dir, { recursive: true, mode: RUNNER_DIR_MODE });
|
|
2662
|
-
const
|
|
2663
|
-
const partial = `${
|
|
2665
|
+
const target2 = join(dir, `${record.id}.json`);
|
|
2666
|
+
const partial = `${target2}.partial`;
|
|
2664
2667
|
writeFileSync(partial, `${JSON.stringify(record, null, 2)}
|
|
2665
2668
|
`, { mode: RUNNER_FILE_MODE });
|
|
2666
|
-
renameSync(partial,
|
|
2669
|
+
renameSync(partial, target2);
|
|
2667
2670
|
}
|
|
2668
2671
|
function isRecord(value) {
|
|
2669
2672
|
if (typeof value !== "object" || value === null) return false;
|
|
@@ -2802,14 +2805,149 @@ function unread(reason, log) {
|
|
|
2802
2805
|
log(`this build starts from no saved state and saves none: ${reason}`);
|
|
2803
2806
|
return { keyCommits: {}, noSave: `its key files could not be read: ${reason}` };
|
|
2804
2807
|
}
|
|
2808
|
+
var RECORD_FILE = "record.json";
|
|
2809
|
+
var DISK_FILE = "disk.ext4";
|
|
2810
|
+
var WORK_ITEM_ID2 = /^[A-Za-z0-9_-]{1,64}$/;
|
|
2811
|
+
function itemDiskDir(configPath, organizationId, workItemId) {
|
|
2812
|
+
if (!WORK_ITEM_ID2.test(workItemId)) throw new Error(`cannot keep a disk for the work item ${JSON.stringify(workItemId)}`);
|
|
2813
|
+
return join(workspacesDir(configPath, organizationId), workItemId);
|
|
2814
|
+
}
|
|
2815
|
+
function isRecord2(value) {
|
|
2816
|
+
if (typeof value !== "object" || value === null) return false;
|
|
2817
|
+
const r = value;
|
|
2818
|
+
return typeof r.workItemId === "string" && typeof r.claimId === "string" && typeof r.queue === "string" && typeof r.imageId === "string" && typeof r.diskBytes === "number" && (r.kind === "disk" || r.kind === "volume" && typeof r.volume === "string") && typeof r.bytes === "number" && typeof r.keptAt === "string" && Number.isFinite(Date.parse(r.keptAt));
|
|
2819
|
+
}
|
|
2820
|
+
function readItemDisk(dir) {
|
|
2821
|
+
let parsed;
|
|
2822
|
+
try {
|
|
2823
|
+
parsed = JSON.parse(readFileSync(join(dir, RECORD_FILE), "utf8"));
|
|
2824
|
+
} catch {
|
|
2825
|
+
return void 0;
|
|
2826
|
+
}
|
|
2827
|
+
if (!isRecord2(parsed) || parsed.workItemId !== basename(dir)) return void 0;
|
|
2828
|
+
if (parsed.kind === "volume") return { dir, record: parsed };
|
|
2829
|
+
const path = join(dir, DISK_FILE);
|
|
2830
|
+
try {
|
|
2831
|
+
if (!statSync(path).isFile()) return void 0;
|
|
2832
|
+
} catch {
|
|
2833
|
+
return void 0;
|
|
2834
|
+
}
|
|
2835
|
+
return { dir, record: parsed, path };
|
|
2836
|
+
}
|
|
2837
|
+
function swapInItemDisk(dir, incoming) {
|
|
2838
|
+
const older = readItemDisk(dir);
|
|
2839
|
+
const outgoing = join(dirname(dir), `.${basename(dir)}.${randomBytes(4).toString("hex")}.old`);
|
|
2840
|
+
let hadOlder = true;
|
|
2841
|
+
try {
|
|
2842
|
+
renameSync(dir, outgoing);
|
|
2843
|
+
} catch (err) {
|
|
2844
|
+
if (err.code !== "ENOENT") throw err;
|
|
2845
|
+
hadOlder = false;
|
|
2846
|
+
}
|
|
2847
|
+
try {
|
|
2848
|
+
renameSync(incoming, dir);
|
|
2849
|
+
} catch (err) {
|
|
2850
|
+
if (hadOlder) renameSync(outgoing, dir);
|
|
2851
|
+
throw err;
|
|
2852
|
+
}
|
|
2853
|
+
rmSync(outgoing, { recursive: true, force: true });
|
|
2854
|
+
return older;
|
|
2855
|
+
}
|
|
2856
|
+
function stagingItemDiskDir(dir) {
|
|
2857
|
+
const parent = dirname(dir);
|
|
2858
|
+
mkdirSync(parent, { recursive: true, mode: 448 });
|
|
2859
|
+
const staging = join(parent, `.${basename(dir)}.${randomBytes(4).toString("hex")}.partial`);
|
|
2860
|
+
mkdirSync(staging, { mode: 448 });
|
|
2861
|
+
return staging;
|
|
2862
|
+
}
|
|
2863
|
+
function listItemDisks(configPath, scope) {
|
|
2864
|
+
const where = workspacesDir(configPath, scope);
|
|
2865
|
+
let names;
|
|
2866
|
+
try {
|
|
2867
|
+
names = readdirSync(where);
|
|
2868
|
+
} catch {
|
|
2869
|
+
return [];
|
|
2870
|
+
}
|
|
2871
|
+
const kept = [];
|
|
2872
|
+
for (const name of names) {
|
|
2873
|
+
if (!WORK_ITEM_ID2.test(name)) continue;
|
|
2874
|
+
const found = readItemDisk(join(where, name));
|
|
2875
|
+
if (found !== void 0) kept.push(found);
|
|
2876
|
+
}
|
|
2877
|
+
return kept;
|
|
2878
|
+
}
|
|
2879
|
+
function listLeftovers(configPath, scope) {
|
|
2880
|
+
const where = workspacesDir(configPath, scope);
|
|
2881
|
+
let names;
|
|
2882
|
+
try {
|
|
2883
|
+
names = readdirSync(where);
|
|
2884
|
+
} catch {
|
|
2885
|
+
return [];
|
|
2886
|
+
}
|
|
2887
|
+
const found = [];
|
|
2888
|
+
for (const name of names) {
|
|
2889
|
+
if (!name.startsWith(".") && (!WORK_ITEM_ID2.test(name) || readItemDisk(join(where, name)) !== void 0)) continue;
|
|
2890
|
+
try {
|
|
2891
|
+
found.push({ path: join(where, name), mtimeMs: lstatSync(join(where, name)).mtimeMs });
|
|
2892
|
+
} catch {
|
|
2893
|
+
}
|
|
2894
|
+
}
|
|
2895
|
+
return found;
|
|
2896
|
+
}
|
|
2897
|
+
|
|
2898
|
+
// src/runner/workspace/restore.ts
|
|
2899
|
+
var DISK_REASON_TEXT = {
|
|
2900
|
+
no_kept_disk: "no disk is kept for the work item on this host",
|
|
2901
|
+
other_image: "the kept disk was made on another image",
|
|
2902
|
+
other_disk_size: "the kept disk was made with another runner.build.disk size",
|
|
2903
|
+
disk_not_restored: "the kept disk could not be copied",
|
|
2904
|
+
checkout_not_switched: "the kept checkout could not be switched to the branch",
|
|
2905
|
+
build_type_cannot_restore: "this build type cannot restore a disk"
|
|
2906
|
+
};
|
|
2907
|
+
function itemDiskDecision(input) {
|
|
2908
|
+
if (input.queue !== "amend") return { state: void 0 };
|
|
2909
|
+
if (!input.builtin) return { state: { restored: false, reason: "build_type_cannot_restore" } };
|
|
2910
|
+
let kept;
|
|
2911
|
+
try {
|
|
2912
|
+
kept = readItemDisk(itemDiskDir(input.configPath, input.organizationId, input.workItemId));
|
|
2913
|
+
} catch {
|
|
2914
|
+
kept = void 0;
|
|
2915
|
+
}
|
|
2916
|
+
if (kept === void 0) return { state: { restored: false, reason: "no_kept_disk" } };
|
|
2917
|
+
if (kept.record.imageId !== input.imageId) return { state: { restored: false, reason: "other_image" } };
|
|
2918
|
+
if (kept.record.diskBytes !== input.diskBytes) return { state: { restored: false, reason: "other_disk_size" } };
|
|
2919
|
+
if (kept.record.kind !== input.backend) return { state: { restored: false, reason: "disk_not_restored" } };
|
|
2920
|
+
return { kept, state: { restored: true, fromClaimId: kept.record.claimId } };
|
|
2921
|
+
}
|
|
2922
|
+
async function discardItemDisk(kept, removeVolume, onError) {
|
|
2923
|
+
try {
|
|
2924
|
+
if (kept.record.kind === "volume" && kept.record.volume !== void 0) {
|
|
2925
|
+
const removed = await removeVolume(kept.record.volume);
|
|
2926
|
+
if (removed !== void 0 && removed.status !== 0 && !/no such volume/i.test(removed.stderr ?? "")) {
|
|
2927
|
+
onError(`could not remove the volume ${kept.record.volume} of the kept disk: ${removed.stderr?.trim() ?? `exit ${removed.status}`}`);
|
|
2928
|
+
return;
|
|
2929
|
+
}
|
|
2930
|
+
}
|
|
2931
|
+
rmSync(kept.dir, { recursive: true, force: true });
|
|
2932
|
+
} catch (err) {
|
|
2933
|
+
onError(`could not remove the kept disk of work item ${kept.record.workItemId}: ${describeCallFailure(err)}`);
|
|
2934
|
+
}
|
|
2935
|
+
}
|
|
2805
2936
|
|
|
2806
2937
|
// src/runner/saved-state/provision.ts
|
|
2807
|
-
var isRetryFresh = (value) => typeof value === "object" && value !== null &&
|
|
2938
|
+
var isRetryFresh = (value) => typeof value === "object" && value !== null && value.retryFresh === true;
|
|
2808
2939
|
async function provisionWithFreshFallback(input) {
|
|
2809
2940
|
const first = await input.attempt(true);
|
|
2810
2941
|
if (!isRetryFresh(first)) return first;
|
|
2811
2942
|
input.log(`${first.reason}; provisioning the build again from a fresh disk`);
|
|
2812
|
-
return await input.attempt(false);
|
|
2943
|
+
if (first.itemDisk === void 0) return await input.attempt(false);
|
|
2944
|
+
const second = await input.attempt(true, first.itemDisk);
|
|
2945
|
+
if (!isRetryFresh(second)) return second;
|
|
2946
|
+
input.log(`${second.reason}; provisioning the build again from a fresh disk`);
|
|
2947
|
+
return await input.attempt(false, first.itemDisk);
|
|
2948
|
+
}
|
|
2949
|
+
function keptDiskPlan() {
|
|
2950
|
+
return { keyCommits: {}, noSave: "it starts from the work item's kept disk" };
|
|
2813
2951
|
}
|
|
2814
2952
|
function clockOf(now) {
|
|
2815
2953
|
return now ?? (() => /* @__PURE__ */ new Date());
|
|
@@ -2833,7 +2971,8 @@ function setupInputFor(input) {
|
|
|
2833
2971
|
return {
|
|
2834
2972
|
timeoutMs: minutes * 6e4,
|
|
2835
2973
|
limitMinutes: minutes,
|
|
2836
|
-
restored: input.plan.restore !== void 0,
|
|
2974
|
+
restored: input.plan.restore !== void 0 || input.fromItemDisk === true,
|
|
2975
|
+
...input.fromItemDisk === true ? { fromItemDisk: true } : {},
|
|
2837
2976
|
keyCommits: input.plan.keyCommits,
|
|
2838
2977
|
...input.withoutBuildVariables === true ? { withoutBuildVariables: true } : {},
|
|
2839
2978
|
// A plan has a key exactly when the build may save a state. A build that starts from one is not asked to save.
|
|
@@ -2856,7 +2995,11 @@ async function discardUnusableState(input) {
|
|
|
2856
2995
|
return { retryFresh: true, reason: `saved state ${input.record.id} could not be used: ${input.reason}` };
|
|
2857
2996
|
}
|
|
2858
2997
|
function settleFailedSteps(steps, on) {
|
|
2859
|
-
return steps.restoreFailed === true ? on.giveUp(steps.reason) : on.bail(steps.reason, steps.aborted === true);
|
|
2998
|
+
return steps.restoreFailed === true ? on.giveUp(steps.reason, steps.fromItemDisk === true) : on.bail(steps.reason, steps.aborted === true);
|
|
2999
|
+
}
|
|
3000
|
+
async function discardUnusableItemDisk(input) {
|
|
3001
|
+
await discardItemDisk(input.kept, input.removeVolume, input.onError);
|
|
3002
|
+
return { retryFresh: true, reason: `the kept disk of claim ${shortClaim(input.kept.record.claimId)} could not be used: ${input.reason}`, itemDisk: input.why };
|
|
2860
3003
|
}
|
|
2861
3004
|
var FREEZE_DEADLINE_SECONDS = 12 * 60;
|
|
2862
3005
|
function freezeScript() {
|
|
@@ -3323,10 +3466,10 @@ function unitNameFor(buildId) {
|
|
|
3323
3466
|
return `outerlayer-build-${buildId}`;
|
|
3324
3467
|
}
|
|
3325
3468
|
function linkIn(buildDir2, name, source) {
|
|
3326
|
-
const
|
|
3327
|
-
rmSync(
|
|
3469
|
+
const target2 = join(buildDir2, name);
|
|
3470
|
+
rmSync(target2, { force: true });
|
|
3328
3471
|
try {
|
|
3329
|
-
linkSync(source,
|
|
3472
|
+
linkSync(source, target2);
|
|
3330
3473
|
} catch (err) {
|
|
3331
3474
|
if (err.code === "EXDEV") {
|
|
3332
3475
|
throw new Error(`${source} and ${buildDir2} are on different file systems, so the build's files cannot be linked into its jail`);
|
|
@@ -3521,10 +3664,10 @@ async function mountGuestHome(runtime, state, agentUser, known) {
|
|
|
3521
3664
|
}
|
|
3522
3665
|
var NODE_IN_GUEST = "/opt/outerlayer/bin/node";
|
|
3523
3666
|
var NO_FOLDER = 3;
|
|
3524
|
-
async function unpackIntoGuest(runtime, state, dir,
|
|
3525
|
-
const script = opts.readOnly === true ? `set -e; rm -rf ${
|
|
3667
|
+
async function unpackIntoGuest(runtime, state, dir, target2, opts = {}) {
|
|
3668
|
+
const script = opts.readOnly === true ? `set -e; rm -rf ${target2}; mkdir -p ${target2}; tar -x --no-same-owner -C ${target2}; chmod -R a+rX,a-w ${target2}` : `set -e; mkdir -p ${target2}; tar -x --no-same-owner -C ${target2}; chmod -R a+rX ${target2}`;
|
|
3526
3669
|
const result = await asRoot(runtime, state, ["/bin/sh", "-c", script], tarOf(dir));
|
|
3527
|
-
if (result.status !== 0) throw new Error(`unpacking into ${
|
|
3670
|
+
if (result.status !== 0) throw new Error(`unpacking into ${target2} in the guest failed (exit ${result.status}): ${firstLine3(result.stderr)}`);
|
|
3528
3671
|
}
|
|
3529
3672
|
async function deliverCommitHooks(input) {
|
|
3530
3673
|
await unpackIntoGuest(input.runtime, input.state, input.hooksDir, GUEST_HOOKS_DIR);
|
|
@@ -3587,11 +3730,11 @@ async function keepSockets(runtime, buildId, listeners, leaseServer, line2, erro
|
|
|
3587
3730
|
}
|
|
3588
3731
|
async function provisionMicroVm(input) {
|
|
3589
3732
|
return provisionWithFreshFallback({
|
|
3590
|
-
attempt: (allowRestore) => provisionMicroVmOnce(input, allowRestore),
|
|
3733
|
+
attempt: (allowRestore, itemDiskFailed) => provisionMicroVmOnce(input, allowRestore, itemDiskFailed),
|
|
3591
3734
|
log: (line2) => input.log.line(`#${input.item.number} ${line2}`)
|
|
3592
3735
|
});
|
|
3593
3736
|
}
|
|
3594
|
-
async function provisionMicroVmOnce(input, allowRestore) {
|
|
3737
|
+
async function provisionMicroVmOnce(input, allowRestore, itemDiskFailed) {
|
|
3595
3738
|
const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
|
|
3596
3739
|
const managed = config.managed === true;
|
|
3597
3740
|
const { vm } = runtime;
|
|
@@ -3656,7 +3799,10 @@ async function provisionMicroVmOnce(input, allowRestore) {
|
|
|
3656
3799
|
if (!obtained.ok) return obtained.aborted === true ? { ok: false, reason: obtained.reason, aborted: true } : fail(obtained.reason);
|
|
3657
3800
|
const { report } = obtained.built;
|
|
3658
3801
|
const tagged = taggedLog(log, tag);
|
|
3659
|
-
const
|
|
3802
|
+
const itemDisk = input.workItemId === void 0 ? { state: void 0 } : itemDiskFailed !== void 0 ? { state: { restored: false, reason: itemDiskFailed } } : itemDiskDecision({ queue: input.queue ?? "implement", builtin: true, backend: "disk", configPath: config.configPath, organizationId: input.organizationId, workItemId: input.workItemId, imageId: report.imageId, diskBytes: limits2.diskBytes });
|
|
3803
|
+
const keptDisk = itemDisk.kept;
|
|
3804
|
+
if (itemDisk.state?.restored === false) tagged(`this agent job starts without a kept disk: ${DISK_REASON_TEXT[itemDisk.state.reason]}`);
|
|
3805
|
+
const plan = keptDisk !== void 0 ? keptDiskPlan() : await planSavedState({
|
|
3660
3806
|
config,
|
|
3661
3807
|
repositories,
|
|
3662
3808
|
base: savedStateBase(report),
|
|
@@ -3728,6 +3874,10 @@ async function provisionMicroVmOnce(input, allowRestore) {
|
|
|
3728
3874
|
await bail(reason);
|
|
3729
3875
|
return discardUnusableState({ record: plan.restore.record, reason, dir: savedStatesPath, removeVolume: async () => void 0, onError: (line2) => log.error(`${tag} ${line2}`) });
|
|
3730
3876
|
};
|
|
3877
|
+
const giveUpItemDisk = async (reason, why) => {
|
|
3878
|
+
await bail(reason);
|
|
3879
|
+
return discardUnusableItemDisk({ kept: keptDisk, reason, why, removeVolume: async () => void 0, onError: (line2) => log.error(`${tag} ${line2}`) });
|
|
3880
|
+
};
|
|
3731
3881
|
if (input.aborted()) return bail("", true);
|
|
3732
3882
|
let imageConfig;
|
|
3733
3883
|
if (madeInBuilder !== void 0) {
|
|
@@ -3779,12 +3929,15 @@ async function provisionMicroVmOnce(input, allowRestore) {
|
|
|
3779
3929
|
if (restore !== void 0 && (restore.kind !== "disk" || restore.path === void 0)) return giveUpState(`the saved state is a ${restore.kind}, not a disk this runner can copy; provisioning the build again from a fresh disk`);
|
|
3780
3930
|
const restoreStartedAt = Date.now();
|
|
3781
3931
|
try {
|
|
3782
|
-
await createBuildDisk({ host: vm.rootfs, runnerDir: runnerDirPath, bytes: limits2.diskBytes, dest: state.buildDisk, memoryBytes: limits2.memoryBytes, source: restore?.path });
|
|
3932
|
+
await createBuildDisk({ host: vm.rootfs, runnerDir: runnerDirPath, bytes: limits2.diskBytes, dest: state.buildDisk, memoryBytes: limits2.memoryBytes, source: keptDisk?.path ?? restore?.path });
|
|
3783
3933
|
} catch (err) {
|
|
3934
|
+
if (keptDisk !== void 0) return giveUpItemDisk(`the kept disk could not be copied: ${describeCallFailure(err)}`, "disk_not_restored");
|
|
3784
3935
|
if (restore !== void 0) return giveUpState(`the saved disk could not be copied: ${describeCallFailure(err)}`);
|
|
3785
3936
|
throw err;
|
|
3786
3937
|
}
|
|
3787
|
-
if (
|
|
3938
|
+
if (keptDisk?.path !== void 0) {
|
|
3939
|
+
tagged(`restored the disk that claim ${shortClaim(keptDisk.record.claimId)} kept: ${formatBytes(allocatedBytes(keptDisk.path))} allocated, copied in ${formatDuration(Date.now() - restoreStartedAt)}`);
|
|
3940
|
+
} else if (restore?.path !== void 0) {
|
|
3788
3941
|
tagged(`restored saved state ${restore.id}: ${formatBytes(allocatedBytes(restore.path))} allocated, copied in ${formatDuration(Date.now() - restoreStartedAt)}`);
|
|
3789
3942
|
}
|
|
3790
3943
|
writeFileSync(
|
|
@@ -3999,15 +4152,15 @@ async function provisionMicroVmOnce(input, allowRestore) {
|
|
|
3999
4152
|
...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
|
|
4000
4153
|
// A build on the universal image, or on an image built from it, chooses and installs its runtimes.
|
|
4001
4154
|
...runtimeStepInput(report, services),
|
|
4002
|
-
setup: setupInputFor({ setupMinutes: config.build?.setupMinutes, plan, saveState, withoutBuildVariables: managed }),
|
|
4155
|
+
setup: setupInputFor({ setupMinutes: config.build?.setupMinutes, plan, saveState, withoutBuildVariables: managed, fromItemDisk: keptDisk !== void 0 }),
|
|
4003
4156
|
afterSave: releaseGhConfig
|
|
4004
4157
|
});
|
|
4005
|
-
if (!steps.ok) return settleFailedSteps(steps, { giveUp: giveUpState, bail });
|
|
4158
|
+
if (!steps.ok) return settleFailedSteps(steps, { giveUp: (reason, fromItemDisk) => fromItemDisk ? giveUpItemDisk(reason, "checkout_not_switched") : giveUpState(reason), bail });
|
|
4006
4159
|
state.startCommit = steps.startCommit;
|
|
4007
4160
|
keepRuntimeStep(state, steps);
|
|
4008
4161
|
keepMissingSetupScripts(state, steps);
|
|
4009
4162
|
if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
|
|
4010
|
-
return { ok: true, workdir: CHECKOUT_DIR, state, environment };
|
|
4163
|
+
return { ok: true, workdir: CHECKOUT_DIR, state, environment, ...itemDisk.state === void 0 ? {} : { disk: itemDisk.state } };
|
|
4011
4164
|
}
|
|
4012
4165
|
async function waitForDocker(runtime, state, agentUser, now) {
|
|
4013
4166
|
const grant = `[ -S ${DOCKER_SOCKET} ] && chgrp "$(id -g ${shellQuote(agentUser)})" ${DOCKER_SOCKET} && chmod 660 ${DOCKER_SOCKET}`;
|
|
@@ -4451,13 +4604,26 @@ async function stopBuildMicroVm(runtime, state, jobDir) {
|
|
|
4451
4604
|
await runtime.vm.firecracker.stop(state.firecracker);
|
|
4452
4605
|
if (runtime.vm.firecracker.alive(state.firecracker)) throw new Error(`the microVM (pid ${state.firecracker.pid}) is still running after it was stopped`);
|
|
4453
4606
|
}
|
|
4454
|
-
async function
|
|
4607
|
+
async function flushBuildDisk(runtime, state) {
|
|
4608
|
+
if (!isRunning(runtime, state)) return { ok: false, reason: "the microVM was gone before its writes could be flushed" };
|
|
4609
|
+
try {
|
|
4610
|
+
const flushed = await asRoot(runtime, state, ["sync", "-f", "/build"]);
|
|
4611
|
+
if (flushed.status === 0) return { ok: true };
|
|
4612
|
+
return { ok: false, reason: `the build disk could not be flushed (exit ${flushed.status})${said(flushed.stderr)}` };
|
|
4613
|
+
} catch (err) {
|
|
4614
|
+
return { ok: false, reason: `the build disk could not be flushed (${describeCallFailure(err)})` };
|
|
4615
|
+
}
|
|
4616
|
+
}
|
|
4617
|
+
async function cleanupMicroVm(runtime, state, jobDir, warn, onBuildDisk, keepDisk) {
|
|
4455
4618
|
const failures = [];
|
|
4619
|
+
let stopFailed = false;
|
|
4456
4620
|
if (jobDir !== void 0) await copyHookErrors(runtime, state, jobDir);
|
|
4621
|
+
const flushed = keepDisk === void 0 ? void 0 : await flushBuildDisk(runtime, state);
|
|
4457
4622
|
if (state.jailed !== void 0) {
|
|
4458
4623
|
try {
|
|
4459
4624
|
await stopJailed(runtime.vm.helper, state);
|
|
4460
4625
|
} catch (err) {
|
|
4626
|
+
stopFailed = true;
|
|
4461
4627
|
failures.push(describeCallFailure(err));
|
|
4462
4628
|
}
|
|
4463
4629
|
}
|
|
@@ -4465,6 +4631,7 @@ async function cleanupMicroVm(runtime, state, jobDir, warn, onBuildDisk) {
|
|
|
4465
4631
|
try {
|
|
4466
4632
|
await runtime.vm.firecracker.stop(state.firecracker);
|
|
4467
4633
|
} catch (err) {
|
|
4634
|
+
stopFailed = true;
|
|
4468
4635
|
failures.push(`the microVM (pid ${state.firecracker.pid}) was not stopped: ${describeCallFailure(err)}`);
|
|
4469
4636
|
}
|
|
4470
4637
|
}
|
|
@@ -4482,6 +4649,16 @@ async function cleanupMicroVm(runtime, state, jobDir, warn, onBuildDisk) {
|
|
|
4482
4649
|
failures.push(`the build disk's peak size was not recorded: ${describeCallFailure(err)}`);
|
|
4483
4650
|
}
|
|
4484
4651
|
}
|
|
4652
|
+
const stillRunning = vmAlive(runtime, state);
|
|
4653
|
+
if (stillRunning && keepDisk !== void 0) warn?.("the build's disk was not kept: the microVM is still running");
|
|
4654
|
+
else if (flushed !== void 0 && !flushed.ok && !stopFailed) warn?.(`the build's disk was not kept: ${flushed.reason}`);
|
|
4655
|
+
if (keepDisk !== void 0 && flushed?.ok === true && !stopFailed && !stillRunning && state.buildDisk !== void 0 && existsSync(state.buildDisk)) {
|
|
4656
|
+
try {
|
|
4657
|
+
await keepDisk();
|
|
4658
|
+
} catch (err) {
|
|
4659
|
+
warn?.(`the build's disk was not kept: ${describeCallFailure(err)}`);
|
|
4660
|
+
}
|
|
4661
|
+
}
|
|
4485
4662
|
if (state.buildDisk !== void 0) deleteBuildDisk(state.buildDisk);
|
|
4486
4663
|
runtime.leases.get(state.buildId)?.close();
|
|
4487
4664
|
runtime.leases.delete(state.buildId);
|
|
@@ -4788,8 +4965,8 @@ function normalize(address) {
|
|
|
4788
4965
|
const zone = address.indexOf("%");
|
|
4789
4966
|
return (zone === -1 ? address : address.slice(0, zone)).toLowerCase();
|
|
4790
4967
|
}
|
|
4791
|
-
function parseTarget(
|
|
4792
|
-
const match = /^\[([^\]]+)\]:(\d+)$/.exec(
|
|
4968
|
+
function parseTarget(target2) {
|
|
4969
|
+
const match = /^\[([^\]]+)\]:(\d+)$/.exec(target2) ?? /^([^:\[\]]+):(\d+)$/.exec(target2);
|
|
4793
4970
|
if (match === null) return void 0;
|
|
4794
4971
|
const port = Number(match[2]);
|
|
4795
4972
|
return port >= 1 && port <= 65535 ? { host: match[1], port } : void 0;
|
|
@@ -5052,29 +5229,29 @@ ${body}`);
|
|
|
5052
5229
|
};
|
|
5053
5230
|
async function tunnelConnect(req, client, head) {
|
|
5054
5231
|
client.on("error", () => client.destroy());
|
|
5055
|
-
const
|
|
5056
|
-
if (
|
|
5232
|
+
const target2 = parseTarget(req.url ?? "");
|
|
5233
|
+
if (target2 === void 0) {
|
|
5057
5234
|
client.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-Length: 0\r\n\r\n");
|
|
5058
5235
|
return;
|
|
5059
5236
|
}
|
|
5060
5237
|
const over = admit();
|
|
5061
5238
|
if (over !== void 0) {
|
|
5062
|
-
const refusal = refuseOverLimit(
|
|
5239
|
+
const refusal = refuseOverLimit(target2.host, over);
|
|
5063
5240
|
reply(client, refusal.status, refusal.body);
|
|
5064
5241
|
return;
|
|
5065
5242
|
}
|
|
5066
5243
|
client.once("close", release);
|
|
5067
|
-
const decision = await decide(
|
|
5244
|
+
const decision = await decide(target2.host, target2.port);
|
|
5068
5245
|
if (!decision.ok) {
|
|
5069
5246
|
reply(client, decision.status, decision.body);
|
|
5070
5247
|
return;
|
|
5071
5248
|
}
|
|
5072
|
-
const upstream = openConnection({ address: decision.address, port:
|
|
5249
|
+
const upstream = openConnection({ address: decision.address, port: target2.port });
|
|
5073
5250
|
upstream.setTimeout(connectTimeoutMs);
|
|
5074
5251
|
const failed = (err) => {
|
|
5075
5252
|
upstream.destroy();
|
|
5076
|
-
logBadGateway(`could not connect to ${
|
|
5077
|
-
reply(client, 502, `could not connect to ${
|
|
5253
|
+
logBadGateway(`could not connect to ${target2.host}:${target2.port}, ${err === void 0 ? "ETIMEDOUT" : errorCode(err)}`);
|
|
5254
|
+
reply(client, 502, `could not connect to ${target2.host}
|
|
5078
5255
|
`);
|
|
5079
5256
|
};
|
|
5080
5257
|
const timedOut = () => failed();
|
|
@@ -5097,7 +5274,7 @@ ${body}`);
|
|
|
5097
5274
|
const metered = limits2.sendBytesPerSecond > 0 ? meter() : void 0;
|
|
5098
5275
|
if (metered === void 0) client.once("end", buildEnded);
|
|
5099
5276
|
else metered.once("end", buildEnded);
|
|
5100
|
-
reached(
|
|
5277
|
+
reached(target2.host);
|
|
5101
5278
|
client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
|
5102
5279
|
if (metered !== void 0) {
|
|
5103
5280
|
if (head.length > 0) metered.write(head);
|
|
@@ -5438,7 +5615,7 @@ function runWithSpawn(spawnFn = spawn) {
|
|
|
5438
5615
|
let stderr = "";
|
|
5439
5616
|
let timer;
|
|
5440
5617
|
let settled = false;
|
|
5441
|
-
const
|
|
5618
|
+
const finish2 = (result) => {
|
|
5442
5619
|
if (settled) return;
|
|
5443
5620
|
settled = true;
|
|
5444
5621
|
if (timer !== void 0) clearTimeout(timer);
|
|
@@ -5448,12 +5625,12 @@ function runWithSpawn(spawnFn = spawn) {
|
|
|
5448
5625
|
try {
|
|
5449
5626
|
child = spawnFn(file, args, { env, stdio: ["ignore", "pipe", "pipe"] });
|
|
5450
5627
|
} catch (err) {
|
|
5451
|
-
|
|
5628
|
+
finish2({ error: err instanceof Error ? err : new Error(String(err)) });
|
|
5452
5629
|
return;
|
|
5453
5630
|
}
|
|
5454
5631
|
timer = setTimeout(() => {
|
|
5455
5632
|
child.kill("SIGKILL");
|
|
5456
|
-
|
|
5633
|
+
finish2({ error: new Error(`it did not finish in ${timeoutMs / 1e3}s`) });
|
|
5457
5634
|
}, timeoutMs);
|
|
5458
5635
|
child.stdout?.on("data", (chunk) => {
|
|
5459
5636
|
stdout += String(chunk);
|
|
@@ -5461,8 +5638,8 @@ function runWithSpawn(spawnFn = spawn) {
|
|
|
5461
5638
|
child.stderr?.on("data", (chunk) => {
|
|
5462
5639
|
stderr += String(chunk);
|
|
5463
5640
|
});
|
|
5464
|
-
child.on("error", (err) =>
|
|
5465
|
-
child.on("close", (status, signal) =>
|
|
5641
|
+
child.on("error", (err) => finish2({ error: err }));
|
|
5642
|
+
child.on("close", (status, signal) => finish2({ status, signal }));
|
|
5466
5643
|
});
|
|
5467
5644
|
}
|
|
5468
5645
|
async function deliverMavenSettings(build, workdir, base, run, timeoutMs = MAVEN_WRITE_TIMEOUT_MS) {
|
|
@@ -5891,6 +6068,288 @@ async function provisionProcess(input) {
|
|
|
5891
6068
|
function cleanupProcess(runtime, state) {
|
|
5892
6069
|
return closeHostBuild(runtime, state);
|
|
5893
6070
|
}
|
|
6071
|
+
var SAVED_FOLDERS = ["home", "work"];
|
|
6072
|
+
function stopProcessesScript() {
|
|
6073
|
+
return [
|
|
6074
|
+
"me=$$; uid=$(id -u); n=0; pass=0",
|
|
6075
|
+
"while [ $pass -lt 25 ]; do",
|
|
6076
|
+
' left=""',
|
|
6077
|
+
" for p in /proc/[0-9]*; do",
|
|
6078
|
+
" pid=${p##*/}",
|
|
6079
|
+
' [ "$pid" = "$me" ] && continue',
|
|
6080
|
+
` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
|
|
6081
|
+
' case "$info" in *"S=Z"*) continue ;; esac',
|
|
6082
|
+
// The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
|
|
6083
|
+
' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
|
|
6084
|
+
" done",
|
|
6085
|
+
' [ -z "$left" ] && break',
|
|
6086
|
+
' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
|
|
6087
|
+
" pass=$((pass+1))",
|
|
6088
|
+
" sleep 0.2",
|
|
6089
|
+
"done",
|
|
6090
|
+
'[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
|
|
6091
|
+
'echo "stopped $n"'
|
|
6092
|
+
].join("\n");
|
|
6093
|
+
}
|
|
6094
|
+
var HELPER_FLAGS = [
|
|
6095
|
+
"--rm",
|
|
6096
|
+
"--network",
|
|
6097
|
+
"none",
|
|
6098
|
+
"--cap-drop",
|
|
6099
|
+
"ALL",
|
|
6100
|
+
"--cap-add",
|
|
6101
|
+
"CHOWN",
|
|
6102
|
+
"--cap-add",
|
|
6103
|
+
"DAC_OVERRIDE",
|
|
6104
|
+
"--cap-add",
|
|
6105
|
+
"DAC_READ_SEARCH",
|
|
6106
|
+
"--cap-add",
|
|
6107
|
+
"FOWNER",
|
|
6108
|
+
"--security-opt",
|
|
6109
|
+
"no-new-privileges",
|
|
6110
|
+
"--read-only",
|
|
6111
|
+
"--user",
|
|
6112
|
+
"0",
|
|
6113
|
+
"--pids-limit",
|
|
6114
|
+
"256"
|
|
6115
|
+
];
|
|
6116
|
+
function helperArgs(input) {
|
|
6117
|
+
return [
|
|
6118
|
+
...HELPER_FLAGS,
|
|
6119
|
+
"--label",
|
|
6120
|
+
`${RUNNER_LABEL}=${input.runnerId}`,
|
|
6121
|
+
"--mount",
|
|
6122
|
+
`type=volume,src=${input.from},dst=/from,readonly`,
|
|
6123
|
+
"--mount",
|
|
6124
|
+
`type=volume,src=${input.to},dst=/to`,
|
|
6125
|
+
"--entrypoint",
|
|
6126
|
+
"sh",
|
|
6127
|
+
input.image,
|
|
6128
|
+
"-c",
|
|
6129
|
+
input.script
|
|
6130
|
+
];
|
|
6131
|
+
}
|
|
6132
|
+
var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
|
|
6133
|
+
var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
|
|
6134
|
+
function failureText3(result) {
|
|
6135
|
+
const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
|
|
6136
|
+
return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
6137
|
+
}
|
|
6138
|
+
async function saveContainerState(input) {
|
|
6139
|
+
const { host } = input;
|
|
6140
|
+
const id = randomBytes(6).toString("hex");
|
|
6141
|
+
const volume = `outerlayer-saved-${id}`;
|
|
6142
|
+
const record = {
|
|
6143
|
+
id,
|
|
6144
|
+
kind: "volume",
|
|
6145
|
+
volume,
|
|
6146
|
+
savedAt: input.now.toISOString(),
|
|
6147
|
+
bytes: 0,
|
|
6148
|
+
baseKey: input.key.baseKey,
|
|
6149
|
+
key: input.key.key,
|
|
6150
|
+
fileHashes: input.key.fileHashes,
|
|
6151
|
+
...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
|
|
6152
|
+
complete: false
|
|
6153
|
+
};
|
|
6154
|
+
const discard = async (reason) => {
|
|
6155
|
+
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
6156
|
+
if (removed.status !== 0) {
|
|
6157
|
+
input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
|
|
6158
|
+
return { ok: false, reason };
|
|
6159
|
+
}
|
|
6160
|
+
deleteRecord(input.dir, id);
|
|
6161
|
+
return { ok: false, reason };
|
|
6162
|
+
};
|
|
6163
|
+
try {
|
|
6164
|
+
const noEnv = { literals: {}, passed: {} };
|
|
6165
|
+
const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
|
|
6166
|
+
if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText3(stopped)})` };
|
|
6167
|
+
input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
|
|
6168
|
+
const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
|
|
6169
|
+
if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText3(flushed)})` };
|
|
6170
|
+
writeRecord2(input.dir, record);
|
|
6171
|
+
const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
|
|
6172
|
+
if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText3(created)})`);
|
|
6173
|
+
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
6174
|
+
if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText3(copied)})`);
|
|
6175
|
+
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
6176
|
+
const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
|
|
6177
|
+
writeRecord2(input.dir, done);
|
|
6178
|
+
return { ok: true, record: done };
|
|
6179
|
+
} catch (err) {
|
|
6180
|
+
return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
|
|
6181
|
+
}
|
|
6182
|
+
}
|
|
6183
|
+
async function restoreContainerState(input) {
|
|
6184
|
+
try {
|
|
6185
|
+
const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
6186
|
+
if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText3(copied)})` };
|
|
6187
|
+
return { ok: true };
|
|
6188
|
+
} catch (err) {
|
|
6189
|
+
return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
|
|
6190
|
+
}
|
|
6191
|
+
}
|
|
6192
|
+
|
|
6193
|
+
// src/runner/workspace/keep.ts
|
|
6194
|
+
function recordOf(input, kind, bytes, volume) {
|
|
6195
|
+
return {
|
|
6196
|
+
workItemId: input.workItemId,
|
|
6197
|
+
claimId: input.claimId,
|
|
6198
|
+
queue: input.queue,
|
|
6199
|
+
imageId: input.imageId,
|
|
6200
|
+
diskBytes: input.diskBytes,
|
|
6201
|
+
kind,
|
|
6202
|
+
...volume === void 0 ? {} : { volume },
|
|
6203
|
+
bytes,
|
|
6204
|
+
keptAt: input.now.toISOString()
|
|
6205
|
+
};
|
|
6206
|
+
}
|
|
6207
|
+
function finish(input, dir, staging, record) {
|
|
6208
|
+
writeFileSync(join(staging, RECORD_FILE), `${JSON.stringify(record, null, 2)}
|
|
6209
|
+
`, { mode: RUNNER_FILE_MODE });
|
|
6210
|
+
const older = swapInItemDisk(dir, staging);
|
|
6211
|
+
if (record.kind === "disk" && older?.record.kind === "volume") {
|
|
6212
|
+
input.log(`the work item's older disk was the Docker volume ${older.record.volume}, which this copy replaces; remove the volume with \`docker volume rm\``);
|
|
6213
|
+
}
|
|
6214
|
+
}
|
|
6215
|
+
async function keepMicroVmDisk(input) {
|
|
6216
|
+
let dir;
|
|
6217
|
+
try {
|
|
6218
|
+
dir = itemDiskDir(input.configPath, input.organizationId, input.workItemId);
|
|
6219
|
+
} catch (err) {
|
|
6220
|
+
return { ok: false, reason: describeCallFailure(err) };
|
|
6221
|
+
}
|
|
6222
|
+
let staging;
|
|
6223
|
+
const helperCopy = join(input.buildDir, HELPER_COPY_NAME);
|
|
6224
|
+
try {
|
|
6225
|
+
staging = stagingItemDiskDir(dir);
|
|
6226
|
+
const target2 = join(staging, DISK_FILE);
|
|
6227
|
+
if (input.helper !== void 0) {
|
|
6228
|
+
rmSync(helperCopy, { force: true });
|
|
6229
|
+
await input.helper.copy(input.buildId);
|
|
6230
|
+
renameSync(helperCopy, target2);
|
|
6231
|
+
} else {
|
|
6232
|
+
await (input.copyDisk ?? copyDiskFile)(input.buildDisk, target2);
|
|
6233
|
+
}
|
|
6234
|
+
chmodSync(target2, RUNNER_FILE_MODE);
|
|
6235
|
+
const record = recordOf(input, "disk", allocatedBytes(target2));
|
|
6236
|
+
finish(input, dir, staging, record);
|
|
6237
|
+
staging = void 0;
|
|
6238
|
+
return { ok: true, record };
|
|
6239
|
+
} catch (err) {
|
|
6240
|
+
return { ok: false, reason: `the build disk could not be copied: ${describeCallFailure(err)}` };
|
|
6241
|
+
} finally {
|
|
6242
|
+
if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
|
|
6243
|
+
rmSync(helperCopy, { force: true });
|
|
6244
|
+
}
|
|
6245
|
+
}
|
|
6246
|
+
async function keepContainerDisk(input) {
|
|
6247
|
+
let dir;
|
|
6248
|
+
try {
|
|
6249
|
+
dir = itemDiskDir(input.configPath, input.organizationId, input.workItemId);
|
|
6250
|
+
} catch (err) {
|
|
6251
|
+
return { ok: false, reason: describeCallFailure(err) };
|
|
6252
|
+
}
|
|
6253
|
+
const { host } = input;
|
|
6254
|
+
const volume = `outerlayer-workspace-${randomBytes(6).toString("hex")}`;
|
|
6255
|
+
let created = false;
|
|
6256
|
+
let staging;
|
|
6257
|
+
const dropNew = async () => {
|
|
6258
|
+
if (!created) return;
|
|
6259
|
+
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
6260
|
+
if (removed.status !== 0) input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })})`);
|
|
6261
|
+
};
|
|
6262
|
+
try {
|
|
6263
|
+
const made = await host.createVolume(volume, { [WORKSPACE_LABEL]: input.workItemId, [RUNNER_LABEL]: input.runnerId });
|
|
6264
|
+
if (made.status !== 0) return { ok: false, reason: `the volume for the kept disk could not be made (${failureText3(made)})` };
|
|
6265
|
+
created = true;
|
|
6266
|
+
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
6267
|
+
if (copied.status !== 0) {
|
|
6268
|
+
await dropNew();
|
|
6269
|
+
return { ok: false, reason: `the copy into the kept disk failed (${failureText3(copied)})` };
|
|
6270
|
+
}
|
|
6271
|
+
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
6272
|
+
const record = recordOf(input, "volume", Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, volume);
|
|
6273
|
+
const older = readItemDisk(dir);
|
|
6274
|
+
if (older?.record.volume !== void 0) {
|
|
6275
|
+
const removed = await host.removeVolume(older.record.volume);
|
|
6276
|
+
if (removed.status !== 0 && !/no such volume/i.test(removed.stderr)) {
|
|
6277
|
+
await dropNew();
|
|
6278
|
+
return { ok: false, reason: `the work item's older disk could not be replaced: its volume ${older.record.volume} was not removed (${failureText3(removed)})` };
|
|
6279
|
+
}
|
|
6280
|
+
}
|
|
6281
|
+
staging = stagingItemDiskDir(dir);
|
|
6282
|
+
finish(input, dir, staging, record);
|
|
6283
|
+
staging = void 0;
|
|
6284
|
+
return { ok: true, record };
|
|
6285
|
+
} catch (err) {
|
|
6286
|
+
await dropNew();
|
|
6287
|
+
return { ok: false, reason: `the build's volume could not be copied: ${describeCallFailure(err)}` };
|
|
6288
|
+
} finally {
|
|
6289
|
+
if (staging !== void 0) rmSync(staging, { recursive: true, force: true });
|
|
6290
|
+
}
|
|
6291
|
+
}
|
|
6292
|
+
|
|
6293
|
+
// src/runner/workspace/finish.ts
|
|
6294
|
+
function noDiskReason(job, config) {
|
|
6295
|
+
if (job.workItemId === void 0) return "the job names no work item";
|
|
6296
|
+
if (config.managed === true && (job.managedVariables?.length ?? 0) > 0) return "the build held a factory build variable, and the runner cannot remove a value that a command stored in an encoded form";
|
|
6297
|
+
return void 0;
|
|
6298
|
+
}
|
|
6299
|
+
function target(input, imageId, diskBytes) {
|
|
6300
|
+
return {
|
|
6301
|
+
configPath: input.config.configPath,
|
|
6302
|
+
...input.job.organizationId === void 0 ? {} : { organizationId: input.job.organizationId },
|
|
6303
|
+
workItemId: input.job.workItemId,
|
|
6304
|
+
claimId: input.job.claimId,
|
|
6305
|
+
queue: input.job.queue,
|
|
6306
|
+
imageId,
|
|
6307
|
+
diskBytes,
|
|
6308
|
+
now: input.now(),
|
|
6309
|
+
log: (line2) => input.log.line(`${input.tag} ${line2}`)
|
|
6310
|
+
};
|
|
6311
|
+
}
|
|
6312
|
+
function microVmDiskKeeper(input) {
|
|
6313
|
+
const state = input.job.microvm;
|
|
6314
|
+
if (state === void 0) return void 0;
|
|
6315
|
+
const refused2 = noDiskReason(input.job, input.config);
|
|
6316
|
+
if (refused2 !== void 0) {
|
|
6317
|
+
if (input.job.workItemId !== void 0) input.log.line(`${input.tag} the build disk is not kept: ${refused2}`);
|
|
6318
|
+
return void 0;
|
|
6319
|
+
}
|
|
6320
|
+
if (state.imageId === void 0 || state.limits === void 0 || state.buildDisk === void 0) return void 0;
|
|
6321
|
+
const { imageId, buildDisk } = state;
|
|
6322
|
+
const diskBytes = state.limits.diskBytes;
|
|
6323
|
+
return async () => {
|
|
6324
|
+
let helper;
|
|
6325
|
+
if (state.jailed !== void 0) {
|
|
6326
|
+
const probe = await input.runtime.vm.helper.probe();
|
|
6327
|
+
if (!probe.ok) throw new Error(`the root helper does not answer (${probe.detail})`);
|
|
6328
|
+
helper = probe.client;
|
|
6329
|
+
}
|
|
6330
|
+
const copyDisk = input.runtime.vm.rootfs.copyDisk?.bind(input.runtime.vm.rootfs);
|
|
6331
|
+
const kept = await keepMicroVmDisk({ ...target(input, imageId, diskBytes), buildId: state.buildId, buildDisk, buildDir: state.buildDir, ...helper === void 0 ? {} : { helper }, ...copyDisk === void 0 ? {} : { copyDisk } });
|
|
6332
|
+
if (!kept.ok) throw new Error(kept.reason);
|
|
6333
|
+
input.log.line(`${input.tag} kept the build disk for the work item: ${formatBytes(kept.record.bytes)} allocated`);
|
|
6334
|
+
};
|
|
6335
|
+
}
|
|
6336
|
+
function containerDiskKeeper(input) {
|
|
6337
|
+
const state = input.job.container;
|
|
6338
|
+
if (state === void 0) return void 0;
|
|
6339
|
+
const refused2 = noDiskReason(input.job, input.config);
|
|
6340
|
+
if (refused2 !== void 0) {
|
|
6341
|
+
if (input.job.workItemId !== void 0) input.log.line(`${input.tag} the build disk is not kept: ${refused2}`);
|
|
6342
|
+
return void 0;
|
|
6343
|
+
}
|
|
6344
|
+
if (state.imageId === void 0) return void 0;
|
|
6345
|
+
const { imageId } = state;
|
|
6346
|
+
const diskBytes = parseSize(input.config.build?.disk ?? DEFAULT_BUILD_DISK) ?? 0;
|
|
6347
|
+
return async () => {
|
|
6348
|
+
const kept = await keepContainerDisk({ ...target(input, imageId, diskBytes), host: input.runtime.host, image: imageId, buildVolume: state.volume, runnerId: runnerLabelValue(input.config.configPath) });
|
|
6349
|
+
if (!kept.ok) throw new Error(kept.reason);
|
|
6350
|
+
input.log.line(`${input.tag} kept the build disk for the work item: ${formatBytes(kept.record.bytes)} allocated`);
|
|
6351
|
+
};
|
|
6352
|
+
}
|
|
5894
6353
|
var SESSION_TAR = "session.tar";
|
|
5895
6354
|
async function guestOf(job, runtime) {
|
|
5896
6355
|
if (job.microvm !== void 0) {
|
|
@@ -6242,7 +6701,7 @@ async function runJudge(input) {
|
|
|
6242
6701
|
log.line(`${tag} judge: ${plan.toJudge.length} of ${list.length} criteria to judge, ${list.length - plan.toJudge.length} settled without the judge`);
|
|
6243
6702
|
}
|
|
6244
6703
|
if (criteria !== void 0 && plan.toJudge.length === 0) {
|
|
6245
|
-
return await
|
|
6704
|
+
return await finish2(check, { criteria, items: plan.settled, citations, head, branch, finder, judgeReport: void 0 });
|
|
6246
6705
|
}
|
|
6247
6706
|
let diff = "";
|
|
6248
6707
|
const startCommit = (job.container ?? job.microvm)?.startCommit;
|
|
@@ -6317,19 +6776,19 @@ async function runJudge(input) {
|
|
|
6317
6776
|
];
|
|
6318
6777
|
if (answer?.ok === true) {
|
|
6319
6778
|
const results2 = new Map(answer.answer.items.map((entry) => [entry.id, entry]));
|
|
6320
|
-
return await
|
|
6779
|
+
return await finish2(check, { criteria, items: judged(results2), citations, head, branch, finder, judgeReport: void 0 });
|
|
6321
6780
|
}
|
|
6322
6781
|
if (reply.kind === "none") {
|
|
6323
|
-
return await
|
|
6782
|
+
return await finish2(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: reply.reason, limitKey: reply.limitKey === true });
|
|
6324
6783
|
}
|
|
6325
6784
|
if (answer !== void 0) {
|
|
6326
|
-
return await
|
|
6785
|
+
return await finish2(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: answer.reason });
|
|
6327
6786
|
}
|
|
6328
6787
|
const failsAll = reply.verdict === "fail" && reply.unmet.size === 0;
|
|
6329
6788
|
const results = new Map(plan.toJudge.map((criterion) => [criterion.id, { result: failsAll || reply.unmet.has(criterion.id) ? "fail" : "pass" }]));
|
|
6330
|
-
return await
|
|
6789
|
+
return await finish2(check, { criteria, items: judged(results), citations, head, branch, finder, judgeReport: report });
|
|
6331
6790
|
}
|
|
6332
|
-
async function
|
|
6791
|
+
async function finish2(check, at) {
|
|
6333
6792
|
const items = inCriteriaOrder(at.items, at.criteria);
|
|
6334
6793
|
const failing = itemsSentence(items, { one: "criterion", many: "criteria" });
|
|
6335
6794
|
const result = failing === void 0 ? "pass" : "fail";
|
|
@@ -6772,12 +7231,12 @@ async function renewLease(config, item, fetchImpl) {
|
|
|
6772
7231
|
}
|
|
6773
7232
|
var ALREADY_RELEASED = "already released";
|
|
6774
7233
|
async function releaseLease(config, item, outcome, detail, fetchImpl) {
|
|
6775
|
-
const send = async (environment2, sent2, log2, resume2) => postJson(
|
|
7234
|
+
const send = async (environment2, sent2, log2, resume2, disk2) => postJson(
|
|
6776
7235
|
config.url,
|
|
6777
7236
|
`/v1/work-items/${item}/claim/release`,
|
|
6778
7237
|
config.apiKey,
|
|
6779
7238
|
config.appId,
|
|
6780
|
-
{ host: config.host, outcome: sent2, stage: detail.stage, exitCode: detail.exitCode, reason: detail.reason, environment: environment2, resume: resume2, log: log2 },
|
|
7239
|
+
{ host: config.host, outcome: sent2, stage: detail.stage, exitCode: detail.exitCode, reason: detail.reason, environment: environment2, resume: resume2, disk: disk2, log: log2 },
|
|
6781
7240
|
fetchImpl,
|
|
6782
7241
|
void 0,
|
|
6783
7242
|
await signerFor(config.configPath)
|
|
@@ -6786,10 +7245,11 @@ async function releaseLease(config, item, outcome, detail, fetchImpl) {
|
|
|
6786
7245
|
let sent = outcome;
|
|
6787
7246
|
let log = detail.log;
|
|
6788
7247
|
let resume = detail.resume;
|
|
7248
|
+
let disk = detail.disk;
|
|
6789
7249
|
let data;
|
|
6790
7250
|
for (; ; ) {
|
|
6791
7251
|
try {
|
|
6792
|
-
({ data } = await send(environment, sent, log, resume));
|
|
7252
|
+
({ data } = await send(environment, sent, log, resume, disk));
|
|
6793
7253
|
break;
|
|
6794
7254
|
} catch (err) {
|
|
6795
7255
|
if (!(err instanceof WorkCommandError) || err.status !== 400 || err.code !== "invalid_request_body") throw err;
|
|
@@ -6798,6 +7258,8 @@ async function releaseLease(config, item, outcome, detail, fetchImpl) {
|
|
|
6798
7258
|
sent = "ok";
|
|
6799
7259
|
} else if (resume !== void 0 && /\bresume\b/.test(err.message)) {
|
|
6800
7260
|
resume = void 0;
|
|
7261
|
+
} else if (disk !== void 0 && /\bdisk\b/.test(err.message)) {
|
|
7262
|
+
disk = void 0;
|
|
6801
7263
|
} else if (log !== void 0 && /\blog\b/.test(err.message)) {
|
|
6802
7264
|
log = void 0;
|
|
6803
7265
|
} else if (environment?.routes !== void 0) {
|
|
@@ -7166,7 +7628,8 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
|
|
|
7166
7628
|
job.buildDiskPeakBytes = bytes;
|
|
7167
7629
|
writeJobFile(join(jobDirPath, "job.json"), job);
|
|
7168
7630
|
log.line(`${tag} build disk peak ${formatBytes(bytes)}`);
|
|
7169
|
-
}
|
|
7631
|
+
},
|
|
7632
|
+
microVmDiskKeeper({ job, config, runtime: vmBuild.runtime, now, log, tag, jobDirPath })
|
|
7170
7633
|
);
|
|
7171
7634
|
} catch (err) {
|
|
7172
7635
|
cleanedUp = false;
|
|
@@ -7179,7 +7642,7 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
|
|
|
7179
7642
|
if (build !== void 0) {
|
|
7180
7643
|
tunnelRecord = build.runtime.tunnels.get(build.state.buildId)?.record();
|
|
7181
7644
|
try {
|
|
7182
|
-
await cleanupContainer(build.runtime, build.state, jobDirPath);
|
|
7645
|
+
await cleanupContainer(build.runtime, build.state, jobDirPath, containerDiskKeeper({ job, config, runtime: build.runtime, now, log, tag, jobDirPath }), (text2) => log.error(`${tag} ${text2}`));
|
|
7183
7646
|
} catch (err) {
|
|
7184
7647
|
cleanedUp = false;
|
|
7185
7648
|
log.error(`${tag} cleanup failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
@@ -7323,7 +7786,7 @@ async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
|
|
|
7323
7786
|
const sentAtMs = now().getTime();
|
|
7324
7787
|
const runnerLog = valuesNotHeld ? `The runner sends no log for this attempt: ${VALUES_NOT_HELD}.
|
|
7325
7788
|
` : releaseLogFor(jobDirPath, job, config);
|
|
7326
|
-
const detail = outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, log: runnerLog };
|
|
7789
|
+
const detail = outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, disk: job.disk, log: runnerLog };
|
|
7327
7790
|
const release = await retrySend(() => releaseLease(claimCalls(config, job), claimItem(config, job), outcome, detail, deps.fetchImpl), {
|
|
7328
7791
|
now: deps.now,
|
|
7329
7792
|
sleep: deps.sleep
|
|
@@ -7394,7 +7857,7 @@ async function retryFailedReleases(jobsDirPath, config, deps) {
|
|
|
7394
7857
|
continue;
|
|
7395
7858
|
}
|
|
7396
7859
|
const runnerLog = readScrubbedReleaseLog(dirPath) ?? releaseLogFor(dirPath, job, config);
|
|
7397
|
-
const detail = job.outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, log: runnerLog };
|
|
7860
|
+
const detail = job.outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, disk: job.disk, log: runnerLog };
|
|
7398
7861
|
const release = await retrySend(() => releaseLease(claimCalls(config, job), claimItem(config, job), job.outcome, detail, deps.fetchImpl), {
|
|
7399
7862
|
now: deps.now,
|
|
7400
7863
|
sleep: deps.sleep
|
|
@@ -7644,15 +8107,108 @@ async function housekeepingDockerIfReachable(deps) {
|
|
|
7644
8107
|
}
|
|
7645
8108
|
return createHousekeepingDocker(deps);
|
|
7646
8109
|
}
|
|
8110
|
+
var DAY_MS = 864e5;
|
|
8111
|
+
var STAGING_GRACE_MS = 36e5;
|
|
8112
|
+
var keptAtOf = (entry) => Date.parse(entry.kept.record.keptAt);
|
|
8113
|
+
var oldestFirst2 = (entries) => [...entries].sort((a, b) => keptAtOf(a) - keptAtOf(b));
|
|
8114
|
+
var bytesOf = (entries) => entries.reduce((sum, entry) => sum + entry.kept.record.bytes, 0);
|
|
8115
|
+
function allEntries(configPath) {
|
|
8116
|
+
return storeScopes(configPath).flatMap((scope) => listItemDisks(configPath, scope).map((kept) => ({ scope, kept })));
|
|
8117
|
+
}
|
|
8118
|
+
async function removeEntry(entry, removeVolume) {
|
|
8119
|
+
const { record, dir } = entry.kept;
|
|
8120
|
+
if (record.kind === "volume" && record.volume !== void 0) {
|
|
8121
|
+
if (removeVolume === void 0) throw new Error("Docker is not available");
|
|
8122
|
+
await removeVolume(record.volume);
|
|
8123
|
+
}
|
|
8124
|
+
rmSync(dir, { recursive: true, force: true });
|
|
8125
|
+
}
|
|
8126
|
+
function treeBytes(path) {
|
|
8127
|
+
try {
|
|
8128
|
+
const stats = lstatSync(path);
|
|
8129
|
+
if (!stats.isDirectory()) return stats.blocks * 512;
|
|
8130
|
+
return readdirSync(path).reduce((sum, name) => sum + treeBytes(join(path, name)), 0);
|
|
8131
|
+
} catch {
|
|
8132
|
+
return 0;
|
|
8133
|
+
}
|
|
8134
|
+
}
|
|
8135
|
+
async function removeItemDisks(input) {
|
|
8136
|
+
const { configPath, limits: limits2, nowMs } = input;
|
|
8137
|
+
let freed = 0;
|
|
8138
|
+
for (const scope of storeScopes(configPath)) {
|
|
8139
|
+
for (const leftover of listLeftovers(configPath, scope)) {
|
|
8140
|
+
const staging = leftover.path.split("/").at(-1).startsWith(".");
|
|
8141
|
+
const limit = staging ? STAGING_GRACE_MS : limits2.keepDays * DAY_MS;
|
|
8142
|
+
if (nowMs - leftover.mtimeMs < limit) continue;
|
|
8143
|
+
const size = treeBytes(leftover.path);
|
|
8144
|
+
try {
|
|
8145
|
+
rmSync(leftover.path, { recursive: true, force: true });
|
|
8146
|
+
freed += size;
|
|
8147
|
+
input.log(`housekeeping: removed ${leftover.path} (${formatBytes(size)}), a kept disk that was never finished or has no record`);
|
|
8148
|
+
} catch (err) {
|
|
8149
|
+
input.error(`housekeeping: could not remove ${leftover.path}: ${describeCallFailure(err)}`);
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
8152
|
+
}
|
|
8153
|
+
const gone = /* @__PURE__ */ new Set();
|
|
8154
|
+
const remove = async (entry, why) => {
|
|
8155
|
+
try {
|
|
8156
|
+
await removeEntry(entry, input.removeVolume);
|
|
8157
|
+
gone.add(entry);
|
|
8158
|
+
freed += entry.kept.record.bytes;
|
|
8159
|
+
input.log(`housekeeping: removed the kept disk of work item ${entry.kept.record.workItemId} (${formatBytes(entry.kept.record.bytes)}), ${why}`);
|
|
8160
|
+
} catch (err) {
|
|
8161
|
+
input.error(`housekeeping: could not remove the kept disk of work item ${entry.kept.record.workItemId}: ${describeCallFailure(err)}`);
|
|
8162
|
+
}
|
|
8163
|
+
};
|
|
8164
|
+
const entries = allEntries(configPath);
|
|
8165
|
+
for (const entry of entries) {
|
|
8166
|
+
const sinceMs = nowMs - keptAtOf(entry);
|
|
8167
|
+
if (sinceMs >= limits2.keepDays * DAY_MS) await remove(entry, `no job replaced it for ${formatDuration(sinceMs)}, past runner.housekeeping.workspaceKeepDays`);
|
|
8168
|
+
}
|
|
8169
|
+
for (const scope of new Set(entries.map((entry) => entry.scope))) {
|
|
8170
|
+
if (scope === void 0) continue;
|
|
8171
|
+
const mine = oldestFirst2(entries.filter((entry) => entry.scope === scope && !gone.has(entry)));
|
|
8172
|
+
let total2 = bytesOf(mine);
|
|
8173
|
+
for (const entry of mine.slice(0, -1)) {
|
|
8174
|
+
if (total2 <= limits2.orgMax) break;
|
|
8175
|
+
const before = gone.size;
|
|
8176
|
+
await remove(entry, `organization ${scope}'s kept disks pass runner.housekeeping.workspaceOrgMax`);
|
|
8177
|
+
if (gone.size > before) total2 -= entry.kept.record.bytes;
|
|
8178
|
+
}
|
|
8179
|
+
}
|
|
8180
|
+
let total = bytesOf(entries.filter((entry) => !gone.has(entry)));
|
|
8181
|
+
for (const entry of oldestFirst2(entries.filter((candidate) => !gone.has(candidate)))) {
|
|
8182
|
+
if (total <= limits2.max) break;
|
|
8183
|
+
const before = gone.size;
|
|
8184
|
+
await remove(entry, "the kept disks pass runner.housekeeping.workspaceMax");
|
|
8185
|
+
if (gone.size > before) total -= entry.kept.record.bytes;
|
|
8186
|
+
}
|
|
8187
|
+
return freed;
|
|
8188
|
+
}
|
|
8189
|
+
async function freeItemDisksUntil(input) {
|
|
8190
|
+
let freed = 0;
|
|
8191
|
+
for (const entry of oldestFirst2(allEntries(input.configPath))) {
|
|
8192
|
+
if (input.enough()) break;
|
|
8193
|
+
try {
|
|
8194
|
+
await removeEntry(entry, input.removeVolume);
|
|
8195
|
+
freed += entry.kept.record.bytes;
|
|
8196
|
+
input.log(`removed the kept disk of work item ${entry.kept.record.workItemId} from ${entry.kept.record.keptAt}`);
|
|
8197
|
+
} catch (err) {
|
|
8198
|
+
input.log(`could not remove the kept disk of work item ${entry.kept.record.workItemId}: ${describeCallFailure(err)}`);
|
|
8199
|
+
}
|
|
8200
|
+
}
|
|
8201
|
+
return freed;
|
|
8202
|
+
}
|
|
7647
8203
|
|
|
7648
8204
|
// src/runner/housekeeping/select.ts
|
|
7649
|
-
var
|
|
8205
|
+
var DAY_MS2 = 864e5;
|
|
7650
8206
|
function spoolMaxAgeMs() {
|
|
7651
8207
|
return 24 * 36e5;
|
|
7652
8208
|
}
|
|
7653
8209
|
function selectJobDirs(entries, nowMs, keepDays) {
|
|
7654
8210
|
return entries.filter(
|
|
7655
|
-
(entry) => (entry.state === "finished" || entry.state === "no_job_file") && nowMs - entry.mtimeMs > keepDays *
|
|
8211
|
+
(entry) => (entry.state === "finished" || entry.state === "no_job_file") && nowMs - entry.mtimeMs > keepDays * DAY_MS2
|
|
7656
8212
|
);
|
|
7657
8213
|
}
|
|
7658
8214
|
function liveBuildIds(jobs) {
|
|
@@ -7888,6 +8444,14 @@ async function runHousekeeping(options) {
|
|
|
7888
8444
|
freed += await removeSavedStates(options, nowMs, savedStatesDir(root));
|
|
7889
8445
|
freed += removeStaleBuilderCaches(options, nowMs);
|
|
7890
8446
|
freed += removeStaleSessions(options, nowMs);
|
|
8447
|
+
freed += await removeItemDisks({
|
|
8448
|
+
configPath: options.configPath,
|
|
8449
|
+
limits: { max: options.workspaceMax ?? Number.POSITIVE_INFINITY, orgMax: options.workspaceOrgMax ?? Number.POSITIVE_INFINITY, keepDays: options.workspaceKeepDays ?? DEFAULT_WORKSPACE_KEEP_DAYS },
|
|
8450
|
+
nowMs,
|
|
8451
|
+
removeVolume: options.docker === void 0 ? void 0 : (name) => options.docker.removeVolume(name),
|
|
8452
|
+
log: (line2) => options.log.line(line2),
|
|
8453
|
+
error: (line2) => options.log.error(line2)
|
|
8454
|
+
});
|
|
7891
8455
|
for (const scope of storeScopes(options.configPath)) {
|
|
7892
8456
|
const spoolPath = spoolDir(orgStoreIn(root, scope));
|
|
7893
8457
|
const spoolFiles = listDir(spoolPath).flatMap((name) => {
|
|
@@ -8100,7 +8664,7 @@ var BRIDGE_ADDRESS = "172.17.0.1";
|
|
|
8100
8664
|
var STEP_PROXY = `http://${BRIDGE_ADDRESS}:${TUNNEL_PORT}`;
|
|
8101
8665
|
var NO_PROXY = "localhost,127.0.0.1,::1";
|
|
8102
8666
|
var TUNNEL_SOCKET = "tunnel.sock";
|
|
8103
|
-
var
|
|
8667
|
+
var RECORD_FILE2 = "image-builder.json";
|
|
8104
8668
|
var BOOT_ATTEMPTS2 = 120;
|
|
8105
8669
|
var BOOT_POLL_MS2 = 500;
|
|
8106
8670
|
var ROOT_PATH2 = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
@@ -8116,8 +8680,8 @@ var GUEST_OUTPUT = "/out/rootfs.ext4";
|
|
|
8116
8680
|
var LEFTOVERS_SCRIPT = "umount /work /out 2>/dev/null; rm -rf /cache/work /cache/rootfs /cache/out; status=$?; sync; exit $status";
|
|
8117
8681
|
var DOCKER_WAIT_SECONDS2 = 90;
|
|
8118
8682
|
var BUILD_CACHE_KEEP = "2GB";
|
|
8119
|
-
function unpackCommand(
|
|
8120
|
-
return ["sh", "-c", 'tar -x --no-same-owner -C "$1"; s=$?; cat >/dev/null; exit $s', "unpack",
|
|
8683
|
+
function unpackCommand(target2) {
|
|
8684
|
+
return ["sh", "-c", 'tar -x --no-same-owner -C "$1"; s=$?; cat >/dev/null; exit $s', "unpack", target2];
|
|
8121
8685
|
}
|
|
8122
8686
|
function engineLogExcerpt(log) {
|
|
8123
8687
|
return `grep -m1 'panic:' ${log} | sed 's/^/${ENGINE_PANIC_PREFIX}/' || true; tail -n 20 ${log}`;
|
|
@@ -8272,7 +8836,7 @@ function readStored(prepared) {
|
|
|
8272
8836
|
}
|
|
8273
8837
|
}
|
|
8274
8838
|
function writeBuilderRecord(dir, record) {
|
|
8275
|
-
writeFileSync(join(dir,
|
|
8839
|
+
writeFileSync(join(dir, RECORD_FILE2), `${JSON.stringify(record)}
|
|
8276
8840
|
`, { mode: 384 });
|
|
8277
8841
|
}
|
|
8278
8842
|
function refusalReason(refusals, logPath) {
|
|
@@ -8476,16 +9040,16 @@ async function buildInBuilder(opts, deps, prepared) {
|
|
|
8476
9040
|
})
|
|
8477
9041
|
);
|
|
8478
9042
|
inputs = builderInput({ checkoutDir: checkout.dir, toolsDir, toolFilesDir, recipeDir }, archive);
|
|
8479
|
-
for (const { target, stream } of inputs) {
|
|
9043
|
+
for (const { target: target2, stream } of inputs) {
|
|
8480
9044
|
let status;
|
|
8481
9045
|
try {
|
|
8482
|
-
status = (await ports.exec(state.vsockPath, exec(unpackCommand(
|
|
9046
|
+
status = (await ports.exec(state.vsockPath, exec(unpackCommand(target2)), { stdin: stream, signal: controller.signal })).status;
|
|
8483
9047
|
} catch (err) {
|
|
8484
9048
|
if (hostFailure !== void 0 || controller.signal.aborted) throw err;
|
|
8485
|
-
throw new StepFailed(`unpacking ${
|
|
9049
|
+
throw new StepFailed(`unpacking ${target2} in the builder failed: ${err instanceof Error ? err.message : String(err)}${await spaceNote()}`);
|
|
8486
9050
|
}
|
|
8487
|
-
if (status !== 0 && hostFailure === void 0 && !controller.signal.aborted) throw new StepFailed(`unpacking ${
|
|
8488
|
-
check(status, `unpacking ${
|
|
9051
|
+
if (status !== 0 && hostFailure === void 0 && !controller.signal.aborted) throw new StepFailed(`unpacking ${target2} in the builder failed (exit ${status})${await spaceNote()}`);
|
|
9052
|
+
check(status, `unpacking ${target2} in the builder`);
|
|
8489
9053
|
}
|
|
8490
9054
|
const script = builderGuestScript({ recipe: recipeParts, recipeTag, buildTag });
|
|
8491
9055
|
const ran = await ports.execStream(state.vsockPath, exec(["/bin/sh", "-c", script]), { signal: controller.signal, onStdout: onOutput, onStderr: onOutput });
|
|
@@ -8640,7 +9204,7 @@ async function sweepImageBuilders(configPath, ports, onError) {
|
|
|
8640
9204
|
const dir = buildDir(configPath, id);
|
|
8641
9205
|
let record;
|
|
8642
9206
|
try {
|
|
8643
|
-
record = JSON.parse(readFileSync(join(dir,
|
|
9207
|
+
record = JSON.parse(readFileSync(join(dir, RECORD_FILE2), "utf8"));
|
|
8644
9208
|
} catch {
|
|
8645
9209
|
continue;
|
|
8646
9210
|
}
|
|
@@ -8680,127 +9244,6 @@ async function sweepImageBuilders(configPath, ports, onError) {
|
|
|
8680
9244
|
}
|
|
8681
9245
|
return swept;
|
|
8682
9246
|
}
|
|
8683
|
-
var SAVED_FOLDERS = ["home", "work"];
|
|
8684
|
-
function stopProcessesScript() {
|
|
8685
|
-
return [
|
|
8686
|
-
"me=$$; uid=$(id -u); n=0; pass=0",
|
|
8687
|
-
"while [ $pass -lt 25 ]; do",
|
|
8688
|
-
' left=""',
|
|
8689
|
-
" for p in /proc/[0-9]*; do",
|
|
8690
|
-
" pid=${p##*/}",
|
|
8691
|
-
' [ "$pid" = "$me" ] && continue',
|
|
8692
|
-
` info=$(sed -n -e 's/^State:[[:space:]]*\\(.\\).*/S=\\1/p' -e 's/^Uid:[[:space:]]*\\([0-9]*\\).*/U=\\1/p' "$p/status" 2>/dev/null) || continue`,
|
|
8693
|
-
' case "$info" in *"S=Z"*) continue ;; esac',
|
|
8694
|
-
// The uid is the last line of `info`, so a match at its end cannot be the longer id 10000 for 1000.
|
|
8695
|
-
' case "$info" in *"U=$uid") left="$left $pid" ;; esac',
|
|
8696
|
-
" done",
|
|
8697
|
-
' [ -z "$left" ] && break',
|
|
8698
|
-
' for pid in $left; do kill -KILL "$pid" 2>/dev/null && n=$((n+1)); done',
|
|
8699
|
-
" pass=$((pass+1))",
|
|
8700
|
-
" sleep 0.2",
|
|
8701
|
-
"done",
|
|
8702
|
-
'[ -z "$left" ] || { echo "$left are still running" >&2; exit 1; }',
|
|
8703
|
-
'echo "stopped $n"'
|
|
8704
|
-
].join("\n");
|
|
8705
|
-
}
|
|
8706
|
-
var HELPER_FLAGS = [
|
|
8707
|
-
"--rm",
|
|
8708
|
-
"--network",
|
|
8709
|
-
"none",
|
|
8710
|
-
"--cap-drop",
|
|
8711
|
-
"ALL",
|
|
8712
|
-
"--cap-add",
|
|
8713
|
-
"CHOWN",
|
|
8714
|
-
"--cap-add",
|
|
8715
|
-
"DAC_OVERRIDE",
|
|
8716
|
-
"--cap-add",
|
|
8717
|
-
"DAC_READ_SEARCH",
|
|
8718
|
-
"--cap-add",
|
|
8719
|
-
"FOWNER",
|
|
8720
|
-
"--security-opt",
|
|
8721
|
-
"no-new-privileges",
|
|
8722
|
-
"--read-only",
|
|
8723
|
-
"--user",
|
|
8724
|
-
"0",
|
|
8725
|
-
"--pids-limit",
|
|
8726
|
-
"256"
|
|
8727
|
-
];
|
|
8728
|
-
function helperArgs(input) {
|
|
8729
|
-
return [
|
|
8730
|
-
...HELPER_FLAGS,
|
|
8731
|
-
"--label",
|
|
8732
|
-
`${RUNNER_LABEL}=${input.runnerId}`,
|
|
8733
|
-
"--mount",
|
|
8734
|
-
`type=volume,src=${input.from},dst=/from,readonly`,
|
|
8735
|
-
"--mount",
|
|
8736
|
-
`type=volume,src=${input.to},dst=/to`,
|
|
8737
|
-
"--entrypoint",
|
|
8738
|
-
"sh",
|
|
8739
|
-
input.image,
|
|
8740
|
-
"-c",
|
|
8741
|
-
input.script
|
|
8742
|
-
];
|
|
8743
|
-
}
|
|
8744
|
-
var SAVE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/; du -sk /to | cut -f1`;
|
|
8745
|
-
var RESTORE_SCRIPT = `set -e; cp -a ${SAVED_FOLDERS.map((name) => `/from/${name}`).join(" ")} /to/`;
|
|
8746
|
-
function failureText3(result) {
|
|
8747
|
-
const line2 = result.stderr.trim().split("\n").at(-1) ?? "";
|
|
8748
|
-
return `exit ${result.status}${line2 === "" ? "" : `: ${line2}`}`;
|
|
8749
|
-
}
|
|
8750
|
-
async function saveContainerState(input) {
|
|
8751
|
-
const { host } = input;
|
|
8752
|
-
const id = randomBytes(6).toString("hex");
|
|
8753
|
-
const volume = `outerlayer-saved-${id}`;
|
|
8754
|
-
const record = {
|
|
8755
|
-
id,
|
|
8756
|
-
kind: "volume",
|
|
8757
|
-
volume,
|
|
8758
|
-
savedAt: input.now.toISOString(),
|
|
8759
|
-
bytes: 0,
|
|
8760
|
-
baseKey: input.key.baseKey,
|
|
8761
|
-
key: input.key.key,
|
|
8762
|
-
fileHashes: input.key.fileHashes,
|
|
8763
|
-
...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
|
|
8764
|
-
complete: false
|
|
8765
|
-
};
|
|
8766
|
-
const discard = async (reason) => {
|
|
8767
|
-
const removed = await host.removeVolume(volume).catch((err) => ({ status: -1, stderr: describeCallFailure(err) }));
|
|
8768
|
-
if (removed.status !== 0) {
|
|
8769
|
-
input.log(`the volume ${volume} could not be removed (${failureText3({ status: removed.status, stderr: removed.stderr })}); its record stays so a later pass retries`);
|
|
8770
|
-
return { ok: false, reason };
|
|
8771
|
-
}
|
|
8772
|
-
deleteRecord(input.dir, id);
|
|
8773
|
-
return { ok: false, reason };
|
|
8774
|
-
};
|
|
8775
|
-
try {
|
|
8776
|
-
const noEnv = { literals: {}, passed: {} };
|
|
8777
|
-
const stopped = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sh", "-c", stopProcessesScript()] });
|
|
8778
|
-
if (stopped.status !== 0) return { ok: false, reason: `the build's processes could not be stopped (${failureText3(stopped)})` };
|
|
8779
|
-
input.log(`${stopped.stdout.trim() || "stopped 0"} process${stopped.stdout.trim() === "stopped 1" ? "" : "es"} before saving the state`);
|
|
8780
|
-
const flushed = await host.exec({ container: input.container, user: input.agentUser, env: noEnv, argv: ["sync"] });
|
|
8781
|
-
if (flushed.status !== 0) return { ok: false, reason: `the build's disk could not be flushed (${failureText3(flushed)})` };
|
|
8782
|
-
writeRecord2(input.dir, record);
|
|
8783
|
-
const created = await host.createVolume(volume, { [SAVED_STATE_LABEL]: id, [RUNNER_LABEL]: input.runnerId });
|
|
8784
|
-
if (created.status !== 0) return await discard(`the volume for the saved state could not be made (${failureText3(created)})`);
|
|
8785
|
-
const copied = await host.run(helperArgs({ image: input.image, from: input.buildVolume, to: volume, script: SAVE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
8786
|
-
if (copied.status !== 0) return await discard(`the copy into the saved state failed (${failureText3(copied)})`);
|
|
8787
|
-
const kilobytes = Number(copied.stdout.trim().split("\n").at(-1));
|
|
8788
|
-
const done = { ...record, bytes: Number.isFinite(kilobytes) ? kilobytes * 1024 : 0, complete: true };
|
|
8789
|
-
writeRecord2(input.dir, done);
|
|
8790
|
-
return { ok: true, record: done };
|
|
8791
|
-
} catch (err) {
|
|
8792
|
-
return await discard(`the state could not be saved: ${describeCallFailure(err)}`);
|
|
8793
|
-
}
|
|
8794
|
-
}
|
|
8795
|
-
async function restoreContainerState(input) {
|
|
8796
|
-
try {
|
|
8797
|
-
const copied = await input.host.run(helperArgs({ image: input.image, from: input.savedVolume, to: input.targetVolume, script: RESTORE_SCRIPT, runnerId: input.runnerId }), { logPath: input.logPath });
|
|
8798
|
-
if (copied.status !== 0) return { ok: false, reason: `the saved volume could not be copied (${failureText3(copied)})` };
|
|
8799
|
-
return { ok: true };
|
|
8800
|
-
} catch (err) {
|
|
8801
|
-
return { ok: false, reason: `the saved volume could not be copied: ${describeCallFailure(err)}` };
|
|
8802
|
-
}
|
|
8803
|
-
}
|
|
8804
9247
|
var noop = () => void 0;
|
|
8805
9248
|
var FRAME_HEADER_BYTES = 9;
|
|
8806
9249
|
var GOAWAY_FRAME = 7;
|
|
@@ -8913,8 +9356,8 @@ function serveSession(transport, services, opts = {}) {
|
|
|
8913
9356
|
});
|
|
8914
9357
|
session.on("stream", (stream, headers) => {
|
|
8915
9358
|
stream.on("error", noop);
|
|
8916
|
-
const
|
|
8917
|
-
if (
|
|
9359
|
+
const target2 = headers[":method"] === "POST" ? targetOfPath(headers[":path"], opts.builtIns) : void 0;
|
|
9360
|
+
if (target2 === void 0 || open >= maxStreams) {
|
|
8918
9361
|
stream.close(http2.constants.NGHTTP2_REFUSED_STREAM);
|
|
8919
9362
|
return;
|
|
8920
9363
|
}
|
|
@@ -8923,7 +9366,7 @@ function serveSession(transport, services, opts = {}) {
|
|
|
8923
9366
|
open--;
|
|
8924
9367
|
});
|
|
8925
9368
|
writeStringsAsBuffers(stream);
|
|
8926
|
-
if (!services.accept(
|
|
9369
|
+
if (!services.accept(target2, stream)) {
|
|
8927
9370
|
stream.close(http2.constants.NGHTTP2_REFUSED_STREAM);
|
|
8928
9371
|
return;
|
|
8929
9372
|
}
|
|
@@ -9373,7 +9816,7 @@ async function drain(req) {
|
|
|
9373
9816
|
function readPrefix(req, gzip) {
|
|
9374
9817
|
return new Promise((resolve2, reject) => {
|
|
9375
9818
|
const held3 = [];
|
|
9376
|
-
const
|
|
9819
|
+
const finish2 = (section2) => {
|
|
9377
9820
|
req.off("data", onData);
|
|
9378
9821
|
req.off("end", onEnd);
|
|
9379
9822
|
req.off("error", reject);
|
|
@@ -9383,9 +9826,9 @@ function readPrefix(req, gzip) {
|
|
|
9383
9826
|
const onData = (chunk) => {
|
|
9384
9827
|
held3.push(chunk);
|
|
9385
9828
|
const section2 = readCommandSection(Buffer.concat(held3), gzip);
|
|
9386
|
-
if (section2.kind !== "need-more")
|
|
9829
|
+
if (section2.kind !== "need-more") finish2(section2);
|
|
9387
9830
|
};
|
|
9388
|
-
const onEnd = () =>
|
|
9831
|
+
const onEnd = () => finish2({ kind: "need-more" });
|
|
9389
9832
|
req.on("data", onData);
|
|
9390
9833
|
req.once("end", onEnd);
|
|
9391
9834
|
req.once("error", reject);
|
|
@@ -9467,7 +9910,7 @@ function createGitRemote(options) {
|
|
|
9467
9910
|
return void 0;
|
|
9468
9911
|
}
|
|
9469
9912
|
}
|
|
9470
|
-
async function push(req, res, route,
|
|
9913
|
+
async function push(req, res, route, target2) {
|
|
9471
9914
|
const encoding = req.headers["content-encoding"];
|
|
9472
9915
|
const gzip = encoding === "gzip";
|
|
9473
9916
|
const { held: held3, section: section2 } = await readPrefix(req, gzip);
|
|
@@ -9492,8 +9935,8 @@ function createGitRemote(options) {
|
|
|
9492
9935
|
let bodyPath;
|
|
9493
9936
|
try {
|
|
9494
9937
|
let fastForward;
|
|
9495
|
-
if (needsFastForwardCheck(commands,
|
|
9496
|
-
const read = await tokenOrRefuse(res, "read",
|
|
9938
|
+
if (needsFastForwardCheck(commands, target2.branch)) {
|
|
9939
|
+
const read = await tokenOrRefuse(res, "read", target2.repository);
|
|
9497
9940
|
if (read === void 0) return void await drain(req);
|
|
9498
9941
|
mkdirSync(options.spoolDir, { recursive: true, mode: 448 });
|
|
9499
9942
|
bodyPath = join(options.spoolDir, randomBytes(8).toString("hex"));
|
|
@@ -9514,21 +9957,21 @@ function createGitRemote(options) {
|
|
|
9514
9957
|
if (size > cap) return void await refuseInBand(`the push is larger than the ${cap} bytes the runner holds to check it`);
|
|
9515
9958
|
try {
|
|
9516
9959
|
fastForward = await isFastForward({
|
|
9517
|
-
mirrorDir:
|
|
9960
|
+
mirrorDir: target2.mirrorDir,
|
|
9518
9961
|
remoteUrl: `${gitUrl}/${route.owner}/${route.name}.git`,
|
|
9519
9962
|
token: read,
|
|
9520
|
-
branch:
|
|
9963
|
+
branch: target2.branch,
|
|
9521
9964
|
bodyPath,
|
|
9522
9965
|
gzip
|
|
9523
9966
|
});
|
|
9524
9967
|
} catch (err) {
|
|
9525
9968
|
log(`fast-forward check could not run: ${err instanceof Error ? err.message : String(err)}`);
|
|
9526
|
-
return void await refuseInBand(`the runner could not check whether this push fast-forwards ${
|
|
9969
|
+
return void await refuseInBand(`the runner could not check whether this push fast-forwards ${target2.branch}`);
|
|
9527
9970
|
}
|
|
9528
9971
|
}
|
|
9529
|
-
const verdict = judgePush({ commands, branch:
|
|
9972
|
+
const verdict = judgePush({ commands, branch: target2.branch, fastForward });
|
|
9530
9973
|
if (!verdict.allowed) return void await refuseInBand(verdict.message);
|
|
9531
|
-
const token = await tokenOrRefuse(res, "push",
|
|
9974
|
+
const token = await tokenOrRefuse(res, "push", target2.repository);
|
|
9532
9975
|
if (token === void 0) return void await drain(req);
|
|
9533
9976
|
options.tokens.markUsed("push");
|
|
9534
9977
|
const headers = upstreamHeaders(req.headers, basicAuthorization(token));
|
|
@@ -9565,15 +10008,15 @@ function createGitRemote(options) {
|
|
|
9565
10008
|
req.resume();
|
|
9566
10009
|
return plain2(res, 403, "this remote is read-only: a check cannot push\n");
|
|
9567
10010
|
}
|
|
9568
|
-
const
|
|
10011
|
+
const target2 = claimed.find((entry) => isOwnRepository(entry.repository, route.owner, route.name));
|
|
9569
10012
|
const pushing = route.kind === "receive-pack" || route.kind === "info-refs" && route.service === "git-receive-pack";
|
|
9570
|
-
if (
|
|
10013
|
+
if (target2 === void 0 && pushing) {
|
|
9571
10014
|
log(`push refused: ${route.owner}/${route.name} is not a repository of this build's claim`);
|
|
9572
10015
|
req.resume();
|
|
9573
10016
|
return plain2(res, 403, `the runner refused this push: ${route.owner}/${route.name} is not a repository of this build's claim
|
|
9574
10017
|
`);
|
|
9575
10018
|
}
|
|
9576
|
-
if (
|
|
10019
|
+
if (target2 !== void 0 && route.kind === "receive-pack") return push(req, res, route, target2);
|
|
9577
10020
|
let kind = route.kind === "info-refs" && route.service === "git-receive-pack" ? "push" : "read";
|
|
9578
10021
|
let body;
|
|
9579
10022
|
if (route.kind === "lfs-batch") {
|
|
@@ -9583,8 +10026,8 @@ function createGitRemote(options) {
|
|
|
9583
10026
|
if (options.readOnly === true && kind === "push") return plain2(res, 403, "this remote is read-only: a check cannot upload\n");
|
|
9584
10027
|
}
|
|
9585
10028
|
let authorization;
|
|
9586
|
-
if (
|
|
9587
|
-
const token = await tokenOrRefuse(res, kind,
|
|
10029
|
+
if (target2 !== void 0) {
|
|
10030
|
+
const token = await tokenOrRefuse(res, kind, target2.repository);
|
|
9588
10031
|
if (token === void 0) return void await drain(req);
|
|
9589
10032
|
authorization = basicAuthorization(token);
|
|
9590
10033
|
if (kind === "read" || route.kind === "lfs-batch") options.tokens.markUsed(kind);
|
|
@@ -9957,11 +10400,11 @@ async function controlPlaneReadToken(tokens, controlPlane) {
|
|
|
9957
10400
|
}
|
|
9958
10401
|
async function provisionContainer(input) {
|
|
9959
10402
|
return provisionWithFreshFallback({
|
|
9960
|
-
attempt: (allowRestore) => provisionContainerOnce(input, allowRestore),
|
|
10403
|
+
attempt: (allowRestore, itemDiskFailed) => provisionContainerOnce(input, allowRestore, itemDiskFailed),
|
|
9961
10404
|
log: (line2) => input.log.line(`#${input.item.number} ${line2}`)
|
|
9962
10405
|
});
|
|
9963
10406
|
}
|
|
9964
|
-
async function provisionContainerOnce(input, allowRestore) {
|
|
10407
|
+
async function provisionContainerOnce(input, allowRestore, itemDiskFailed) {
|
|
9965
10408
|
const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
|
|
9966
10409
|
const { host } = runtime;
|
|
9967
10410
|
const tag = itemTag({ item: item.number, organizationId: input.organizationId, claimId: input.claimId });
|
|
@@ -10021,7 +10464,11 @@ async function provisionContainerOnce(input, allowRestore) {
|
|
|
10021
10464
|
const built = obtained.built;
|
|
10022
10465
|
const { report } = built;
|
|
10023
10466
|
const tagged = taggedLog(log, tag);
|
|
10024
|
-
const
|
|
10467
|
+
const diskBytes = parseSize(config.build?.disk ?? DEFAULT_BUILD_DISK) ?? 0;
|
|
10468
|
+
const itemDisk = input.workItemId === void 0 ? { state: void 0 } : itemDiskFailed !== void 0 ? { state: { restored: false, reason: itemDiskFailed } } : itemDiskDecision({ queue: input.queue ?? "implement", builtin: true, backend: "volume", configPath: config.configPath, organizationId: input.organizationId, workItemId: input.workItemId, imageId: report.imageId, diskBytes });
|
|
10469
|
+
const keptDisk = itemDisk.kept;
|
|
10470
|
+
if (itemDisk.state?.restored === false) tagged(`this agent job starts without a kept disk: ${DISK_REASON_TEXT[itemDisk.state.reason]}`);
|
|
10471
|
+
const plan = keptDisk !== void 0 ? keptDiskPlan() : await planSavedState({
|
|
10025
10472
|
config,
|
|
10026
10473
|
repositories,
|
|
10027
10474
|
base: savedStateBase(report),
|
|
@@ -10029,7 +10476,7 @@ async function provisionContainerOnce(input, allowRestore) {
|
|
|
10029
10476
|
tokens,
|
|
10030
10477
|
now: clockOf(input.now)(),
|
|
10031
10478
|
allowRestore,
|
|
10032
|
-
diskBytes
|
|
10479
|
+
diskBytes,
|
|
10033
10480
|
log: tagged
|
|
10034
10481
|
});
|
|
10035
10482
|
const savedStatesPath = savedStatesDir(runnerDir(config.configPath));
|
|
@@ -10100,6 +10547,10 @@ async function provisionContainerOnce(input, allowRestore) {
|
|
|
10100
10547
|
onError: (line2) => log.error(`${tag} ${line2}`)
|
|
10101
10548
|
});
|
|
10102
10549
|
};
|
|
10550
|
+
const giveUpItemDisk = async (reason, why) => {
|
|
10551
|
+
await bail(reason);
|
|
10552
|
+
return discardUnusableItemDisk({ kept: keptDisk, reason, why, removeVolume: (volume2) => host.removeVolume(volume2), onError: (line2) => log.error(`${tag} ${line2}`) });
|
|
10553
|
+
};
|
|
10103
10554
|
if (input.aborted()) return bail("", true);
|
|
10104
10555
|
mkdirSync(buildDir2, { recursive: true });
|
|
10105
10556
|
const ghDir = ghConfigDir(buildDir2);
|
|
@@ -10151,7 +10602,12 @@ async function provisionContainerOnce(input, allowRestore) {
|
|
|
10151
10602
|
};
|
|
10152
10603
|
const prepared = await host.run(prepareRunArgs(runInput), { logPath });
|
|
10153
10604
|
if (prepared.status !== 0) return dockerFailed("preparing the build's volume", prepared);
|
|
10154
|
-
if (
|
|
10605
|
+
if (keptDisk !== void 0) {
|
|
10606
|
+
const restoreStartedAt = Date.now();
|
|
10607
|
+
const restored = await restoreContainerState({ host, image: report.imageId, savedVolume: keptDisk.record.volume, targetVolume: name, runnerId, logPath });
|
|
10608
|
+
if (!restored.ok) return giveUpItemDisk(restored.reason, "disk_not_restored");
|
|
10609
|
+
tagged(`restored the disk that claim ${shortClaim(keptDisk.record.claimId)} kept: ${formatBytes(keptDisk.record.bytes)} allocated, copied in ${formatDuration(Date.now() - restoreStartedAt)}`);
|
|
10610
|
+
} else if (plan.restore !== void 0) {
|
|
10155
10611
|
const restored = await restoreContainerState({ host, image: report.imageId, savedVolume: plan.restore.record.volume, targetVolume: name, runnerId, logPath });
|
|
10156
10612
|
if (!restored.ok) return giveUpState(restored.reason);
|
|
10157
10613
|
}
|
|
@@ -10247,14 +10703,14 @@ async function provisionContainerOnce(input, allowRestore) {
|
|
|
10247
10703
|
...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
|
|
10248
10704
|
// A build on the universal image, or on an image built from it, chooses and installs its runtimes.
|
|
10249
10705
|
...runtimeStepInput(report, services),
|
|
10250
|
-
setup: setupInputFor({ setupMinutes: config.build?.setupMinutes, plan, saveState })
|
|
10706
|
+
setup: setupInputFor({ setupMinutes: config.build?.setupMinutes, plan, saveState, fromItemDisk: keptDisk !== void 0 })
|
|
10251
10707
|
});
|
|
10252
|
-
if (!steps.ok) return settleFailedSteps(steps, { giveUp: giveUpState, bail });
|
|
10708
|
+
if (!steps.ok) return settleFailedSteps(steps, { giveUp: (reason, fromItemDisk) => fromItemDisk ? giveUpItemDisk(reason, "checkout_not_switched") : giveUpState(reason), bail });
|
|
10253
10709
|
state.startCommit = steps.startCommit;
|
|
10254
10710
|
keepRuntimeStep(state, steps);
|
|
10255
10711
|
keepMissingSetupScripts(state, steps);
|
|
10256
10712
|
if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
|
|
10257
|
-
return { ok: true, workdir: CHECKOUT_DIR, state, environment };
|
|
10713
|
+
return { ok: true, workdir: CHECKOUT_DIR, state, environment, ...itemDisk.state === void 0 ? {} : { disk: itemDisk.state } };
|
|
10258
10714
|
}
|
|
10259
10715
|
function endedAtLimit(result) {
|
|
10260
10716
|
return result.status === 124 && /no answer within \d+s/.test(result.stderr);
|
|
@@ -10559,7 +11015,7 @@ async function copyHookErrors2(runtime, state, jobDir) {
|
|
|
10559
11015
|
} catch {
|
|
10560
11016
|
}
|
|
10561
11017
|
}
|
|
10562
|
-
async function cleanupContainer(runtime, state, jobDir) {
|
|
11018
|
+
async function cleanupContainer(runtime, state, jobDir, keepDisk, warn) {
|
|
10563
11019
|
const failures = [];
|
|
10564
11020
|
if (jobDir !== void 0) await copyHookErrors2(runtime, state, jobDir);
|
|
10565
11021
|
await leaveBuild(runtime, state);
|
|
@@ -10572,6 +11028,13 @@ async function cleanupContainer(runtime, state, jobDir) {
|
|
|
10572
11028
|
}
|
|
10573
11029
|
const helperLeft = await removeNetworkHelper(runtime, state);
|
|
10574
11030
|
if (helperLeft !== void 0) failures.push(helperLeft);
|
|
11031
|
+
if (keepDisk !== void 0 && removedContainer.status === 0) {
|
|
11032
|
+
try {
|
|
11033
|
+
await keepDisk();
|
|
11034
|
+
} catch (err) {
|
|
11035
|
+
warn?.(`the build's disk was not kept: ${describeCallFailure(err)}`);
|
|
11036
|
+
}
|
|
11037
|
+
}
|
|
10575
11038
|
let volumeGone = false;
|
|
10576
11039
|
for (let attempt = 0; attempt < VOLUME_REMOVE_ATTEMPTS && !volumeGone; attempt++) {
|
|
10577
11040
|
const removed = await runtime.host.removeVolume(state.volume);
|
|
@@ -11282,4 +11745,4 @@ async function runHostChecks(input) {
|
|
|
11282
11745
|
}
|
|
11283
11746
|
}
|
|
11284
11747
|
|
|
11285
|
-
export { ADOPTED_FETCH_RETRY_WINDOW_MS, ALREADY_RELEASED, AttemptLogBook, EXIT_CODE_FILE, GUEST_USAGE_INTERVAL_MS, HELPER_ENV, KIND_LABEL, LEASE_SECONDS, MAX_CHECK_ITEMS_BYTES, MICROVM_COMMAND_DIR, MIN_ENGINE_MAJOR, PLACEHOLDER_MARKER, RENEW_INTERVAL_SECONDS, RunnerInitError, agentCredentialCheck, appendCommandLog, attemptNote, checksIn, claimPermissionProblem, commandItems, commandLogOffset, containerCommandExec, createContainerRuntime, createHousekeepingDocker, createLeaseChannel, currentSkillCommand, decideHostBuildSkill, deliverCommitHooks, deliverContainerHooks, deliverExecMaven, deliverJobFiles, describeHookEnd, describeMachineDocker, detectMachine, ensureFirecrackerAndJailer, ensureNetworkReady, ensureTunnel, execArgv, execCommandLine, fetchManagedBuildSettings, fetchManagedClaudeKey, filesAreCopied, finishJob, firstLine3 as firstLine, foldSample, formerSkillProblem, formerSkillQueues, gitRemoteFor, helperFirecrackerNote, helperSocketPath, holdManagedBuildSettings, holdManagedClaudeKey, hookLimitsFor, hostCommandEnvironment, hostEnvironment, housekeepingDockerIfReachable, inTeardown, isContainerTracked, isGroupAlive, isIdle, isJudgeCheck, isMicroVmTracked, isPlaceholderCommand, isReadable, isTimedOut, isTracked, isUnfinished, itemScope, itemTag, jobEnv, judgeProbe, keepGhConfigCurrent, leaseExpiresAtMs, leaseTimeLeftMs, lifecycleFailureReason, lifecycleSteps, listAdoptableJobs, listTrackedJobs, listUnreadableJobs, listenOnBroker, makeRecorder, managedBuildSettingsFor, measureDisk, microVmArchSupported, microVmAssetsPinned, microVmCommandLine, microVmOomKills, missingPermissionFix, missingPermissionOf, needsAgentCredential, needsBuildSkill, needsRenew, outcomeFromExitCode, parseCpuList, permissionModeProblem, permissionModeProblems, placeholderProblem, placeholderQueues, prepareBranch, prepareResume, prepareRunnerInit, probeCommand, probeHelper, probeMicroVmCommand, provisionContainer, provisionEnvironment, provisionMicroVm, provisionProcess, reachDockerStep, readConfigObject, readCpuUsec, readDefinitions, readProvisionOut, readRecords, recoverLostJobs, releaseLease, removeOldestUntil, renewLease, resolveHostBuild, restartSender, restoreSessionIntoContainer, restoreSessionIntoMicroVm, retryFailedReleases, runCommand, runHook, runHostChecks, runHousekeeping, runRunnerInit, runWithSpawn, sampleGuest, savedStatesDir, scrubReason, serveLease, setStage, setUpRunnerCgroup, shellQuote, startHostServices, startMicroVmCommand, startSender, startServicesFor, stopOrphanedMicroVms, stoppedHelperReason, stripTerminalControls, supportsContainerBuilds, supportsMicroVmBuilds, sweepImageBuilders, testProofIn, timeoutAfter, unfinishedJobs, usableCpus, vmAlive, withTunnelLog, workdirProblem, writeGuestGhConfig, writeRunnerInit };
|
|
11748
|
+
export { ADOPTED_FETCH_RETRY_WINDOW_MS, ALREADY_RELEASED, AttemptLogBook, EXIT_CODE_FILE, GUEST_USAGE_INTERVAL_MS, HELPER_ENV, KIND_LABEL, LEASE_SECONDS, MAX_CHECK_ITEMS_BYTES, MICROVM_COMMAND_DIR, MIN_ENGINE_MAJOR, PLACEHOLDER_MARKER, RENEW_INTERVAL_SECONDS, RunnerInitError, agentCredentialCheck, appendCommandLog, attemptNote, checksIn, claimPermissionProblem, commandItems, commandLogOffset, containerCommandExec, createContainerRuntime, createHousekeepingDocker, createLeaseChannel, currentSkillCommand, decideHostBuildSkill, deliverCommitHooks, deliverContainerHooks, deliverExecMaven, deliverJobFiles, describeHookEnd, describeMachineDocker, detectMachine, ensureFirecrackerAndJailer, ensureNetworkReady, ensureTunnel, execArgv, execCommandLine, fetchManagedBuildSettings, fetchManagedClaudeKey, filesAreCopied, finishJob, firstLine3 as firstLine, foldSample, formerSkillProblem, formerSkillQueues, freeItemDisksUntil, gitRemoteFor, helperFirecrackerNote, helperSocketPath, holdManagedBuildSettings, holdManagedClaudeKey, hookLimitsFor, hostCommandEnvironment, hostEnvironment, housekeepingDockerIfReachable, inTeardown, isContainerTracked, isGroupAlive, isIdle, isJudgeCheck, isMicroVmTracked, isPlaceholderCommand, isReadable, isTimedOut, isTracked, isUnfinished, itemScope, itemTag, jobEnv, judgeProbe, keepGhConfigCurrent, leaseExpiresAtMs, leaseTimeLeftMs, lifecycleFailureReason, lifecycleSteps, listAdoptableJobs, listTrackedJobs, listUnreadableJobs, listenOnBroker, makeRecorder, managedBuildSettingsFor, measureDisk, microVmArchSupported, microVmAssetsPinned, microVmCommandLine, microVmOomKills, missingPermissionFix, missingPermissionOf, needsAgentCredential, needsBuildSkill, needsRenew, outcomeFromExitCode, parseCpuList, permissionModeProblem, permissionModeProblems, placeholderProblem, placeholderQueues, prepareBranch, prepareResume, prepareRunnerInit, probeCommand, probeHelper, probeMicroVmCommand, provisionContainer, provisionEnvironment, provisionMicroVm, provisionProcess, reachDockerStep, readConfigObject, readCpuUsec, readDefinitions, readProvisionOut, readRecords, recoverLostJobs, releaseLease, removeOldestUntil, renewLease, resolveHostBuild, restartSender, restoreSessionIntoContainer, restoreSessionIntoMicroVm, retryFailedReleases, runCommand, runHook, runHostChecks, runHousekeeping, runRunnerInit, runWithSpawn, sampleGuest, savedStatesDir, scrubReason, serveLease, setStage, setUpRunnerCgroup, shellQuote, startHostServices, startMicroVmCommand, startSender, startServicesFor, stopOrphanedMicroVms, stoppedHelperReason, stripTerminalControls, supportsContainerBuilds, supportsMicroVmBuilds, sweepImageBuilders, testProofIn, timeoutAfter, unfinishedJobs, usableCpus, vmAlive, withTunnelLog, workdirProblem, writeGuestGhConfig, writeRunnerInit };
|