@outerlayer/cli 0.36.0-next.20261030203048 → 0.36.0-next.20261031112440
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -14
- package/dist/{agent-setup-RZE6O737.js → agent-setup-UYTS2PHZ.js} +11 -10
- package/dist/{artifact-link-cmd-DLKYKNWF.js → artifact-link-cmd-W7LRR76D.js} +2 -2
- package/dist/{build-GVNOXEDV.js → build-KM6UILOZ.js} +4 -4
- package/dist/{build-disk-H5VDB4G2.js → build-disk-GCEB5EJC.js} +4 -4
- package/dist/build-info.json +1 -1
- package/dist/{builder-QW6N3XBA.js → builder-NRPJGNXU.js} +3 -3
- package/dist/{caller-TUO3VAKG.js → caller-XHNAY4E2.js} +6 -6
- package/dist/{check-E2ZTWJPM.js → check-MENIW2MN.js} +43 -44
- package/dist/{chunk-OSQEG4HG.js → chunk-25GQP6NR.js} +7026 -2552
- package/dist/{chunk-QLWEXEEA.js → chunk-34YFGWHR.js} +1 -1
- package/dist/{chunk-MJEUD4PD.js → chunk-3GFZSLSP.js} +4 -4
- package/dist/{chunk-YQDXFFOK.js → chunk-3PQTOKML.js} +1 -1
- package/dist/{chunk-HSSECQOA.js → chunk-3QLHEGXK.js} +2 -2
- package/dist/{chunk-OUMMLC45.js → chunk-3SQZ5LIK.js} +2 -2
- package/dist/{chunk-RUML4JVG.js → chunk-3YRJUX6J.js} +1 -1
- package/dist/{chunk-LOI3ESFV.js → chunk-4AMSCAHT.js} +6 -151
- package/dist/{chunk-OJ2KWQPQ.js → chunk-4SFHEARD.js} +1 -1
- package/dist/{chunk-3XIAGOPV.js → chunk-5AJZGYWC.js} +4 -4
- package/dist/{chunk-D4BIUQPA.js → chunk-5H57X7YC.js} +1 -1
- package/dist/{chunk-BBUIVGIR.js → chunk-5UBNSGK3.js} +3 -3
- package/dist/{chunk-7ABJKQAQ.js → chunk-6IJZL7X2.js} +2 -2
- package/dist/{chunk-O6V6T6HD.js → chunk-6T2JWCSZ.js} +12 -12
- package/dist/{chunk-7GX4X25F.js → chunk-7O7LJWQO.js} +1 -1
- package/dist/{chunk-TGVIFTKI.js → chunk-AVJWMOEY.js} +1 -1
- package/dist/{chunk-TUN3TYBX.js → chunk-BFF6AJXM.js} +8 -8
- package/dist/{chunk-A2QQGMQ7.js → chunk-C4YULCXX.js} +7 -7
- package/dist/chunk-F3ZMVUAS.js +40 -0
- package/dist/{chunk-OFOEDDN6.js → chunk-FRDYAKH3.js} +2 -2
- package/dist/{chunk-5TJQBHKO.js → chunk-GDIVOYDA.js} +4 -4
- package/dist/{chunk-O3OREOCP.js → chunk-GDMAN42R.js} +2 -2
- package/dist/{chunk-JII4O3EZ.js → chunk-GU25RNKD.js} +3 -3
- package/dist/{chunk-6KLNELLP.js → chunk-IVY3PD7P.js} +3 -3
- package/dist/{chunk-AILO7SNG.js → chunk-JFMSFHER.js} +13 -211
- package/dist/{chunk-FCJWPFDS.js → chunk-JPROSTGX.js} +4 -4
- package/dist/{chunk-3IQE34I3.js → chunk-K2YG2HZN.js} +19 -393
- package/dist/{chunk-4A7GNO4U.js → chunk-LESWXCKZ.js} +1 -1
- package/dist/{chunk-YZXI37NW.js → chunk-LHFG5YUG.js} +2 -2
- package/dist/{chunk-WNM5GXJ3.js → chunk-LRCGXRQI.js} +2 -2
- package/dist/{chunk-KSMN2FUF.js → chunk-LZ7PAU55.js} +6 -2
- package/dist/{chunk-VOURWC2H.js → chunk-MMDGO77D.js} +1 -1
- package/dist/{chunk-BIAXDR3K.js → chunk-MMXPNEEQ.js} +8 -8
- package/dist/{chunk-BR2WJ6BX.js → chunk-MVSC7GCE.js} +2 -2
- package/dist/{chunk-7K7TGHDF.js → chunk-MXWPLGO6.js} +74 -707
- package/dist/{chunk-EA57H6AT.js → chunk-OLENS2DD.js} +1 -1
- package/dist/{chunk-WQXEJ43L.js → chunk-OPF6OJAT.js} +4 -4
- package/dist/{chunk-XHDJ75TC.js → chunk-PDQWXH3F.js} +5 -5
- package/dist/{chunk-QH6RZLDV.js → chunk-PJZKKFQO.js} +1 -1
- package/dist/{chunk-GGNG6CVE.js → chunk-PN6TNCQI.js} +14 -7
- package/dist/{chunk-YTNZ237E.js → chunk-RGH5GYU2.js} +7 -7
- package/dist/{chunk-UCRCIRQQ.js → chunk-S6OD32NC.js} +5 -5
- package/dist/{chunk-D3H37WUD.js → chunk-T3AJOPUR.js} +2 -2
- package/dist/{chunk-UHD5ZR46.js → chunk-U553JKMO.js} +4 -4
- package/dist/{chunk-IZVBCDSH.js → chunk-VG2NIWR4.js} +4 -4
- package/dist/{chunk-ID57YZP2.js → chunk-VJYNMGCE.js} +2 -2
- package/dist/{chunk-QZSA5R3L.js → chunk-VUG4IZ7H.js} +1 -1
- package/dist/{chunk-WXR6AVZ7.js → chunk-XTVVKNRP.js} +41 -10
- package/dist/{claude-launcher-FK5ZQWXC.js → claude-launcher-YV3TDXBU.js} +5 -5
- package/dist/{cli-YMCBXWRM.js → cli-TAFO3WOQ.js} +116 -118
- package/dist/{command-AYXNOGIU.js → command-TRSFSBT5.js} +45 -25
- package/dist/{config-NVGVPL3L.js → config-YCQV3OQK.js} +5 -5
- package/dist/{connect-cmd-KO2TIY6J.js → connect-cmd-IRJ3MBPY.js} +8 -8
- package/dist/{container-start-KS52ID4M.js → container-start-UY2YVY3L.js} +39 -40
- package/dist/{context-cleanup-2MTGLUYS.js → context-cleanup-TPHCVVFV.js} +3 -3
- package/dist/{context-show-cmd-XLER7QAI.js → context-show-cmd-YQEQ5B34.js} +9 -9
- package/dist/{control-plane-launch-RUJOCYEI.js → control-plane-launch-UWFQWMWB.js} +11 -6
- package/dist/{demo-voice-cmd-GLN5QZJP.js → demo-voice-cmd-QAH7KPVO.js} +15 -16
- package/dist/dist-2SPMAD7Z.js +7 -0
- package/dist/{docker-3AZ3ZFLA.js → docker-2247NBIK.js} +3 -3
- package/dist/{doctor-TLIRZM2C.js → doctor-VVFPHFS4.js} +46 -27
- package/dist/{emit-artifact-cmd-CNC5CPKI.js → emit-artifact-cmd-3YHOU4TI.js} +13 -14
- package/dist/{emit-cmd-6NZOT5DS.js → emit-cmd-HNXRGC5Z.js} +7 -6
- package/dist/{emit-criteria-cmd-BNXWF46E.js → emit-criteria-cmd-NN52H53B.js} +10 -10
- package/dist/{emit-demo-cmd-UXTH6XBO.js → emit-demo-cmd-V7CP333I.js} +15 -16
- package/dist/{emit-finding-cmd-5HZE7W6C.js → emit-finding-cmd-J5UZGNP4.js} +10 -10
- package/dist/{emit-result-cmd-KAEWXQIH.js → emit-result-cmd-IWMACBFI.js} +12 -13
- package/dist/{emit-step-cmd-HSKB3SNR.js → emit-step-cmd-SRWFWEU7.js} +10 -10
- package/dist/{find-unit-CNYGB5AN.js → find-unit-FEYD4IHC.js} +7 -7
- package/dist/{findings-cmd-2OQMIVHA.js → findings-cmd-BWTLCXDR.js} +10 -10
- package/dist/{fleet-readiness-PCQSCHQ5.js → fleet-readiness-FAKMWCJV.js} +8 -8
- package/dist/{governing-source-7XCL3AYM.js → governing-source-6ANJ4HQI.js} +14 -13
- package/dist/{guest-init-2TTQKFFZ.js → guest-init-WPUS45IN.js} +2 -2
- package/dist/{hook-fast-EMZVAUJ5.js → hook-fast-3HPWMSFB.js} +1 -1
- package/dist/{host-checks-TRVDAYBL.js → host-checks-TX22YEFD.js} +37 -38
- package/dist/index.js +4 -4
- package/dist/{init-OOYCYL2C.js → init-36AY3ZQZ.js} +44 -24
- package/dist/init-IYL3YXEM.js +78 -0
- package/dist/{init-cmd-4PVOILZD.js → init-cmd-JK2OGQU5.js} +18 -18
- package/dist/{init-update-QMSQEZQU.js → init-update-4IQPGOA6.js} +24 -31
- package/dist/{layout-YJ2RD5H2.js → layout-TLUOMM6F.js} +4 -4
- package/dist/{login-browser-TWUHD2DS.js → login-browser-ODTJEGDN.js} +2 -2
- package/dist/{loop-EIR2YPA2.js → loop-U72FDFQR.js} +42 -43
- package/dist/machine-S2TYJJPP.js +78 -0
- package/dist/{machine-skills-FWIZV5HO.js → machine-skills-VRUJVBQV.js} +4 -4
- package/dist/{runner-env-IDGTVTYW.js → runner-env-OPRWKIKI.js} +7 -7
- package/dist/{runner-identity-RJYVFQ5K.js → runner-identity-OY75LG6G.js} +3 -3
- package/dist/{runner-key-BKMFJ7RA.js → runner-key-6WN67PRU.js} +44 -24
- package/dist/{session-sender-QDZWNAY7.js → session-sender-EINASB3R.js} +13 -14
- package/dist/{spec-cmd-OYQEUAMI.js → spec-cmd-253AGRFF.js} +11 -11
- package/dist/{starter-pack-6FOYP6DY.js → starter-pack-7QNUT74O.js} +5 -5
- package/dist/{status-BZ37VEAJ.js → status-WCQG6SX4.js} +37 -38
- package/dist/{sync-cmd-MNXFOIHW.js → sync-cmd-Z6PU33GC.js} +12 -13
- package/dist/{update-steps-VPBGYVW5.js → update-steps-BOKCWGQ6.js} +38 -39
- package/dist/{watch-E7RGSZF6.js → watch-XNBXSIEI.js} +12 -13
- package/dist/{work-claim-cmd-HX7DGFY3.js → work-claim-cmd-2A6D5EYO.js} +15 -15
- package/dist/{work-cmd-IEC6XKC6.js → work-cmd-JZUBXXJN.js} +10 -10
- package/dist/{work-comment-cmd-X65LS27O.js → work-comment-cmd-L4R6ZXLO.js} +12 -12
- package/dist/{work-demo-cmd-D7WGXJOC.js → work-demo-cmd-YNPN2PHQ.js} +10 -10
- package/dist/{work-logs-cmd-FQH2UVMI.js → work-logs-cmd-CSQJB5MD.js} +10 -10
- package/dist/{work-open-pr-cmd-EFAFERSF.js → work-open-pr-cmd-Y7YNI2PF.js} +11 -11
- package/dist/{work-pr-cmd-E55GPZKO.js → work-pr-cmd-QGAOPNLV.js} +10 -10
- package/dist/{work-proofs-cmd-B5X3AJQY.js → work-proofs-cmd-RPVVD2SC.js} +12 -12
- package/dist/{work-proofs-record-DDOAQOPX.js → work-proofs-record-ULCY5QEP.js} +19 -20
- package/dist/{work-stop-cmd-ANPTSTYS.js → work-stop-cmd-LF4OTNPL.js} +10 -10
- package/package.json +1 -1
- package/dist/chunk-PHFZMU32.js +0 -3924
- package/dist/chunk-TBT347UY.js +0 -41
- package/dist/dist-4Q4Y6YJL.js +0 -7
- package/dist/init-HEZDVJPU.js +0 -58
- package/dist/machine-23Y2EHH7.js +0 -58
- package/dist/{chunk-H2GESWLW.js → chunk-55PYF6S2.js} +1 -1
package/dist/chunk-PHFZMU32.js
DELETED
|
@@ -1,3924 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { createRequire } from 'node:module';
|
|
3
|
-
import { itemsSentence, criteriaOf, hostItemsOf, loadCitationFinder, testFileMatcher, MAX_TEST_FILE_BYTES, MAX_TEST_FILES, criterionInputs, uncitedIds, planCriteria, parseJudgeAnswer, inCriteriaOrder, oneLineReason, judgedItems, IdScan, firstInPathOrder, isSearchCandidate } from './chunk-5TJQBHKO.js';
|
|
4
|
-
import { readsJobState, READ_JOB_STATE_FORMATS, jobStateFormatOf } from './chunk-ZVB57WI6.js';
|
|
5
|
-
import { HELPER_PATH, DOCKERD_LOG, HELPER_RECORD } from './chunk-H2GESWLW.js';
|
|
6
|
-
import { writeDockerClientConfig } from './chunk-MKSZHT6S.js';
|
|
7
|
-
import { zodProblem } from './chunk-I3LTIAOR.js';
|
|
8
|
-
import { senderPidfile } from './chunk-BBUIVGIR.js';
|
|
9
|
-
import { installScrubContext } from './chunk-WXR6AVZ7.js';
|
|
10
|
-
import { upgradedStarterText } from './chunk-UHD5ZR46.js';
|
|
11
|
-
import { ADDED_USER, shellQuote, GH_CONFIG_YML, ghHostsYml, firstLine, provisionWithFreshFallback, itemTag, hasClaudeCredential, AssetError, obtainImage, taggedLog, planSavedState, clockOf, savedStateBase, savedStatesDir, ensureRootfs, makeBuildsDirPrivate, gitRemoteFor, startServicesFor, createLeaseChannel, createBuildDisk, machineConfig, startJailedInto, resolveAgentUser, AgentUserError, agentUidOf, userIdChangedReason, runBuildSteps, setupInputFor, runtimeStepInput, settleFailedSteps, keepRuntimeStep, keepMissingSetupScripts, restoreSession, packKeptSession, createContainerRuntime, containerSenderCommand, hostCommandEnvironment, execArgv, signalInContainer, startHostServices, LEASE_SECONDS, itemScope, managedBuildSettingsFor, stopContainer, cleanupContainer, recordProofsInContainer, PROOFS_LOG, outOfMemoryReason, closeHostBuild, dropManagedClaudeKey, dropManagedBuildSettings, startCheckContainer, ranRuntimeStep, provisionRuntimes, lifecycleSteps, lifecycleFailureReason, runSetupScripts, writeEvalInput, DeliveryFailure, EVAL_INPUT_NAME, syncInContainer, copyTranscriptsFromContainer, senderAliveScript, discardUnusableState, MANAGED_UNJAILED_REASON, uidFromProbe, saveAndPrune, managedMachineLimits, withFactoryHosts, helperFirecrackerNote, said, managedClaudeKeyValuesFor, managedBuildSettingValuesFor, managedClaudeKeyValues, managedBuildSettingValues, readResumeRecords, copyOutSession, keepSession, keptSessionDir, stopJailed, deleteBuildDisk, RECORD_PROOFS_ARGV, managedClaudeKeyFor, ROOT_DISK_ID_PREFIX, copyTranscripts, writeRecord, ensureContainerRunning, sessionGuestOfContainer, containerJudgeEnvironment, deleteRecord, senderSignalScript, copyJudgeTree } from './chunk-OSQEG4HG.js';
|
|
12
|
-
import { fetchCommit, readBranchHead, diffBetween } from './chunk-4A7GNO4U.js';
|
|
13
|
-
import { isSameProcess, startTimeOf } from './chunk-JBJTC3UU.js';
|
|
14
|
-
import { waitForAgent, vmExecArgs } from './chunk-4PH4DVV4.js';
|
|
15
|
-
import { socketPathProblem, vsockLinks, LEASE_LINK, linkVsockSockets } from './chunk-UZGQT6OF.js';
|
|
16
|
-
import { describeCallFailure, resolveGoverningSource, prefetchGovernedContext, refusalCode, removeStagedContext, stageGovernedContext, pinnedSourceOf, commandWithShim, logStamp, createRunnerLog, formatDuration, removeKeptContext, retrySend, formatClock, keptContextTree, WORKFLOW_PERMISSION_REASON } from './chunk-KSMN2FUF.js';
|
|
17
|
-
import { claimCalls, spoolHomeFor, itemCalls, claimItem, leftByRunnerKey } from './chunk-RUML4JVG.js';
|
|
18
|
-
import { imageBuildOf, isBuiltinContainer, isBuiltinProcess, isBuiltinVm, egressSummary, DEFAULT_CHECK_TIME_LIMIT_MINUTES, DEFAULT_SETUP_MINUTES, optionalEgress } from './chunk-HSSECQOA.js';
|
|
19
|
-
import { formatBytes, DEFAULT_CACHE_DISK, allocatedBytes } from './chunk-TGVIFTKI.js';
|
|
20
|
-
import { HASH_ARGV, parseProgramHash, JUDGE_PROGRAM, writeJobFile, readJobFile, parseSize, identityScript, identityArgs, parseIdentity, mountableHome, homeCopyScript, homeCopyArgs, childEnvironment, withAnswerProtocol, judgeCommand, MAX_REPORT_BYTES, JUDGE_MAX_TURNS, JUDGE_DIR, COMMAND_DIR, MAX_ITEMS_BYTES } from './chunk-QH6RZLDV.js';
|
|
21
|
-
import { CLAUDE_CODE_VERSION, FIRECRACKER_VERSION } from './chunk-7SO2VHTQ.js';
|
|
22
|
-
import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-UCRCIRQQ.js';
|
|
23
|
-
import { planArtifactUpload } from './chunk-YQDXFFOK.js';
|
|
24
|
-
import { readCloudConfig } from './chunk-HWVMRETT.js';
|
|
25
|
-
import { signerFor } from './chunk-4DJZSNVT.js';
|
|
26
|
-
import { AGENTS_SKILLS_BASE, OUTERLAYER_SKILLS_BASE, resolvePackRoot } from './chunk-7GX4X25F.js';
|
|
27
|
-
import { scrubText } from './chunk-7K7TGHDF.js';
|
|
28
|
-
import { MAINTAINED_SKILLS } from './chunk-CDDED64E.js';
|
|
29
|
-
import { resolveVersion } from './chunk-RA3O54FC.js';
|
|
30
|
-
import { parsePolicy } from './chunk-AILO7SNG.js';
|
|
31
|
-
import { CheckItemsSchema } from './chunk-QZSA5R3L.js';
|
|
32
|
-
import { homeFromEnv } from './chunk-352DHUTP.js';
|
|
33
|
-
import { RUNNER_FILE_MODE, runnerDir, buildDir, orgStoreIn, vsockPath, jobConsolePath, orgHomeDir } from './chunk-WT7J2627.js';
|
|
34
|
-
import { JOB_MOUNT, repositoriesOf, primaryOf, buildEnvironment, CHECKOUT_DIR, claudeCredential, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, fdName, BUILD_HOME, CLAUDE_CREDENTIAL_VARIABLES, currentClaudeCredential, removedClaudeCredentialValues, managedClaudeCredential, checkEnvironment, EVAL_INPUT_DIR, withMiseVariables } from './chunk-GANAOCHR.js';
|
|
35
|
-
import { RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON } from './chunk-MESOOLF4.js';
|
|
36
|
-
import { randomBytes, createHash } from 'crypto';
|
|
37
|
-
import { rmSync, existsSync, statSync, appendFileSync, mkdirSync, chmodSync, writeFileSync, openSync, closeSync, readFileSync, readdirSync, createReadStream, lstatSync, realpathSync, writeSync, renameSync, fstatSync, readSync } from 'fs';
|
|
38
|
-
import { join, dirname, delimiter, sep } from 'path';
|
|
39
|
-
import { spawn, spawnSync } from 'child_process';
|
|
40
|
-
import { once } from 'events';
|
|
41
|
-
import { createServer } from 'net';
|
|
42
|
-
|
|
43
|
-
createRequire(import.meta.url);
|
|
44
|
-
var failureText = (status, stderr) => {
|
|
45
|
-
const line = stderr.trim().split("\n").at(-1) ?? "";
|
|
46
|
-
return `exit ${status}${line === "" ? "" : `: ${line}`}`;
|
|
47
|
-
};
|
|
48
|
-
async function saveMicroVmState(input) {
|
|
49
|
-
const id = randomBytes(6).toString("hex");
|
|
50
|
-
const path = join(input.dir, `${id}.tar`);
|
|
51
|
-
const partial = `${path}.partial`;
|
|
52
|
-
const record = {
|
|
53
|
-
id,
|
|
54
|
-
kind: "tar",
|
|
55
|
-
path,
|
|
56
|
-
savedAt: input.now.toISOString(),
|
|
57
|
-
bytes: 0,
|
|
58
|
-
baseKey: input.key.baseKey,
|
|
59
|
-
key: input.key.key,
|
|
60
|
-
fileHashes: input.key.fileHashes,
|
|
61
|
-
...input.agentUid === void 0 ? {} : { agentUid: input.agentUid },
|
|
62
|
-
complete: false
|
|
63
|
-
};
|
|
64
|
-
const discard = (reason) => {
|
|
65
|
-
rmSync(partial, { force: true });
|
|
66
|
-
rmSync(path, { force: true });
|
|
67
|
-
deleteRecord(input.dir, id);
|
|
68
|
-
return { ok: false, reason };
|
|
69
|
-
};
|
|
70
|
-
try {
|
|
71
|
-
const stopped = await input.guest.run(["/bin/sh", "-c", guestStopScript()]);
|
|
72
|
-
if (stopped.status !== 0) return { ok: false, reason: `the guest's processes could not be stopped (${failureText(stopped.status, stopped.stderr)})` };
|
|
73
|
-
const flushed = await input.guest.run(["sync", "-f", "/build"]);
|
|
74
|
-
if (flushed.status !== 0) return { ok: false, reason: `the build disk could not be flushed (${failureText(flushed.status, flushed.stderr)})` };
|
|
75
|
-
mkdirSync(input.dir, { recursive: true, mode: 448 });
|
|
76
|
-
writeRecord(input.dir, record);
|
|
77
|
-
const fd = openSync(partial, "w", 384);
|
|
78
|
-
let stderr = "";
|
|
79
|
-
let result;
|
|
80
|
-
try {
|
|
81
|
-
result = await input.guest.stream(["tar", "--numeric-owner", "-C", "/build", "-cf", "-", "home", "work"], {
|
|
82
|
-
onStdout: (chunk) => void writeSync(fd, chunk),
|
|
83
|
-
onStderr: (chunk) => void (stderr = (stderr + chunk.toString()).slice(-400))
|
|
84
|
-
});
|
|
85
|
-
} finally {
|
|
86
|
-
closeSync(fd);
|
|
87
|
-
}
|
|
88
|
-
if (result.status !== 0) return discard(`the guest's tar failed (${failureText(result.status, stderr)})`);
|
|
89
|
-
renameSync(partial, path);
|
|
90
|
-
const done = { ...record, bytes: statSync(path).size, complete: true };
|
|
91
|
-
writeRecord(input.dir, done);
|
|
92
|
-
return { ok: true, record: done };
|
|
93
|
-
} catch (err) {
|
|
94
|
-
return discard(`the state could not be saved: ${describeCallFailure(err)}`);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
async function restoreMicroVmState(input) {
|
|
98
|
-
const { path } = input.record;
|
|
99
|
-
if (path === void 0 || !existsSync(path)) return { ok: false, reason: "the saved tar file is gone" };
|
|
100
|
-
try {
|
|
101
|
-
let stderr = "";
|
|
102
|
-
const result = await input.guest.stream(["tar", "-x", "--numeric-owner", "-C", "/build"], {
|
|
103
|
-
stdin: createReadStream(path),
|
|
104
|
-
onStderr: (chunk) => void (stderr = (stderr + chunk.toString()).slice(-400))
|
|
105
|
-
});
|
|
106
|
-
return result.status === 0 ? { ok: true } : { ok: false, reason: `tar could not unpack the saved state (${failureText(result.status, stderr)})` };
|
|
107
|
-
} catch (err) {
|
|
108
|
-
return { ok: false, reason: `the saved state could not be unpacked: ${describeCallFailure(err)}` };
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
async function serveLease(socketPath, channel, makeServer = createServer, listenOn) {
|
|
112
|
-
const server = makeServer((socket) => channel.accept(socket));
|
|
113
|
-
if (listenOn === void 0) {
|
|
114
|
-
rmSync(socketPath, { force: true });
|
|
115
|
-
server.listen(socketPath);
|
|
116
|
-
await once(server, "listening");
|
|
117
|
-
} else {
|
|
118
|
-
await listenOn(server);
|
|
119
|
-
}
|
|
120
|
-
server.on("error", () => void 0);
|
|
121
|
-
return {
|
|
122
|
-
...channel,
|
|
123
|
-
listener: server,
|
|
124
|
-
close() {
|
|
125
|
-
channel.close();
|
|
126
|
-
server.close();
|
|
127
|
-
rmSync(socketPath, { force: true });
|
|
128
|
-
}
|
|
129
|
-
};
|
|
130
|
-
}
|
|
131
|
-
var PROBE_TIMEOUT_MS = 3e4;
|
|
132
|
-
var PROBE_BYTES = 256 * 1024;
|
|
133
|
-
function probeScript(dir, alive) {
|
|
134
|
-
return `d=${dir}
|
|
135
|
-
code=$(cat "$d/exit-code" 2>/dev/null | tr -d ' \\r\\n')
|
|
136
|
-
pid=$(cat "$d/command.pid" 2>/dev/null | tr -d ' \\r\\n')
|
|
137
|
-
alive=0
|
|
138
|
-
if [ -n "$pid" ] && ${alive}; then alive=1; fi
|
|
139
|
-
size=$(wc -c < "$d/command.log" 2>/dev/null || echo 0)
|
|
140
|
-
size=$(echo $size)
|
|
141
|
-
echo "OL-COMMAND exit=$code alive=$alive size=$size"
|
|
142
|
-
n=$((size - $1))
|
|
143
|
-
if [ "$n" -gt "$2" ]; then n=$2; fi
|
|
144
|
-
if [ "$n" -gt 0 ]; then tail -c +$(($1 + 1)) "$d/command.log" | head -c "$n" | base64 | tr -d '\\n'; fi
|
|
145
|
-
echo
|
|
146
|
-
`;
|
|
147
|
-
}
|
|
148
|
-
var PROBE_SCRIPT = probeScript(COMMAND_DIR, 'grep -qa build-command "/proc/$pid/cmdline" 2>/dev/null');
|
|
149
|
-
function probeArgv(offset, script = PROBE_SCRIPT) {
|
|
150
|
-
return ["/bin/sh", "-c", script, "probe", String(offset), String(PROBE_BYTES)];
|
|
151
|
-
}
|
|
152
|
-
function parseProbe(stdout) {
|
|
153
|
-
const [header = "", body = ""] = stdout.split("\n");
|
|
154
|
-
const match = /^OL-COMMAND exit=(\d*) alive=([01]) size=(\d+)$/.exec(header);
|
|
155
|
-
if (match === null) return void 0;
|
|
156
|
-
return {
|
|
157
|
-
...match[1] === "" ? {} : { exitCode: Number(match[1]) },
|
|
158
|
-
alive: match[2] === "1",
|
|
159
|
-
size: Number(match[3]),
|
|
160
|
-
bytes: Buffer.from(body.trim(), "base64")
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
async function probeCommand(runtime, state, offset) {
|
|
164
|
-
const result = await runtime.host.exec({
|
|
165
|
-
container: state.container,
|
|
166
|
-
user: state.agentUser ?? ADDED_USER,
|
|
167
|
-
env: { literals: {}, passed: {} },
|
|
168
|
-
argv: probeArgv(offset)
|
|
169
|
-
}, { timeoutMs: PROBE_TIMEOUT_MS });
|
|
170
|
-
return result.status === 0 ? parseProbe(result.stdout) : void 0;
|
|
171
|
-
}
|
|
172
|
-
async function copyCommandLogTail(runtime, state, jobDir) {
|
|
173
|
-
for (let reads = 0; reads < MAX_TAIL_READS; reads++) {
|
|
174
|
-
const offset = commandLogOffset(jobDir, state.commandLogBase);
|
|
175
|
-
const probe = await probeCommand(runtime, state, offset);
|
|
176
|
-
if (probe === void 0) return false;
|
|
177
|
-
appendCommandLog(jobDir, probe.bytes);
|
|
178
|
-
if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) return true;
|
|
179
|
-
}
|
|
180
|
-
return false;
|
|
181
|
-
}
|
|
182
|
-
var MAX_TAIL_READS = 8;
|
|
183
|
-
function commandLogOffset(jobDir, base) {
|
|
184
|
-
const path = join(jobDir, "command.log");
|
|
185
|
-
return existsSync(path) ? Math.max(0, statSync(path).size - (base ?? 0)) : 0;
|
|
186
|
-
}
|
|
187
|
-
function appendCommandLog(jobDir, bytes) {
|
|
188
|
-
if (bytes.length > 0) appendFileSync(join(jobDir, "command.log"), bytes, { mode: RUNNER_FILE_MODE });
|
|
189
|
-
}
|
|
190
|
-
function judgeProbe(probe, missedBefore) {
|
|
191
|
-
if (probe.exitCode !== void 0) return { kind: "exited", code: probe.exitCode };
|
|
192
|
-
if (probe.alive) return { kind: "running" };
|
|
193
|
-
return missedBefore ? { kind: "lost" } : { kind: "recheck" };
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
// src/runner/microvm/command-state.ts
|
|
197
|
-
var MICROVM_COMMAND_DIR = "/build/.command";
|
|
198
|
-
var SCRIPT = probeScript(MICROVM_COMMAND_DIR, '[ -d "/proc/$pid" ]');
|
|
199
|
-
var GUEST_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
200
|
-
var STOP_COMMAND_SCRIPT = `pid=$(cat ${MICROVM_COMMAND_DIR}/command.pid 2>/dev/null | tr -d ' \\r\\n'); [ -n "$pid" ] && kill -KILL "-$pid" 2>/dev/null; true`;
|
|
201
|
-
var EXEC_TIMEOUT_MS = 3e4;
|
|
202
|
-
var ABORT_GRACE_MS = 5e3;
|
|
203
|
-
async function withinTime(exec) {
|
|
204
|
-
const controller = new AbortController();
|
|
205
|
-
let abortTimer;
|
|
206
|
-
let graceTimer;
|
|
207
|
-
const deadline = new Promise((_resolve, reject) => {
|
|
208
|
-
abortTimer = setTimeout(() => {
|
|
209
|
-
controller.abort();
|
|
210
|
-
graceTimer = setTimeout(() => reject(new Error("the guest did not end the exec after it was aborted")), ABORT_GRACE_MS);
|
|
211
|
-
}, EXEC_TIMEOUT_MS);
|
|
212
|
-
});
|
|
213
|
-
try {
|
|
214
|
-
return await Promise.race([exec(controller.signal), deadline]);
|
|
215
|
-
} finally {
|
|
216
|
-
clearTimeout(abortTimer);
|
|
217
|
-
clearTimeout(graceTimer);
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
async function probeMicroVmCommand(runtime, state, offset) {
|
|
221
|
-
try {
|
|
222
|
-
const result = await withinTime(
|
|
223
|
-
(signal) => runtime.vm.exec(state.vsockPath, { argv: probeArgv(offset, SCRIPT), user: state.agentUser ?? ADDED_USER, cwd: "/", env: { PATH: GUEST_PATH } }, { signal })
|
|
224
|
-
);
|
|
225
|
-
return result.status === 0 ? parseProbe(result.stdout) : void 0;
|
|
226
|
-
} catch {
|
|
227
|
-
return void 0;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
var MAX_TAIL_READS2 = 8;
|
|
231
|
-
async function copyMicroVmLogTail(runtime, state, jobDir) {
|
|
232
|
-
for (let reads = 0; reads < MAX_TAIL_READS2; reads++) {
|
|
233
|
-
const offset = commandLogOffset(jobDir, state.commandLogBase);
|
|
234
|
-
const probe = await probeMicroVmCommand(runtime, state, offset);
|
|
235
|
-
if (probe === void 0) return false;
|
|
236
|
-
appendCommandLog(jobDir, probe.bytes);
|
|
237
|
-
if (probe.bytes.length === 0 || offset + probe.bytes.length >= probe.size) return true;
|
|
238
|
-
}
|
|
239
|
-
return false;
|
|
240
|
-
}
|
|
241
|
-
async function stopMicroVmCommand(runtime, state) {
|
|
242
|
-
await withinTime(
|
|
243
|
-
(signal) => runtime.vm.exec(state.vsockPath, { argv: ["/bin/sh", "-c", STOP_COMMAND_SCRIPT], user: "root", cwd: "/", env: { PATH: GUEST_PATH } }, { signal })
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
// src/runner/microvm/hooks.ts
|
|
248
|
-
var MANAGED_NO_BUILDER_DISK_REASON = "this release of the CLI pins no builder root disk, and a managed runner builds its images in a builder microVM, never on the host. The build did not start; update the CLI";
|
|
249
|
-
var DEFAULT_MEMORY = "8g";
|
|
250
|
-
var DEFAULT_DISK = "40g";
|
|
251
|
-
var DEFAULT_CPUS = 4;
|
|
252
|
-
var BOOT_ATTEMPTS = 120;
|
|
253
|
-
var BOOT_POLL_MS = 500;
|
|
254
|
-
var HOOK_ERRORS_LIMIT = 1024 * 1024;
|
|
255
|
-
var HOOK_ERRORS_IN_GUEST = "/build/home/.outerlayer/spool/hook-errors.log";
|
|
256
|
-
var DOCKER_WAIT_SECONDS = 60;
|
|
257
|
-
var DOCKER_SOCKET = "/var/run/docker.sock";
|
|
258
|
-
var DOCKER_LOG_LINES = 20;
|
|
259
|
-
var GUEST_GH_DIR = `${JOB_MOUNT}/gh`;
|
|
260
|
-
var GUEST_HOOKS_DIR = `${JOB_MOUNT}/git-hooks`;
|
|
261
|
-
var GUEST_CONTEXT_DIR = `${JOB_MOUNT}/context`;
|
|
262
|
-
var GUEST_PATH2 = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
263
|
-
function buildMemoryBytes(config) {
|
|
264
|
-
return parseSize(config.build?.memory ?? DEFAULT_MEMORY) ?? parseSize(DEFAULT_MEMORY);
|
|
265
|
-
}
|
|
266
|
-
function machineLimits(config, jobDir) {
|
|
267
|
-
const settings = managedBuildSettingsFor(jobDir);
|
|
268
|
-
if (settings !== void 0) return managedMachineLimits(settings);
|
|
269
|
-
return {
|
|
270
|
-
cpus: config.build?.cpus ?? DEFAULT_CPUS,
|
|
271
|
-
memoryBytes: buildMemoryBytes(config),
|
|
272
|
-
diskBytes: parseSize(config.build?.disk ?? DEFAULT_DISK) ?? parseSize(DEFAULT_DISK)
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
function recordedLimits(limits) {
|
|
276
|
-
return { memoryBytes: limits.memoryBytes, diskBytes: limits.diskBytes };
|
|
277
|
-
}
|
|
278
|
-
function memoryTextOf(config, jobDir, limits) {
|
|
279
|
-
return managedBuildSettingsFor(jobDir) === void 0 ? config.build?.memory : `${limits.memoryBytes / 1024 ** 3}g`;
|
|
280
|
-
}
|
|
281
|
-
function builderMachineFor(config, jobDir, assets, limits) {
|
|
282
|
-
return {
|
|
283
|
-
kernel: assets.kernel,
|
|
284
|
-
firecracker: assets.firecracker,
|
|
285
|
-
limits: { ...limits, cacheDiskBytes: parseSize(config.build?.cacheDisk ?? DEFAULT_CACHE_DISK) ?? parseSize(DEFAULT_CACHE_DISK) },
|
|
286
|
-
memory: memoryTextOf(config, jobDir, limits),
|
|
287
|
-
allowHosts: withFactoryHosts(config.build?.allowHosts, managedBuildSettingsFor(jobDir)),
|
|
288
|
-
...optionalEgress(config),
|
|
289
|
-
managed: config.managed === true,
|
|
290
|
-
timeLimitMinutes: config.timeLimitMinutes
|
|
291
|
-
};
|
|
292
|
-
}
|
|
293
|
-
function factoryVariablesOf(config, jobDir) {
|
|
294
|
-
const held = managedBuildSettingsFor(jobDir)?.variables;
|
|
295
|
-
if (held !== void 0) return held;
|
|
296
|
-
return config.managed === true ? [] : void 0;
|
|
297
|
-
}
|
|
298
|
-
function execEnvironment(image, env) {
|
|
299
|
-
const merged = { ...image, ...env.literals };
|
|
300
|
-
for (const [name, value] of Object.entries(env.passed)) if (value !== void 0) merged[name] = value;
|
|
301
|
-
return merged;
|
|
302
|
-
}
|
|
303
|
-
function tarOf(dir) {
|
|
304
|
-
const packed = spawnSync("tar", ["-C", dir, "-cf", "-", "."], { maxBuffer: 1024 * 1024 * 1024 });
|
|
305
|
-
if (packed.status !== 0) throw new Error(`tar could not pack ${dir}: ${packed.stderr.toString("utf8").trim()}`);
|
|
306
|
-
return packed.stdout;
|
|
307
|
-
}
|
|
308
|
-
var ROOT_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
309
|
-
function guestPortFor(runtime, state) {
|
|
310
|
-
return {
|
|
311
|
-
run: (argv) => asRoot(runtime, state, argv),
|
|
312
|
-
stream: (argv, io) => runtime.vm.execStream(state.vsockPath, { argv, user: "root", cwd: "/", env: { PATH: ROOT_PATH } }, io)
|
|
313
|
-
};
|
|
314
|
-
}
|
|
315
|
-
function guestExecFor(input) {
|
|
316
|
-
return async (argv, workdir, opts) => {
|
|
317
|
-
const signal = opts?.timeoutMs === void 0 ? void 0 : AbortSignal.timeout(opts.timeoutMs);
|
|
318
|
-
const ran = await input.vm.exec(input.vsockPath, input.request(argv, workdir, opts?.variables), signal === void 0 ? void 0 : { signal });
|
|
319
|
-
if (opts?.logPath !== void 0) appendFileSync(opts.logPath, ran.stdout + ran.stderr);
|
|
320
|
-
return signal?.aborted === true ? { ...ran, timedOut: true } : ran;
|
|
321
|
-
};
|
|
322
|
-
}
|
|
323
|
-
function stepExecFor(input) {
|
|
324
|
-
const exec = guestExecFor(input);
|
|
325
|
-
return (argv, workdir, opts) => exec(argv, workdir, { ...opts?.quiet === true ? {} : { logPath: input.logPath }, timeoutMs: opts?.timeoutMs, variables: opts?.variables });
|
|
326
|
-
}
|
|
327
|
-
function asRoot(runtime, state, argv, stdin) {
|
|
328
|
-
return runtime.vm.exec(state.vsockPath, { argv, user: "root", cwd: "/", env: { PATH: ROOT_PATH } }, stdin === void 0 ? void 0 : { stdin });
|
|
329
|
-
}
|
|
330
|
-
async function writeGuestGhConfig(runtime, state, token) {
|
|
331
|
-
const script = [
|
|
332
|
-
"set -e",
|
|
333
|
-
"umask 022",
|
|
334
|
-
`mkdir -p ${GUEST_GH_DIR}`,
|
|
335
|
-
`cat > ${GUEST_GH_DIR}/hosts.yml.partial`,
|
|
336
|
-
`mv ${GUEST_GH_DIR}/hosts.yml.partial ${GUEST_GH_DIR}/hosts.yml`,
|
|
337
|
-
`printf '%s' ${shellQuote(GH_CONFIG_YML)} > ${GUEST_GH_DIR}/config.yml.partial`,
|
|
338
|
-
`mv ${GUEST_GH_DIR}/config.yml.partial ${GUEST_GH_DIR}/config.yml`
|
|
339
|
-
].join("\n");
|
|
340
|
-
const result = await asRoot(runtime, state, ["/bin/sh", "-c", script], ghHostsYml(token));
|
|
341
|
-
if (result.status !== 0) throw new Error(`writing gh's configuration in the guest failed (exit ${result.status}): ${firstLine(result.stderr)}`);
|
|
342
|
-
}
|
|
343
|
-
async function mountGuestHome(runtime, state, agentUser, known) {
|
|
344
|
-
let home = known;
|
|
345
|
-
if (home === void 0) {
|
|
346
|
-
if (agentUser === ADDED_USER) return void 0;
|
|
347
|
-
const read = await asRoot(runtime, state, [NODE_IN_GUEST, "-e", identityScript(), ...identityArgs(agentUser)]);
|
|
348
|
-
const found = parseIdentity(read.stdout);
|
|
349
|
-
if (read.status !== 0 || found === void 0) throw new Error(`reading the build's user from the image failed (exit ${read.status})${said(read.stderr)}`);
|
|
350
|
-
home = mountableHome(found?.home);
|
|
351
|
-
if (home === void 0) return void 0;
|
|
352
|
-
}
|
|
353
|
-
const copied = await runtime.vm.exec(state.vsockPath, { argv: [NODE_IN_GUEST, "-e", homeCopyScript(), ...homeCopyArgs(home)], user: agentUser, cwd: "/", env: { PATH: GUEST_PATH2, ...state.imageEnv, NODE_OPTIONS: "" } });
|
|
354
|
-
if (copied.status !== 0) throw new Error(`copying ${home} into the build's home failed (exit ${copied.status})${said(copied.stderr)}`);
|
|
355
|
-
const folder = shellQuote(home);
|
|
356
|
-
const mounted = await asRoot(runtime, state, ["/bin/sh", "-c", `[ -d ${folder} ] || exit ${NO_FOLDER}; exec ${HELPER_PATH} mount --bind --source ${BUILD_HOME} --target ${folder}`]);
|
|
357
|
-
if (mounted.status === NO_FOLDER) return void 0;
|
|
358
|
-
if (mounted.status !== 0) throw new Error(`mounting the build's home at ${home} failed (exit ${mounted.status})${said(mounted.stderr)}`);
|
|
359
|
-
return home;
|
|
360
|
-
}
|
|
361
|
-
var NODE_IN_GUEST = "/opt/outerlayer/bin/node";
|
|
362
|
-
var NO_FOLDER = 3;
|
|
363
|
-
async function unpackIntoGuest(runtime, state, dir, target, opts = {}) {
|
|
364
|
-
const script = opts.readOnly === true ? `set -e; rm -rf ${target}; mkdir -p ${target}; tar -x --no-same-owner -C ${target}; chmod -R a+rX,a-w ${target}` : `set -e; mkdir -p ${target}; tar -x --no-same-owner -C ${target}; chmod -R a+rX ${target}`;
|
|
365
|
-
const result = await asRoot(runtime, state, ["/bin/sh", "-c", script], tarOf(dir));
|
|
366
|
-
if (result.status !== 0) throw new Error(`unpacking into ${target} in the guest failed (exit ${result.status}): ${firstLine(result.stderr)}`);
|
|
367
|
-
}
|
|
368
|
-
async function deliverCommitHooks(input) {
|
|
369
|
-
await unpackIntoGuest(input.runtime, input.state, input.hooksDir, GUEST_HOOKS_DIR);
|
|
370
|
-
}
|
|
371
|
-
function vmAlive(runtime, state) {
|
|
372
|
-
if (state.jailed !== void 0) return runtime.vm.helper.alive(state.jailed.unit);
|
|
373
|
-
return state.firecracker !== void 0 && runtime.vm.firecracker.alive(state.firecracker);
|
|
374
|
-
}
|
|
375
|
-
async function helperFor(runtime, log, managed) {
|
|
376
|
-
const probe = await runtime.vm.helper.probe();
|
|
377
|
-
if (probe.ok) {
|
|
378
|
-
if (probe.firecracker !== FIRECRACKER_VERSION) log(`${helperFirecrackerNote(probe.firecracker)}; run \`outerlayer runner install --service --replace\` to update it`);
|
|
379
|
-
return probe.client;
|
|
380
|
-
}
|
|
381
|
-
if (managed) return void 0;
|
|
382
|
-
if (probe.reason === "unsupported") log(probe.detail);
|
|
383
|
-
else if (probe.unreachable === true) log(`${probe.detail}, so this build runs without the jailer`);
|
|
384
|
-
return void 0;
|
|
385
|
-
}
|
|
386
|
-
function silence(runtime, state, lastError, console) {
|
|
387
|
-
if (!vmAlive(runtime, state)) return `it exited before its exec agent answered; ${console}`;
|
|
388
|
-
const last = lastError === void 0 ? "" : `; the last connection to its exec agent failed: ${describeCallFailure(lastError)}`;
|
|
389
|
-
return `its exec agent did not answer; ${console}${last}`;
|
|
390
|
-
}
|
|
391
|
-
function consoleNote(state, consolePath) {
|
|
392
|
-
return state.jailed === void 0 ? `its console is in ${consolePath}` : `its console output is in the journal of the unit ${state.jailed.unit}`;
|
|
393
|
-
}
|
|
394
|
-
async function helperKills(runtime, state) {
|
|
395
|
-
const probe = await runtime.vm.helper.probe();
|
|
396
|
-
return probe.ok ? (await probe.client.state(state.buildId)).oomKills : 0;
|
|
397
|
-
}
|
|
398
|
-
async function microVmOomKills(runtime, state) {
|
|
399
|
-
if (state.cgroup !== void 0 && runtime.vm.cgroup.oomKills(state.cgroup) > 0) return 1;
|
|
400
|
-
if (state.jailed !== void 0) return helperKills(runtime, state);
|
|
401
|
-
return 0;
|
|
402
|
-
}
|
|
403
|
-
var socketNameOf = (file) => file.replace(/\.sock$/, "");
|
|
404
|
-
async function keepSockets(runtime, buildId, listeners, leaseServer, line, error) {
|
|
405
|
-
const store = runtime.vm.fdStore;
|
|
406
|
-
const status = await store.status();
|
|
407
|
-
if (!status.ok) {
|
|
408
|
-
line(`this build ends with its runner: ${status.reason}`);
|
|
409
|
-
return [];
|
|
410
|
-
}
|
|
411
|
-
const sockets = Object.entries(listeners);
|
|
412
|
-
if (leaseServer !== void 0) sockets.push([LEASE_LINK.socket, leaseServer]);
|
|
413
|
-
const stored = [];
|
|
414
|
-
try {
|
|
415
|
-
for (const [file, server] of sockets) {
|
|
416
|
-
const name = fdName(buildId, socketNameOf(file));
|
|
417
|
-
await store.push(name, server);
|
|
418
|
-
stored.push(name);
|
|
419
|
-
}
|
|
420
|
-
} catch (err) {
|
|
421
|
-
error(`this build ends with its runner: its sockets could not be kept for the next one: ${describeCallFailure(err)}`);
|
|
422
|
-
for (const name of stored) await store.remove(name).catch((removeErr) => error(`the descriptor ${name} could not be taken out of the store: ${describeCallFailure(removeErr)}`));
|
|
423
|
-
return [];
|
|
424
|
-
}
|
|
425
|
-
return stored;
|
|
426
|
-
}
|
|
427
|
-
async function provisionMicroVm(input) {
|
|
428
|
-
return provisionWithFreshFallback({
|
|
429
|
-
attempt: (allowRestore) => provisionMicroVmOnce(input, allowRestore),
|
|
430
|
-
log: (line) => input.log.line(`#${input.item.number} ${line}`)
|
|
431
|
-
});
|
|
432
|
-
}
|
|
433
|
-
async function provisionMicroVmOnce(input, allowRestore) {
|
|
434
|
-
const { config, runtime, item, itemKey, jobDir, log, tokens } = input;
|
|
435
|
-
const { vm } = runtime;
|
|
436
|
-
const tag = itemTag({ item: item.number, organizationId: input.organizationId, claimId: input.claimId });
|
|
437
|
-
const logPath = join(jobDir, "provision.log");
|
|
438
|
-
const fail = (reason) => ({ ok: false, reason });
|
|
439
|
-
if (!hasClaudeCredential(config, runtime.hostEnv, jobDir)) return fail(AGENT_CREDENTIAL_MISSING);
|
|
440
|
-
const repositories = repositoriesOf({ repositories: input.repositories, repository: item.specRepository, branch: input.branch });
|
|
441
|
-
const primary = primaryOf(repositories);
|
|
442
|
-
if (primary === void 0) return fail("the work item names no repository to clone");
|
|
443
|
-
const repository = primary.repository;
|
|
444
|
-
const runnerDirPath = runnerDir(config.configPath);
|
|
445
|
-
const tooDeep = socketPathProblem(buildDir(config.configPath, "0".repeat(12)));
|
|
446
|
-
if (tooDeep !== void 0) return fail(tooDeep);
|
|
447
|
-
let assets;
|
|
448
|
-
try {
|
|
449
|
-
assets = await vm.assets(runnerDirPath);
|
|
450
|
-
} catch (err) {
|
|
451
|
-
return fail(err instanceof AssetError ? err.message : `the microVM's assets could not be had: ${describeCallFailure(err)}`);
|
|
452
|
-
}
|
|
453
|
-
const governing = await resolveGoverningSource({ config, repository, fetchImpl: input.fetchImpl });
|
|
454
|
-
if (!governing.ok) return fail(governing.reason);
|
|
455
|
-
const governingSource = governing.source;
|
|
456
|
-
const controlPlaneUrl = (controlPlane) => runtime.remoteFor?.(controlPlane) ?? `${runtime.github.gitUrl}/${controlPlane}.git`;
|
|
457
|
-
if (governingSource.kind === "governed") {
|
|
458
|
-
const unfetched = await prefetchGovernedContext({
|
|
459
|
-
configPath: config.configPath,
|
|
460
|
-
...input.organizationId === void 0 ? {} : { organizationId: input.organizationId },
|
|
461
|
-
repository,
|
|
462
|
-
source: governingSource,
|
|
463
|
-
controlPlaneUrl,
|
|
464
|
-
controlPlaneToken: () => tokens.controlPlane(),
|
|
465
|
-
log,
|
|
466
|
-
tag
|
|
467
|
-
});
|
|
468
|
-
if (unfetched !== void 0) return fail(unfetched);
|
|
469
|
-
}
|
|
470
|
-
const limits = machineLimits(config, jobDir);
|
|
471
|
-
const memoryText = memoryTextOf(config, jobDir, limits);
|
|
472
|
-
let buildsInBuilder = imageBuildOf(config) === "vm";
|
|
473
|
-
if (buildsInBuilder && !vm.builderDiskPinned()) {
|
|
474
|
-
if (config.managed === true) return fail(MANAGED_NO_BUILDER_DISK_REASON);
|
|
475
|
-
log.line(`${tag} this release of the CLI pins no builder root disk, so the image builds on the host, as it does with runner.build.imageBuild set to host`);
|
|
476
|
-
buildsInBuilder = false;
|
|
477
|
-
}
|
|
478
|
-
const obtained = await obtainImage({
|
|
479
|
-
config,
|
|
480
|
-
runtime,
|
|
481
|
-
tokens,
|
|
482
|
-
repository: governingSource.kind === "governed" ? governingSource.repository : repository,
|
|
483
|
-
pinned: governingSource.kind === "governed" ? { commit: governingSource.sha, ref: governingSource.ref } : void 0,
|
|
484
|
-
governedRepository: repository,
|
|
485
|
-
...input.organizationId === void 0 ? {} : { organizationId: input.organizationId },
|
|
486
|
-
tag,
|
|
487
|
-
log,
|
|
488
|
-
aborted: input.aborted,
|
|
489
|
-
deadlineMs: input.deadlineMs,
|
|
490
|
-
now: input.now,
|
|
491
|
-
guestHelper: { path: assets.helper, sha256: assets.helperSha256 },
|
|
492
|
-
...buildsInBuilder ? { inBuilder: builderMachineFor(config, jobDir, assets, limits) } : {}
|
|
493
|
-
});
|
|
494
|
-
if (!obtained.ok) return obtained.aborted === true ? { ok: false, reason: obtained.reason, aborted: true } : fail(obtained.reason);
|
|
495
|
-
const { report } = obtained.built;
|
|
496
|
-
const tagged = taggedLog(log, tag);
|
|
497
|
-
const plan = await planSavedState({
|
|
498
|
-
config,
|
|
499
|
-
repositories,
|
|
500
|
-
base: savedStateBase(report),
|
|
501
|
-
runtime,
|
|
502
|
-
tokens,
|
|
503
|
-
now: clockOf(input.now)(),
|
|
504
|
-
allowRestore,
|
|
505
|
-
log: tagged
|
|
506
|
-
});
|
|
507
|
-
const savedStatesPath = savedStatesDir(runnerDirPath);
|
|
508
|
-
const madeInBuilder = "rootfs" in obtained.built ? obtained.built.rootfs : void 0;
|
|
509
|
-
const buildId = randomBytes(6).toString("hex");
|
|
510
|
-
const buildDir2 = buildDir(config.configPath, buildId);
|
|
511
|
-
const state = {
|
|
512
|
-
buildId,
|
|
513
|
-
buildDir: buildDir2,
|
|
514
|
-
repository,
|
|
515
|
-
imageId: report.imageId,
|
|
516
|
-
containerEnv: report.apply.containerEnv,
|
|
517
|
-
builtIns: true,
|
|
518
|
-
limits: recordedLimits(limits),
|
|
519
|
-
lifecycle: report.apply.lifecycle,
|
|
520
|
-
...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository, ref: governingSource.ref, commit: governingSource.sha } } : {},
|
|
521
|
-
...report.docker === true ? { docker: true } : {}
|
|
522
|
-
};
|
|
523
|
-
const envInput = {
|
|
524
|
-
config,
|
|
525
|
-
item: item.number,
|
|
526
|
-
itemKey,
|
|
527
|
-
repository,
|
|
528
|
-
repositories,
|
|
529
|
-
hostEnv: runtime.hostEnv,
|
|
530
|
-
containerEnv: report.apply.containerEnv,
|
|
531
|
-
builtIns: true,
|
|
532
|
-
factoryVariables: factoryVariablesOf(config, jobDir)
|
|
533
|
-
};
|
|
534
|
-
const env = buildEnvironment(envInput);
|
|
535
|
-
if (env.skippedTokenVariables.length > 0) {
|
|
536
|
-
log.line(`${tag} not passing ${env.skippedTokenVariables.join(", ")} from runner.build.variables into the build: its repository token comes from the runner`);
|
|
537
|
-
}
|
|
538
|
-
const environment = {
|
|
539
|
-
isolation: "vm",
|
|
540
|
-
recipe: { sourceCommit: report.sourceCommit, key: report.recipeKey, path: report.recipe },
|
|
541
|
-
imageId: report.imageId,
|
|
542
|
-
ignoredProperties: report.ignored.map(({ property, source }) => ({ property, source })),
|
|
543
|
-
variables: env.variables,
|
|
544
|
-
cliVersion: input.cliVersion
|
|
545
|
-
};
|
|
546
|
-
input.save(state, environment);
|
|
547
|
-
const bail = async (reason, aborted = false) => {
|
|
548
|
-
let why = reason;
|
|
549
|
-
if (!aborted && (state.cgroup !== void 0 || state.jailed !== void 0)) {
|
|
550
|
-
try {
|
|
551
|
-
if (await microVmOomKills(runtime, state) > 0) why = outOfMemoryReason(memoryText);
|
|
552
|
-
} catch (err) {
|
|
553
|
-
log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
try {
|
|
557
|
-
await cleanupMicroVm(runtime, state);
|
|
558
|
-
} catch (err) {
|
|
559
|
-
log.error(`${tag} cleanup failed: ${describeCallFailure(err)}`);
|
|
560
|
-
}
|
|
561
|
-
return aborted ? { ok: false, reason: "ended before provision finished", aborted: true } : fail(why);
|
|
562
|
-
};
|
|
563
|
-
const giveUpState = async (reason) => {
|
|
564
|
-
await bail(reason);
|
|
565
|
-
return discardUnusableState({ record: plan.restore.record, reason, dir: savedStatesPath, removeVolume: async () => void 0, onError: (line) => log.error(`${tag} ${line}`) });
|
|
566
|
-
};
|
|
567
|
-
if (input.aborted()) return bail("", true);
|
|
568
|
-
let imageConfig;
|
|
569
|
-
if (madeInBuilder !== void 0) {
|
|
570
|
-
state.rootfs = madeInBuilder.path;
|
|
571
|
-
imageConfig = { env: madeInBuilder.env, user: madeInBuilder.user };
|
|
572
|
-
} else {
|
|
573
|
-
try {
|
|
574
|
-
state.rootfs = await ensureRootfs({ host: vm.rootfs, runnerDir: orgStoreIn(runnerDirPath, input.organizationId), imageId: report.imageId, memoryBytes: limits.memoryBytes });
|
|
575
|
-
imageConfig = await vm.rootfs.imageConfig(report.imageId);
|
|
576
|
-
} catch (err) {
|
|
577
|
-
return bail(`the image could not be converted to a root disk: ${describeCallFailure(err)}`);
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
state.imageEnv = imageConfig.env;
|
|
581
|
-
state.buildDisk = join(buildDir2, "build.ext4");
|
|
582
|
-
state.vsockPath = vsockPath(buildDir2);
|
|
583
|
-
input.save(state, environment);
|
|
584
|
-
if (input.aborted()) return bail("", true);
|
|
585
|
-
mkdirSync(buildDir2, { recursive: true, mode: 448 });
|
|
586
|
-
chmodSync(buildDir2, 448);
|
|
587
|
-
makeBuildsDirPrivate(buildDir2);
|
|
588
|
-
let services;
|
|
589
|
-
try {
|
|
590
|
-
const gitRemote = gitRemoteFor(runtime, { repositories, tokens, buildDir: buildDir2, organizationId: input.organizationId, log: (line) => log.line(`${tag} ${line}`) });
|
|
591
|
-
services = await startServicesFor(runtime, config, buildDir2, jobDir, itemKey, true, gitRemote, "socket", void 0, void 0, (line) => log.line(`${tag} ${line}`));
|
|
592
|
-
runtime.tunnels.set(buildId, services);
|
|
593
|
-
} catch (err) {
|
|
594
|
-
return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
|
|
595
|
-
}
|
|
596
|
-
const links = vsockLinks({ destinations: config.build?.destinations?.length ?? 0, builtIns: true });
|
|
597
|
-
const served = input.leaseTimeLeftMs === void 0 ? links : [...links, LEASE_LINK];
|
|
598
|
-
let leaseServer;
|
|
599
|
-
if (input.leaseTimeLeftMs !== void 0) {
|
|
600
|
-
try {
|
|
601
|
-
const channel = createLeaseChannel(input.leaseTimeLeftMs);
|
|
602
|
-
const served2 = await serveLease(join(buildDir2, LEASE_LINK.socket), { ...channel, accept: (stream) => runtime.leases.get(buildId)?.accept(stream) });
|
|
603
|
-
runtime.leases.set(buildId, { ...channel, close: served2.close });
|
|
604
|
-
leaseServer = served2.listener;
|
|
605
|
-
} catch (err) {
|
|
606
|
-
return bail(`the lease stream could not start: ${describeCallFailure(err)}`);
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
const configPath = join(buildDir2, "vm.json");
|
|
610
|
-
const helper = await helperFor(runtime, (line) => log.line(`${tag} ${line}`), config.managed === true);
|
|
611
|
-
if (helper === void 0 && config.managed === true) return bail(MANAGED_UNJAILED_REASON);
|
|
612
|
-
try {
|
|
613
|
-
if (helper === void 0) linkVsockSockets({ buildDir: buildDir2, vsockPath: state.vsockPath, links: served });
|
|
614
|
-
await createBuildDisk({ host: vm.rootfs, runnerDir: runnerDirPath, bytes: limits.diskBytes, dest: state.buildDisk, memoryBytes: limits.memoryBytes });
|
|
615
|
-
writeFileSync(
|
|
616
|
-
configPath,
|
|
617
|
-
JSON.stringify(
|
|
618
|
-
machineConfig(limits, {
|
|
619
|
-
kernel: assets.kernel,
|
|
620
|
-
rootfs: state.rootfs,
|
|
621
|
-
buildDisk: state.buildDisk,
|
|
622
|
-
vsock: state.vsockPath,
|
|
623
|
-
relayPorts: links.map((link) => link.port),
|
|
624
|
-
...input.leaseTimeLeftMs === void 0 ? {} : { leasePort: LEASE_LINK.port },
|
|
625
|
-
docker: report.docker
|
|
626
|
-
}),
|
|
627
|
-
null,
|
|
628
|
-
2
|
|
629
|
-
),
|
|
630
|
-
{ mode: 384 }
|
|
631
|
-
);
|
|
632
|
-
} catch (err) {
|
|
633
|
-
return bail(`the build's disk and machine could not be made: ${describeCallFailure(err)}`);
|
|
634
|
-
}
|
|
635
|
-
if (input.aborted()) return bail("", true);
|
|
636
|
-
if (helper === void 0) {
|
|
637
|
-
try {
|
|
638
|
-
state.cgroup = vm.cgroup.create(buildId, limits.memoryBytes);
|
|
639
|
-
} catch (err) {
|
|
640
|
-
return bail(`the build's memory limit could not be set: ${describeCallFailure(err)}`);
|
|
641
|
-
}
|
|
642
|
-
if (state.cgroup !== void 0) input.save(state, environment);
|
|
643
|
-
}
|
|
644
|
-
const consolePath = jobConsolePath(jobDir);
|
|
645
|
-
if (helper !== void 0) {
|
|
646
|
-
try {
|
|
647
|
-
await startJailedInto(
|
|
648
|
-
state,
|
|
649
|
-
{ client: helper, kind: "build", kernel: assets.kernel, rootfs: state.rootfs, sockets: served.map((link) => link.socket), timeLimitMinutes: config.timeLimitMinutes },
|
|
650
|
-
() => input.save(state, environment)
|
|
651
|
-
);
|
|
652
|
-
} catch (err) {
|
|
653
|
-
return bail(`the microVM did not start under the jailer: ${describeCallFailure(err)}`);
|
|
654
|
-
}
|
|
655
|
-
const kept = await keepSockets(runtime, buildId, services.listeners, leaseServer, (line) => log.line(`${tag} ${line}`), (line) => log.error(`${tag} ${line}`));
|
|
656
|
-
if (kept.length > 0) {
|
|
657
|
-
state.fdStore = kept;
|
|
658
|
-
input.save(state, environment);
|
|
659
|
-
}
|
|
660
|
-
} else {
|
|
661
|
-
try {
|
|
662
|
-
state.firecracker = await vm.firecracker.spawn({ binary: assets.firecracker, configPath, consolePath, cgroup: state.cgroup });
|
|
663
|
-
} catch (err) {
|
|
664
|
-
return bail(`Firecracker did not start: ${describeCallFailure(err)}`);
|
|
665
|
-
}
|
|
666
|
-
input.save(state, environment);
|
|
667
|
-
}
|
|
668
|
-
if (input.aborted()) return bail("", true);
|
|
669
|
-
let lastError;
|
|
670
|
-
const ready = await waitForAgent(state.vsockPath, { attempts: BOOT_ATTEMPTS, delayMs: BOOT_POLL_MS, sleep: runtime.sleep, onFailure: (err) => lastError = err });
|
|
671
|
-
if (!ready) {
|
|
672
|
-
return bail(`the microVM did not start: ${silence(runtime, state, lastError, consoleNote(state, consolePath))}`);
|
|
673
|
-
}
|
|
674
|
-
let pushes = Promise.resolve();
|
|
675
|
-
let firstWriteFailure;
|
|
676
|
-
let firstWriteSettled = false;
|
|
677
|
-
const reportWrite = (err) => {
|
|
678
|
-
if (!firstWriteSettled && firstWriteFailure === void 0) firstWriteFailure = err;
|
|
679
|
-
log.error(`${tag} the read token could not be written for gh in the guest: ${describeCallFailure(err)}`);
|
|
680
|
-
};
|
|
681
|
-
try {
|
|
682
|
-
await tokens.token("read");
|
|
683
|
-
} catch (err) {
|
|
684
|
-
return bail(
|
|
685
|
-
err instanceof WorkCommandError ? `the build's read token could not be had: ${refusalCode(err)}` : `the build's gh configuration could not be written: ${describeCallFailure(err)}`
|
|
686
|
-
);
|
|
687
|
-
}
|
|
688
|
-
for (const entry of repositories) {
|
|
689
|
-
if (entry.repository === primary.repository) continue;
|
|
690
|
-
try {
|
|
691
|
-
await tokens.token("read", entry.repository);
|
|
692
|
-
} catch (err) {
|
|
693
|
-
return bail(`the build's read token for ${entry.repository} could not be had: ${refusalCode(err)}`);
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
tokens.onRead((token) => {
|
|
697
|
-
pushes = pushes.then(async () => {
|
|
698
|
-
if (!vmAlive(runtime, state)) return;
|
|
699
|
-
await writeGuestGhConfig(runtime, state, token);
|
|
700
|
-
}, void 0).catch(reportWrite);
|
|
701
|
-
});
|
|
702
|
-
await pushes;
|
|
703
|
-
firstWriteSettled = true;
|
|
704
|
-
if (firstWriteFailure !== void 0) return bail(`the build's gh configuration could not be written: ${describeCallFailure(firstWriteFailure)}`);
|
|
705
|
-
tokens.markUsed("read");
|
|
706
|
-
environment.credentials = tokens.used();
|
|
707
|
-
const uidOf = async (user) => {
|
|
708
|
-
const probe = await asRoot(runtime, state, ["id", "-u", user]);
|
|
709
|
-
return uidFromProbe(probe);
|
|
710
|
-
};
|
|
711
|
-
let agentUser;
|
|
712
|
-
try {
|
|
713
|
-
agentUser = await resolveAgentUser({ remoteUser: report.apply.remoteUser, imageUser: imageConfig.user, probe: uidOf });
|
|
714
|
-
} catch (err) {
|
|
715
|
-
if (err instanceof AgentUserError) return bail(err.message);
|
|
716
|
-
throw err;
|
|
717
|
-
}
|
|
718
|
-
state.agentUser = agentUser;
|
|
719
|
-
try {
|
|
720
|
-
const home = await mountGuestHome(runtime, state, agentUser);
|
|
721
|
-
if (home !== void 0) state.home = home;
|
|
722
|
-
} catch (err) {
|
|
723
|
-
return bail(describeCallFailure(err));
|
|
724
|
-
}
|
|
725
|
-
const agentUid = agentUidOf(await asRoot(runtime, state, ["id", "-u", agentUser]));
|
|
726
|
-
const guest = guestPortFor(runtime, state);
|
|
727
|
-
if (plan.restore !== void 0) {
|
|
728
|
-
const changed = userIdChangedReason(plan.restore.record.agentUid, agentUid);
|
|
729
|
-
if (changed !== void 0) return giveUpState(changed);
|
|
730
|
-
const restored = await restoreMicroVmState({ guest, record: plan.restore.record });
|
|
731
|
-
if (!restored.ok) return giveUpState(restored.reason);
|
|
732
|
-
}
|
|
733
|
-
const tools = await asRoot(runtime, state, ["cat", "/opt/outerlayer/tools.json"]);
|
|
734
|
-
const agentVersion = agentVersionOf(tools.status === 0 ? tools.stdout : "");
|
|
735
|
-
environment.agentVersion = agentVersion;
|
|
736
|
-
const program = await asRoot(runtime, state, [...HASH_ARGV]);
|
|
737
|
-
const hashed = program.status === 0 ? parseProgramHash(program.stdout) : void 0;
|
|
738
|
-
if (hashed !== void 0) state.judgeProgram = { path: JUDGE_PROGRAM, sha256: hashed };
|
|
739
|
-
input.save(state, environment);
|
|
740
|
-
if (report.docker === true) {
|
|
741
|
-
const failure = await waitForDocker(runtime, state, agentUser, input.now ?? (() => /* @__PURE__ */ new Date()));
|
|
742
|
-
if (failure !== void 0) return bail(failure);
|
|
743
|
-
}
|
|
744
|
-
const saveState = () => saveAndPrune({
|
|
745
|
-
save: () => saveMicroVmState({ guest, dir: savedStatesPath, key: plan.key, agentUid, now: clockOf(input.now)(), log: tagged }),
|
|
746
|
-
dir: savedStatesPath,
|
|
747
|
-
removeVolume: async () => void 0,
|
|
748
|
-
log: tagged
|
|
749
|
-
});
|
|
750
|
-
const request = (argv, workdir, variables) => ({
|
|
751
|
-
argv,
|
|
752
|
-
user: agentUser,
|
|
753
|
-
cwd: workdir,
|
|
754
|
-
env: execEnvironment(state.imageEnv, withMiseVariables(env, variables))
|
|
755
|
-
});
|
|
756
|
-
const steps = await runBuildSteps({
|
|
757
|
-
repository,
|
|
758
|
-
branch: primary.branch,
|
|
759
|
-
repositories,
|
|
760
|
-
...governingSource.kind === "governed" ? { governing: { repository: governingSource.repository } } : {},
|
|
761
|
-
agentVersion,
|
|
762
|
-
lifecycle: report.apply.lifecycle,
|
|
763
|
-
exec: stepExecFor({ vm, vsockPath: state.vsockPath, request, logPath }),
|
|
764
|
-
log,
|
|
765
|
-
tag,
|
|
766
|
-
aborted: input.aborted,
|
|
767
|
-
stageContext: () => stageGovernedContext({
|
|
768
|
-
config,
|
|
769
|
-
...input.organizationId === void 0 ? {} : { organizationId: input.organizationId },
|
|
770
|
-
repository,
|
|
771
|
-
source: pinnedSourceOf(governingSource),
|
|
772
|
-
buildId,
|
|
773
|
-
buildDir: buildDir2,
|
|
774
|
-
fetchImpl: input.fetchImpl,
|
|
775
|
-
controlPlaneUrl,
|
|
776
|
-
controlPlaneToken: () => tokens.controlPlane(),
|
|
777
|
-
log,
|
|
778
|
-
tag
|
|
779
|
-
}),
|
|
780
|
-
// The guest cannot read the build's directory, so the staged files are unpacked into its job directory.
|
|
781
|
-
deliverContext: async (staged) => {
|
|
782
|
-
try {
|
|
783
|
-
await unpackIntoGuest(runtime, state, staged.dir, GUEST_CONTEXT_DIR);
|
|
784
|
-
return { ok: true };
|
|
785
|
-
} catch (err) {
|
|
786
|
-
return { ok: false, reason: `context_unavailable: ${describeCallFailure(err)}` };
|
|
787
|
-
}
|
|
788
|
-
},
|
|
789
|
-
removeStagedContext: () => removeStagedContext(buildDir2),
|
|
790
|
-
keepIn: jobDir,
|
|
791
|
-
...input.requireBuildSkill === true ? { requireBuildSkill: true } : {},
|
|
792
|
-
// A build on the universal image, or on an image built from it, chooses and installs its runtimes.
|
|
793
|
-
...runtimeStepInput(report, services),
|
|
794
|
-
setup: setupInputFor({ setupMinutes: config.build?.setupMinutes, plan, saveState })
|
|
795
|
-
});
|
|
796
|
-
if (!steps.ok) return settleFailedSteps(steps, { giveUp: giveUpState, bail });
|
|
797
|
-
state.startCommit = steps.startCommit;
|
|
798
|
-
keepRuntimeStep(state, steps);
|
|
799
|
-
keepMissingSetupScripts(state, steps);
|
|
800
|
-
if (steps.controlPlane !== void 0) environment.controlPlane = steps.controlPlane;
|
|
801
|
-
return { ok: true, workdir: CHECKOUT_DIR, state, environment };
|
|
802
|
-
}
|
|
803
|
-
async function waitForDocker(runtime, state, agentUser, now) {
|
|
804
|
-
const grant = `[ -S ${DOCKER_SOCKET} ] && chgrp "$(id -g ${shellQuote(agentUser)})" ${DOCKER_SOCKET} && chmod 660 ${DOCKER_SOCKET}`;
|
|
805
|
-
const request = { argv: ["timeout", "5", "docker", "info"], user: agentUser, cwd: "/", env: { PATH: ROOT_PATH } };
|
|
806
|
-
const deadline = now().getTime() + DOCKER_WAIT_SECONDS * 1e3;
|
|
807
|
-
let lastError = "";
|
|
808
|
-
for (; ; ) {
|
|
809
|
-
const granted = await asRoot(runtime, state, ["/bin/sh", "-c", grant]);
|
|
810
|
-
if (granted.status !== 0) {
|
|
811
|
-
if (firstLine(granted.stderr) !== "") lastError = `giving the agent's group the Docker socket failed: ${firstLine(granted.stderr)}`;
|
|
812
|
-
} else {
|
|
813
|
-
const answered = await runtime.vm.exec(state.vsockPath, request);
|
|
814
|
-
if (answered.status === 0) return writeDockerClientConfig(asRoot.bind(null, runtime, state), agentUser);
|
|
815
|
-
lastError = firstLine(answered.stderr) === "" ? "" : `docker info said: ${firstLine(answered.stderr)}`;
|
|
816
|
-
}
|
|
817
|
-
if (now().getTime() >= deadline) break;
|
|
818
|
-
await runtime.sleep(1e3);
|
|
819
|
-
}
|
|
820
|
-
const tail = await asRoot(runtime, state, ["tail", "-n", String(DOCKER_LOG_LINES), DOCKERD_LOG]).catch(() => void 0);
|
|
821
|
-
const log = tail !== void 0 && tail.status === 0 ? tail.stdout.trim() : "";
|
|
822
|
-
const detail = [lastError, log === "" ? "" : `the daemon's log ends: ${log}`].filter((part) => part !== "");
|
|
823
|
-
return `the Docker daemon in the microVM did not answer docker info within ${DOCKER_WAIT_SECONDS} seconds${detail.length === 0 ? "" : `; ${detail.join("; ")}`}`;
|
|
824
|
-
}
|
|
825
|
-
function agentVersionOf(toolsJson) {
|
|
826
|
-
try {
|
|
827
|
-
const version = JSON.parse(toolsJson).claudeCodeVersion;
|
|
828
|
-
if (typeof version === "string" && version !== "") return version;
|
|
829
|
-
} catch {
|
|
830
|
-
}
|
|
831
|
-
return CLAUDE_CODE_VERSION;
|
|
832
|
-
}
|
|
833
|
-
function vmExecFor(input, argv) {
|
|
834
|
-
const state = input.job.microvm;
|
|
835
|
-
const built = buildEnvironment({
|
|
836
|
-
config: claimCalls(input.config, input.job),
|
|
837
|
-
item: input.job.item,
|
|
838
|
-
itemKey: input.job.itemKey,
|
|
839
|
-
branch: input.job.branch,
|
|
840
|
-
repository: state.repository,
|
|
841
|
-
repositories: input.job.repositories,
|
|
842
|
-
hostEnv: input.runtime.hostEnv,
|
|
843
|
-
containerEnv: state.containerEnv ?? {},
|
|
844
|
-
unpinnedRuntimes: state.unpinnedRuntimes,
|
|
845
|
-
missingSetupScripts: state.missingSetupScripts,
|
|
846
|
-
miseVariables: state.miseVariables,
|
|
847
|
-
builtIns: state.builtIns === true,
|
|
848
|
-
identity: input.job.commitIdentity,
|
|
849
|
-
factoryVariables: factoryVariablesOf(input.config, input.jobDir)
|
|
850
|
-
});
|
|
851
|
-
const env = withExtraEnv(built, input.extraEnv);
|
|
852
|
-
const options = {
|
|
853
|
-
socket: state.vsockPath,
|
|
854
|
-
user: state.agentUser ?? ADDED_USER,
|
|
855
|
-
cwd: CHECKOUT_DIR,
|
|
856
|
-
env: { ...state.imageEnv, ...env.literals },
|
|
857
|
-
pass: Object.keys(env.passed).filter((name) => env.passed[name] !== void 0),
|
|
858
|
-
argv
|
|
859
|
-
};
|
|
860
|
-
return { options, env: { ...childEnvironment(input.runtime.hostEnv), ...env.passed } };
|
|
861
|
-
}
|
|
862
|
-
function withExtraEnv(env, extra) {
|
|
863
|
-
return { ...env, literals: { ...env.literals, ...extra } };
|
|
864
|
-
}
|
|
865
|
-
function microVmCommandLine(input) {
|
|
866
|
-
const { options, env } = vmExecFor(input, ["/bin/sh", "-c", commandWithShim(input.job, input.cmdline)]);
|
|
867
|
-
return { line: [process.execPath, input.cli, "vm-exec", ...vmExecArgs(options)].map(shellQuote).join(" "), options, env };
|
|
868
|
-
}
|
|
869
|
-
async function startMicroVmCommand(input) {
|
|
870
|
-
const state = input.job.microvm;
|
|
871
|
-
const env = withExtraEnv(buildEnvironment({
|
|
872
|
-
config: claimCalls(input.config, input.job),
|
|
873
|
-
item: input.job.item,
|
|
874
|
-
itemKey: input.job.itemKey,
|
|
875
|
-
branch: input.job.branch,
|
|
876
|
-
repository: state.repository,
|
|
877
|
-
repositories: input.job.repositories,
|
|
878
|
-
hostEnv: input.runtime.hostEnv,
|
|
879
|
-
containerEnv: state.containerEnv ?? {},
|
|
880
|
-
unpinnedRuntimes: state.unpinnedRuntimes,
|
|
881
|
-
missingSetupScripts: state.missingSetupScripts,
|
|
882
|
-
miseVariables: state.miseVariables,
|
|
883
|
-
builtIns: state.builtIns === true,
|
|
884
|
-
identity: input.job.commitIdentity,
|
|
885
|
-
factoryVariables: factoryVariablesOf(input.config, input.jobDir)
|
|
886
|
-
}), input.extraEnv);
|
|
887
|
-
const started = await input.runtime.vm.exec(state.vsockPath, {
|
|
888
|
-
argv: ["/bin/sh", "-c", commandWithShim(input.job, input.cmdline)],
|
|
889
|
-
user: state.agentUser ?? ADDED_USER,
|
|
890
|
-
cwd: CHECKOUT_DIR,
|
|
891
|
-
env: execEnvironment(state.imageEnv, env),
|
|
892
|
-
detach: state.commandDir ?? MICROVM_COMMAND_DIR
|
|
893
|
-
});
|
|
894
|
-
if (started.status !== 0) {
|
|
895
|
-
const said2 = firstLine(started.stderr);
|
|
896
|
-
throw new Error(`the guest's exec agent exited ${started.status}${said2 === "" ? "" : `: ${said2}`}`);
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
async function endMicroVmCommand(runtime, state, jobDir, onError) {
|
|
900
|
-
if (!vmAlive(runtime, state)) return;
|
|
901
|
-
await copyMicroVmLogTail(runtime, state, jobDir).catch((err) => onError(`copying the end of the command's log failed: ${describeCallFailure(err)} \xB7 continuing`));
|
|
902
|
-
await stopMicroVmCommand(runtime, state).catch((err) => onError(`ending the command in the microVM failed: ${describeCallFailure(err)} \xB7 continuing`));
|
|
903
|
-
}
|
|
904
|
-
function microVmSenderCommand(input) {
|
|
905
|
-
const { options, env } = vmExecFor(input, ["outerlayer", "session-sender"]);
|
|
906
|
-
return { command: process.execPath, args: [input.cli, "vm-exec", ...vmExecArgs(options)], env };
|
|
907
|
-
}
|
|
908
|
-
function isRunning(runtime, state) {
|
|
909
|
-
return vmAlive(runtime, state);
|
|
910
|
-
}
|
|
911
|
-
async function syncInMicroVm(input) {
|
|
912
|
-
return runCliInMicroVm(input, ["outerlayer", "sync"], "sync.log", "the microVM is gone, so its sessions cannot be read");
|
|
913
|
-
}
|
|
914
|
-
async function recordProofsInMicroVm(input) {
|
|
915
|
-
return runCliInMicroVm(input, [...RECORD_PROOFS_ARGV], PROOFS_LOG, "the microVM is gone, so its proofs cannot be recorded");
|
|
916
|
-
}
|
|
917
|
-
async function runCliInMicroVm(input, argv, logName, goneReason) {
|
|
918
|
-
const { runtime, job } = input;
|
|
919
|
-
const state = job.microvm;
|
|
920
|
-
if (!isRunning(runtime, state)) return { ok: false, reason: goneReason };
|
|
921
|
-
try {
|
|
922
|
-
const env = buildEnvironment({
|
|
923
|
-
config: claimCalls(input.config, job),
|
|
924
|
-
item: job.item,
|
|
925
|
-
itemKey: job.itemKey,
|
|
926
|
-
branch: job.branch,
|
|
927
|
-
repository: state.repository,
|
|
928
|
-
repositories: job.repositories,
|
|
929
|
-
hostEnv: runtime.hostEnv,
|
|
930
|
-
containerEnv: state.containerEnv ?? {},
|
|
931
|
-
unpinnedRuntimes: state.unpinnedRuntimes,
|
|
932
|
-
missingSetupScripts: state.missingSetupScripts,
|
|
933
|
-
miseVariables: state.miseVariables,
|
|
934
|
-
builtIns: state.builtIns === true,
|
|
935
|
-
identity: job.commitIdentity,
|
|
936
|
-
factoryVariables: factoryVariablesOf(input.config, input.jobDir)
|
|
937
|
-
});
|
|
938
|
-
const result = await runtime.vm.exec(state.vsockPath, {
|
|
939
|
-
argv,
|
|
940
|
-
user: state.agentUser ?? ADDED_USER,
|
|
941
|
-
cwd: job.workdir ?? CHECKOUT_DIR,
|
|
942
|
-
env: execEnvironment(state.imageEnv, env)
|
|
943
|
-
});
|
|
944
|
-
appendFileSync(join(input.jobDir, logName), result.stdout + result.stderr);
|
|
945
|
-
return result.status === 0 ? { ok: true } : { ok: false, reason: `exit ${result.status}` };
|
|
946
|
-
} catch (err) {
|
|
947
|
-
return { ok: false, reason: describeCallFailure(err) };
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
async function copyHookErrors(runtime, state, jobDir) {
|
|
951
|
-
if (!isRunning(runtime, state)) return;
|
|
952
|
-
try {
|
|
953
|
-
const log = await asRoot(runtime, state, ["head", "-c", String(HOOK_ERRORS_LIMIT), HOOK_ERRORS_IN_GUEST]);
|
|
954
|
-
if (log.status === 0 && log.stdout !== "") writeFileSync(join(jobDir, "hook-errors.log"), log.stdout);
|
|
955
|
-
} catch {
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
async function copyTranscriptsFromMicroVm(runtime, state) {
|
|
959
|
-
if (!isRunning(runtime, state)) return { ok: false, reason: "the build's microVM is gone" };
|
|
960
|
-
return copyTranscripts((argv) => asRoot(runtime, state, argv));
|
|
961
|
-
}
|
|
962
|
-
function guestStopScript(paths = { proc: "/proc", record: HELPER_RECORD }) {
|
|
963
|
-
return [
|
|
964
|
-
`record=${shellQuote(paths.record)}`,
|
|
965
|
-
'[ -r "$record" ] || exit 2',
|
|
966
|
-
'keep=" "',
|
|
967
|
-
'while read -r kpid kstart; do keep="$keep$kpid:$kstart "; done < "$record"',
|
|
968
|
-
"n=0",
|
|
969
|
-
"while [ $n -lt 50 ]; do",
|
|
970
|
-
' left=""',
|
|
971
|
-
// The list of /proc entries is made once per pass, so the commands this script runs are not in it.
|
|
972
|
-
` for p in ${shellQuote(paths.proc)}/[0-9]*; do`,
|
|
973
|
-
" pid=${p##*/}",
|
|
974
|
-
' [ "$pid" = 1 ] && continue',
|
|
975
|
-
' [ "$pid" = "$$" ] && continue',
|
|
976
|
-
' stat=$(cat "$p/stat" 2>/dev/null) || continue',
|
|
977
|
-
// Field 2 of stat is the command name, which can hold spaces, so the fields are counted from its closing ")":
|
|
978
|
-
// $1 is the state, $7 the flags, ${18} the thread count and ${20} the start time.
|
|
979
|
-
' set -- ${stat##*") "}',
|
|
980
|
-
// 0x200000 is PF_KTHREAD.
|
|
981
|
-
" [ $(($7 & 2097152)) -ne 0 ] && continue",
|
|
982
|
-
' [ "$1" = Z ] && [ "${18}" -le 1 ] && continue',
|
|
983
|
-
' case "$keep" in *" $pid:${20} "*) continue ;; esac',
|
|
984
|
-
' left="$left $pid"',
|
|
985
|
-
" done",
|
|
986
|
-
' [ -z "$left" ] && exit 0',
|
|
987
|
-
" for pid in $left; do kill -KILL $pid 2>/dev/null; done",
|
|
988
|
-
" n=$((n+1))",
|
|
989
|
-
" sleep 0.2",
|
|
990
|
-
"done",
|
|
991
|
-
"exit 1"
|
|
992
|
-
].join("\n");
|
|
993
|
-
}
|
|
994
|
-
var GUEST_JUDGE_DIR = JUDGE_DIR;
|
|
995
|
-
function microVmJudgeEnvironment(input) {
|
|
996
|
-
const { runtime, job } = input;
|
|
997
|
-
const state = job.microvm;
|
|
998
|
-
const agentUser = state.agentUser ?? ADDED_USER;
|
|
999
|
-
const needsVm = () => {
|
|
1000
|
-
if ((state.firecracker !== void 0 || state.jailed !== void 0) && !isRunning(runtime, state)) throw new Error("the build's microVM is gone");
|
|
1001
|
-
};
|
|
1002
|
-
const readJudgeFile = async (name, maxBytes) => {
|
|
1003
|
-
needsVm();
|
|
1004
|
-
const file = `${GUEST_JUDGE_DIR}/${name}`;
|
|
1005
|
-
const read = await asRoot(runtime, state, ["/bin/sh", "-c", `[ -f ${file} ] && [ ! -L ${file} ] && head -c ${maxBytes + 1} ${file}`]);
|
|
1006
|
-
return read.status === 0 && Buffer.byteLength(read.stdout) <= maxBytes ? read.stdout : void 0;
|
|
1007
|
-
};
|
|
1008
|
-
return {
|
|
1009
|
-
stop: async () => {
|
|
1010
|
-
needsVm();
|
|
1011
|
-
const stopped = await asRoot(runtime, state, ["/bin/sh", "-c", guestStopScript()]);
|
|
1012
|
-
if (stopped.status === 2) throw new Error(`the guest has no record of the runner's helper processes at ${HELPER_RECORD}, so nothing was stopped`);
|
|
1013
|
-
if (stopped.status !== 0) throw new Error(`processes of the build were still running after they were killed (exit ${stopped.status})`);
|
|
1014
|
-
},
|
|
1015
|
-
hashProgram: async () => {
|
|
1016
|
-
needsVm();
|
|
1017
|
-
const ran = await asRoot(runtime, state, [...HASH_ARGV]);
|
|
1018
|
-
const sha256 = ran.status === 0 ? parseProgramHash(ran.stdout) : void 0;
|
|
1019
|
-
return sha256 === void 0 ? { ok: false, reason: firstLine(ran.stderr) || `the hash could not be taken (exit ${ran.status})` } : { ok: true, sha256 };
|
|
1020
|
-
},
|
|
1021
|
-
prepare: async (files, tree) => {
|
|
1022
|
-
needsVm();
|
|
1023
|
-
const staging = join(input.jobDir, "judge-input");
|
|
1024
|
-
rmSync(staging, { recursive: true, force: true });
|
|
1025
|
-
mkdirSync(staging, { recursive: true });
|
|
1026
|
-
try {
|
|
1027
|
-
for (const [fileName, text2] of Object.entries(files)) writeFileSync(join(staging, fileName), text2);
|
|
1028
|
-
if (tree !== void 0) copyJudgeTree(tree, join(staging, "tree"));
|
|
1029
|
-
const dir = GUEST_JUDGE_DIR;
|
|
1030
|
-
const script = [
|
|
1031
|
-
"set -e",
|
|
1032
|
-
`grep -q " ${dir} tmpfs " /proc/mounts || ${HELPER_PATH} mount --target ${dir} --source tmpfs --type tmpfs --options mode=0755`,
|
|
1033
|
-
`find ${dir} -mindepth 1 -delete`,
|
|
1034
|
-
`tar -x --no-same-owner -C ${dir}`,
|
|
1035
|
-
`mkdir -p ${dir}/home/tmp`,
|
|
1036
|
-
`chmod -R a+rX ${dir}`,
|
|
1037
|
-
`chown -R ${shellQuote(agentUser)} ${dir}/home`,
|
|
1038
|
-
`chmod 1777 ${dir}`
|
|
1039
|
-
].join("\n");
|
|
1040
|
-
const result = await asRoot(runtime, state, ["/bin/sh", "-c", script], tarOf(staging));
|
|
1041
|
-
if (result.status !== 0) throw new Error(`preparing ${dir} in the guest failed (exit ${result.status}): ${firstLine(result.stderr)}`);
|
|
1042
|
-
} finally {
|
|
1043
|
-
rmSync(staging, { recursive: true, force: true });
|
|
1044
|
-
}
|
|
1045
|
-
},
|
|
1046
|
-
run: async (command, opts) => {
|
|
1047
|
-
needsVm();
|
|
1048
|
-
const ran = await runtime.vm.exec(state.vsockPath, { argv: command.argv, user: agentUser, cwd: command.cwd, env: command.env });
|
|
1049
|
-
appendFileSync(opts.logPath, ran.stdout + ran.stderr);
|
|
1050
|
-
return ran;
|
|
1051
|
-
},
|
|
1052
|
-
readReport: () => readJudgeFile("report.html", MAX_REPORT_BYTES),
|
|
1053
|
-
readItems: () => readJudgeFile("items.json", MAX_ITEMS_BYTES)
|
|
1054
|
-
};
|
|
1055
|
-
}
|
|
1056
|
-
async function startCheckMicroVm(input) {
|
|
1057
|
-
const { config, runtime, job, jobDir, tokens, log } = input;
|
|
1058
|
-
const { vm } = runtime;
|
|
1059
|
-
const tag = `${itemTag(job)}`;
|
|
1060
|
-
const limits = machineLimits(config, jobDir);
|
|
1061
|
-
const built = job.microvm;
|
|
1062
|
-
const repository = built?.repository ?? job.repository;
|
|
1063
|
-
if (built === void 0 || built.imageId === void 0 || repository === void 0 || job.branch === void 0) {
|
|
1064
|
-
return { ok: false, reason: "the build's image, repository or branch is not recorded" };
|
|
1065
|
-
}
|
|
1066
|
-
const branch = job.branch;
|
|
1067
|
-
const runnerDirPath = runnerDir(config.configPath);
|
|
1068
|
-
const tooDeep = socketPathProblem(buildDir(config.configPath, "0".repeat(12)));
|
|
1069
|
-
if (tooDeep !== void 0) return { ok: false, reason: tooDeep };
|
|
1070
|
-
const buildId = randomBytes(6).toString("hex");
|
|
1071
|
-
const buildDir2 = buildDir(config.configPath, buildId);
|
|
1072
|
-
const agentUser = built.agentUser ?? ADDED_USER;
|
|
1073
|
-
const state = {
|
|
1074
|
-
buildId,
|
|
1075
|
-
buildDir: buildDir2,
|
|
1076
|
-
repository,
|
|
1077
|
-
imageId: built.imageId,
|
|
1078
|
-
agentUser,
|
|
1079
|
-
containerEnv: built.containerEnv,
|
|
1080
|
-
imageEnv: built.imageEnv,
|
|
1081
|
-
lifecycle: built.lifecycle,
|
|
1082
|
-
...built.docker === true ? { docker: true } : {},
|
|
1083
|
-
...built.home === void 0 ? {} : { home: built.home }
|
|
1084
|
-
};
|
|
1085
|
-
const checksDir = join(jobDir, "checks");
|
|
1086
|
-
mkdirSync(checksDir, { recursive: true });
|
|
1087
|
-
const consolePath = jobConsolePath(checksDir);
|
|
1088
|
-
input.save(state);
|
|
1089
|
-
const fail = async (reason) => {
|
|
1090
|
-
try {
|
|
1091
|
-
await cleanupMicroVm(runtime, state);
|
|
1092
|
-
} catch (err) {
|
|
1093
|
-
log.error(`${tag} cleanup of the check environment failed: ${describeCallFailure(err)}`);
|
|
1094
|
-
}
|
|
1095
|
-
return { ok: false, reason };
|
|
1096
|
-
};
|
|
1097
|
-
const abandoned = () => input.aborted?.() === true;
|
|
1098
|
-
const ABANDONED = "the check environment was abandoned before it was ready";
|
|
1099
|
-
let assets;
|
|
1100
|
-
try {
|
|
1101
|
-
assets = await vm.assets(runnerDirPath);
|
|
1102
|
-
} catch (err) {
|
|
1103
|
-
return fail(err instanceof AssetError ? err.message : `the microVM's assets could not be had: ${describeCallFailure(err)}`);
|
|
1104
|
-
}
|
|
1105
|
-
if (abandoned()) return fail(ABANDONED);
|
|
1106
|
-
if (built.imageId.startsWith(ROOT_DISK_ID_PREFIX)) {
|
|
1107
|
-
if (built.rootfs === void 0 || !existsSync(built.rootfs)) return fail("the build's root disk is no longer stored, so the check environment cannot start from it");
|
|
1108
|
-
state.rootfs = built.rootfs;
|
|
1109
|
-
} else {
|
|
1110
|
-
try {
|
|
1111
|
-
state.rootfs = await ensureRootfs({ host: vm.rootfs, runnerDir: orgStoreIn(runnerDirPath, job.organizationId), imageId: built.imageId, memoryBytes: limits.memoryBytes });
|
|
1112
|
-
} catch (err) {
|
|
1113
|
-
return fail(`the image could not be converted to a root disk: ${describeCallFailure(err)}`);
|
|
1114
|
-
}
|
|
1115
|
-
}
|
|
1116
|
-
if (abandoned()) return fail(ABANDONED);
|
|
1117
|
-
state.buildDisk = join(buildDir2, "build.ext4");
|
|
1118
|
-
state.vsockPath = vsockPath(buildDir2);
|
|
1119
|
-
input.save(state);
|
|
1120
|
-
mkdirSync(buildDir2, { recursive: true, mode: 448 });
|
|
1121
|
-
try {
|
|
1122
|
-
const gitRemote = gitRemoteFor(runtime, {
|
|
1123
|
-
repositories: repositoriesOf({ repositories: job.repositories, repository, branch }),
|
|
1124
|
-
tokens,
|
|
1125
|
-
buildDir: buildDir2,
|
|
1126
|
-
organizationId: job.organizationId,
|
|
1127
|
-
log: (line) => log.line(`${tag} ${line}`),
|
|
1128
|
-
readOnly: true
|
|
1129
|
-
});
|
|
1130
|
-
runtime.tunnels.set(buildId, await startServicesFor(runtime, config, buildDir2, checksDir, void 0, false, gitRemote, "socket", void 0, void 0, void 0, jobDir));
|
|
1131
|
-
} catch (err) {
|
|
1132
|
-
return fail(`the tunnel could not start: ${describeCallFailure(err)}`);
|
|
1133
|
-
}
|
|
1134
|
-
if (abandoned()) return fail(ABANDONED);
|
|
1135
|
-
const links = vsockLinks({ destinations: config.build?.destinations?.length ?? 0, builtIns: false });
|
|
1136
|
-
const configPath = join(buildDir2, "vm.json");
|
|
1137
|
-
const helper = await helperFor(runtime, (line) => log.line(`${tag} ${line}`), config.managed === true);
|
|
1138
|
-
if (helper === void 0 && config.managed === true) return fail(MANAGED_UNJAILED_REASON);
|
|
1139
|
-
try {
|
|
1140
|
-
if (helper === void 0) linkVsockSockets({ buildDir: buildDir2, vsockPath: state.vsockPath, links });
|
|
1141
|
-
await createBuildDisk({ host: vm.rootfs, runnerDir: runnerDirPath, bytes: limits.diskBytes, dest: state.buildDisk, memoryBytes: limits.memoryBytes });
|
|
1142
|
-
writeFileSync(
|
|
1143
|
-
configPath,
|
|
1144
|
-
JSON.stringify(
|
|
1145
|
-
machineConfig(limits, { kernel: assets.kernel, rootfs: state.rootfs, buildDisk: state.buildDisk, vsock: state.vsockPath, relayPorts: links.map((link) => link.port), docker: built.docker }),
|
|
1146
|
-
null,
|
|
1147
|
-
2
|
|
1148
|
-
),
|
|
1149
|
-
{ mode: 384 }
|
|
1150
|
-
);
|
|
1151
|
-
} catch (err) {
|
|
1152
|
-
return fail(`the check environment's disk and machine could not be made: ${describeCallFailure(err)}`);
|
|
1153
|
-
}
|
|
1154
|
-
if (abandoned()) return fail(ABANDONED);
|
|
1155
|
-
if (helper === void 0) {
|
|
1156
|
-
try {
|
|
1157
|
-
state.cgroup = vm.cgroup.create(buildId, limits.memoryBytes);
|
|
1158
|
-
} catch (err) {
|
|
1159
|
-
return fail(`the check environment's memory limit could not be set: ${describeCallFailure(err)}`);
|
|
1160
|
-
}
|
|
1161
|
-
if (state.cgroup !== void 0) input.save(state);
|
|
1162
|
-
}
|
|
1163
|
-
const boot = async () => {
|
|
1164
|
-
if (helper !== void 0) {
|
|
1165
|
-
try {
|
|
1166
|
-
await startJailedInto(
|
|
1167
|
-
state,
|
|
1168
|
-
{ client: helper, kind: "check", kernel: assets.kernel, rootfs: state.rootfs, sockets: links.map((link) => link.socket), timeLimitMinutes: config.timeLimitMinutes },
|
|
1169
|
-
() => input.save(state)
|
|
1170
|
-
);
|
|
1171
|
-
} catch (err) {
|
|
1172
|
-
return `the microVM did not start under the jailer: ${describeCallFailure(err)}`;
|
|
1173
|
-
}
|
|
1174
|
-
} else {
|
|
1175
|
-
try {
|
|
1176
|
-
state.firecracker = await vm.firecracker.spawn({ binary: assets.firecracker, configPath, consolePath, cgroup: state.cgroup });
|
|
1177
|
-
} catch (err) {
|
|
1178
|
-
return `Firecracker did not start: ${describeCallFailure(err)}`;
|
|
1179
|
-
}
|
|
1180
|
-
input.save(state);
|
|
1181
|
-
}
|
|
1182
|
-
if (abandoned()) return ABANDONED;
|
|
1183
|
-
let lastError;
|
|
1184
|
-
const ready = await waitForAgent(state.vsockPath, { attempts: BOOT_ATTEMPTS, delayMs: BOOT_POLL_MS, sleep: runtime.sleep, onFailure: (err) => lastError = err });
|
|
1185
|
-
if (!ready) {
|
|
1186
|
-
return `the microVM did not start: ${silence(runtime, state, lastError, consoleNote(state, consolePath))}`;
|
|
1187
|
-
}
|
|
1188
|
-
if (state.home !== void 0) {
|
|
1189
|
-
try {
|
|
1190
|
-
await mountGuestHome(runtime, state, agentUser, state.home);
|
|
1191
|
-
} catch (err) {
|
|
1192
|
-
return describeCallFailure(err);
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
if (built.docker === true) return waitForDocker(runtime, state, agentUser, () => /* @__PURE__ */ new Date());
|
|
1196
|
-
return void 0;
|
|
1197
|
-
};
|
|
1198
|
-
const failed = await boot();
|
|
1199
|
-
if (failed !== void 0) return fail(failed);
|
|
1200
|
-
if (abandoned()) return fail(ABANDONED);
|
|
1201
|
-
const env = checkEnvironment({
|
|
1202
|
-
config,
|
|
1203
|
-
item: job.item,
|
|
1204
|
-
repository,
|
|
1205
|
-
hostEnv: runtime.hostEnv,
|
|
1206
|
-
containerEnv: built.containerEnv ?? {},
|
|
1207
|
-
// The checks of a managed build get its factory's variables, as its command did, and none of the runner's own.
|
|
1208
|
-
factoryVariables: factoryVariablesOf(config, jobDir)
|
|
1209
|
-
});
|
|
1210
|
-
const evalInput = { dir: join(checksDir, EVAL_INPUT_NAME), delivered: false };
|
|
1211
|
-
const deliverEvalInput = async () => {
|
|
1212
|
-
await unpackIntoGuest(runtime, state, evalInput.dir, EVAL_INPUT_DIR, { readOnly: true });
|
|
1213
|
-
evalInput.delivered = true;
|
|
1214
|
-
};
|
|
1215
|
-
return {
|
|
1216
|
-
ok: true,
|
|
1217
|
-
environment: {
|
|
1218
|
-
state,
|
|
1219
|
-
evalInput: { dir: evalInput.dir, deliver: deliverEvalInput },
|
|
1220
|
-
exec: guestExecFor({
|
|
1221
|
-
vm,
|
|
1222
|
-
vsockPath: state.vsockPath,
|
|
1223
|
-
request: (argv, workdir, variables) => ({ argv, user: agentUser, cwd: workdir, env: execEnvironment(state.imageEnv, withMiseVariables(env, variables)) })
|
|
1224
|
-
}),
|
|
1225
|
-
restart: async () => {
|
|
1226
|
-
try {
|
|
1227
|
-
if (state.jailed !== void 0) await stopJailed(vm.helper, state);
|
|
1228
|
-
else await vm.firecracker.stop(state.firecracker);
|
|
1229
|
-
rmSync(state.vsockPath, { force: true });
|
|
1230
|
-
if (await boot() !== void 0) return false;
|
|
1231
|
-
if (evalInput.delivered) await deliverEvalInput();
|
|
1232
|
-
return true;
|
|
1233
|
-
} catch {
|
|
1234
|
-
return false;
|
|
1235
|
-
}
|
|
1236
|
-
}
|
|
1237
|
-
}
|
|
1238
|
-
};
|
|
1239
|
-
}
|
|
1240
|
-
async function stopBuildMicroVm(runtime, state, jobDir) {
|
|
1241
|
-
await copyHookErrors(runtime, state, jobDir);
|
|
1242
|
-
if (state.jailed !== void 0) {
|
|
1243
|
-
await stopJailed(runtime.vm.helper, state);
|
|
1244
|
-
return;
|
|
1245
|
-
}
|
|
1246
|
-
if (state.firecracker === void 0) return;
|
|
1247
|
-
await runtime.vm.firecracker.stop(state.firecracker);
|
|
1248
|
-
if (runtime.vm.firecracker.alive(state.firecracker)) throw new Error(`the microVM (pid ${state.firecracker.pid}) is still running after it was stopped`);
|
|
1249
|
-
}
|
|
1250
|
-
async function cleanupMicroVm(runtime, state, jobDir, warn, onBuildDisk) {
|
|
1251
|
-
const failures = [];
|
|
1252
|
-
if (jobDir !== void 0) await copyHookErrors(runtime, state, jobDir);
|
|
1253
|
-
if (state.jailed !== void 0) {
|
|
1254
|
-
try {
|
|
1255
|
-
await stopJailed(runtime.vm.helper, state);
|
|
1256
|
-
} catch (err) {
|
|
1257
|
-
failures.push(describeCallFailure(err));
|
|
1258
|
-
}
|
|
1259
|
-
}
|
|
1260
|
-
if (state.firecracker !== void 0) {
|
|
1261
|
-
try {
|
|
1262
|
-
await runtime.vm.firecracker.stop(state.firecracker);
|
|
1263
|
-
} catch (err) {
|
|
1264
|
-
failures.push(`the microVM (pid ${state.firecracker.pid}) was not stopped: ${describeCallFailure(err)}`);
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
if (state.cgroup !== void 0) {
|
|
1268
|
-
try {
|
|
1269
|
-
runtime.vm.cgroup.remove(state.cgroup);
|
|
1270
|
-
} catch (err) {
|
|
1271
|
-
failures.push(`the build's cgroup ${state.cgroup} was not removed: ${describeCallFailure(err)}`);
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
if (state.buildDisk !== void 0 && existsSync(state.buildDisk)) {
|
|
1275
|
-
try {
|
|
1276
|
-
onBuildDisk?.(allocatedBytes(state.buildDisk));
|
|
1277
|
-
} catch (err) {
|
|
1278
|
-
failures.push(`the build disk's peak size was not recorded: ${describeCallFailure(err)}`);
|
|
1279
|
-
}
|
|
1280
|
-
}
|
|
1281
|
-
if (state.buildDisk !== void 0) deleteBuildDisk(state.buildDisk);
|
|
1282
|
-
runtime.leases.get(state.buildId)?.close();
|
|
1283
|
-
runtime.leases.delete(state.buildId);
|
|
1284
|
-
const tunnel = runtime.tunnels.get(state.buildId);
|
|
1285
|
-
if (tunnel !== void 0) {
|
|
1286
|
-
runtime.tunnels.delete(state.buildId);
|
|
1287
|
-
await tunnel.close();
|
|
1288
|
-
}
|
|
1289
|
-
for (const name of state.fdStore ?? []) {
|
|
1290
|
-
await runtime.vm.fdStore.remove(name).catch((err) => warn?.(`the descriptor ${name} could not be taken out of the store: ${describeCallFailure(err)}`));
|
|
1291
|
-
}
|
|
1292
|
-
rmSync(state.buildDir, { recursive: true, force: true });
|
|
1293
|
-
if (failures.length > 0) throw new Error(failures.join("; "));
|
|
1294
|
-
}
|
|
1295
|
-
async function stopOrphanedMicroVms(input) {
|
|
1296
|
-
const stopped = [];
|
|
1297
|
-
const report = (line) => input.onError?.(line);
|
|
1298
|
-
const jailedByJobs = [];
|
|
1299
|
-
const kept = input.keep ?? /* @__PURE__ */ new Set();
|
|
1300
|
-
for (const { job } of input.jobs) {
|
|
1301
|
-
if (job.microvm !== void 0 && kept.has(job.microvm.buildId)) continue;
|
|
1302
|
-
const states = [job.microvm, job.checkMicrovm].filter((state) => state !== void 0);
|
|
1303
|
-
if (states.length === 0) continue;
|
|
1304
|
-
if (job.pid !== void 0 && job.start !== void 0 && isSameProcess(job.pid, job.start)) {
|
|
1305
|
-
try {
|
|
1306
|
-
process.kill(-job.pid, "SIGKILL");
|
|
1307
|
-
} catch {
|
|
1308
|
-
}
|
|
1309
|
-
const deadline = Date.now() + 2e3;
|
|
1310
|
-
while (isSameProcess(job.pid, job.start) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1311
|
-
}
|
|
1312
|
-
for (const state of states) {
|
|
1313
|
-
if (state.jailed !== void 0) jailedByJobs.push(state.jailed.unit);
|
|
1314
|
-
if (state.firecracker !== void 0 && input.runtime.vm.firecracker.alive(state.firecracker)) {
|
|
1315
|
-
try {
|
|
1316
|
-
await input.runtime.vm.firecracker.stop(state.firecracker);
|
|
1317
|
-
stopped.push(state.buildId);
|
|
1318
|
-
} catch (err) {
|
|
1319
|
-
report(`the microVM of build ${state.buildId} (pid ${state.firecracker.pid}) was not stopped: ${describeCallFailure(err)}`);
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
}
|
|
1323
|
-
}
|
|
1324
|
-
const probe = await input.runtime.vm.helper.probe();
|
|
1325
|
-
if (!probe.ok) {
|
|
1326
|
-
if (jailedByJobs.length > 0) report(`the units ${jailedByJobs.join(", ")} could not be stopped, because there is no helper to ask: ${probe.detail}`);
|
|
1327
|
-
return stopped;
|
|
1328
|
-
}
|
|
1329
|
-
let builds;
|
|
1330
|
-
try {
|
|
1331
|
-
builds = await probe.client.list();
|
|
1332
|
-
} catch (err) {
|
|
1333
|
-
report(`the helper's list of builds could not be read: ${describeCallFailure(err)}`);
|
|
1334
|
-
return stopped;
|
|
1335
|
-
}
|
|
1336
|
-
for (const build of builds) {
|
|
1337
|
-
if (kept.has(build)) continue;
|
|
1338
|
-
try {
|
|
1339
|
-
await probe.client.stop(build);
|
|
1340
|
-
} catch (err) {
|
|
1341
|
-
report(`the unit of build ${build} was not stopped: ${describeCallFailure(err)}`);
|
|
1342
|
-
continue;
|
|
1343
|
-
}
|
|
1344
|
-
if (!stopped.includes(build)) stopped.push(build);
|
|
1345
|
-
}
|
|
1346
|
-
return stopped;
|
|
1347
|
-
}
|
|
1348
|
-
function sessionGuestOfMicroVm(runtime, state, as) {
|
|
1349
|
-
const request = (argv) => ({ argv, user: as === "root" ? "root" : state.agentUser ?? ADDED_USER, cwd: "/", env: { PATH: ROOT_PATH } });
|
|
1350
|
-
return {
|
|
1351
|
-
run: (argv) => runtime.vm.exec(state.vsockPath, request(argv)),
|
|
1352
|
-
stream: (argv, io) => runtime.vm.execStream(state.vsockPath, request(argv), io)
|
|
1353
|
-
};
|
|
1354
|
-
}
|
|
1355
|
-
async function restoreSessionIntoMicroVm(runtime, state, keptDir, meta) {
|
|
1356
|
-
const restored = await restoreSession({ guest: sessionGuestOfMicroVm(runtime, state, "agent"), tar: packKeptSession(keptDir, meta) });
|
|
1357
|
-
if (!restored.ok) throw new Error(restored.reason);
|
|
1358
|
-
}
|
|
1359
|
-
|
|
1360
|
-
// src/runner/verdict.ts
|
|
1361
|
-
var VERDICT_SETTLE_MS = 5 * 6e4;
|
|
1362
|
-
var VERDICT_READ_EVERY_MS = 3e4;
|
|
1363
|
-
var REASON_MAX = 200;
|
|
1364
|
-
var FAILING_IDS_MAX = 5;
|
|
1365
|
-
function text(value) {
|
|
1366
|
-
return typeof value === "string" ? value : "";
|
|
1367
|
-
}
|
|
1368
|
-
function fields(value) {
|
|
1369
|
-
return Object(value);
|
|
1370
|
-
}
|
|
1371
|
-
function failingItems(checks) {
|
|
1372
|
-
const byName = /* @__PURE__ */ new Map();
|
|
1373
|
-
if (!Array.isArray(checks)) return byName;
|
|
1374
|
-
for (const entry of checks) {
|
|
1375
|
-
const check = fields(entry);
|
|
1376
|
-
if (check.result !== "fail" || !Array.isArray(check.items)) continue;
|
|
1377
|
-
const ids = check.items.map(fields).filter((item) => item.result === "fail" && typeof item.id === "string" && item.id !== "").map((item) => item.id);
|
|
1378
|
-
byName.set(text(check.name), ids);
|
|
1379
|
-
}
|
|
1380
|
-
return byName;
|
|
1381
|
-
}
|
|
1382
|
-
function parseEvaluation(value, checks) {
|
|
1383
|
-
if (value === void 0) return void 0;
|
|
1384
|
-
const failing = failingItems(checks);
|
|
1385
|
-
const raw = fields(value);
|
|
1386
|
-
const verdict = raw.verdict;
|
|
1387
|
-
if (verdict !== "pass" && verdict !== "flag" && verdict !== "unverifiable" && verdict !== "waiting") return null;
|
|
1388
|
-
const open = [];
|
|
1389
|
-
for (const row of Array.isArray(raw.open) ? raw.open : []) {
|
|
1390
|
-
const r = fields(row);
|
|
1391
|
-
if (r.status !== "flag" && r.status !== "pending") continue;
|
|
1392
|
-
const parsed = { label: text(r.label), status: r.status, sentence: text(r.sentence) };
|
|
1393
|
-
if (typeof r.check === "string" && r.check !== "") parsed.check = { id: r.check, failing: failing.get(r.check) ?? [] };
|
|
1394
|
-
open.push(parsed);
|
|
1395
|
-
}
|
|
1396
|
-
return { verdict, sentence: text(raw.sentence), waitingOn: raw.waitingOn, open };
|
|
1397
|
-
}
|
|
1398
|
-
async function readWorkItem(config, item, fetchImpl) {
|
|
1399
|
-
const { data } = await getJson(config.url, `/v1/work-items/${item}`, config.apiKey, config.appId, fetchImpl, await signerFor(config.configPath));
|
|
1400
|
-
return data;
|
|
1401
|
-
}
|
|
1402
|
-
function declaredPullRequest(workItem, repository) {
|
|
1403
|
-
const declared = fields(workItem).pullRequests;
|
|
1404
|
-
if (!Array.isArray(declared)) return void 0;
|
|
1405
|
-
const wanted = repository.toLowerCase();
|
|
1406
|
-
const matching = declared.map((entry) => fields(entry)).filter((entry) => typeof entry.repository === "string" && entry.repository.toLowerCase() === wanted && typeof entry.prNumber === "number" && Number.isInteger(entry.prNumber) && entry.prNumber > 0);
|
|
1407
|
-
const chosen = matching.find((entry) => entry.state === "open") ?? matching[0];
|
|
1408
|
-
return chosen === void 0 ? void 0 : chosen.prNumber;
|
|
1409
|
-
}
|
|
1410
|
-
async function readItemEvaluation(config, item, fetchImpl) {
|
|
1411
|
-
try {
|
|
1412
|
-
const body = fields(await readWorkItem(config, item, fetchImpl));
|
|
1413
|
-
const hasPullRequest = Array.isArray(body.pullRequests) ? body.pullRequests.length > 0 : void 0;
|
|
1414
|
-
return { ok: true, evaluation: parseEvaluation(body.evaluation, body.checks), hasPullRequest };
|
|
1415
|
-
} catch (err) {
|
|
1416
|
-
return { ok: false, reason: describeCallFailure(err) };
|
|
1417
|
-
}
|
|
1418
|
-
}
|
|
1419
|
-
function rowText(row) {
|
|
1420
|
-
const name = row.label !== "" && row.sentence !== "" && row.label !== row.sentence ? `${row.label}: ${row.sentence}` : row.sentence || row.label;
|
|
1421
|
-
if (row.check === void 0) return row.status === "pending" ? `${name} (no result)` : name;
|
|
1422
|
-
const ids = row.check.failing;
|
|
1423
|
-
if (ids.length === 0) return name;
|
|
1424
|
-
const shown = ids.slice(0, FAILING_IDS_MAX).join(", ");
|
|
1425
|
-
return `${name} (failing: ${ids.length > FAILING_IDS_MAX ? `${shown} and ${ids.length - FAILING_IDS_MAX} more` : shown})`;
|
|
1426
|
-
}
|
|
1427
|
-
function fit(prefix, parts) {
|
|
1428
|
-
let text2 = prefix + parts.join("; ");
|
|
1429
|
-
for (let kept = parts.length - 1; text2.length > REASON_MAX && kept > 0; kept--) {
|
|
1430
|
-
text2 = `${prefix}${parts.slice(0, kept).join("; ")}; and ${parts.length - kept} more`;
|
|
1431
|
-
}
|
|
1432
|
-
return text2.length > REASON_MAX ? `${text2.slice(0, REASON_MAX - 1)}\u2026` : text2;
|
|
1433
|
-
}
|
|
1434
|
-
function decideFromEvaluation(read) {
|
|
1435
|
-
if (!read.ok) return { outcome: "incomplete", reason: fit("the item's status could not be read: ", [read.reason]) };
|
|
1436
|
-
if (read.hasPullRequest === false) {
|
|
1437
|
-
return { outcome: "incomplete", reason: "the build opened no pull request for the item" };
|
|
1438
|
-
}
|
|
1439
|
-
const evaluation = read.evaluation;
|
|
1440
|
-
if (evaluation === void 0) return { outcome: "ok" };
|
|
1441
|
-
if (evaluation === null) return { outcome: "incomplete", reason: "the item has no evidence evaluation yet" };
|
|
1442
|
-
if (evaluation.open.length > 0) return { outcome: "incomplete", reason: fit("checks not passing: ", evaluation.open.map(rowText)) };
|
|
1443
|
-
if (evaluation.verdict === "pass") return { outcome: "ok" };
|
|
1444
|
-
if (evaluation.verdict === "waiting" && evaluation.waitingOn === "links") {
|
|
1445
|
-
return { outcome: "incomplete", reason: "the item's evidence was still waiting on session links to be confirmed" };
|
|
1446
|
-
}
|
|
1447
|
-
if (evaluation.verdict === "waiting" && evaluation.waitingOn === "claim") {
|
|
1448
|
-
return { outcome: "incomplete", reason: "the item's required checks had no result when the build ended" };
|
|
1449
|
-
}
|
|
1450
|
-
return { outcome: "incomplete", reason: fit("the item's verdict is not passing: ", [evaluation.sentence || evaluation.verdict]) };
|
|
1451
|
-
}
|
|
1452
|
-
function incompleteReason(reason, judgeNotRun) {
|
|
1453
|
-
return judgeNotRun === void 0 ? reason : `the judge did not run (${judgeNotRun}) \xB7 ${reason}`;
|
|
1454
|
-
}
|
|
1455
|
-
function unfinishedRelease(verdictReason, learned) {
|
|
1456
|
-
return learned === WORKFLOW_PERMISSION_REASON ? { outcome: "failed", reason: WORKFLOW_PERMISSION_REASON } : { outcome: "incomplete", reason: verdictReason };
|
|
1457
|
-
}
|
|
1458
|
-
function waitMs(ms) {
|
|
1459
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1460
|
-
}
|
|
1461
|
-
async function settleVerdict(deps) {
|
|
1462
|
-
const settleMs = deps.settleMs ?? VERDICT_SETTLE_MS;
|
|
1463
|
-
const everyMs = deps.everyMs ?? VERDICT_READ_EVERY_MS;
|
|
1464
|
-
const startedMs = deps.now().getTime();
|
|
1465
|
-
for (; ; ) {
|
|
1466
|
-
const decision = decideFromEvaluation(await deps.read());
|
|
1467
|
-
if (decision.outcome === "ok") return decision;
|
|
1468
|
-
if (deps.now().getTime() - startedMs + everyMs > settleMs) return decision;
|
|
1469
|
-
await deps.sleep(everyMs);
|
|
1470
|
-
}
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
// src/runner/microvm/guest-usage.ts
|
|
1474
|
-
var GUEST_USAGE_INTERVAL_MS = 15e3;
|
|
1475
|
-
var MS_PER_TICK = 10;
|
|
1476
|
-
function usageScript(procDir = "/proc") {
|
|
1477
|
-
return [
|
|
1478
|
-
`echo OL-USAGE-MEMINFO; cat ${procDir}/meminfo`,
|
|
1479
|
-
`echo OL-USAGE-STAT; cat ${procDir}/stat`,
|
|
1480
|
-
`if [ -r ${procDir}/pressure/memory ]; then echo OL-USAGE-PSI-MEMORY; cat ${procDir}/pressure/memory; fi`,
|
|
1481
|
-
`if [ -r ${procDir}/pressure/cpu ]; then echo OL-USAGE-PSI-CPU; cat ${procDir}/pressure/cpu; fi`
|
|
1482
|
-
].join("; ");
|
|
1483
|
-
}
|
|
1484
|
-
function section(stdout, name) {
|
|
1485
|
-
const lines = stdout.split("\n");
|
|
1486
|
-
const start = lines.indexOf(`OL-USAGE-${name}`);
|
|
1487
|
-
if (start === -1) return void 0;
|
|
1488
|
-
const end = lines.findIndex((line, index) => index > start && line.startsWith("OL-USAGE-"));
|
|
1489
|
-
return lines.slice(start + 1, end === -1 ? void 0 : end).join("\n");
|
|
1490
|
-
}
|
|
1491
|
-
function meminfoKb(text2, key) {
|
|
1492
|
-
const match = new RegExp(`^${key}:\\s+(\\d+)\\s+kB\\s*$`, "m").exec(text2);
|
|
1493
|
-
return match === null ? void 0 : Number(match[1]);
|
|
1494
|
-
}
|
|
1495
|
-
function pressureTotal(text2) {
|
|
1496
|
-
if (text2 === void 0) return void 0;
|
|
1497
|
-
const match = /^some\s.*\btotal=(\d+)\s*$/m.exec(text2);
|
|
1498
|
-
return match === null ? void 0 : Number(match[1]);
|
|
1499
|
-
}
|
|
1500
|
-
function parseGuestSample(stdout) {
|
|
1501
|
-
const meminfo = section(stdout, "MEMINFO");
|
|
1502
|
-
const stat = section(stdout, "STAT");
|
|
1503
|
-
if (meminfo === void 0 || stat === void 0) return void 0;
|
|
1504
|
-
const totalKb = meminfoKb(meminfo, "MemTotal");
|
|
1505
|
-
const availableKb = meminfoKb(meminfo, "MemAvailable");
|
|
1506
|
-
const aggregate = /^cpu\s+(\d+(?:\s+\d+){7,})\s*$/m.exec(stat);
|
|
1507
|
-
if (totalKb === void 0 || availableKb === void 0 || totalKb <= 0 || availableKb > totalKb || aggregate === null) return void 0;
|
|
1508
|
-
const [user, nice, system, , , irq, softirq, steal] = aggregate[1].split(/\s+/).map(Number);
|
|
1509
|
-
const vcpus = stat.split("\n").filter((line) => /^cpu\d+\s/.test(line)).length;
|
|
1510
|
-
if (vcpus < 1) return void 0;
|
|
1511
|
-
const memoryPressureUs = pressureTotal(section(stdout, "PSI-MEMORY"));
|
|
1512
|
-
const cpuPressureUs = pressureTotal(section(stdout, "PSI-CPU"));
|
|
1513
|
-
return {
|
|
1514
|
-
memoryBytes: totalKb * 1024,
|
|
1515
|
-
vcpus,
|
|
1516
|
-
usedBytes: (totalKb - availableKb) * 1024,
|
|
1517
|
-
cpuBusyMs: (user + nice + system + irq + softirq + steal) * MS_PER_TICK,
|
|
1518
|
-
...memoryPressureUs === void 0 ? {} : { memoryPressureUs },
|
|
1519
|
-
...cpuPressureUs === void 0 ? {} : { cpuPressureUs }
|
|
1520
|
-
};
|
|
1521
|
-
}
|
|
1522
|
-
function foldSample(usage, sample) {
|
|
1523
|
-
return { peakMemoryBytes: Math.max(usage?.peakMemoryBytes ?? 0, sample.usedBytes), last: sample };
|
|
1524
|
-
}
|
|
1525
|
-
function usageRecord(usage) {
|
|
1526
|
-
if (usage === void 0) return void 0;
|
|
1527
|
-
const { last } = usage;
|
|
1528
|
-
return {
|
|
1529
|
-
guestMemoryBytes: last.memoryBytes,
|
|
1530
|
-
guestVcpus: last.vcpus,
|
|
1531
|
-
peakMemoryBytes: usage.peakMemoryBytes,
|
|
1532
|
-
cpuBusyMs: last.cpuBusyMs,
|
|
1533
|
-
...last.memoryPressureUs === void 0 ? {} : { memoryPressureUs: last.memoryPressureUs },
|
|
1534
|
-
...last.cpuPressureUs === void 0 ? {} : { cpuPressureUs: last.cpuPressureUs }
|
|
1535
|
-
};
|
|
1536
|
-
}
|
|
1537
|
-
async function sampleGuest(runtime, state) {
|
|
1538
|
-
if (state.vsockPath === void 0) return void 0;
|
|
1539
|
-
try {
|
|
1540
|
-
const result = await withinTime(
|
|
1541
|
-
(signal) => runtime.vm.exec(state.vsockPath, { argv: ["/bin/sh", "-c", usageScript()], user: "root", cwd: "/", env: { PATH: GUEST_PATH } }, { signal })
|
|
1542
|
-
);
|
|
1543
|
-
return result.status === 0 ? parseGuestSample(result.stdout) : void 0;
|
|
1544
|
-
} catch {
|
|
1545
|
-
return void 0;
|
|
1546
|
-
}
|
|
1547
|
-
}
|
|
1548
|
-
var STOP_WAIT_MS = 3e4;
|
|
1549
|
-
var POLL_MS = 200;
|
|
1550
|
-
var ASK_AGAIN_MS = 1e3;
|
|
1551
|
-
var EXIT_REPORT_WAIT_MS = 2e3;
|
|
1552
|
-
var exitsLogged = /* @__PURE__ */ new Map();
|
|
1553
|
-
var BUILD_HOME2 = "/build/home";
|
|
1554
|
-
async function startSender(input) {
|
|
1555
|
-
const { config, job, jobDir, deps } = input;
|
|
1556
|
-
const tag = `${itemTag(job)}`;
|
|
1557
|
-
if (job.itemKey === void 0) return void 0;
|
|
1558
|
-
const spawnFn = deps.spawnFn ?? spawn;
|
|
1559
|
-
try {
|
|
1560
|
-
const launch = launchFor(config, job, jobDir, deps);
|
|
1561
|
-
const fd = openSync(join(jobDir, "sender.log"), "a", RUNNER_FILE_MODE);
|
|
1562
|
-
let pid;
|
|
1563
|
-
let exitLogged;
|
|
1564
|
-
try {
|
|
1565
|
-
const child = spawnFn(launch.command, launch.args, { env: launch.env, cwd: launch.cwd, detached: true, stdio: ["ignore", fd, fd] });
|
|
1566
|
-
child.on("error", (err) => deps.log.error(`${tag} session sender failed: ${describeCallFailure(err)}`));
|
|
1567
|
-
exitLogged = new Promise((resolve) => {
|
|
1568
|
-
child.on("exit", (code, signal) => {
|
|
1569
|
-
deps.log.line(`${tag} session sender exited \xB7 ${signal ?? `code ${code}`} \xB7 see sender.log`);
|
|
1570
|
-
resolve();
|
|
1571
|
-
});
|
|
1572
|
-
});
|
|
1573
|
-
child.unref?.();
|
|
1574
|
-
pid = child.pid;
|
|
1575
|
-
} finally {
|
|
1576
|
-
closeSync(fd);
|
|
1577
|
-
}
|
|
1578
|
-
if (pid === void 0) {
|
|
1579
|
-
deps.log.line(`${tag} session sender did not start: no pid`);
|
|
1580
|
-
return void 0;
|
|
1581
|
-
}
|
|
1582
|
-
exitsLogged.set(pid, exitLogged);
|
|
1583
|
-
deps.log.line(`${tag} session sender started \xB7 pid ${pid}`);
|
|
1584
|
-
return { pid, start: startTimeOf(pid) ?? void 0 };
|
|
1585
|
-
} catch (err) {
|
|
1586
|
-
deps.log.error(`${tag} session sender did not start: ${describeCallFailure(err)} \xB7 continuing`);
|
|
1587
|
-
return void 0;
|
|
1588
|
-
}
|
|
1589
|
-
}
|
|
1590
|
-
function launchFor(config, job, jobDir, deps) {
|
|
1591
|
-
if (job.microvm !== void 0) {
|
|
1592
|
-
const runtime = deps.runtime ?? createContainerRuntime();
|
|
1593
|
-
const cli = deps.cliPath ?? join(runtime.cliBuild().distDir, "index.js");
|
|
1594
|
-
return { ...microVmSenderCommand({ job, jobDir, runtime, config, cli }), cwd: jobDir };
|
|
1595
|
-
}
|
|
1596
|
-
if (job.container !== void 0) {
|
|
1597
|
-
return { ...containerSenderCommand({ job, runtime: deps.runtime ?? createContainerRuntime(), config }), cwd: jobDir };
|
|
1598
|
-
}
|
|
1599
|
-
const env = jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDir, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job));
|
|
1600
|
-
const build = job.build;
|
|
1601
|
-
const hostEnv = build === void 0 ? env : hostCommandEnvironment(env, build, config, job.commitIdentity?.hooksPath);
|
|
1602
|
-
if (build?.exec !== void 0) {
|
|
1603
|
-
return { command: build.exec, args: execArgv(job.workdir, ["outerlayer", "session-sender"]), env: hostEnv, cwd: jobDir };
|
|
1604
|
-
}
|
|
1605
|
-
return { command: process.execPath, args: [deps.cliPath ?? process.argv[1] ?? "outerlayer", "session-sender"], env: hostEnv, cwd: job.workdir ?? jobDir };
|
|
1606
|
-
}
|
|
1607
|
-
async function restartSender(input) {
|
|
1608
|
-
const { job, deps } = input;
|
|
1609
|
-
const state = job.container;
|
|
1610
|
-
if (state !== void 0 && job.itemKey !== void 0) {
|
|
1611
|
-
const runtime = deps.runtime ?? createContainerRuntime();
|
|
1612
|
-
const pidfile = senderPidfile(BUILD_HOME2, Number(job.item));
|
|
1613
|
-
const sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
1614
|
-
const running = async () => {
|
|
1615
|
-
const probe = await runtime.host.exec({ container: state.container, user: state.agentUser ?? ADDED_USER, env: { literals: {}, passed: {} }, argv: ["/bin/sh", "-c", senderAliveScript(pidfile)] });
|
|
1616
|
-
return probe.status === 0 && probe.stdout.trim() === "alive";
|
|
1617
|
-
};
|
|
1618
|
-
try {
|
|
1619
|
-
await signalInContainer(runtime, state, pidfile, "TERM");
|
|
1620
|
-
const deadline = (deps.waitMs ?? STOP_WAIT_MS) / POLL_MS;
|
|
1621
|
-
for (let waited = 0; await running() && waited < deadline; waited += 1) await sleep(POLL_MS);
|
|
1622
|
-
if (await running()) {
|
|
1623
|
-
deps.log.error(`${itemTag(job)} the session sender an earlier runner left did not stop in time \xB7 killing it`);
|
|
1624
|
-
await signalInContainer(runtime, state, pidfile, "KILL");
|
|
1625
|
-
}
|
|
1626
|
-
} catch (err) {
|
|
1627
|
-
deps.log.error(`${itemTag(job)} stopping the session sender an earlier runner left failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
1628
|
-
}
|
|
1629
|
-
}
|
|
1630
|
-
return startSender(input);
|
|
1631
|
-
}
|
|
1632
|
-
function isLive(sender) {
|
|
1633
|
-
if (sender.start !== void 0) return isSameProcess(sender.pid, sender.start);
|
|
1634
|
-
try {
|
|
1635
|
-
process.kill(sender.pid, 0);
|
|
1636
|
-
return true;
|
|
1637
|
-
} catch {
|
|
1638
|
-
return false;
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
function signalHost(sender, signal) {
|
|
1642
|
-
if (!isLive(sender)) return;
|
|
1643
|
-
try {
|
|
1644
|
-
process.kill(-sender.pid, signal);
|
|
1645
|
-
} catch {
|
|
1646
|
-
try {
|
|
1647
|
-
process.kill(sender.pid, signal);
|
|
1648
|
-
} catch {
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
}
|
|
1652
|
-
async function signalInGuest(runtime, state, pidfile, signal) {
|
|
1653
|
-
const { status } = await runtime.vm.exec(state.vsockPath, {
|
|
1654
|
-
argv: ["/bin/sh", "-c", senderSignalScript(pidfile, signal)],
|
|
1655
|
-
user: state.agentUser ?? ADDED_USER,
|
|
1656
|
-
cwd: "/",
|
|
1657
|
-
env: { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" }
|
|
1658
|
-
});
|
|
1659
|
-
return status === 0;
|
|
1660
|
-
}
|
|
1661
|
-
async function exitReported(pid) {
|
|
1662
|
-
const logged = exitsLogged.get(pid);
|
|
1663
|
-
let timer;
|
|
1664
|
-
const bound = new Promise((resolve) => {
|
|
1665
|
-
timer = setTimeout(resolve, EXIT_REPORT_WAIT_MS);
|
|
1666
|
-
timer.unref?.();
|
|
1667
|
-
});
|
|
1668
|
-
await Promise.race([logged, bound]);
|
|
1669
|
-
clearTimeout(timer);
|
|
1670
|
-
exitsLogged.delete(pid);
|
|
1671
|
-
}
|
|
1672
|
-
async function stopSender(input) {
|
|
1673
|
-
const { job, deps } = input;
|
|
1674
|
-
const sender = job.sender;
|
|
1675
|
-
if (sender === void 0) return;
|
|
1676
|
-
const tag = `${itemTag(job)}`;
|
|
1677
|
-
const sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
1678
|
-
const inContainer = job.container === void 0 ? void 0 : { state: job.container, runtime: deps.runtime ?? createContainerRuntime(), pidfile: senderPidfile(BUILD_HOME2, Number(job.item)) };
|
|
1679
|
-
const inGuest = job.microvm === void 0 ? void 0 : { state: job.microvm, runtime: deps.runtime ?? createContainerRuntime(), pidfile: senderPidfile(BUILD_HOME2, Number(job.item)) };
|
|
1680
|
-
const signalInside = async (signal) => {
|
|
1681
|
-
if (inContainer !== void 0) return signalInContainer(inContainer.runtime, inContainer.state, inContainer.pidfile, signal).catch(() => void 0);
|
|
1682
|
-
if (inGuest !== void 0) return signalInGuest(inGuest.runtime, inGuest.state, inGuest.pidfile, signal).catch(() => void 0);
|
|
1683
|
-
return void 0;
|
|
1684
|
-
};
|
|
1685
|
-
try {
|
|
1686
|
-
let reached = true;
|
|
1687
|
-
if (inContainer !== void 0 || inGuest !== void 0) reached = await signalInside("TERM");
|
|
1688
|
-
else signalHost(sender, "SIGTERM");
|
|
1689
|
-
const deadline = (deps.waitMs ?? STOP_WAIT_MS) / POLL_MS;
|
|
1690
|
-
for (let waited = 0; isLive(sender) && waited < deadline; waited += 1) {
|
|
1691
|
-
await sleep(POLL_MS);
|
|
1692
|
-
if (!reached && (waited + 1) % (ASK_AGAIN_MS / POLL_MS) === 0) reached = await signalInside("TERM");
|
|
1693
|
-
}
|
|
1694
|
-
if (isLive(sender)) {
|
|
1695
|
-
deps.log.error(`${tag} session sender did not stop in time \xB7 killing it`);
|
|
1696
|
-
await signalInside("KILL");
|
|
1697
|
-
signalHost(sender, "SIGKILL");
|
|
1698
|
-
}
|
|
1699
|
-
await exitReported(sender.pid);
|
|
1700
|
-
} catch (err) {
|
|
1701
|
-
deps.log.error(`${tag} stopping the session sender failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
1702
|
-
}
|
|
1703
|
-
}
|
|
1704
|
-
var RUNNER_LOG_FILE = "runner.log";
|
|
1705
|
-
var RUNNER_LOG_KEEP_BYTES = 65536;
|
|
1706
|
-
function isTailedStage(stage) {
|
|
1707
|
-
const tailed = ["provision", "sync", "cleanup", "report"];
|
|
1708
|
-
return tailed.includes(stage);
|
|
1709
|
-
}
|
|
1710
|
-
var PROVISION_PROBLEM = /\b(warn(ing)?|error|fail(ed|ure)?|refused)\b/i;
|
|
1711
|
-
var READ_LIMIT_BYTES = 1048576;
|
|
1712
|
-
var ITEM_LINE = /^#(\d+(?:@[A-Za-z0-9_-]+)?)(?![A-Za-z0-9_@-])/;
|
|
1713
|
-
function keyOf(item, scope) {
|
|
1714
|
-
return scope === void 0 ? String(Number(item)) : `${Number(item)}@${scope}`;
|
|
1715
|
-
}
|
|
1716
|
-
var AttemptLogBook = class {
|
|
1717
|
-
constructor(now = () => /* @__PURE__ */ new Date()) {
|
|
1718
|
-
this.now = now;
|
|
1719
|
-
this.dirs = /* @__PURE__ */ new Map();
|
|
1720
|
-
this.lastByKey = /* @__PURE__ */ new Map();
|
|
1721
|
-
}
|
|
1722
|
-
/** Starts keeping lines for `item` in `jobDirPath`. Idempotent. A fleet build passes the `scope` of its organization. */
|
|
1723
|
-
open(item, jobDirPath, scope) {
|
|
1724
|
-
this.dirs.set(keyOf(item, scope), jobDirPath);
|
|
1725
|
-
}
|
|
1726
|
-
/** Stops keeping lines for `item`; its file stays on disk until housekeeping removes the job directory. */
|
|
1727
|
-
close(item, scope) {
|
|
1728
|
-
const key = keyOf(item, scope);
|
|
1729
|
-
this.dirs.delete(key);
|
|
1730
|
-
for (const slot of [...this.lastByKey.keys()]) {
|
|
1731
|
-
if (slot.startsWith(`${key}:`)) this.lastByKey.delete(slot);
|
|
1732
|
-
}
|
|
1733
|
-
}
|
|
1734
|
-
/** The log with every line it writes for an open item also kept in that item's file. */
|
|
1735
|
-
wrap(log) {
|
|
1736
|
-
return {
|
|
1737
|
-
line: (text2) => {
|
|
1738
|
-
log.line(text2);
|
|
1739
|
-
this.keepItemLine(text2);
|
|
1740
|
-
},
|
|
1741
|
-
error: (text2) => {
|
|
1742
|
-
log.error(text2);
|
|
1743
|
-
this.keepItemLine(text2);
|
|
1744
|
-
}
|
|
1745
|
-
};
|
|
1746
|
-
}
|
|
1747
|
-
/** One line for `item` that does not itself start with its number. */
|
|
1748
|
-
note(item, text2, scope) {
|
|
1749
|
-
this.append(keyOf(item, scope), text2);
|
|
1750
|
-
}
|
|
1751
|
-
/** One line, written only when `text` differs from the last one written under `key` for the item. */
|
|
1752
|
-
noteChanged(item, key, text2, scope) {
|
|
1753
|
-
const slot = `${keyOf(item, scope)}:${key}`;
|
|
1754
|
-
if (this.lastByKey.get(slot) === text2) return;
|
|
1755
|
-
this.lastByKey.set(slot, text2);
|
|
1756
|
-
this.append(keyOf(item, scope), text2);
|
|
1757
|
-
}
|
|
1758
|
-
keepItemLine(text2) {
|
|
1759
|
-
const match = ITEM_LINE.exec(text2);
|
|
1760
|
-
if (match === null) return;
|
|
1761
|
-
const [number = "", scope] = match[1].split("@");
|
|
1762
|
-
this.append(keyOf(number, scope), text2);
|
|
1763
|
-
}
|
|
1764
|
-
append(item, text2) {
|
|
1765
|
-
const dir = this.dirs.get(item);
|
|
1766
|
-
if (dir === void 0) return;
|
|
1767
|
-
try {
|
|
1768
|
-
appendFileSync(join(dir, RUNNER_LOG_FILE), `${logStamp(this.now())} ${text2.replace(/[\r\n]+/g, " ")}
|
|
1769
|
-
`, { mode: 384 });
|
|
1770
|
-
} catch {
|
|
1771
|
-
}
|
|
1772
|
-
}
|
|
1773
|
-
};
|
|
1774
|
-
function readEnd(path, maxBytes) {
|
|
1775
|
-
let fd;
|
|
1776
|
-
try {
|
|
1777
|
-
fd = openSync(path, "r");
|
|
1778
|
-
const size = fstatSync(fd).size;
|
|
1779
|
-
const length = Math.min(size, maxBytes);
|
|
1780
|
-
const buffer = Buffer.alloc(length);
|
|
1781
|
-
readSync(fd, buffer, 0, length, size - length);
|
|
1782
|
-
return buffer.toString("utf8");
|
|
1783
|
-
} catch {
|
|
1784
|
-
return void 0;
|
|
1785
|
-
} finally {
|
|
1786
|
-
if (fd !== void 0) closeSync(fd);
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
function lastLines(text2, count) {
|
|
1790
|
-
const lines = text2.split("\n");
|
|
1791
|
-
if (lines.at(-1) === "") lines.pop();
|
|
1792
|
-
return lines.slice(-200);
|
|
1793
|
-
}
|
|
1794
|
-
function capRunnerLog(text2) {
|
|
1795
|
-
const bytes = Buffer.from(text2, "utf8");
|
|
1796
|
-
if (bytes.length <= RUNNER_LOG_KEEP_BYTES) return text2;
|
|
1797
|
-
let start = bytes.length - RUNNER_LOG_KEEP_BYTES;
|
|
1798
|
-
if (bytes[start - 1] !== 10) {
|
|
1799
|
-
const newline = bytes.indexOf(10, start);
|
|
1800
|
-
if (newline + 1 < bytes.length) {
|
|
1801
|
-
start = newline + 1;
|
|
1802
|
-
} else {
|
|
1803
|
-
while ((bytes[start] & 192) === 128) start++;
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
return `[runner log truncated: the first ${start} bytes were cut]
|
|
1807
|
-
${bytes.subarray(start).toString("utf8")}`;
|
|
1808
|
-
}
|
|
1809
|
-
function buildReleaseLog(jobDirPath, endStage, scrub) {
|
|
1810
|
-
const parts = [];
|
|
1811
|
-
const own = readEnd(join(jobDirPath, RUNNER_LOG_FILE), READ_LIMIT_BYTES);
|
|
1812
|
-
if (own !== void 0 && own.trim() !== "") parts.push(own.endsWith("\n") ? own.slice(0, -1) : own);
|
|
1813
|
-
if (isTailedStage(endStage)) {
|
|
1814
|
-
const file = `${endStage}.log`;
|
|
1815
|
-
const stage = readEnd(join(jobDirPath, file), READ_LIMIT_BYTES);
|
|
1816
|
-
const tail = stage === void 0 ? [] : lastLines(stage);
|
|
1817
|
-
if (tail.length > 0) parts.push(`--- the last ${tail.length} lines of ${file} ---`, ...tail);
|
|
1818
|
-
}
|
|
1819
|
-
if (endStage !== "provision") {
|
|
1820
|
-
const provision = readEnd(join(jobDirPath, "provision.log"), READ_LIMIT_BYTES);
|
|
1821
|
-
const problems = provision === void 0 ? [] : provision.split("\n").filter((line) => PROVISION_PROBLEM.test(line)).slice(-50);
|
|
1822
|
-
if (problems.length > 0) parts.push(`--- provision problems (${problems.length} lines of provision.log) ---`, ...problems);
|
|
1823
|
-
}
|
|
1824
|
-
if (parts.length === 0) return void 0;
|
|
1825
|
-
return capRunnerLog(scrub(parts.join("\n")) + "\n");
|
|
1826
|
-
}
|
|
1827
|
-
var ENDED = "ended before provision finished";
|
|
1828
|
-
async function provisionProcess(input) {
|
|
1829
|
-
const { config, runtime, item, repository, jobDir } = input;
|
|
1830
|
-
if (claudeCredential(runtime.hostEnv) === void 0) return { ok: false, reason: AGENT_CREDENTIAL_MISSING };
|
|
1831
|
-
if (repository === void 0 || repository === null || repository === "") {
|
|
1832
|
-
return { ok: false, reason: "the work item names no repository to clone" };
|
|
1833
|
-
}
|
|
1834
|
-
const buildId = randomBytes(6).toString("hex");
|
|
1835
|
-
const buildDir2 = buildDir(config.configPath, buildId);
|
|
1836
|
-
const workDir = join(jobDir, "work");
|
|
1837
|
-
const state = { buildId, buildDir: buildDir2, workDir };
|
|
1838
|
-
const environment = sharedUserEnvironment(input.cliVersion);
|
|
1839
|
-
input.save(state, environment);
|
|
1840
|
-
const bail = async (reason) => {
|
|
1841
|
-
try {
|
|
1842
|
-
await cleanupProcess(runtime, state);
|
|
1843
|
-
} catch (err) {
|
|
1844
|
-
input.log.error(`#${item.number} cleanup failed: ${describeCallFailure(err)}`);
|
|
1845
|
-
}
|
|
1846
|
-
return { ok: false, reason };
|
|
1847
|
-
};
|
|
1848
|
-
if (input.aborted()) return bail(ENDED);
|
|
1849
|
-
mkdirSync(buildDir2, { recursive: true });
|
|
1850
|
-
chmodSync(buildDir2, 493);
|
|
1851
|
-
try {
|
|
1852
|
-
await startHostServices({ runtime, config, build: state, jobDir, itemKey: input.itemKey, process: true, routeLog: (line) => input.log.line(`#${item.number} ${line}`) });
|
|
1853
|
-
} catch (err) {
|
|
1854
|
-
return bail(`the tunnel could not start: ${describeCallFailure(err)}`);
|
|
1855
|
-
}
|
|
1856
|
-
input.save(state, environment);
|
|
1857
|
-
if (input.aborted()) return bail(ENDED);
|
|
1858
|
-
const checkout = join(workDir, "checkout");
|
|
1859
|
-
const remote = runtime.remoteFor?.(repository) ?? `https://github.com/${repository}.git`;
|
|
1860
|
-
const cloned = await runtime.clone({ remote, dest: checkout });
|
|
1861
|
-
if (!cloned.ok) return bail(cloned.reason);
|
|
1862
|
-
if (input.aborted()) return bail(ENDED);
|
|
1863
|
-
return { ok: true, workdir: checkout, state, environment };
|
|
1864
|
-
}
|
|
1865
|
-
function cleanupProcess(runtime, state) {
|
|
1866
|
-
return closeHostBuild(runtime, state);
|
|
1867
|
-
}
|
|
1868
|
-
var SESSION_TAR = "session.tar";
|
|
1869
|
-
async function guestOf(job, runtime) {
|
|
1870
|
-
if (job.microvm !== void 0) {
|
|
1871
|
-
if (!vmAlive(runtime, job.microvm)) return { reason: "the build's microVM is gone" };
|
|
1872
|
-
return { guest: sessionGuestOfMicroVm(runtime, job.microvm, "root") };
|
|
1873
|
-
}
|
|
1874
|
-
if (job.container !== void 0) {
|
|
1875
|
-
const why = await ensureContainerRunning(runtime, job.container);
|
|
1876
|
-
return why === void 0 ? { guest: sessionGuestOfContainer(runtime, job.container) } : { reason: why };
|
|
1877
|
-
}
|
|
1878
|
-
return { reason: "the build is not a container or microVM build" };
|
|
1879
|
-
}
|
|
1880
|
-
async function keepEndedSession(input) {
|
|
1881
|
-
const { job, log, tag } = input;
|
|
1882
|
-
if (job.sessionId === void 0 || job.workItemId === void 0 || job.sessionKept === true) return false;
|
|
1883
|
-
if (job.microvm === void 0 && job.container === void 0) return false;
|
|
1884
|
-
const found = await guestOf(job, input.runtime);
|
|
1885
|
-
if ("reason" in found) {
|
|
1886
|
-
log.error(`${tag} the session was not kept: ${found.reason} \xB7 continuing`);
|
|
1887
|
-
return false;
|
|
1888
|
-
}
|
|
1889
|
-
let changed = false;
|
|
1890
|
-
const records = await readResumeRecords(found.guest);
|
|
1891
|
-
if (records.unread !== void 0) log.error(`${tag} the launcher's records could not be read, so a fallback to a fresh session is not reported: ${records.unread} \xB7 continuing`);
|
|
1892
|
-
if (records.launchedAt !== void 0) {
|
|
1893
|
-
job.sessionLaunchedAt = records.launchedAt;
|
|
1894
|
-
changed = true;
|
|
1895
|
-
}
|
|
1896
|
-
if (records.failed && job.resume?.resumed === true) {
|
|
1897
|
-
job.resume = { resumed: false, reason: "session_not_loaded" };
|
|
1898
|
-
changed = true;
|
|
1899
|
-
log.line(`${tag} agent job started fresh because the saved session could not be loaded`);
|
|
1900
|
-
}
|
|
1901
|
-
const tarPath = join(input.jobDirPath, SESSION_TAR);
|
|
1902
|
-
try {
|
|
1903
|
-
const copied = await copyOutSession({ guest: found.guest, sessionId: job.sessionId, tarPath });
|
|
1904
|
-
if (!copied.ok) {
|
|
1905
|
-
log.error(`${tag} the session was not kept: ${copied.reason} \xB7 continuing`);
|
|
1906
|
-
return changed;
|
|
1907
|
-
}
|
|
1908
|
-
await keepSession({
|
|
1909
|
-
dir: keptSessionDir(input.configPath, job.organizationId, job.workItemId),
|
|
1910
|
-
meta: { sessionId: job.sessionId, claimId: job.claimId, queue: job.queue, keptAt: input.now().toISOString() },
|
|
1911
|
-
tar: createReadStream(tarPath)
|
|
1912
|
-
});
|
|
1913
|
-
log.line(`${tag} session ${job.sessionId.slice(0, 8)} kept for the next agent job (${formatBytes(statSync(tarPath).size)})`);
|
|
1914
|
-
job.sessionKept = true;
|
|
1915
|
-
return true;
|
|
1916
|
-
} catch (err) {
|
|
1917
|
-
log.error(`${tag} the session was not kept: ${describeCallFailure(err)} \xB7 continuing`);
|
|
1918
|
-
return changed;
|
|
1919
|
-
} finally {
|
|
1920
|
-
rmSync(tarPath, { force: true });
|
|
1921
|
-
}
|
|
1922
|
-
}
|
|
1923
|
-
|
|
1924
|
-
// src/runner/judge-verdict.ts
|
|
1925
|
-
function firstLine2(text2) {
|
|
1926
|
-
return text2.trim().split(/\r\n|\r|\n/)[0]?.trim() ?? "";
|
|
1927
|
-
}
|
|
1928
|
-
function parseOutput(stdout) {
|
|
1929
|
-
try {
|
|
1930
|
-
const parsed = JSON.parse(stdout);
|
|
1931
|
-
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
1932
|
-
const { subtype, result } = parsed;
|
|
1933
|
-
return { subtype: typeof subtype === "string" ? subtype : void 0, result: typeof result === "string" ? result : "" };
|
|
1934
|
-
} catch {
|
|
1935
|
-
return void 0;
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
var RESULT_LINE = /^result:\s*(pass|fail)\s*$/i;
|
|
1939
|
-
var UNMET_LINE = /^unmet:\s*(\S+)\s*$/i;
|
|
1940
|
-
function readJudgeReply(run, minutes) {
|
|
1941
|
-
if (run.kind === "timeout") return { kind: "none", reason: `the judge ran out of time after ${minutes}`, limitKey: true };
|
|
1942
|
-
const output = parseOutput(run.stdout);
|
|
1943
|
-
if (output === void 0) {
|
|
1944
|
-
const said2 = firstLine2(run.stderr);
|
|
1945
|
-
return { kind: "none", reason: `the judge exited ${run.status} and printed no result${said2 === "" ? "" : `: ${said2}`}` };
|
|
1946
|
-
}
|
|
1947
|
-
if (output.subtype === "error_max_turns") return { kind: "none", reason: `the judge reached the turn limit of ${JUDGE_MAX_TURNS} turns before it gave a result` };
|
|
1948
|
-
const lines = output.result.split(/\r\n|\r|\n/).map((line) => line.trim()).filter((line) => line !== "");
|
|
1949
|
-
const verdict = RESULT_LINE.exec(lines.at(-1) ?? "")?.[1]?.toLowerCase();
|
|
1950
|
-
if (verdict === void 0) return { kind: "none", reason: "the judge gave no `result:` line" };
|
|
1951
|
-
return { kind: "answered", verdict, unmet: new Set(lines.flatMap((line) => UNMET_LINE.exec(line)?.[1] ?? [])) };
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
// src/runner/judge-report.ts
|
|
1955
|
-
function escape(text2) {
|
|
1956
|
-
return text2.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
1957
|
-
}
|
|
1958
|
-
var RESULT_LABEL = { pass: "satisfied", fail: "not satisfied", skipped: "not judged" };
|
|
1959
|
-
function refsCell(item) {
|
|
1960
|
-
return (item.refs ?? []).map((ref) => {
|
|
1961
|
-
const label = escape(ref.line === void 0 ? ref.path : `${ref.path}:${ref.line}`);
|
|
1962
|
-
return ref.href === void 0 ? `<code>${label}</code>` : `<a href="${escape(ref.href)}"><code>${label}</code></a>`;
|
|
1963
|
-
}).join("<br>");
|
|
1964
|
-
}
|
|
1965
|
-
function renderJudgeReport(input) {
|
|
1966
|
-
const count = (result) => input.items.filter((item) => item.result === result).length;
|
|
1967
|
-
const summary = `${count("pass")} satisfied, ${count("fail")} not satisfied, ${count("skipped")} not judged`;
|
|
1968
|
-
const rows = input.items.map(
|
|
1969
|
-
(item) => `<tr class="${item.result}"><td><code>${escape(item.id)}</code></td><td>${RESULT_LABEL[item.result]}</td><td>${escape(item.reason ?? "")}</td><td>${refsCell(item)}</td></tr>`
|
|
1970
|
-
).join("\n");
|
|
1971
|
-
return `<!doctype html>
|
|
1972
|
-
<html lang="en">
|
|
1973
|
-
<head>
|
|
1974
|
-
<meta charset="utf-8">
|
|
1975
|
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
1976
|
-
<title>${escape(input.check)}</title>
|
|
1977
|
-
<style>
|
|
1978
|
-
:root { color-scheme: light dark; --fail: #b42318; --pass: #067647; --muted: #667085; }
|
|
1979
|
-
body { font: 14px/1.5 system-ui, sans-serif; margin: 24px; max-width: 1100px; }
|
|
1980
|
-
table { border-collapse: collapse; width: 100%; }
|
|
1981
|
-
th, td { text-align: left; vertical-align: top; padding: 6px 10px; border-bottom: 1px solid color-mix(in srgb, currentColor 15%, transparent); }
|
|
1982
|
-
tr.fail td:nth-child(2) { color: var(--fail); font-weight: 600; }
|
|
1983
|
-
tr.pass td:nth-child(2) { color: var(--pass); }
|
|
1984
|
-
tr.skipped td:nth-child(2) { color: var(--muted); }
|
|
1985
|
-
.meta { color: var(--muted); }
|
|
1986
|
-
</style>
|
|
1987
|
-
${input.extraHead ?? ""}
|
|
1988
|
-
</head>
|
|
1989
|
-
<body>
|
|
1990
|
-
<h1>${escape(input.check)}: ${input.result}</h1>
|
|
1991
|
-
<p>${escape(summary)}.${input.reason === void 0 ? "" : ` ${escape(input.reason)}.`}</p>
|
|
1992
|
-
<p class="meta">Judged at commit <code>${escape(input.commit)}</code>.</p>
|
|
1993
|
-
<table>
|
|
1994
|
-
<thead><tr><th>Item</th><th>Result</th><th>Reason</th><th>Tests</th></tr></thead>
|
|
1995
|
-
<tbody>
|
|
1996
|
-
${rows}
|
|
1997
|
-
</tbody>
|
|
1998
|
-
</table>
|
|
1999
|
-
</body>
|
|
2000
|
-
</html>
|
|
2001
|
-
`;
|
|
2002
|
-
}
|
|
2003
|
-
|
|
2004
|
-
// src/runner/judge.ts
|
|
2005
|
-
var LIMIT_KEY_TAIL = " (runner.checks.timeLimitMinutes)";
|
|
2006
|
-
var MAX_DIFF_BYTES = 4 * 1024 * 1024;
|
|
2007
|
-
var NO_CLAUDE = "the image has no Claude Code for the judge";
|
|
2008
|
-
var REPORT_CAPTION = "The host's criteria judge, each criterion judged against the test that cites it";
|
|
2009
|
-
function testFilesIn(root, isTest, limits = { files: MAX_TEST_FILES, bytes: MAX_TEST_FILE_BYTES }) {
|
|
2010
|
-
const files = [];
|
|
2011
|
-
const walk = (dir, prefix) => {
|
|
2012
|
-
for (const name of readdirSync(dir).sort()) {
|
|
2013
|
-
if (files.length >= limits.files) return;
|
|
2014
|
-
if (prefix === "" && name === ".git") continue;
|
|
2015
|
-
const path = join(dir, name);
|
|
2016
|
-
const rel = prefix === "" ? name : `${prefix}/${name}`;
|
|
2017
|
-
const stat = lstatSync(path);
|
|
2018
|
-
if (stat.isDirectory()) walk(path, rel);
|
|
2019
|
-
else if (stat.isFile() && isTest(rel) && stat.size <= limits.bytes) files.push({ path: rel, contents: readFileSync(path, "utf8") });
|
|
2020
|
-
}
|
|
2021
|
-
};
|
|
2022
|
-
walk(root, "");
|
|
2023
|
-
return files;
|
|
2024
|
-
}
|
|
2025
|
-
function findIdsOutsideTestsIn(root, input, limits = { files: MAX_TEST_FILES, bytes: MAX_TEST_FILE_BYTES }) {
|
|
2026
|
-
const candidates = [];
|
|
2027
|
-
const walk = (dir, prefix) => {
|
|
2028
|
-
for (const name of readdirSync(dir)) {
|
|
2029
|
-
if (prefix === "" && name === ".git") continue;
|
|
2030
|
-
const path = join(dir, name);
|
|
2031
|
-
const rel = prefix === "" ? name : `${prefix}/${name}`;
|
|
2032
|
-
const stat = lstatSync(path);
|
|
2033
|
-
if (stat.isDirectory()) walk(path, rel);
|
|
2034
|
-
else if (stat.isFile() && isSearchCandidate({ path: rel, size: stat.size }, input, limits.bytes)) candidates.push({ path: rel, size: stat.size });
|
|
2035
|
-
}
|
|
2036
|
-
};
|
|
2037
|
-
walk(root, "");
|
|
2038
|
-
const scan = new IdScan(input.finder, input.ids);
|
|
2039
|
-
for (const { path } of firstInPathOrder(candidates, limits.files)) {
|
|
2040
|
-
if (scan.complete) break;
|
|
2041
|
-
let contents;
|
|
2042
|
-
try {
|
|
2043
|
-
contents = readFileSync(join(root, path), "utf8");
|
|
2044
|
-
} catch {
|
|
2045
|
-
continue;
|
|
2046
|
-
}
|
|
2047
|
-
scan.offer(path, contents);
|
|
2048
|
-
}
|
|
2049
|
-
return scan.result();
|
|
2050
|
-
}
|
|
2051
|
-
function proofsFileIn(root) {
|
|
2052
|
-
const path = join(root, ".outerlayer-proofs.yaml");
|
|
2053
|
-
try {
|
|
2054
|
-
const stat = lstatSync(path);
|
|
2055
|
-
return stat.isFile() && stat.size <= MAX_TEST_FILE_BYTES ? readFileSync(path, "utf8") : void 0;
|
|
2056
|
-
} catch {
|
|
2057
|
-
return void 0;
|
|
2058
|
-
}
|
|
2059
|
-
}
|
|
2060
|
-
async function hostPassedNames(config, item, fetchImpl) {
|
|
2061
|
-
const workItemId = Object(await readWorkItem(config, item, fetchImpl)).workItemId;
|
|
2062
|
-
if (typeof workItemId !== "string") throw new WorkCommandError("the work item read named no id");
|
|
2063
|
-
const signer = await signerFor(config.configPath);
|
|
2064
|
-
const latest = /* @__PURE__ */ new Map();
|
|
2065
|
-
let since = null;
|
|
2066
|
-
do {
|
|
2067
|
-
const query = since === null ? "" : `?since=${encodeURIComponent(since)}`;
|
|
2068
|
-
const page = await getJson(config.url, `/v1/work-items/${encodeURIComponent(workItemId)}/changes${query}`, config.apiKey, config.appId, fetchImpl, signer);
|
|
2069
|
-
for (const entry of Array.isArray(page.data) ? page.data : []) {
|
|
2070
|
-
const { name, result, source } = Object(entry);
|
|
2071
|
-
if (source === "host" && typeof name === "string" && typeof result === "string") latest.set(name, result);
|
|
2072
|
-
}
|
|
2073
|
-
since = page.pagination?.cursor ?? null;
|
|
2074
|
-
} while (since !== null);
|
|
2075
|
-
return [...latest].filter(([, result]) => result === "pass").map(([name]) => name);
|
|
2076
|
-
}
|
|
2077
|
-
function environmentFor(input) {
|
|
2078
|
-
const { config, job, jobDir, deps } = input;
|
|
2079
|
-
if (deps.judgeEnvironment !== void 0) return deps.judgeEnvironment;
|
|
2080
|
-
if (job.container !== void 0) return containerJudgeEnvironment({ runtime: deps.runtime, config, job, jobDir, log: (line) => deps.log.line(`${itemTag(job)} ${line}`) });
|
|
2081
|
-
if (job.microvm !== void 0) return microVmJudgeEnvironment({ runtime: deps.runtime, job, jobDir });
|
|
2082
|
-
return void 0;
|
|
2083
|
-
}
|
|
2084
|
-
async function runJudge(input) {
|
|
2085
|
-
const { config, job, jobDir, deps } = input;
|
|
2086
|
-
const { runtime, log } = deps;
|
|
2087
|
-
const tag = `${itemTag(job)}`;
|
|
2088
|
-
const now = deps.now ?? (() => /* @__PURE__ */ new Date());
|
|
2089
|
-
const repository = job.repository ?? job.container?.repository;
|
|
2090
|
-
if (repository === void 0) return;
|
|
2091
|
-
const repo = repository;
|
|
2092
|
-
let itemRead;
|
|
2093
|
-
const readItem = () => itemRead ??= readWorkItem(itemCalls(config, job), job.item, deps.fetchImpl).catch((err) => {
|
|
2094
|
-
itemRead = void 0;
|
|
2095
|
-
throw err;
|
|
2096
|
-
});
|
|
2097
|
-
const tokens = runtime.credentials.get(jobDir);
|
|
2098
|
-
let readToken;
|
|
2099
|
-
try {
|
|
2100
|
-
readToken = await tokens?.token("read");
|
|
2101
|
-
} catch (err) {
|
|
2102
|
-
log.error(`${tag} no read token for the judge: ${describeCallFailure(err)}`);
|
|
2103
|
-
}
|
|
2104
|
-
const scratch = join(jobDir, "judge-scratch");
|
|
2105
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
2106
|
-
const definitionsDir = join(scratch, "definitions");
|
|
2107
|
-
mkdirSync(definitionsDir, { recursive: true });
|
|
2108
|
-
let definitions;
|
|
2109
|
-
try {
|
|
2110
|
-
definitions = await readDefinitions(input, repository, definitionsDir, readToken, tokens);
|
|
2111
|
-
} catch (err) {
|
|
2112
|
-
definitions = { ok: false, reason: describeCallFailure(err) };
|
|
2113
|
-
} finally {
|
|
2114
|
-
removeStagedContext(definitionsDir);
|
|
2115
|
-
}
|
|
2116
|
-
if (!definitions.ok) {
|
|
2117
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
2118
|
-
log.line(`${tag} judge not run: the definitions could not be read: ${definitions.reason}`);
|
|
2119
|
-
job.judgeNotRun = `the definitions could not be read: ${definitions.reason}`;
|
|
2120
|
-
const passed = await retrySend(() => hostPassedNames(itemCalls(config, job), job.item, deps.fetchImpl), { now: deps.now });
|
|
2121
|
-
if (!passed.ok) {
|
|
2122
|
-
log.error(`${tag} the item's host results could not be read, so none was failed: ${passed.reason}`);
|
|
2123
|
-
return;
|
|
2124
|
-
}
|
|
2125
|
-
const record2 = makeRecorder({ config, job, jobDir, deps });
|
|
2126
|
-
for (const name of passed.value) await record2(name, "fail", `the definitions could not be read: ${definitions.reason}`, { idTag: "judge-unread" });
|
|
2127
|
-
return;
|
|
2128
|
-
}
|
|
2129
|
-
for (const id of definitions.unrecorded) {
|
|
2130
|
-
log.line(`${tag} judge not run for ${id}: the check is not run.where: host with a judge or a command, so this host records no result for it`);
|
|
2131
|
-
}
|
|
2132
|
-
const checks = definitions.checks.filter(isJudgeCheck);
|
|
2133
|
-
if (checks.length === 0) {
|
|
2134
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
2135
|
-
return;
|
|
2136
|
-
}
|
|
2137
|
-
log.line(`${tag} judge started (${checks.map((check) => check.name).join(", ")})`);
|
|
2138
|
-
const startedMs = now().getTime();
|
|
2139
|
-
const record = makeRecorder({ config, job, jobDir, deps });
|
|
2140
|
-
const fail = (check, reason) => {
|
|
2141
|
-
log.error(`${tag} judge failed: ${reason}`);
|
|
2142
|
-
return record(check.name, "fail", reason);
|
|
2143
|
-
};
|
|
2144
|
-
try {
|
|
2145
|
-
for (const check of checks) {
|
|
2146
|
-
try {
|
|
2147
|
-
await judgeOne(check);
|
|
2148
|
-
} catch (err) {
|
|
2149
|
-
await fail(check, `the judge could not be run: ${describeCallFailure(err)}`);
|
|
2150
|
-
}
|
|
2151
|
-
}
|
|
2152
|
-
} finally {
|
|
2153
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
2154
|
-
log.line(`${tag} judge done (${formatDuration(now().getTime() - startedMs)})`);
|
|
2155
|
-
}
|
|
2156
|
-
async function judgeOne(check) {
|
|
2157
|
-
const environment = environmentFor(input);
|
|
2158
|
-
if (environment === void 0) return await fail(check, `${NO_CLAUDE}: this build ran outside a container or microVM`);
|
|
2159
|
-
const expected = (job.container ?? job.microvm)?.judgeProgram;
|
|
2160
|
-
if (expected === void 0) return await fail(check, NO_CLAUDE);
|
|
2161
|
-
const brief = check.judge.text;
|
|
2162
|
-
if (brief === void 0) return await fail(check, `the brief ${check.judge.brief} is not in the definitions repository`);
|
|
2163
|
-
if (tokens === void 0) return await fail(check, "the build's repository tokens are no longer held, so the branch cannot be fetched");
|
|
2164
|
-
const credential = config.managed === true && managedClaudeKeyFor(jobDir) !== void 0 ? managedClaudeCredential() : claudeCredential(runtime.hostEnv);
|
|
2165
|
-
if (credential === void 0) return await fail(check, "the host holds no Claude credential for the judge");
|
|
2166
|
-
const branch = job.branch;
|
|
2167
|
-
if (branch === void 0) return await fail(check, "the build pushed no branch: the claim names none");
|
|
2168
|
-
const remote = runtime.remoteFor?.(repo) ?? `${runtime.github.gitUrl}/${repo}.git`;
|
|
2169
|
-
try {
|
|
2170
|
-
await environment.stop();
|
|
2171
|
-
} catch (err) {
|
|
2172
|
-
return await fail(check, `the build's processes could not be stopped: ${describeCallFailure(err)}`);
|
|
2173
|
-
}
|
|
2174
|
-
const hashed = await environment.hashProgram();
|
|
2175
|
-
if (!hashed.ok) return await fail(check, `${NO_CLAUDE}: ${hashed.reason}`);
|
|
2176
|
-
if (hashed.sha256 !== expected.sha256) return await fail(check, "the judge program changed during the build");
|
|
2177
|
-
const read = await (deps.readBranch ?? readBranchHead)({ remote, dir: scratch, branch, token: readToken });
|
|
2178
|
-
if (read.kind === "absent") return await fail(check, `the build pushed no branch: ${branch} does not exist on ${repo}`);
|
|
2179
|
-
if (read.kind === "unreadable") return await fail(check, `the branch ${branch} could not be read: ${read.reason}`);
|
|
2180
|
-
const head = read.sha;
|
|
2181
|
-
let criteria;
|
|
2182
|
-
let previous = [];
|
|
2183
|
-
if (check.judge.subjects === "criteria") {
|
|
2184
|
-
let workItem;
|
|
2185
|
-
try {
|
|
2186
|
-
workItem = await readItem();
|
|
2187
|
-
} catch (err) {
|
|
2188
|
-
return await fail(check, `the item's criteria could not be read: ${describeCallFailure(err)}`);
|
|
2189
|
-
}
|
|
2190
|
-
criteria = criteriaOf(workItem);
|
|
2191
|
-
if (criteria === void 0) return await fail(check, "no criteria are recorded for the item");
|
|
2192
|
-
previous = hostItemsOf(workItem, check.name);
|
|
2193
|
-
}
|
|
2194
|
-
const branchDir = join(scratch, "branch");
|
|
2195
|
-
const fetched = await (deps.fetchCommit ?? fetchCommit)({ remote, dir: branchDir, commit: head, token: readToken });
|
|
2196
|
-
if (!fetched.ok) return await fail(check, `the build's tests could not be fetched: ${fetched.reason}`);
|
|
2197
|
-
const finder = await loadCitationFinder();
|
|
2198
|
-
const list = criteria ?? [];
|
|
2199
|
-
const isTest = testFileMatcher(finder, proofsFileIn(branchDir));
|
|
2200
|
-
const testFiles = testFilesIn(branchDir, isTest);
|
|
2201
|
-
const citations = finder.buildCitations({ ids: list.map((criterion) => criterion.id), files: testFiles, repository: repo, sha: head });
|
|
2202
|
-
const testFileText = new Map(testFiles.map((file) => [file.path, file.contents]));
|
|
2203
|
-
const inputs = new Map(list.map((criterion) => [criterion.id, criterionInputs({ brief, criterion, citations, testFiles: testFileText })]));
|
|
2204
|
-
const uncited = uncitedIds(list, citations);
|
|
2205
|
-
let foundIn;
|
|
2206
|
-
if (uncited.length > 0) {
|
|
2207
|
-
try {
|
|
2208
|
-
foundIn = (deps.findIdsOutside ?? findIdsOutsideTestsIn)(branchDir, { isTest, isSearchable: finder.isSearchableFile, finder, ids: uncited });
|
|
2209
|
-
} catch (err) {
|
|
2210
|
-
log.error(`${tag} the search for the ids of the criteria that no test cites failed: ${describeCallFailure(err)}`);
|
|
2211
|
-
}
|
|
2212
|
-
}
|
|
2213
|
-
const plan = planCriteria({ criteria: list, citations, inputs, previous, noTestFiles: testFiles.length === 0, foundIn });
|
|
2214
|
-
if (plan.toJudge.length < list.length) {
|
|
2215
|
-
log.line(`${tag} judge: ${plan.toJudge.length} of ${list.length} criteria to judge, ${list.length - plan.toJudge.length} settled without the judge`);
|
|
2216
|
-
}
|
|
2217
|
-
if (criteria !== void 0 && plan.toJudge.length === 0) {
|
|
2218
|
-
return await finish(check, { criteria, items: plan.settled, citations, head, branch, finder, judgeReport: void 0 });
|
|
2219
|
-
}
|
|
2220
|
-
let diff = "";
|
|
2221
|
-
const startCommit = (job.container ?? job.microvm)?.startCommit;
|
|
2222
|
-
if (startCommit !== void 0) {
|
|
2223
|
-
const output = join(scratch, "diff.patch");
|
|
2224
|
-
const made = await (deps.fetchDiff ?? ((request) => diffBetween(request)))({ remote, dir: join(scratch, "diff"), base: startCommit, head, token: readToken, output });
|
|
2225
|
-
if (!made.ok) return await fail(check, `the build's diff could not be read: ${made.reason}`);
|
|
2226
|
-
const bytes = readFileSync(output);
|
|
2227
|
-
diff = bytes.length > MAX_DIFF_BYTES ? `${bytes.subarray(0, MAX_DIFF_BYTES).toString("utf8")}
|
|
2228
|
-
[the diff was cut at ${MAX_DIFF_BYTES} bytes]
|
|
2229
|
-
` : bytes.toString("utf8");
|
|
2230
|
-
}
|
|
2231
|
-
await environment.prepare(
|
|
2232
|
-
{
|
|
2233
|
-
...criteria === void 0 ? {} : {
|
|
2234
|
-
// Only the criteria the judge is asked about. The others are settled.
|
|
2235
|
-
"criteria.json": `${JSON.stringify({ criteria: plan.toJudge }, null, 2)}
|
|
2236
|
-
`,
|
|
2237
|
-
"citations.json": `${JSON.stringify(citations, null, 2)}
|
|
2238
|
-
`
|
|
2239
|
-
},
|
|
2240
|
-
"diff.patch": diff
|
|
2241
|
-
},
|
|
2242
|
-
branchDir
|
|
2243
|
-
);
|
|
2244
|
-
const limitMinutes = config.checks?.timeLimitMinutes ?? DEFAULT_CHECK_TIME_LIMIT_MINUTES;
|
|
2245
|
-
const minutes = `${limitMinutes} ${limitMinutes === 1 ? "minute" : "minutes"}`;
|
|
2246
|
-
const clear = async () => {
|
|
2247
|
-
await environment.clear?.().catch((err) => log.error(`${tag} what the judge left in its folder could not be removed: ${describeCallFailure(err)}`));
|
|
2248
|
-
};
|
|
2249
|
-
const prompt = withAnswerProtocol(brief, check.judge.subjects);
|
|
2250
|
-
const running = environment.run(judgeCommand({ brief: prompt, credential }), { logPath: join(jobDir, "judge.log") });
|
|
2251
|
-
running.catch(() => void 0);
|
|
2252
|
-
const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
|
|
2253
|
-
let outcome;
|
|
2254
|
-
try {
|
|
2255
|
-
outcome = await Promise.race([running, timer.done]);
|
|
2256
|
-
} catch (err) {
|
|
2257
|
-
await clear();
|
|
2258
|
-
return await fail(check, `the judge could not be run: ${describeCallFailure(err)}`);
|
|
2259
|
-
} finally {
|
|
2260
|
-
timer.cancel();
|
|
2261
|
-
}
|
|
2262
|
-
let report;
|
|
2263
|
-
let answerText;
|
|
2264
|
-
const readOutputs = async () => {
|
|
2265
|
-
try {
|
|
2266
|
-
report = await environment.readReport();
|
|
2267
|
-
} catch (err) {
|
|
2268
|
-
log.error(`${tag} the judge's report could not be read: ${describeCallFailure(err)}`);
|
|
2269
|
-
}
|
|
2270
|
-
try {
|
|
2271
|
-
answerText = await environment.readItems();
|
|
2272
|
-
} catch (err) {
|
|
2273
|
-
log.error(`${tag} the judge's items could not be read: ${describeCallFailure(err)}`);
|
|
2274
|
-
}
|
|
2275
|
-
};
|
|
2276
|
-
if (outcome === "timeout") {
|
|
2277
|
-
await readOutputs();
|
|
2278
|
-
await environment.stop().catch((err) => log.error(`${tag} the judge could not be stopped after its time ran out: ${describeCallFailure(err)}`));
|
|
2279
|
-
} else {
|
|
2280
|
-
await readOutputs();
|
|
2281
|
-
}
|
|
2282
|
-
await clear();
|
|
2283
|
-
const run = outcome === "timeout" ? { kind: "timeout" } : { kind: "exited", status: outcome.status, stdout: outcome.stdout, stderr: outcome.stderr };
|
|
2284
|
-
const reply = readJudgeReply(run, minutes);
|
|
2285
|
-
const answer = parseJudgeAnswer(answerText);
|
|
2286
|
-
if (criteria === void 0) return await finishWhole(check, { reply, answer, report, head, branch });
|
|
2287
|
-
const judged = (results2) => [
|
|
2288
|
-
...plan.settled,
|
|
2289
|
-
...judgedItems({ toJudge: plan.toJudge, results: results2, citations, inputs })
|
|
2290
|
-
];
|
|
2291
|
-
if (answer?.ok === true) {
|
|
2292
|
-
const results2 = new Map(answer.answer.items.map((entry) => [entry.id, entry]));
|
|
2293
|
-
return await finish(check, { criteria, items: judged(results2), citations, head, branch, finder, judgeReport: void 0 });
|
|
2294
|
-
}
|
|
2295
|
-
if (reply.kind === "none") {
|
|
2296
|
-
return await finish(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: reply.reason, limitKey: reply.limitKey === true });
|
|
2297
|
-
}
|
|
2298
|
-
if (answer !== void 0) {
|
|
2299
|
-
return await finish(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: answer.reason });
|
|
2300
|
-
}
|
|
2301
|
-
const failsAll = reply.verdict === "fail" && reply.unmet.size === 0;
|
|
2302
|
-
const results = new Map(plan.toJudge.map((criterion) => [criterion.id, { result: failsAll || reply.unmet.has(criterion.id) ? "fail" : "pass" }]));
|
|
2303
|
-
return await finish(check, { criteria, items: judged(results), citations, head, branch, finder, judgeReport: report });
|
|
2304
|
-
}
|
|
2305
|
-
async function finish(check, at) {
|
|
2306
|
-
const items = inCriteriaOrder(at.items, at.criteria);
|
|
2307
|
-
const failing = itemsSentence(items, { one: "criterion", many: "criteria" });
|
|
2308
|
-
const result = failing === void 0 ? "pass" : "fail";
|
|
2309
|
-
const reason = at.reason ?? failing;
|
|
2310
|
-
const report = at.judgeReport !== void 0 && at.judgeReport !== "" ? at.judgeReport : renderJudgeReport({ check: check.name, result, reason, items, commit: at.head, extraHead: at.finder.renderBlock(at.citations) });
|
|
2311
|
-
const link = await uploadReport(check, report, { branch: at.branch, head: at.head });
|
|
2312
|
-
await record(check.name, result, reason, { keyTail: at.limitKey === true ? LIMIT_KEY_TAIL : "", link, items: sendable(items) });
|
|
2313
|
-
}
|
|
2314
|
-
async function finishWhole(check, at) {
|
|
2315
|
-
const where = { head: at.head, branch: at.branch };
|
|
2316
|
-
if (at.answer?.ok === true) {
|
|
2317
|
-
const items = at.answer.answer.items.map((entry) => {
|
|
2318
|
-
const reason2 = oneLineReason(entry.reason);
|
|
2319
|
-
return { id: entry.id, result: entry.result, ...reason2 === void 0 ? {} : { reason: reason2 } };
|
|
2320
|
-
});
|
|
2321
|
-
const failing = itemsSentence(items, { one: "check item", many: "check items" });
|
|
2322
|
-
const said2 = at.answer.answer.result ?? (at.reply.kind === "answered" ? at.reply.verdict : void 0);
|
|
2323
|
-
const result = said2 === "pass" && failing === void 0 ? "pass" : "fail";
|
|
2324
|
-
const reason = failing ?? (said2 === "fail" ? "the judge failed the change" : said2 === void 0 ? at.reply.kind === "none" ? at.reply.reason : "the judge gave no result" : void 0);
|
|
2325
|
-
const report = renderJudgeReport({ check: check.name, result, reason, items, commit: at.head });
|
|
2326
|
-
return await recordWhole(check, { ...where, result, reason, items, report });
|
|
2327
|
-
}
|
|
2328
|
-
if (at.reply.kind === "none") return await recordWhole(check, { ...where, result: "fail", reason: at.reply.reason, report: at.report, limitKey: at.reply.limitKey === true });
|
|
2329
|
-
if (at.answer !== void 0) return await recordWhole(check, { ...where, result: "fail", reason: at.answer.reason, report: at.report });
|
|
2330
|
-
const failed = at.reply.verdict === "fail";
|
|
2331
|
-
return await recordWhole(check, { ...where, result: at.reply.verdict, reason: failed ? "the judge failed the change" : void 0, report: at.report });
|
|
2332
|
-
}
|
|
2333
|
-
async function recordWhole(check, at) {
|
|
2334
|
-
const link = at.report === void 0 ? void 0 : await uploadReport(check, at.report, { branch: at.branch, head: at.head });
|
|
2335
|
-
await record(check.name, at.result, at.reason, { keyTail: at.limitKey === true ? LIMIT_KEY_TAIL : "", link, items: sendable(at.items) });
|
|
2336
|
-
}
|
|
2337
|
-
function sendable(items) {
|
|
2338
|
-
if (items === void 0) return void 0;
|
|
2339
|
-
const parsed = CheckItemsSchema.safeParse(items);
|
|
2340
|
-
if (parsed.success) return parsed.data;
|
|
2341
|
-
log.error(`${tag} the judge's check items were left off the result: ${parsed.error.issues.map((issue) => issue.message).join("; ")}`);
|
|
2342
|
-
return void 0;
|
|
2343
|
-
}
|
|
2344
|
-
async function uploadReport(check, report, at) {
|
|
2345
|
-
const bytes = Buffer.from(report, "utf8");
|
|
2346
|
-
if (bytes.length === 0 || bytes.length > MAX_REPORT_BYTES) {
|
|
2347
|
-
log.error(`${tag} the judge's report was not uploaded: it is ${bytes.length === 0 ? "empty" : `larger than ${MAX_REPORT_BYTES} bytes`}`);
|
|
2348
|
-
return void 0;
|
|
2349
|
-
}
|
|
2350
|
-
const clientArtifactId = `host-judge-${createHash("sha256").update(`${job.claimId}:${job.startedAt}:${check.name}`).digest("hex").slice(0, 40)}`;
|
|
2351
|
-
const calls = itemCalls(config, job);
|
|
2352
|
-
let prNumber;
|
|
2353
|
-
if (config.fleet === true) {
|
|
2354
|
-
try {
|
|
2355
|
-
prNumber = declaredPullRequest(await readItem(), repo);
|
|
2356
|
-
} catch (err) {
|
|
2357
|
-
log.error(`${tag} the item's pull requests could not be read for the judge's report: ${describeCallFailure(err)}`);
|
|
2358
|
-
}
|
|
2359
|
-
}
|
|
2360
|
-
const request = {
|
|
2361
|
-
schemaVersion: 1,
|
|
2362
|
-
artifact: {
|
|
2363
|
-
clientArtifactId,
|
|
2364
|
-
filename: "criteria-judge-report.html",
|
|
2365
|
-
mediaType: "text/html",
|
|
2366
|
-
bytes: bytes.length,
|
|
2367
|
-
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
2368
|
-
caption: REPORT_CAPTION,
|
|
2369
|
-
criterionId: check.name,
|
|
2370
|
-
emittedAt: now().toISOString(),
|
|
2371
|
-
repository: repo,
|
|
2372
|
-
gitRepo: repo,
|
|
2373
|
-
gitBranch: at.branch,
|
|
2374
|
-
commitSha: at.head,
|
|
2375
|
-
...prNumber === void 0 ? {} : { prNumber }
|
|
2376
|
-
},
|
|
2377
|
-
blob: { data: bytes.toString("base64") }
|
|
2378
|
-
};
|
|
2379
|
-
const signer = await signerFor(config.configPath);
|
|
2380
|
-
const sent = await retrySend(() => postJson(calls.url, "/v1/artifacts", calls.apiKey, calls.appId, request, deps.fetchImpl, void 0, signer), { now: deps.now });
|
|
2381
|
-
if (!sent.ok) {
|
|
2382
|
-
log.error(`${tag} the judge's report could not be uploaded: ${sent.reason}`);
|
|
2383
|
-
return void 0;
|
|
2384
|
-
}
|
|
2385
|
-
const url = sent.value.data?.url;
|
|
2386
|
-
return typeof url === "string" && /^https?:\/\/\S+$/.test(url) ? url : void 0;
|
|
2387
|
-
}
|
|
2388
|
-
}
|
|
2389
|
-
|
|
2390
|
-
// src/runner/job.ts
|
|
2391
|
-
var RENEW_INTERVAL_SECONDS = 300;
|
|
2392
|
-
var IDLE_CPU_USEC_PER_MINUTE_FLOOR = 2e6;
|
|
2393
|
-
function jobEnv(config, item, itemKey, jobDirPath, baseEnv = process.env, branch, author, spoolHome) {
|
|
2394
|
-
const secrets = new Set(headerVariables(config.build?.destinations));
|
|
2395
|
-
const passthrough = {};
|
|
2396
|
-
for (const [key, value] of Object.entries(baseEnv)) {
|
|
2397
|
-
if (value === void 0) continue;
|
|
2398
|
-
if (key.startsWith("OUTERLAYER_")) continue;
|
|
2399
|
-
if (secrets.has(key)) continue;
|
|
2400
|
-
passthrough[key] = value;
|
|
2401
|
-
}
|
|
2402
|
-
const path = cliFirstPath(passthrough.PATH);
|
|
2403
|
-
return {
|
|
2404
|
-
...passthrough,
|
|
2405
|
-
...path === void 0 ? {} : { PATH: path },
|
|
2406
|
-
OUTERLAYER_URL: config.url,
|
|
2407
|
-
...itemKey === void 0 ? {} : { OUTERLAYER_API_KEY: itemKey },
|
|
2408
|
-
OUTERLAYER_APP_ID: config.appId,
|
|
2409
|
-
OUTERLAYER_WORK: String(item),
|
|
2410
|
-
OUTERLAYER_JOB_DIR: jobDirPath,
|
|
2411
|
-
// The branch the claim named. A hook that makes its own checkout puts it
|
|
2412
|
-
// on this branch, and the build's commands push to it.
|
|
2413
|
-
...branch === void 0 ? {} : { OUTERLAYER_BRANCH: branch },
|
|
2414
|
-
// A fleet build's artifacts are spooled under its organization's folder, not the user's home.
|
|
2415
|
-
...spoolHome === void 0 ? {} : { OUTERLAYER_SPOOL_HOME: spoolHome },
|
|
2416
|
-
// Git prefers these to any configuration, so a host's own git identity never authors a commit of the build.
|
|
2417
|
-
...authorEnvironment(author)
|
|
2418
|
-
};
|
|
2419
|
-
}
|
|
2420
|
-
function cliFirstPath(currentPath) {
|
|
2421
|
-
const entry = process.argv[1];
|
|
2422
|
-
const cliDir = entry === void 0 || entry === "" ? void 0 : dirname(entry);
|
|
2423
|
-
if (cliDir === void 0) return currentPath;
|
|
2424
|
-
const parts = (currentPath ?? "").split(delimiter).filter((p) => p !== "");
|
|
2425
|
-
if (parts[0] === cliDir) return currentPath;
|
|
2426
|
-
return [cliDir, ...parts.filter((p) => p !== cliDir)].join(delimiter);
|
|
2427
|
-
}
|
|
2428
|
-
var HOOK_ENDED_EXIT_CODE = 124;
|
|
2429
|
-
function runHook(hookPath, env, cwd, logPath, spawnFn = spawn, limits, args = []) {
|
|
2430
|
-
return new Promise((resolve, reject) => {
|
|
2431
|
-
let fd;
|
|
2432
|
-
try {
|
|
2433
|
-
fd = openSync(logPath, "a", RUNNER_FILE_MODE);
|
|
2434
|
-
} catch (err) {
|
|
2435
|
-
reject(err instanceof Error ? err : new Error(String(err)));
|
|
2436
|
-
return;
|
|
2437
|
-
}
|
|
2438
|
-
let timer;
|
|
2439
|
-
let endedBy;
|
|
2440
|
-
const settle = () => {
|
|
2441
|
-
if (timer) clearInterval(timer);
|
|
2442
|
-
try {
|
|
2443
|
-
closeSync(fd);
|
|
2444
|
-
} catch {
|
|
2445
|
-
}
|
|
2446
|
-
};
|
|
2447
|
-
const child = spawnFn(hookPath, args, { env, cwd, detached: true, stdio: ["ignore", fd, fd] });
|
|
2448
|
-
if (limits !== void 0 && child.pid !== void 0) {
|
|
2449
|
-
const pid = child.pid;
|
|
2450
|
-
const readClock = limits.now ?? (() => /* @__PURE__ */ new Date());
|
|
2451
|
-
let lastActivityAtMs = readClock().getTime();
|
|
2452
|
-
let lastSeen = readLogState(logPath);
|
|
2453
|
-
timer = setInterval(() => {
|
|
2454
|
-
const nowMs = readClock().getTime();
|
|
2455
|
-
const seen = readLogState(logPath);
|
|
2456
|
-
if (seen.size !== lastSeen.size || seen.mtimeMs !== lastSeen.mtimeMs) {
|
|
2457
|
-
lastSeen = seen;
|
|
2458
|
-
lastActivityAtMs = nowMs;
|
|
2459
|
-
}
|
|
2460
|
-
const reason = nowMs >= limits.endBy ? "time limit" : nowMs - lastActivityAtMs >= limits.idleMs ? "idle" : void 0;
|
|
2461
|
-
if (reason === void 0) return;
|
|
2462
|
-
endedBy = reason;
|
|
2463
|
-
if (timer) clearInterval(timer);
|
|
2464
|
-
void stopProcessGroup(pid);
|
|
2465
|
-
}, limits.pollMs ?? 1e3);
|
|
2466
|
-
timer.unref?.();
|
|
2467
|
-
}
|
|
2468
|
-
child.on("error", (err) => {
|
|
2469
|
-
settle();
|
|
2470
|
-
reject(hookSpawnError(hookPath, err));
|
|
2471
|
-
});
|
|
2472
|
-
child.on("exit", (code, signal) => {
|
|
2473
|
-
settle();
|
|
2474
|
-
if (endedBy !== void 0) {
|
|
2475
|
-
resolve({ code: HOOK_ENDED_EXIT_CODE, endedBy });
|
|
2476
|
-
return;
|
|
2477
|
-
}
|
|
2478
|
-
if (signal !== null) {
|
|
2479
|
-
resolve({ code: 1, signal });
|
|
2480
|
-
return;
|
|
2481
|
-
}
|
|
2482
|
-
resolve({ code: code ?? 1 });
|
|
2483
|
-
});
|
|
2484
|
-
});
|
|
2485
|
-
}
|
|
2486
|
-
function hookSpawnError(hookPath, err) {
|
|
2487
|
-
const code = err?.code;
|
|
2488
|
-
if (code === "ENOENT") {
|
|
2489
|
-
return new Error(
|
|
2490
|
-
`${hookPath}: no such executable. A hook is run directly, not through a shell, so the path must name an executable file with no arguments and no variables to expand.`
|
|
2491
|
-
);
|
|
2492
|
-
}
|
|
2493
|
-
if (code === "EACCES") {
|
|
2494
|
-
return new Error(`${hookPath}: not executable. Give the file the execute bit, or name one that has it.`);
|
|
2495
|
-
}
|
|
2496
|
-
return err instanceof Error ? err : new Error(String(err));
|
|
2497
|
-
}
|
|
2498
|
-
function workdirProblem(workdir) {
|
|
2499
|
-
let stats;
|
|
2500
|
-
try {
|
|
2501
|
-
stats = statSync(workdir);
|
|
2502
|
-
} catch {
|
|
2503
|
-
return "does not exist";
|
|
2504
|
-
}
|
|
2505
|
-
return stats.isDirectory() ? void 0 : "is not a directory";
|
|
2506
|
-
}
|
|
2507
|
-
function readProvisionOut(jobDirPath) {
|
|
2508
|
-
const path = join(jobDirPath, "provision.out");
|
|
2509
|
-
if (!existsSync(path)) return { ok: false };
|
|
2510
|
-
let text2;
|
|
2511
|
-
try {
|
|
2512
|
-
text2 = readFileSync(path, "utf8");
|
|
2513
|
-
} catch {
|
|
2514
|
-
return { ok: false };
|
|
2515
|
-
}
|
|
2516
|
-
const kv = {};
|
|
2517
|
-
for (const line of text2.split("\n")) {
|
|
2518
|
-
const eq = line.indexOf("=");
|
|
2519
|
-
if (eq === -1) continue;
|
|
2520
|
-
kv[line.slice(0, eq).trim()] = line.slice(eq + 1).trim();
|
|
2521
|
-
}
|
|
2522
|
-
if (!kv.workdir) return { ok: false };
|
|
2523
|
-
return {
|
|
2524
|
-
ok: true,
|
|
2525
|
-
workdir: kv.workdir,
|
|
2526
|
-
...kv.cgroup ? { cgroup: kv.cgroup } : {},
|
|
2527
|
-
...kv.isolation !== void 0 ? { isolation: kv.isolation } : {},
|
|
2528
|
-
...kv.exec !== void 0 ? { exec: kv.exec } : {},
|
|
2529
|
-
...kv.broker !== void 0 ? { broker: kv.broker } : {},
|
|
2530
|
-
...kv.recipe !== void 0 ? { recipe: kv.recipe } : {}
|
|
2531
|
-
};
|
|
2532
|
-
}
|
|
2533
|
-
var EXIT_CODE_FILE = "exit-code";
|
|
2534
|
-
function commandWithExitCodeTrailer(cmdline) {
|
|
2535
|
-
return [
|
|
2536
|
-
"__ol_record_status() {",
|
|
2537
|
-
" __ol_status=$?",
|
|
2538
|
-
' if [ -n "$OUTERLAYER_JOB_DIR" ]; then',
|
|
2539
|
-
` echo "$__ol_status" > "$OUTERLAYER_JOB_DIR/${EXIT_CODE_FILE}.partial" 2>/dev/null &&`,
|
|
2540
|
-
` mv -f "$OUTERLAYER_JOB_DIR/${EXIT_CODE_FILE}.partial" "$OUTERLAYER_JOB_DIR/${EXIT_CODE_FILE}" 2>/dev/null`,
|
|
2541
|
-
" fi",
|
|
2542
|
-
' return "$__ol_status"',
|
|
2543
|
-
"}",
|
|
2544
|
-
"trap __ol_record_status EXIT",
|
|
2545
|
-
cmdline
|
|
2546
|
-
].join("\n");
|
|
2547
|
-
}
|
|
2548
|
-
function readExitCode(jobDirPath) {
|
|
2549
|
-
let text2;
|
|
2550
|
-
try {
|
|
2551
|
-
text2 = readFileSync(join(jobDirPath, EXIT_CODE_FILE), "utf8");
|
|
2552
|
-
} catch {
|
|
2553
|
-
return void 0;
|
|
2554
|
-
}
|
|
2555
|
-
const trimmed = text2.trim();
|
|
2556
|
-
return /^\d+$/.test(trimmed) ? Number(trimmed) : void 0;
|
|
2557
|
-
}
|
|
2558
|
-
function outcomeFromExitCode(jobDirPath) {
|
|
2559
|
-
const code = readExitCode(jobDirPath);
|
|
2560
|
-
if (code === void 0) return void 0;
|
|
2561
|
-
return { outcome: code === 0 ? "ok" : "failed", code };
|
|
2562
|
-
}
|
|
2563
|
-
var REASON_FILE = "reason";
|
|
2564
|
-
var REASON_MAX2 = 200;
|
|
2565
|
-
function cutReasonLine(line, max = REASON_MAX2) {
|
|
2566
|
-
const cut = line.slice(0, max);
|
|
2567
|
-
const last = cut.charCodeAt(cut.length - 1);
|
|
2568
|
-
return last >= 55296 && last <= 56319 ? cut.slice(0, -1) : cut;
|
|
2569
|
-
}
|
|
2570
|
-
function scrubList(value) {
|
|
2571
|
-
return Array.isArray(value) ? value : [];
|
|
2572
|
-
}
|
|
2573
|
-
function mergeScrubConfig(a, b) {
|
|
2574
|
-
if (!a && !b) return void 0;
|
|
2575
|
-
return {
|
|
2576
|
-
literals: [...scrubList(a?.literals), ...scrubList(b?.literals)],
|
|
2577
|
-
patterns: [...scrubList(a?.patterns), ...scrubList(b?.patterns)]
|
|
2578
|
-
};
|
|
2579
|
-
}
|
|
2580
|
-
function releaseScrub(config, home, jobDir) {
|
|
2581
|
-
const merged = mergeScrubConfig(config.scrub, readCloudConfig(home).scrub);
|
|
2582
|
-
const secrets = [...headerVariables(config.build?.destinations), ...CLAUDE_CREDENTIAL_VARIABLES].map((name) => process.env[name] ?? "");
|
|
2583
|
-
const managed = config.fleet === true ? jobDir === void 0 ? [] : [...managedClaudeKeyValuesFor(jobDir), ...managedBuildSettingValuesFor(jobDir)] : [...managedClaudeKeyValues(), ...managedBuildSettingValues()];
|
|
2584
|
-
const held = [currentClaudeCredential(process.env)?.value, ...removedClaudeCredentialValues(), ...managed].filter((value) => value !== void 0);
|
|
2585
|
-
return { literals: [...scrubList(merged?.literals), ...secrets, ...held], patterns: scrubList(merged?.patterns) };
|
|
2586
|
-
}
|
|
2587
|
-
function transcriptScrub(config, secrets, jobDir) {
|
|
2588
|
-
const base = releaseScrub(config, homeFromEnv(), jobDir);
|
|
2589
|
-
return { literals: [...scrubList(base.literals), ...secrets.filter((secret) => secret !== void 0 && secret !== "")], patterns: scrubList(base.patterns) };
|
|
2590
|
-
}
|
|
2591
|
-
function stripUnsendableReasonChars(text2) {
|
|
2592
|
-
return text2.replace(/\u0000/g, "").replace(/\p{Cs}/gu, "");
|
|
2593
|
-
}
|
|
2594
|
-
function stripTerminalControls(text2) {
|
|
2595
|
-
return text2.replace(/\u001b\[[0-?]*[ -\/]*[@-~]/g, "").replace(/\u001b\][^\u0007\u001b\r\n]*(?:\u0007|\u001b\\)?/g, "").replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, "");
|
|
2596
|
-
}
|
|
2597
|
-
function releaseLogFor(jobDirPath, job, config) {
|
|
2598
|
-
const home = homeFromEnv();
|
|
2599
|
-
installScrubContext(home, transcriptScrub(config, [job.itemKey, config.apiKey], jobDirPath));
|
|
2600
|
-
return buildReleaseLog(jobDirPath, job.endStage, (text2) => scrubText(stripTerminalControls(stripUnsendableReasonChars(text2))));
|
|
2601
|
-
}
|
|
2602
|
-
function readReasonLine(jobDirPath, config, home = homeFromEnv()) {
|
|
2603
|
-
let raw;
|
|
2604
|
-
try {
|
|
2605
|
-
raw = readFileSync(join(jobDirPath, REASON_FILE), "utf8");
|
|
2606
|
-
} catch {
|
|
2607
|
-
return void 0;
|
|
2608
|
-
}
|
|
2609
|
-
return scrubReason(raw.split(/\r\n|\r|\n/)[0] ?? "", config, home, jobDirPath);
|
|
2610
|
-
}
|
|
2611
|
-
function scrubReason(line, config, home = homeFromEnv(), jobDir) {
|
|
2612
|
-
installScrubContext(home, releaseScrub(config, home, jobDir));
|
|
2613
|
-
const oneLine = scrubText(stripUnsendableReasonChars(line)).replace(/[\r\n]+/g, " ");
|
|
2614
|
-
const trimmed = cutReasonLine(oneLine).trim();
|
|
2615
|
-
return trimmed.length > 0 ? trimmed : void 0;
|
|
2616
|
-
}
|
|
2617
|
-
function scrubTunnelRecord(record, config, jobDir) {
|
|
2618
|
-
if (record === void 0) return void 0;
|
|
2619
|
-
const home = homeFromEnv();
|
|
2620
|
-
installScrubContext(home, releaseScrub(config, home, jobDir));
|
|
2621
|
-
return {
|
|
2622
|
-
hosts: record.hosts.map((host) => scrubText(host)),
|
|
2623
|
-
hostsMore: record.hostsMore,
|
|
2624
|
-
refusalsMore: record.refusalsMore,
|
|
2625
|
-
refusals: record.refusals.map((r) => ({ ...r, host: scrubText(r.host) })),
|
|
2626
|
-
destinations: (record.destinations ?? []).map((name) => scrubText(name)),
|
|
2627
|
-
...record.egress === void 0 ? {} : { egress: record.egress },
|
|
2628
|
-
...record.routes === void 0 ? {} : { routes: Object.fromEntries(Object.entries(record.routes).map(([name, counts]) => [scrubText(name), counts])) }
|
|
2629
|
-
};
|
|
2630
|
-
}
|
|
2631
|
-
function runCommand(cmdline, env, cwd, logPath, spawnFn = spawn) {
|
|
2632
|
-
const fd = openSync(logPath, "a", RUNNER_FILE_MODE);
|
|
2633
|
-
const child = spawnFn("/bin/sh", ["-c", commandWithExitCodeTrailer(cmdline)], {
|
|
2634
|
-
env,
|
|
2635
|
-
cwd,
|
|
2636
|
-
detached: true,
|
|
2637
|
-
stdio: ["ignore", fd, fd]
|
|
2638
|
-
});
|
|
2639
|
-
const pid = child.pid;
|
|
2640
|
-
if (pid === void 0) {
|
|
2641
|
-
try {
|
|
2642
|
-
closeSync(fd);
|
|
2643
|
-
} catch {
|
|
2644
|
-
}
|
|
2645
|
-
throw new Error("command failed to spawn \u2014 no pid");
|
|
2646
|
-
}
|
|
2647
|
-
const onExit = new Promise((resolve) => {
|
|
2648
|
-
const close = () => {
|
|
2649
|
-
try {
|
|
2650
|
-
closeSync(fd);
|
|
2651
|
-
} catch {
|
|
2652
|
-
}
|
|
2653
|
-
};
|
|
2654
|
-
child.on("exit", (code, signal) => {
|
|
2655
|
-
close();
|
|
2656
|
-
resolve({ code, signal });
|
|
2657
|
-
});
|
|
2658
|
-
child.on("error", (err) => {
|
|
2659
|
-
close();
|
|
2660
|
-
resolve({ code: null, signal: null, error: err instanceof Error ? err : new Error(String(err)) });
|
|
2661
|
-
});
|
|
2662
|
-
});
|
|
2663
|
-
return { pid, start: startTimeOf(pid), onExit };
|
|
2664
|
-
}
|
|
2665
|
-
function isGroupAlive(pid) {
|
|
2666
|
-
try {
|
|
2667
|
-
process.kill(-pid, 0);
|
|
2668
|
-
return true;
|
|
2669
|
-
} catch {
|
|
2670
|
-
return false;
|
|
2671
|
-
}
|
|
2672
|
-
}
|
|
2673
|
-
async function stopProcessGroup(pid, opts = {}) {
|
|
2674
|
-
const graceMs = opts.graceMs ?? 15e3;
|
|
2675
|
-
const pollMs = opts.pollMs ?? 200;
|
|
2676
|
-
try {
|
|
2677
|
-
process.kill(-pid, "SIGTERM");
|
|
2678
|
-
} catch {
|
|
2679
|
-
return;
|
|
2680
|
-
}
|
|
2681
|
-
const deadline = Date.now() + graceMs;
|
|
2682
|
-
while (isGroupAlive(pid) && Date.now() < deadline) {
|
|
2683
|
-
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
2684
|
-
}
|
|
2685
|
-
if (isGroupAlive(pid)) {
|
|
2686
|
-
try {
|
|
2687
|
-
process.kill(-pid, "SIGKILL");
|
|
2688
|
-
} catch {
|
|
2689
|
-
}
|
|
2690
|
-
const killDeadline = Date.now() + 2e3;
|
|
2691
|
-
while (isGroupAlive(pid) && Date.now() < killDeadline) {
|
|
2692
|
-
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
2693
|
-
}
|
|
2694
|
-
}
|
|
2695
|
-
}
|
|
2696
|
-
function isTimedOut(startedAtMs, nowMs, timeLimitMinutes) {
|
|
2697
|
-
return nowMs - startedAtMs > timeLimitMinutes * 6e4;
|
|
2698
|
-
}
|
|
2699
|
-
function needsRenew(lastRenewAtMs, nowMs, intervalSeconds = RENEW_INTERVAL_SECONDS) {
|
|
2700
|
-
return nowMs - lastRenewAtMs >= intervalSeconds * 1e3;
|
|
2701
|
-
}
|
|
2702
|
-
function isIdle(logMtimeMs, nowMs, idleLimitMinutes, cpuUsecPerMinuteOverWindow) {
|
|
2703
|
-
const windowMs = nowMs - logMtimeMs;
|
|
2704
|
-
if (windowMs <= idleLimitMinutes * 6e4) return false;
|
|
2705
|
-
if (cpuUsecPerMinuteOverWindow === null) return true;
|
|
2706
|
-
return cpuUsecPerMinuteOverWindow < IDLE_CPU_USEC_PER_MINUTE_FLOOR;
|
|
2707
|
-
}
|
|
2708
|
-
function readCpuUsec(cgroupPath) {
|
|
2709
|
-
try {
|
|
2710
|
-
const text2 = readFileSync(join(cgroupPath, "cpu.stat"), "utf8");
|
|
2711
|
-
const match = /^usage_usec (\d+)$/m.exec(text2);
|
|
2712
|
-
return match ? Number(match[1]) : null;
|
|
2713
|
-
} catch {
|
|
2714
|
-
return null;
|
|
2715
|
-
}
|
|
2716
|
-
}
|
|
2717
|
-
async function renewLease(config, item, fetchImpl) {
|
|
2718
|
-
try {
|
|
2719
|
-
await postJson(
|
|
2720
|
-
config.url,
|
|
2721
|
-
`/v1/work-items/${item}/claim/renew`,
|
|
2722
|
-
config.apiKey,
|
|
2723
|
-
config.appId,
|
|
2724
|
-
{ host: config.host, seconds: LEASE_SECONDS },
|
|
2725
|
-
fetchImpl,
|
|
2726
|
-
void 0,
|
|
2727
|
-
await signerFor(config.configPath)
|
|
2728
|
-
);
|
|
2729
|
-
return "renewed";
|
|
2730
|
-
} catch (err) {
|
|
2731
|
-
if (err instanceof WorkCommandError && isSignatureRefusal(err.code)) return "retry_later";
|
|
2732
|
-
if (err instanceof WorkCommandError && !err.retryable) return "lease_lost";
|
|
2733
|
-
return "retry_later";
|
|
2734
|
-
}
|
|
2735
|
-
}
|
|
2736
|
-
var ALREADY_RELEASED = "already released";
|
|
2737
|
-
async function releaseLease(config, item, outcome, detail, fetchImpl) {
|
|
2738
|
-
const send = async (environment2, sent2, log2, resume2) => postJson(
|
|
2739
|
-
config.url,
|
|
2740
|
-
`/v1/work-items/${item}/claim/release`,
|
|
2741
|
-
config.apiKey,
|
|
2742
|
-
config.appId,
|
|
2743
|
-
{ host: config.host, outcome: sent2, stage: detail.stage, exitCode: detail.exitCode, reason: detail.reason, environment: environment2, resume: resume2, log: log2 },
|
|
2744
|
-
fetchImpl,
|
|
2745
|
-
void 0,
|
|
2746
|
-
await signerFor(config.configPath)
|
|
2747
|
-
);
|
|
2748
|
-
let environment = detail.environment;
|
|
2749
|
-
let sent = outcome;
|
|
2750
|
-
let log = detail.log;
|
|
2751
|
-
let resume = detail.resume;
|
|
2752
|
-
let data;
|
|
2753
|
-
for (; ; ) {
|
|
2754
|
-
try {
|
|
2755
|
-
({ data } = await send(environment, sent, log, resume));
|
|
2756
|
-
break;
|
|
2757
|
-
} catch (err) {
|
|
2758
|
-
if (!(err instanceof WorkCommandError) || err.status !== 400 || err.code !== "invalid_request_body") throw err;
|
|
2759
|
-
const namesOutcome = /\boutcome\b/.test(err.message);
|
|
2760
|
-
if (sent === "incomplete" && namesOutcome) {
|
|
2761
|
-
sent = "ok";
|
|
2762
|
-
} else if (resume !== void 0 && /\bresume\b/.test(err.message)) {
|
|
2763
|
-
resume = void 0;
|
|
2764
|
-
} else if (log !== void 0 && /\blog\b/.test(err.message)) {
|
|
2765
|
-
log = void 0;
|
|
2766
|
-
} else if (environment?.routes !== void 0) {
|
|
2767
|
-
const { routes: _omittedRoutes, ...withoutRoutes } = environment;
|
|
2768
|
-
environment = withoutRoutes;
|
|
2769
|
-
} else if (environment?.guestUsage !== void 0) {
|
|
2770
|
-
const { guestUsage: _omittedUsage, ...withoutUsage } = environment;
|
|
2771
|
-
environment = withoutUsage;
|
|
2772
|
-
} else if (environment?.controlPlane !== void 0) {
|
|
2773
|
-
const { controlPlane: _omittedPlane, ...withoutPlane } = environment;
|
|
2774
|
-
environment = withoutPlane;
|
|
2775
|
-
} else if (environment?.commitIdentity !== void 0) {
|
|
2776
|
-
const { commitIdentity: _omitted, ...withoutIdentity } = environment;
|
|
2777
|
-
environment = withoutIdentity;
|
|
2778
|
-
} else if (sent === "incomplete") {
|
|
2779
|
-
sent = "ok";
|
|
2780
|
-
} else {
|
|
2781
|
-
throw err;
|
|
2782
|
-
}
|
|
2783
|
-
}
|
|
2784
|
-
}
|
|
2785
|
-
return {
|
|
2786
|
-
releasedAt: typeof data.releasedAt === "string" ? data.releasedAt : void 0,
|
|
2787
|
-
outcome: data.outcome === void 0 ? void 0 : data.outcome
|
|
2788
|
-
};
|
|
2789
|
-
}
|
|
2790
|
-
var RELEASE_SKEW_MS = 6e4;
|
|
2791
|
-
function wasAlreadyReleased(result, sentOutcome, sentAtMs) {
|
|
2792
|
-
const stored = result.outcome ?? void 0;
|
|
2793
|
-
if (stored !== void 0 && stored !== sentOutcome) return true;
|
|
2794
|
-
if (result.releasedAt === void 0) return false;
|
|
2795
|
-
const releasedAtMs = Date.parse(result.releasedAt);
|
|
2796
|
-
return Number.isFinite(releasedAtMs) && releasedAtMs < sentAtMs - RELEASE_SKEW_MS;
|
|
2797
|
-
}
|
|
2798
|
-
function runSync(config, item, itemKey, workdir, jobDirPath, spawnFn = spawn, branch, author, spoolHome) {
|
|
2799
|
-
return new Promise((resolve) => {
|
|
2800
|
-
const env = jobEnv(config, item, itemKey, jobDirPath, process.env, branch, author, spoolHome);
|
|
2801
|
-
let child;
|
|
2802
|
-
try {
|
|
2803
|
-
child = spawnFn(process.execPath, [process.argv[1] ?? "outerlayer", "sync"], {
|
|
2804
|
-
cwd: workdir,
|
|
2805
|
-
env,
|
|
2806
|
-
stdio: "ignore"
|
|
2807
|
-
});
|
|
2808
|
-
} catch (err) {
|
|
2809
|
-
resolve({ ok: false, reason: describeCallFailure(err) });
|
|
2810
|
-
return;
|
|
2811
|
-
}
|
|
2812
|
-
child.on("exit", (code) => {
|
|
2813
|
-
resolve(code === 0 || code === null ? { ok: true } : { ok: false, reason: `exit ${code}` });
|
|
2814
|
-
});
|
|
2815
|
-
child.on("error", (err) => {
|
|
2816
|
-
resolve({ ok: false, reason: describeCallFailure(err) });
|
|
2817
|
-
});
|
|
2818
|
-
});
|
|
2819
|
-
}
|
|
2820
|
-
async function runSyncViaExec(config, job, jobDirPath, spawnFn = spawn) {
|
|
2821
|
-
const build = job.build;
|
|
2822
|
-
const env = hostCommandEnvironment(jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDirPath, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job)), build, config, job.commitIdentity?.hooksPath);
|
|
2823
|
-
try {
|
|
2824
|
-
const result = await runHook(
|
|
2825
|
-
build.exec,
|
|
2826
|
-
env,
|
|
2827
|
-
jobDirPath,
|
|
2828
|
-
join(jobDirPath, "sync.log"),
|
|
2829
|
-
spawnFn,
|
|
2830
|
-
void 0,
|
|
2831
|
-
execArgv(job.workdir, ["outerlayer", "sync"])
|
|
2832
|
-
);
|
|
2833
|
-
return result.code === 0 ? { ok: true } : { ok: false, reason: `exit ${result.code}` };
|
|
2834
|
-
} catch (err) {
|
|
2835
|
-
return { ok: false, reason: describeCallFailure(err) };
|
|
2836
|
-
}
|
|
2837
|
-
}
|
|
2838
|
-
function runRecordProofs(config, job, jobDirPath, spawnFn = spawn) {
|
|
2839
|
-
return new Promise((resolve) => {
|
|
2840
|
-
const env = jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDirPath, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job));
|
|
2841
|
-
let fd;
|
|
2842
|
-
let child;
|
|
2843
|
-
try {
|
|
2844
|
-
fd = openSync(join(jobDirPath, PROOFS_LOG), "a", RUNNER_FILE_MODE);
|
|
2845
|
-
} catch (err) {
|
|
2846
|
-
resolve({ ok: false, reason: describeCallFailure(err) });
|
|
2847
|
-
return;
|
|
2848
|
-
}
|
|
2849
|
-
try {
|
|
2850
|
-
child = spawnFn(process.execPath, [process.argv[1] ?? "outerlayer", ...RECORD_PROOFS_ARGV.slice(1)], { cwd: job.workdir, env, stdio: ["ignore", fd, fd] });
|
|
2851
|
-
} catch (err) {
|
|
2852
|
-
closeSync(fd);
|
|
2853
|
-
resolve({ ok: false, reason: describeCallFailure(err) });
|
|
2854
|
-
return;
|
|
2855
|
-
}
|
|
2856
|
-
child.on("exit", (code) => {
|
|
2857
|
-
closeSync(fd);
|
|
2858
|
-
resolve(code === 0 ? { ok: true } : { ok: false, reason: `exit ${code ?? "by signal"}` });
|
|
2859
|
-
});
|
|
2860
|
-
child.on("error", (err) => {
|
|
2861
|
-
closeSync(fd);
|
|
2862
|
-
resolve({ ok: false, reason: describeCallFailure(err) });
|
|
2863
|
-
});
|
|
2864
|
-
});
|
|
2865
|
-
}
|
|
2866
|
-
async function runRecordProofsViaExec(config, job, jobDirPath, spawnFn = spawn) {
|
|
2867
|
-
const build = job.build;
|
|
2868
|
-
const env = hostCommandEnvironment(jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDirPath, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job)), build, config, job.commitIdentity?.hooksPath);
|
|
2869
|
-
try {
|
|
2870
|
-
const result = await runHook(build.exec, env, jobDirPath, join(jobDirPath, PROOFS_LOG), spawnFn, void 0, execArgv(job.workdir, [...RECORD_PROOFS_ARGV]));
|
|
2871
|
-
return result.code === 0 ? { ok: true } : { ok: false, reason: `exit ${result.code}` };
|
|
2872
|
-
} catch (err) {
|
|
2873
|
-
return { ok: false, reason: describeCallFailure(err) };
|
|
2874
|
-
}
|
|
2875
|
-
}
|
|
2876
|
-
var RECORDED_PROOFS = /* @__PURE__ */ new Set(["screenshot", "video", "report", "log", "file"]);
|
|
2877
|
-
async function declaresRecordedProof(config, item, fetchImpl) {
|
|
2878
|
-
try {
|
|
2879
|
-
return (criteriaOf(await readWorkItem(config, item, fetchImpl)) ?? []).some((criterion) => criterion.proof !== null && RECORDED_PROOFS.has(criterion.proof));
|
|
2880
|
-
} catch {
|
|
2881
|
-
return true;
|
|
2882
|
-
}
|
|
2883
|
-
}
|
|
2884
|
-
var MAX_NAMED_SPOOLED = 20;
|
|
2885
|
-
function spooledArtifactIds(home) {
|
|
2886
|
-
try {
|
|
2887
|
-
return planArtifactUpload({ home }).candidates.map((pending) => pending.record.artifactId);
|
|
2888
|
-
} catch {
|
|
2889
|
-
return [];
|
|
2890
|
-
}
|
|
2891
|
-
}
|
|
2892
|
-
function leaseExpiry(job, fallback) {
|
|
2893
|
-
const base = Date.parse(job.lastRenewAt ?? job.startedAt);
|
|
2894
|
-
return new Date((Number.isFinite(base) ? base : fallback.getTime()) + LEASE_SECONDS * 1e3);
|
|
2895
|
-
}
|
|
2896
|
-
var TEARDOWN_STAGES = ["proofs", "judge", "checks", "sync", "cleanup", "report"];
|
|
2897
|
-
function inTeardown(job) {
|
|
2898
|
-
return job.stage !== void 0 && TEARDOWN_STAGES.includes(job.stage);
|
|
2899
|
-
}
|
|
2900
|
-
function setStage(job, jobDirPath, stage) {
|
|
2901
|
-
job.stage = stage;
|
|
2902
|
-
writeJobFile(join(jobDirPath, "job.json"), job);
|
|
2903
|
-
}
|
|
2904
|
-
function readTunnelRecord(path) {
|
|
2905
|
-
try {
|
|
2906
|
-
return JSON.parse(readFileSync(path, "utf8"));
|
|
2907
|
-
} catch {
|
|
2908
|
-
return void 0;
|
|
2909
|
-
}
|
|
2910
|
-
}
|
|
2911
|
-
var VALUES_NOT_HELD = "the runner restarted and could not read this build's variables from its factory again, so it cannot remove their values from what it uploads";
|
|
2912
|
-
var SCRUBBED_RELEASE_LOG = "release-log.txt";
|
|
2913
|
-
function readScrubbedReleaseLog(jobDirPath) {
|
|
2914
|
-
try {
|
|
2915
|
-
return readFileSync(join(jobDirPath, SCRUBBED_RELEASE_LOG), "utf8");
|
|
2916
|
-
} catch {
|
|
2917
|
-
return void 0;
|
|
2918
|
-
}
|
|
2919
|
-
}
|
|
2920
|
-
async function finishJob(job, jobDirPath, outcome, config, deps, ending) {
|
|
2921
|
-
const spawnFn = deps.spawnFn ?? spawn;
|
|
2922
|
-
const now = deps.now ?? (() => /* @__PURE__ */ new Date());
|
|
2923
|
-
const log = deps.log ?? createRunnerLog(now);
|
|
2924
|
-
const jobFilePath = join(jobDirPath, "job.json");
|
|
2925
|
-
const logPath = join(jobDirPath, "command.log");
|
|
2926
|
-
const tag = `${itemTag(job)}`;
|
|
2927
|
-
deps.attemptLogs?.open(job.item, jobDirPath, itemScope(job));
|
|
2928
|
-
const valuesNotHeld = (job.managedVariables?.length ?? 0) > 0 && managedBuildSettingsFor(jobDirPath) === void 0;
|
|
2929
|
-
if (job.sender !== void 0) {
|
|
2930
|
-
await stopSender({ job, deps: { spawnFn, runtime: deps.container, log } });
|
|
2931
|
-
delete job.sender;
|
|
2932
|
-
writeJobFile(jobFilePath, job);
|
|
2933
|
-
}
|
|
2934
|
-
const build = job.container === void 0 ? void 0 : { state: job.container, runtime: deps.container ?? createContainerRuntime() };
|
|
2935
|
-
const vmBuild = job.microvm === void 0 ? void 0 : { state: job.microvm, runtime: deps.container ?? createContainerRuntime() };
|
|
2936
|
-
const commandMayRun = job.pid !== void 0 && isGroupAlive(job.pid) || isContainerTracked(job) && job.exitCode === void 0;
|
|
2937
|
-
if (vmBuild !== void 0) {
|
|
2938
|
-
const sample = await sampleGuest(vmBuild.runtime, vmBuild.state);
|
|
2939
|
-
if (sample !== void 0) {
|
|
2940
|
-
job.guestUsage = foldSample(job.guestUsage, sample);
|
|
2941
|
-
writeJobFile(jobFilePath, job);
|
|
2942
|
-
}
|
|
2943
|
-
}
|
|
2944
|
-
if (vmBuild !== void 0 && isMicroVmTracked(job) && job.exitCode === void 0) {
|
|
2945
|
-
await endMicroVmCommand(vmBuild.runtime, vmBuild.state, jobDirPath, (line) => log.error(`${tag} ${line}`));
|
|
2946
|
-
}
|
|
2947
|
-
if (build !== void 0 && commandMayRun) {
|
|
2948
|
-
if (isContainerTracked(job) && build.state.stopped !== true) {
|
|
2949
|
-
await copyCommandLogTail(build.runtime, build.state, jobDirPath).catch((err) => {
|
|
2950
|
-
log.error(`${tag} copying the end of the command's log failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
2951
|
-
});
|
|
2952
|
-
}
|
|
2953
|
-
await stopContainer(build.runtime, build.state).catch((err) => {
|
|
2954
|
-
log.error(`${tag} stopping the container failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
2955
|
-
});
|
|
2956
|
-
writeJobFile(jobFilePath, job);
|
|
2957
|
-
}
|
|
2958
|
-
if (job.pid !== void 0 && isGroupAlive(job.pid)) {
|
|
2959
|
-
await stopProcessGroup(job.pid);
|
|
2960
|
-
}
|
|
2961
|
-
const sessionRuntime = (build ?? vmBuild)?.runtime;
|
|
2962
|
-
if (sessionRuntime !== void 0) {
|
|
2963
|
-
if (!valuesNotHeld) {
|
|
2964
|
-
if (await keepEndedSession({ job, jobDirPath, runtime: sessionRuntime, configPath: config.configPath, log, tag, now })) writeJobFile(jobFilePath, job);
|
|
2965
|
-
} else if (job.sessionId !== void 0 && job.workItemId !== void 0 && job.sessionKept !== true) {
|
|
2966
|
-
log.error(`${tag} the session was not kept: ${VALUES_NOT_HELD} \xB7 continuing`);
|
|
2967
|
-
}
|
|
2968
|
-
}
|
|
2969
|
-
if (outcome !== void 0 && job.endStage === void 0) {
|
|
2970
|
-
const staleTeardownStage = inTeardown(job);
|
|
2971
|
-
job.endStage = ending?.stage ?? (staleTeardownStage ? job.exitCode !== void 0 ? "command" : void 0 : job.stage);
|
|
2972
|
-
job.endExitCode = ending?.exitCode ?? (outcome === "ok" || outcome === "failed" ? job.exitCode : void 0);
|
|
2973
|
-
writeJobFile(jobFilePath, job);
|
|
2974
|
-
}
|
|
2975
|
-
if (outcome !== void 0 && ending?.reason !== void 0) {
|
|
2976
|
-
job.reason = RUNNER_WORDED_REASONS.has(ending.reason) ? ending.reason : scrubReason(ending.reason, config, void 0, jobDirPath);
|
|
2977
|
-
writeJobFile(jobFilePath, job);
|
|
2978
|
-
}
|
|
2979
|
-
const uploadSessions = async (workdir) => {
|
|
2980
|
-
if (valuesNotHeld) {
|
|
2981
|
-
log.error(`${tag} sync skipped: ${VALUES_NOT_HELD}`);
|
|
2982
|
-
return;
|
|
2983
|
-
}
|
|
2984
|
-
if (job.container === void 0 && job.microvm === void 0 && job.build?.exec === void 0 && workdirProblem(workdir) !== void 0) {
|
|
2985
|
-
log.line(`${tag} sync skipped: workdir ${workdir} is gone`);
|
|
2986
|
-
} else if (job.itemKey === void 0) {
|
|
2987
|
-
log.line(`${tag} sync skipped: the job file holds no item key`);
|
|
2988
|
-
} else {
|
|
2989
|
-
log.line(`${tag} sync started`);
|
|
2990
|
-
const startedMs = now().getTime();
|
|
2991
|
-
const sync = build !== void 0 ? await syncInContainer({ config, runtime: build.runtime, job, jobDir: jobDirPath, log: (line) => log.line(`${tag} ${line}`) }) : vmBuild !== void 0 ? await syncInMicroVm({ config, runtime: vmBuild.runtime, job, jobDir: jobDirPath }) : job.build?.exec !== void 0 ? await runSyncViaExec(config, job, jobDirPath, spawnFn) : await runSync(claimCalls(config, job), job.item, job.itemKey, workdir, jobDirPath, spawnFn, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job));
|
|
2992
|
-
const took = formatDuration(now().getTime() - startedMs);
|
|
2993
|
-
if (sync.ok) log.line(`${tag} sync ok (${took})`);
|
|
2994
|
-
else log.error(`${tag} sync failed: ${sync.reason ?? "no reason given"} \xB7 continuing`);
|
|
2995
|
-
const left = spooledArtifactIds(deps.home ?? (job.organizationId === void 0 ? homeFromEnv() : orgHomeDir(config.configPath, job.organizationId)));
|
|
2996
|
-
if (left.length > 0) {
|
|
2997
|
-
job.spooledArtifacts = left.slice(0, MAX_NAMED_SPOOLED);
|
|
2998
|
-
const named = job.spooledArtifacts.join(", ");
|
|
2999
|
-
log.error(
|
|
3000
|
-
`${tag} ${left.length} spooled artifact(s) not uploaded: ${named}${left.length > MAX_NAMED_SPOOLED ? `, and ${left.length - MAX_NAMED_SPOOLED} more` : ""}`
|
|
3001
|
-
);
|
|
3002
|
-
}
|
|
3003
|
-
}
|
|
3004
|
-
};
|
|
3005
|
-
let buildVmStopLogged = false;
|
|
3006
|
-
const endBuildMicroVm = vmBuild === void 0 ? void 0 : async () => {
|
|
3007
|
-
if (job.workdir && job.sessionsSynced !== true) {
|
|
3008
|
-
await uploadSessions(job.workdir);
|
|
3009
|
-
job.sessionsSynced = true;
|
|
3010
|
-
writeJobFile(jobFilePath, job);
|
|
3011
|
-
}
|
|
3012
|
-
await stopBuildMicroVm(vmBuild.runtime, vmBuild.state, jobDirPath);
|
|
3013
|
-
log.line(`${tag} build microVM stopped`);
|
|
3014
|
-
buildVmStopLogged = true;
|
|
3015
|
-
};
|
|
3016
|
-
const collectSessions = build !== void 0 ? () => copyTranscriptsFromContainer(build.runtime, build.state) : vmBuild !== void 0 ? () => copyTranscriptsFromMicroVm(vmBuild.runtime, vmBuild.state) : void 0;
|
|
3017
|
-
if (job.checkContainer !== void 0 && job.checksRan === true) {
|
|
3018
|
-
try {
|
|
3019
|
-
const runtime = deps.container ?? createContainerRuntime();
|
|
3020
|
-
const again = isContainerTracked(job) && outcome === "ok" && !job.cleanupRan;
|
|
3021
|
-
if (again) await runtime.host.stop(job.checkContainer.container);
|
|
3022
|
-
await cleanupContainer(runtime, job.checkContainer);
|
|
3023
|
-
delete job.checkContainer;
|
|
3024
|
-
if (again) job.checksRan = false;
|
|
3025
|
-
writeJobFile(jobFilePath, job);
|
|
3026
|
-
} catch (err) {
|
|
3027
|
-
log.error(`${tag} removing an earlier attempt's check environment failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3028
|
-
}
|
|
3029
|
-
}
|
|
3030
|
-
if (job.checkMicrovm !== void 0 && job.checksRan === true) {
|
|
3031
|
-
try {
|
|
3032
|
-
const runtime = deps.container ?? createContainerRuntime();
|
|
3033
|
-
const again = isMicroVmTracked(job) && outcome === "ok" && !job.cleanupRan;
|
|
3034
|
-
await cleanupMicroVm(runtime, job.checkMicrovm);
|
|
3035
|
-
delete job.checkMicrovm;
|
|
3036
|
-
if (again) job.checksRan = false;
|
|
3037
|
-
writeJobFile(jobFilePath, job);
|
|
3038
|
-
} catch (err) {
|
|
3039
|
-
log.error(`${tag} removing an earlier attempt's check environment failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3040
|
-
}
|
|
3041
|
-
}
|
|
3042
|
-
if (outcome === "ok" && !job.cleanupRan && job.proofsRan !== true && !valuesNotHeld && job.workdir !== void 0 && job.itemKey !== void 0 && await declaresRecordedProof(itemCalls(config, job), job.item, deps.fetchImpl)) {
|
|
3043
|
-
setStage(job, jobDirPath, "proofs");
|
|
3044
|
-
job.proofsRan = true;
|
|
3045
|
-
writeJobFile(jobFilePath, job);
|
|
3046
|
-
log.line(`${tag} proofs started`);
|
|
3047
|
-
const startedMs = now().getTime();
|
|
3048
|
-
try {
|
|
3049
|
-
const recorded = build !== void 0 ? await recordProofsInContainer({ config, runtime: build.runtime, job, jobDir: jobDirPath, log: (line) => log.line(`${tag} ${line}`) }) : vmBuild !== void 0 ? await recordProofsInMicroVm({ config, runtime: vmBuild.runtime, job, jobDir: jobDirPath }) : job.build?.exec !== void 0 ? await runRecordProofsViaExec(config, job, jobDirPath, spawnFn) : await runRecordProofs(config, job, jobDirPath, spawnFn);
|
|
3050
|
-
const took = formatDuration(now().getTime() - startedMs);
|
|
3051
|
-
if (recorded.ok) log.line(`${tag} proofs ok (${took}) \xB7 see ${PROOFS_LOG}`);
|
|
3052
|
-
else log.error(`${tag} proofs not all current (${took}): ${recorded.reason ?? "no reason given"} \xB7 see ${PROOFS_LOG} \xB7 continuing`);
|
|
3053
|
-
} catch (err) {
|
|
3054
|
-
log.error(`${tag} the host's proof recording stopped: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3055
|
-
}
|
|
3056
|
-
}
|
|
3057
|
-
if (valuesNotHeld && outcome === "ok" && !job.cleanupRan) log.error(`${tag} the host's judge and checks skipped: ${VALUES_NOT_HELD}`);
|
|
3058
|
-
if (outcome === "ok" && !job.cleanupRan && job.judgeRan !== true && !valuesNotHeld) {
|
|
3059
|
-
setStage(job, jobDirPath, "judge");
|
|
3060
|
-
job.judgeRan = true;
|
|
3061
|
-
writeJobFile(jobFilePath, job);
|
|
3062
|
-
try {
|
|
3063
|
-
await runJudge({
|
|
3064
|
-
config,
|
|
3065
|
-
job,
|
|
3066
|
-
jobDir: jobDirPath,
|
|
3067
|
-
deps: { fetchImpl: deps.fetchImpl, runtime: deps.container ?? createContainerRuntime(), log, now: deps.now, endBuildMicroVm, collectSessions, ...deps.hostChecks }
|
|
3068
|
-
});
|
|
3069
|
-
} catch (err) {
|
|
3070
|
-
log.error(`${tag} the host's judge stopped: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3071
|
-
}
|
|
3072
|
-
}
|
|
3073
|
-
if (outcome === "ok" && !job.cleanupRan && job.checksRan !== true && !valuesNotHeld) {
|
|
3074
|
-
setStage(job, jobDirPath, "checks");
|
|
3075
|
-
job.checksRan = true;
|
|
3076
|
-
writeJobFile(jobFilePath, job);
|
|
3077
|
-
try {
|
|
3078
|
-
await runHostChecks({
|
|
3079
|
-
config,
|
|
3080
|
-
job,
|
|
3081
|
-
jobDir: jobDirPath,
|
|
3082
|
-
deps: { fetchImpl: deps.fetchImpl, runtime: deps.container ?? createContainerRuntime(), log, now: deps.now, endBuildMicroVm, collectSessions, ...deps.hostChecks }
|
|
3083
|
-
});
|
|
3084
|
-
} catch (err) {
|
|
3085
|
-
log.error(`${tag} the host's checks stopped: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3086
|
-
}
|
|
3087
|
-
}
|
|
3088
|
-
removeKeptContext(jobDirPath);
|
|
3089
|
-
let outOfMemory;
|
|
3090
|
-
if ((vmBuild?.state.cgroup !== void 0 || vmBuild?.state.jailed !== void 0) && outcome !== void 0 && outcome !== "ok" && ending?.reason === void 0) {
|
|
3091
|
-
try {
|
|
3092
|
-
if (await microVmOomKills(vmBuild.runtime, vmBuild.state) > 0) {
|
|
3093
|
-
outOfMemory = outOfMemoryReason(config.build?.memory);
|
|
3094
|
-
}
|
|
3095
|
-
} catch (err) {
|
|
3096
|
-
log.error(`${tag} could not read the build's memory events: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3097
|
-
}
|
|
3098
|
-
if (outOfMemory !== void 0) {
|
|
3099
|
-
job.reason = scrubReason(outOfMemory, config, void 0, jobDirPath);
|
|
3100
|
-
writeJobFile(jobFilePath, job);
|
|
3101
|
-
}
|
|
3102
|
-
}
|
|
3103
|
-
const hookLimits = () => hookLimitsFor(config, now().getTime(), deps.now);
|
|
3104
|
-
let tunnelRecord;
|
|
3105
|
-
if (!job.cleanupRan) {
|
|
3106
|
-
if (job.workdir) {
|
|
3107
|
-
setStage(job, jobDirPath, "sync");
|
|
3108
|
-
if (job.sessionsSynced === true) log.line(`${tag} sync done before the build's microVM stopped`);
|
|
3109
|
-
else await uploadSessions(job.workdir);
|
|
3110
|
-
}
|
|
3111
|
-
setStage(job, jobDirPath, "cleanup");
|
|
3112
|
-
try {
|
|
3113
|
-
rmSync(join(jobDirPath, REASON_FILE));
|
|
3114
|
-
} catch {
|
|
3115
|
-
}
|
|
3116
|
-
log.line(`${tag} cleanup started`);
|
|
3117
|
-
const cleanupStartedMs = now().getTime();
|
|
3118
|
-
const hostMade = job.build !== void 0 && job.build.workDir === void 0;
|
|
3119
|
-
if (vmBuild !== void 0) {
|
|
3120
|
-
let cleanedUp = true;
|
|
3121
|
-
tunnelRecord = vmBuild.runtime.tunnels.get(vmBuild.state.buildId)?.record();
|
|
3122
|
-
try {
|
|
3123
|
-
await cleanupMicroVm(
|
|
3124
|
-
vmBuild.runtime,
|
|
3125
|
-
vmBuild.state,
|
|
3126
|
-
jobDirPath,
|
|
3127
|
-
(text2) => log.error(`${tag} ${text2}`),
|
|
3128
|
-
(bytes) => {
|
|
3129
|
-
job.buildDiskPeakBytes = bytes;
|
|
3130
|
-
writeJobFile(join(jobDirPath, "job.json"), job);
|
|
3131
|
-
log.line(`${tag} build disk peak ${formatBytes(bytes)}`);
|
|
3132
|
-
}
|
|
3133
|
-
);
|
|
3134
|
-
} catch (err) {
|
|
3135
|
-
cleanedUp = false;
|
|
3136
|
-
log.error(`${tag} cleanup failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3137
|
-
}
|
|
3138
|
-
if (cleanedUp && !buildVmStopLogged) log.line(`${tag} build microVM stopped`);
|
|
3139
|
-
if (cleanedUp) log.line(`${tag} cleanup ok (${formatDuration(now().getTime() - cleanupStartedMs)})`);
|
|
3140
|
-
} else if (build !== void 0 || !hostMade && isBuiltinContainer(config.hooks.cleanup)) {
|
|
3141
|
-
let cleanedUp = true;
|
|
3142
|
-
if (build !== void 0) {
|
|
3143
|
-
tunnelRecord = build.runtime.tunnels.get(build.state.buildId)?.record();
|
|
3144
|
-
try {
|
|
3145
|
-
await cleanupContainer(build.runtime, build.state, jobDirPath);
|
|
3146
|
-
} catch (err) {
|
|
3147
|
-
cleanedUp = false;
|
|
3148
|
-
log.error(`${tag} cleanup failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3149
|
-
}
|
|
3150
|
-
}
|
|
3151
|
-
if (cleanedUp) log.line(`${tag} cleanup ok (${formatDuration(now().getTime() - cleanupStartedMs)})`);
|
|
3152
|
-
} else if (!hostMade && (job.build !== void 0 || isBuiltinProcess(config.hooks.cleanup))) {
|
|
3153
|
-
let cleanedUp = true;
|
|
3154
|
-
if (job.build !== void 0) {
|
|
3155
|
-
const runtime = deps.container ?? createContainerRuntime();
|
|
3156
|
-
tunnelRecord = runtime.tunnels.get(job.build.buildId)?.record();
|
|
3157
|
-
try {
|
|
3158
|
-
await cleanupProcess(runtime, job.build);
|
|
3159
|
-
} catch (err) {
|
|
3160
|
-
cleanedUp = false;
|
|
3161
|
-
log.error(`${tag} cleanup failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3162
|
-
}
|
|
3163
|
-
}
|
|
3164
|
-
if (cleanedUp) log.line(`${tag} cleanup ok (${formatDuration(now().getTime() - cleanupStartedMs)})`);
|
|
3165
|
-
} else {
|
|
3166
|
-
if (!isBuiltinContainer(config.hooks.cleanup) && !isBuiltinProcess(config.hooks.cleanup) && !isBuiltinVm(config.hooks.cleanup)) {
|
|
3167
|
-
const cleanupEnv = {
|
|
3168
|
-
...jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDirPath, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job)),
|
|
3169
|
-
...job.build === void 0 ? {} : { OUTERLAYER_BUILD_DIR: job.build.buildDir },
|
|
3170
|
-
OUTERLAYER_OUTCOME: outcome ?? "",
|
|
3171
|
-
OUTERLAYER_LOG: logPath
|
|
3172
|
-
};
|
|
3173
|
-
const cleanup = await runHook(
|
|
3174
|
-
config.hooks.cleanup,
|
|
3175
|
-
cleanupEnv,
|
|
3176
|
-
jobDirPath,
|
|
3177
|
-
join(jobDirPath, "cleanup.log"),
|
|
3178
|
-
spawnFn,
|
|
3179
|
-
hookLimits()
|
|
3180
|
-
).catch((err) => {
|
|
3181
|
-
log.error(`${tag} cleanup failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3182
|
-
return { code: -1 };
|
|
3183
|
-
});
|
|
3184
|
-
const cleanupTook = formatDuration(now().getTime() - cleanupStartedMs);
|
|
3185
|
-
if (cleanup.endedBy !== void 0) {
|
|
3186
|
-
log.error(`${tag} ${describeHookEnd("cleanup", cleanup.endedBy, config)} \xB7 continuing`);
|
|
3187
|
-
} else if (cleanup.code > 0) {
|
|
3188
|
-
log.error(`${tag} cleanup exited ${cleanup.code} (${cleanupTook}) \xB7 continuing`);
|
|
3189
|
-
} else if (cleanup.code === 0) {
|
|
3190
|
-
log.line(`${tag} cleanup ok (${cleanupTook})`);
|
|
3191
|
-
}
|
|
3192
|
-
}
|
|
3193
|
-
if (job.build !== void 0) {
|
|
3194
|
-
const runtime = deps.container ?? createContainerRuntime();
|
|
3195
|
-
tunnelRecord = runtime.tunnels.get(job.build.buildId)?.record();
|
|
3196
|
-
try {
|
|
3197
|
-
await closeHostBuild(runtime, job.build);
|
|
3198
|
-
} catch (err) {
|
|
3199
|
-
log.error(`${tag} closing the build's tunnel failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3200
|
-
}
|
|
3201
|
-
}
|
|
3202
|
-
}
|
|
3203
|
-
job.cleanupRan = true;
|
|
3204
|
-
writeJobFile(jobFilePath, job);
|
|
3205
|
-
}
|
|
3206
|
-
let verdictReason;
|
|
3207
|
-
if (outcome === "ok") {
|
|
3208
|
-
const decision = await settleVerdict({
|
|
3209
|
-
read: () => readItemEvaluation(itemCalls(config, job), job.item, deps.fetchImpl),
|
|
3210
|
-
now,
|
|
3211
|
-
sleep: deps.sleep ?? waitMs
|
|
3212
|
-
});
|
|
3213
|
-
if (decision.outcome === "incomplete") {
|
|
3214
|
-
const release2 = unfinishedRelease(
|
|
3215
|
-
scrubReason(incompleteReason(decision.reason, job.judgeNotRun), config, void 0, jobDirPath),
|
|
3216
|
-
deps.container?.credentials.get(jobDirPath)?.reason
|
|
3217
|
-
);
|
|
3218
|
-
outcome = release2.outcome;
|
|
3219
|
-
verdictReason = release2.reason;
|
|
3220
|
-
log.error(`${tag} the command exited 0 but the item is not done: ${verdictReason}`);
|
|
3221
|
-
}
|
|
3222
|
-
}
|
|
3223
|
-
if (config.hooks.report) {
|
|
3224
|
-
setStage(job, jobDirPath, "report");
|
|
3225
|
-
log.line(`${tag} report started \xB7 phase end`);
|
|
3226
|
-
const reportStartedMs = now().getTime();
|
|
3227
|
-
const reportEnv = {
|
|
3228
|
-
...jobEnv(claimCalls(config, job), job.item, job.itemKey, jobDirPath, process.env, job.branch, job.commitIdentity?.author, spoolHomeFor(config, job)),
|
|
3229
|
-
OUTERLAYER_PHASE: "end",
|
|
3230
|
-
OUTERLAYER_OUTCOME: outcome ?? "",
|
|
3231
|
-
OUTERLAYER_LOG: logPath
|
|
3232
|
-
};
|
|
3233
|
-
const report = await runHook(
|
|
3234
|
-
config.hooks.report,
|
|
3235
|
-
reportEnv,
|
|
3236
|
-
jobDirPath,
|
|
3237
|
-
join(jobDirPath, "report.log"),
|
|
3238
|
-
spawnFn,
|
|
3239
|
-
hookLimits()
|
|
3240
|
-
).catch((err) => {
|
|
3241
|
-
log.error(`${tag} report failed: ${describeCallFailure(err)} \xB7 continuing`);
|
|
3242
|
-
return { code: -1 };
|
|
3243
|
-
});
|
|
3244
|
-
const reportTook = formatDuration(now().getTime() - reportStartedMs);
|
|
3245
|
-
if (report.endedBy !== void 0) {
|
|
3246
|
-
log.error(`${tag} ${describeHookEnd("report", report.endedBy, config)} \xB7 continuing`);
|
|
3247
|
-
} else if (report.code === 0) {
|
|
3248
|
-
log.line(`${tag} report ok (${reportTook})`);
|
|
3249
|
-
} else if (report.code > 0) {
|
|
3250
|
-
log.error(`${tag} report exited ${report.code} (${reportTook}) \xB7 continuing`);
|
|
3251
|
-
}
|
|
3252
|
-
}
|
|
3253
|
-
const isolated = job.container !== void 0 || job.microvm !== void 0;
|
|
3254
|
-
const held = deps.container?.credentials.get(jobDirPath);
|
|
3255
|
-
if (held !== void 0) {
|
|
3256
|
-
await held.revokeAll();
|
|
3257
|
-
deps.container.credentials.delete(jobDirPath);
|
|
3258
|
-
}
|
|
3259
|
-
dropManagedClaudeKey(jobDirPath);
|
|
3260
|
-
if (outcome !== void 0) {
|
|
3261
|
-
const fromHook = readReasonLine(jobDirPath, config);
|
|
3262
|
-
if (verdictReason !== void 0) {
|
|
3263
|
-
job.reason = verdictReason;
|
|
3264
|
-
} else if (outOfMemory !== void 0) {
|
|
3265
|
-
job.reason = scrubReason(outOfMemory, config, void 0, jobDirPath);
|
|
3266
|
-
} else if (ending?.reason === void 0) {
|
|
3267
|
-
job.reason = (outcome === "ok" ? void 0 : held?.reason) ?? (isolated ? fromHook ?? job.reason : fromHook);
|
|
3268
|
-
}
|
|
3269
|
-
if (outcome === "failed" && job.reason === void 0 && job.commandEndReason !== void 0) {
|
|
3270
|
-
job.reason = scrubReason(job.commandEndReason, config, void 0, jobDirPath);
|
|
3271
|
-
}
|
|
3272
|
-
const cliVersion = deps.cliVersion ?? resolveVersion();
|
|
3273
|
-
const base = job.environment ?? (job.microvm !== void 0 ? { isolation: "vm", cliVersion } : job.container === void 0 ? sharedUserEnvironment(cliVersion) : { isolation: "container", cliVersion });
|
|
3274
|
-
if (isolated || job.build !== void 0) {
|
|
3275
|
-
tunnelRecord ??= readTunnelRecord(join(jobDirPath, "tunnel.json"));
|
|
3276
|
-
job.environment = withTunnelRecord(base, scrubTunnelRecord(tunnelRecord, config, jobDirPath), !isolated);
|
|
3277
|
-
} else {
|
|
3278
|
-
job.environment = base;
|
|
3279
|
-
}
|
|
3280
|
-
job.environment = { ...job.environment, commitIdentity: commitIdentityRecord(job.commitIdentity) };
|
|
3281
|
-
const guestUsage = job.microvm === void 0 ? void 0 : usageRecord(job.guestUsage);
|
|
3282
|
-
if (guestUsage !== void 0) job.environment = { ...job.environment, guestUsage };
|
|
3283
|
-
else delete job.environment.guestUsage;
|
|
3284
|
-
writeJobFile(jobFilePath, job);
|
|
3285
|
-
}
|
|
3286
|
-
const sentAtMs = now().getTime();
|
|
3287
|
-
const runnerLog = valuesNotHeld ? `The runner sends no log for this attempt: ${VALUES_NOT_HELD}.
|
|
3288
|
-
` : releaseLogFor(jobDirPath, job, config);
|
|
3289
|
-
const detail = outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, log: runnerLog };
|
|
3290
|
-
const release = await retrySend(() => releaseLease(claimCalls(config, job), claimItem(config, job), outcome, detail, deps.fetchImpl), {
|
|
3291
|
-
now: deps.now,
|
|
3292
|
-
sleep: deps.sleep
|
|
3293
|
-
});
|
|
3294
|
-
if (!release.ok && release.retryable && job.managedVariables !== void 0 && runnerLog !== void 0) {
|
|
3295
|
-
writeFileSync(join(jobDirPath, SCRUBBED_RELEASE_LOG), runnerLog, { mode: RUNNER_FILE_MODE });
|
|
3296
|
-
}
|
|
3297
|
-
dropManagedBuildSettings(jobDirPath);
|
|
3298
|
-
const alreadyReleased = release.ok && wasAlreadyReleased(release.value, outcome, sentAtMs);
|
|
3299
|
-
const endedAt = now();
|
|
3300
|
-
job.outcome = outcome;
|
|
3301
|
-
job.endedAt = endedAt.toISOString();
|
|
3302
|
-
job.releaseFailed = release.ok ? alreadyReleased ? ALREADY_RELEASED : void 0 : release.reason;
|
|
3303
|
-
writeJobFile(jobFilePath, job);
|
|
3304
|
-
deps.attemptLogs?.close(job.item, itemScope(job));
|
|
3305
|
-
const total = formatDuration(endedAt.getTime() - new Date(job.startedAt).getTime());
|
|
3306
|
-
if (alreadyReleased) {
|
|
3307
|
-
log.line(
|
|
3308
|
-
`${tag} lease was already released (outcome ${release.ok ? release.value.outcome ?? "none" : "unknown"}) \xB7 this attempt ended as ${outcome ?? "not attempted"}`
|
|
3309
|
-
);
|
|
3310
|
-
} else if (release.ok) {
|
|
3311
|
-
log.line(
|
|
3312
|
-
outcome === void 0 ? `${tag} released \xB7 not attempted (provision exit 75) \xB7 ${total}` : `${tag} released \xB7 outcome ${outcome} \xB7 ${total}${egressText(tunnelRecord)}`
|
|
3313
|
-
);
|
|
3314
|
-
} else {
|
|
3315
|
-
const lapse = release.code === "machine_revoked" ? "the gateway ended the lease when the machine was revoked" : `the lease expires at ${formatClock(leaseExpiry(job, endedAt), endedAt)}`;
|
|
3316
|
-
log.error(`${tag} release failed after ${release.attempts} attempt(s): ${release.reason}; ${lapse}${egressText(tunnelRecord)}`);
|
|
3317
|
-
}
|
|
3318
|
-
return job;
|
|
3319
|
-
}
|
|
3320
|
-
function egressText(record) {
|
|
3321
|
-
if (record === void 0) return "";
|
|
3322
|
-
return record.egress === void 0 ? " \xB7 tunnel counts not recorded" : ` \xB7 ${egressSummary(record.egress)}`;
|
|
3323
|
-
}
|
|
3324
|
-
function readLogState(logPath) {
|
|
3325
|
-
try {
|
|
3326
|
-
const stats = statSync(logPath);
|
|
3327
|
-
return { size: stats.size, mtimeMs: stats.mtimeMs };
|
|
3328
|
-
} catch {
|
|
3329
|
-
return { size: -1, mtimeMs: -1 };
|
|
3330
|
-
}
|
|
3331
|
-
}
|
|
3332
|
-
function describeHookEnd(stage, endedBy, config) {
|
|
3333
|
-
return endedBy === "idle" ? `${stage} idle for ${config.idleLimitMinutes}m, ended` : `${stage} past the ${config.timeLimitMinutes}m time limit, ended`;
|
|
3334
|
-
}
|
|
3335
|
-
function hookLimitsFor(config, fromMs, now) {
|
|
3336
|
-
return {
|
|
3337
|
-
idleMs: config.idleLimitMinutes * 6e4,
|
|
3338
|
-
endBy: fromMs + config.timeLimitMinutes * 6e4,
|
|
3339
|
-
now
|
|
3340
|
-
};
|
|
3341
|
-
}
|
|
3342
|
-
async function retryFailedReleases(jobsDirPath, config, deps) {
|
|
3343
|
-
if (!existsSync(jobsDirPath)) return [];
|
|
3344
|
-
const results = [];
|
|
3345
|
-
for (const entry of readdirSync(jobsDirPath, { withFileTypes: true })) {
|
|
3346
|
-
if (!entry.isDirectory()) continue;
|
|
3347
|
-
const dirPath = join(jobsDirPath, entry.name);
|
|
3348
|
-
const jobFilePath = join(dirPath, "job.json");
|
|
3349
|
-
const job = readJobFile(jobFilePath);
|
|
3350
|
-
if (!job || job.releaseFailed === void 0 || job.releaseFailed === ALREADY_RELEASED || isUnfinished(job)) {
|
|
3351
|
-
continue;
|
|
3352
|
-
}
|
|
3353
|
-
if (leftByRunnerKey(config, job)) {
|
|
3354
|
-
rmSync(join(dirPath, SCRUBBED_RELEASE_LOG), { force: true });
|
|
3355
|
-
job.releaseFailed = void 0;
|
|
3356
|
-
writeJobFile(jobFilePath, job);
|
|
3357
|
-
continue;
|
|
3358
|
-
}
|
|
3359
|
-
const runnerLog = readScrubbedReleaseLog(dirPath) ?? releaseLogFor(dirPath, job, config);
|
|
3360
|
-
const detail = job.outcome === void 0 ? { log: runnerLog } : { stage: job.endStage, exitCode: job.endExitCode, reason: job.reason, environment: job.environment, resume: job.resume, log: runnerLog };
|
|
3361
|
-
const release = await retrySend(() => releaseLease(claimCalls(config, job), claimItem(config, job), job.outcome, detail, deps.fetchImpl), {
|
|
3362
|
-
now: deps.now,
|
|
3363
|
-
sleep: deps.sleep
|
|
3364
|
-
});
|
|
3365
|
-
if (release.ok || !release.retryable) {
|
|
3366
|
-
rmSync(join(dirPath, SCRUBBED_RELEASE_LOG), { force: true });
|
|
3367
|
-
job.releaseFailed = void 0;
|
|
3368
|
-
writeJobFile(jobFilePath, job);
|
|
3369
|
-
} else {
|
|
3370
|
-
job.releaseFailed = release.reason;
|
|
3371
|
-
writeJobFile(jobFilePath, job);
|
|
3372
|
-
}
|
|
3373
|
-
results.push({ job, released: release.ok });
|
|
3374
|
-
}
|
|
3375
|
-
return results;
|
|
3376
|
-
}
|
|
3377
|
-
function isUnfinished(job) {
|
|
3378
|
-
return job.outcome === void 0 && job.endedAt === void 0;
|
|
3379
|
-
}
|
|
3380
|
-
function unfinishedJobs(jobsDirPath) {
|
|
3381
|
-
if (!existsSync(jobsDirPath)) return [];
|
|
3382
|
-
const found = [];
|
|
3383
|
-
for (const entry of readdirSync(jobsDirPath, { withFileTypes: true })) {
|
|
3384
|
-
if (!entry.isDirectory()) continue;
|
|
3385
|
-
const dirPath = join(jobsDirPath, entry.name);
|
|
3386
|
-
const job = readJobFile(join(dirPath, "job.json"));
|
|
3387
|
-
if (job && isUnfinished(job)) found.push({ dirPath, job });
|
|
3388
|
-
}
|
|
3389
|
-
return found;
|
|
3390
|
-
}
|
|
3391
|
-
function isContainerTracked(job) {
|
|
3392
|
-
return job.container?.tracking === "container";
|
|
3393
|
-
}
|
|
3394
|
-
function isMicroVmTracked(job) {
|
|
3395
|
-
return job.microvm?.tracking === "microvm";
|
|
3396
|
-
}
|
|
3397
|
-
function isTracked(job) {
|
|
3398
|
-
return isContainerTracked(job) || isMicroVmTracked(job);
|
|
3399
|
-
}
|
|
3400
|
-
function trackingOf(job) {
|
|
3401
|
-
return job.container?.tracking ?? job.microvm?.tracking;
|
|
3402
|
-
}
|
|
3403
|
-
function isReadable(job) {
|
|
3404
|
-
return readsJobState(job, READ_JOB_STATE_FORMATS);
|
|
3405
|
-
}
|
|
3406
|
-
function listUnreadableJobs(jobsDirPath) {
|
|
3407
|
-
return unfinishedJobs(jobsDirPath).flatMap(({ dirPath, job }) => isReadable(job) ? [] : [{ dirPath, job, format: jobStateFormatOf(job) }]);
|
|
3408
|
-
}
|
|
3409
|
-
function listTrackedJobs(jobsDirPath) {
|
|
3410
|
-
return unfinishedJobs(jobsDirPath).flatMap(({ dirPath, job }) => {
|
|
3411
|
-
if (!isReadable(job)) return [];
|
|
3412
|
-
const tracking = trackingOf(job);
|
|
3413
|
-
if (tracking === void 0) return [];
|
|
3414
|
-
if ((tracking === "container" || tracking === "microvm") && inTeardown(job)) return [];
|
|
3415
|
-
return [{ dirPath, job, known: tracking === "container" || tracking === "microvm" }];
|
|
3416
|
-
});
|
|
3417
|
-
}
|
|
3418
|
-
function listAdoptableJobs(jobsDirPath) {
|
|
3419
|
-
return unfinishedJobs(jobsDirPath).filter(
|
|
3420
|
-
({ job }) => isReadable(job) && job.microvm === void 0 && job.pid !== void 0 && job.start !== void 0 && isSameProcess(job.pid, job.start)
|
|
3421
|
-
);
|
|
3422
|
-
}
|
|
3423
|
-
async function recoverLostJobs(jobsDirPath, config, deps) {
|
|
3424
|
-
const recovered = [];
|
|
3425
|
-
for (const { dirPath, job } of unfinishedJobs(jobsDirPath)) {
|
|
3426
|
-
if (!isReadable(job)) continue;
|
|
3427
|
-
if (leftByRunnerKey(config, job)) {
|
|
3428
|
-
(deps.log ?? createRunnerLog(deps.now ?? (() => /* @__PURE__ */ new Date()))).error(
|
|
3429
|
-
`${itemTag(job)} left by a runner key: a fleet runner cannot release its lease, which lapses by itself \xB7 closing its job file`
|
|
3430
|
-
);
|
|
3431
|
-
job.outcome = "interrupted";
|
|
3432
|
-
job.endedAt = (deps.now ?? (() => /* @__PURE__ */ new Date()))().toISOString();
|
|
3433
|
-
job.reason = "an unfinished build from a runner key; a fleet runner cannot release its lease";
|
|
3434
|
-
writeJobFile(join(dirPath, "job.json"), job);
|
|
3435
|
-
recovered.push(job);
|
|
3436
|
-
continue;
|
|
3437
|
-
}
|
|
3438
|
-
const tracking = trackingOf(job);
|
|
3439
|
-
if (tracking !== void 0 && tracking !== "container" && tracking !== "microvm") {
|
|
3440
|
-
recovered.push(await finishJob(job, dirPath, "interrupted", config, deps, { reason: UNKNOWN_TRACKING_REASON }));
|
|
3441
|
-
continue;
|
|
3442
|
-
}
|
|
3443
|
-
if ((tracking === "container" || tracking === "microvm") && !inTeardown(job)) continue;
|
|
3444
|
-
const alive = job.pid !== void 0 && job.start !== void 0 && isSameProcess(job.pid, job.start);
|
|
3445
|
-
if (alive) continue;
|
|
3446
|
-
const recorded = outcomeFromExitCode(dirPath);
|
|
3447
|
-
if (recorded !== void 0) job.exitCode = recorded.code;
|
|
3448
|
-
recovered.push(await finishJob(job, dirPath, recorded?.outcome ?? "interrupted", config, deps));
|
|
3449
|
-
}
|
|
3450
|
-
return recovered;
|
|
3451
|
-
}
|
|
3452
|
-
|
|
3453
|
-
// src/runner/host-checks.ts
|
|
3454
|
-
var MAX_VALIDATOR_FILES = 20;
|
|
3455
|
-
var VALIDATORS_DIR = ".outerlayer/validators";
|
|
3456
|
-
var POLICY_FILE = ".outerlayer/policy.yaml";
|
|
3457
|
-
var LIMIT_KEY_TAIL2 = " (runner.checks.timeLimitMinutes)";
|
|
3458
|
-
var MAX_CHECK_ITEMS_BYTES = 1024 * 1024;
|
|
3459
|
-
function commandItems(read, scrub) {
|
|
3460
|
-
if (read.status !== 0 || read.stdout.trim() === "") return { ok: true, items: void 0 };
|
|
3461
|
-
if (Buffer.byteLength(read.stdout) > MAX_CHECK_ITEMS_BYTES) return { ok: false, reason: `its check items file is longer than ${MAX_CHECK_ITEMS_BYTES} bytes` };
|
|
3462
|
-
let raw;
|
|
3463
|
-
try {
|
|
3464
|
-
raw = JSON.parse(read.stdout);
|
|
3465
|
-
} catch {
|
|
3466
|
-
return { ok: false, reason: "its check items file (OUTERLAYER_CHECK_ITEMS) is not JSON" };
|
|
3467
|
-
}
|
|
3468
|
-
const parsed = CheckItemsSchema.safeParse(raw);
|
|
3469
|
-
if (!parsed.success) {
|
|
3470
|
-
return { ok: false, reason: `its check items file does not match the check item shape: ${zodProblem(parsed.error.issues)}` };
|
|
3471
|
-
}
|
|
3472
|
-
return { ok: true, items: parsed.data.map((item) => item.reason === void 0 ? item : { ...item, reason: scrub(item.reason) }) };
|
|
3473
|
-
}
|
|
3474
|
-
function isJudgeCheck(check) {
|
|
3475
|
-
return "judge" in check;
|
|
3476
|
-
}
|
|
3477
|
-
function lastLine(text2) {
|
|
3478
|
-
return text2.split(/\r\n|\r|\n/).map((line) => line.trim()).filter((line) => line !== "").at(-1);
|
|
3479
|
-
}
|
|
3480
|
-
function readInside(root, rel) {
|
|
3481
|
-
const path = join(root, rel);
|
|
3482
|
-
try {
|
|
3483
|
-
if (!lstatSync(path).isFile()) return void 0;
|
|
3484
|
-
const inside = realpathSync(path);
|
|
3485
|
-
const base = realpathSync(root);
|
|
3486
|
-
return inside.startsWith(base + sep) ? readFileSync(inside, "utf8") : void 0;
|
|
3487
|
-
} catch {
|
|
3488
|
-
return void 0;
|
|
3489
|
-
}
|
|
3490
|
-
}
|
|
3491
|
-
var MAINTAINED_BRIEF = /^\.(?:agents|outerlayer)\/skills\/(outerlayer-[a-z]+)\/([A-Za-z0-9_./-]+)$/;
|
|
3492
|
-
function readBrief(root, brief) {
|
|
3493
|
-
const own = readInside(root, brief);
|
|
3494
|
-
if (own !== void 0) return own;
|
|
3495
|
-
const match = MAINTAINED_BRIEF.exec(brief);
|
|
3496
|
-
if (match === null || match[2].split("/").includes("..") || !MAINTAINED_SKILLS.includes(match[1])) return void 0;
|
|
3497
|
-
try {
|
|
3498
|
-
return readFileSync(join(resolvePackRoot(), "maintained", match[1], match[2]), "utf8");
|
|
3499
|
-
} catch {
|
|
3500
|
-
return void 0;
|
|
3501
|
-
}
|
|
3502
|
-
}
|
|
3503
|
-
function validatorFilesIn(root) {
|
|
3504
|
-
const dir = join(root, VALIDATORS_DIR);
|
|
3505
|
-
let names;
|
|
3506
|
-
try {
|
|
3507
|
-
if (!lstatSync(dir).isDirectory()) return [];
|
|
3508
|
-
names = readdirSync(dir);
|
|
3509
|
-
} catch {
|
|
3510
|
-
return [];
|
|
3511
|
-
}
|
|
3512
|
-
const files = [];
|
|
3513
|
-
for (const name of names.filter((entry) => /\.ya?ml$/.test(entry)).sort().slice(0, MAX_VALIDATOR_FILES)) {
|
|
3514
|
-
const path = `${VALIDATORS_DIR}/${name}`;
|
|
3515
|
-
const content = readInside(root, path);
|
|
3516
|
-
if (content !== void 0) files.push({ path, content });
|
|
3517
|
-
}
|
|
3518
|
-
return files;
|
|
3519
|
-
}
|
|
3520
|
-
var CRITERIA_CHECK = "acceptance-criteria";
|
|
3521
|
-
function asHostReads(file, base, log, tag) {
|
|
3522
|
-
const upgraded = upgradedStarterText(file.path, file.content, { base });
|
|
3523
|
-
if (upgraded === void 0) return file;
|
|
3524
|
-
log.line(`${tag} ${file.path} is an unedited copy an earlier release wrote, so it is read as this CLI writes it`);
|
|
3525
|
-
return { path: file.path, content: upgraded };
|
|
3526
|
-
}
|
|
3527
|
-
function testProofIn(root) {
|
|
3528
|
-
const policyText = readInside(root, POLICY_FILE);
|
|
3529
|
-
return parsePolicy(policyText === void 0 ? null : { path: POLICY_FILE, content: policyText }, []).testProof;
|
|
3530
|
-
}
|
|
3531
|
-
function checksIn(root, log, tag) {
|
|
3532
|
-
const base = existsSync(join(root, AGENTS_SKILLS_BASE)) ? AGENTS_SKILLS_BASE : OUTERLAYER_SKILLS_BASE;
|
|
3533
|
-
const policyText = readInside(root, POLICY_FILE);
|
|
3534
|
-
const policy = policyText === void 0 ? null : asHostReads({ path: POLICY_FILE, content: policyText }, base, log, tag);
|
|
3535
|
-
const loaded = parsePolicy(policy, validatorFilesIn(root).map((file) => asHostReads(file, base, log, tag)));
|
|
3536
|
-
for (const problem of loaded.problems) log.error(`${tag} ${problem.file}: ${problem.problem}`);
|
|
3537
|
-
const hostChecks = loaded.customs.filter((def) => def.runWhere === "host" && def.level !== "off" && def.declaresEmit !== null);
|
|
3538
|
-
const checks = [];
|
|
3539
|
-
for (const def of hostChecks) {
|
|
3540
|
-
if (def.judge !== void 0) checks.push({ name: def.declaresEmit, judge: { brief: def.judge.brief, text: readBrief(root, def.judge.brief), subjects: def.judge.subjects } });
|
|
3541
|
-
else if (def.command !== void 0) checks.push({ name: def.declaresEmit, command: def.command });
|
|
3542
|
-
}
|
|
3543
|
-
const unrecorded = loaded.customs.filter((def) => def.level !== "off" && def.declaresEmit === CRITERIA_CHECK && !checks.some((check) => check.name === CRITERIA_CHECK)).map((def) => def.id);
|
|
3544
|
-
return { checks, unrecorded };
|
|
3545
|
-
}
|
|
3546
|
-
async function readDefinitions(input, repository, scratch, token, tokens) {
|
|
3547
|
-
const { config, job, deps } = input;
|
|
3548
|
-
const tag = `${itemTag(job)}`;
|
|
3549
|
-
const stage = deps.stage ?? stageGovernedContext;
|
|
3550
|
-
const controlPlaneUrl = (controlPlane) => deps.runtime.remoteFor?.(controlPlane) ?? `${deps.runtime.github.gitUrl}/${controlPlane}.git`;
|
|
3551
|
-
const governing = (job.container ?? job.microvm)?.governing;
|
|
3552
|
-
const kept = keptContextTree(input.jobDir, governing);
|
|
3553
|
-
if (kept !== void 0) return { ok: true, ...checksIn(kept, deps.log, tag) };
|
|
3554
|
-
const staged = await stage({
|
|
3555
|
-
config: claimCalls(config, job),
|
|
3556
|
-
...job.organizationId === void 0 ? {} : { organizationId: job.organizationId },
|
|
3557
|
-
repository,
|
|
3558
|
-
...governing === void 0 ? {} : { source: { repository: governing.repository, ref: governing.ref, sha: governing.commit } },
|
|
3559
|
-
buildId: `checks-${randomBytes(4).toString("hex")}`,
|
|
3560
|
-
buildDir: scratch,
|
|
3561
|
-
fetchImpl: deps.fetchImpl,
|
|
3562
|
-
controlPlaneUrl,
|
|
3563
|
-
controlPlaneToken: tokens === void 0 ? void 0 : () => tokens.controlPlane(),
|
|
3564
|
-
log: deps.log,
|
|
3565
|
-
tag
|
|
3566
|
-
});
|
|
3567
|
-
if (staged.kind === "refused") return { ok: false, reason: staged.reason };
|
|
3568
|
-
if (staged.kind === "staged") {
|
|
3569
|
-
try {
|
|
3570
|
-
return { ok: true, ...checksIn(join(staged.dir, "tree"), deps.log, tag) };
|
|
3571
|
-
} finally {
|
|
3572
|
-
rmSync(staged.dir, { recursive: true, force: true });
|
|
3573
|
-
}
|
|
3574
|
-
}
|
|
3575
|
-
const repo = join(scratch, "repo");
|
|
3576
|
-
const remote = deps.runtime.remoteFor?.(repository) ?? `${deps.runtime.github.gitUrl}/${repository}.git`;
|
|
3577
|
-
const started = (job.container ?? job.microvm)?.startCommit;
|
|
3578
|
-
const commit = started !== void 0 && started !== "" ? started : job.environment?.recipe?.sourceCommit ?? await deps.runtime.readHead(remote, scratch);
|
|
3579
|
-
if (commit === void 0) return { ok: false, reason: "the repository's default branch could not be read" };
|
|
3580
|
-
const fetched = await (deps.fetchCommit ?? fetchCommit)({ remote, dir: repo, commit, token });
|
|
3581
|
-
if (!fetched.ok) return { ok: false, reason: fetched.reason };
|
|
3582
|
-
deps.log.line(`${tag} check definitions read at ${commit.slice(0, 12)}`);
|
|
3583
|
-
return { ok: true, ...checksIn(repo, deps.log, tag) };
|
|
3584
|
-
}
|
|
3585
|
-
function timeoutAfter(ms, sleep) {
|
|
3586
|
-
if (sleep !== void 0) return { done: sleep(ms).then(() => "timeout"), cancel: () => void 0 };
|
|
3587
|
-
let timer;
|
|
3588
|
-
const done = new Promise((resolve) => {
|
|
3589
|
-
timer = setTimeout(() => resolve("timeout"), ms);
|
|
3590
|
-
timer.unref();
|
|
3591
|
-
});
|
|
3592
|
-
return { done, cancel: () => clearTimeout(timer) };
|
|
3593
|
-
}
|
|
3594
|
-
function noCleanEnvironment(job) {
|
|
3595
|
-
const ran = job.build?.workDir !== void 0 ? "builtin:process" : "a provision hook of the host's own";
|
|
3596
|
-
return `the host has no clean environment for checks: this build ran under ${ran}, and checks run only in a new container or microVM from a build's image`;
|
|
3597
|
-
}
|
|
3598
|
-
function makeRecorder(input) {
|
|
3599
|
-
const { config, job, deps, jobDir } = input;
|
|
3600
|
-
const tag = `${itemTag(job)}`;
|
|
3601
|
-
const now = deps.now ?? (() => /* @__PURE__ */ new Date());
|
|
3602
|
-
const calls = itemCalls(config, job);
|
|
3603
|
-
let signer;
|
|
3604
|
-
return async (name, result, sentence, opts = {}) => {
|
|
3605
|
-
signer ??= await signerFor(config.configPath);
|
|
3606
|
-
const keyTail = opts.keyTail ?? "";
|
|
3607
|
-
const body = sentence === void 0 ? void 0 : cutReasonLine(`${scrubReason(sentence, config, void 0, jobDir) ?? "check failed"}${keyTail}`);
|
|
3608
|
-
const clientEmitId = `host-${createHash("sha256").update(`${job.claimId}:${job.startedAt}:${name}${opts.idTag === void 0 ? "" : `:${opts.idTag}`}`).digest("hex").slice(0, 40)}`;
|
|
3609
|
-
const sent = await retrySend(
|
|
3610
|
-
() => postJson(
|
|
3611
|
-
calls.url,
|
|
3612
|
-
"/v1/emitted-results",
|
|
3613
|
-
calls.apiKey,
|
|
3614
|
-
calls.appId,
|
|
3615
|
-
{
|
|
3616
|
-
schemaVersion: 1,
|
|
3617
|
-
// A field left undefined is not sent: the request is JSON.
|
|
3618
|
-
emit: { clientEmitId, name, result, body, link: opts.link, items: opts.items, emittedAt: now().toISOString(), item: { number: Number(job.item) } }
|
|
3619
|
-
},
|
|
3620
|
-
deps.fetchImpl,
|
|
3621
|
-
void 0,
|
|
3622
|
-
signer
|
|
3623
|
-
),
|
|
3624
|
-
{ now: deps.now }
|
|
3625
|
-
);
|
|
3626
|
-
if (sent.ok) deps.log.line(`${tag} check ${name} ${result}${body === void 0 ? "" : `: ${body}`}`);
|
|
3627
|
-
else deps.log.error(`${tag} check ${name} ${result} could not be recorded: ${sent.reason}`);
|
|
3628
|
-
};
|
|
3629
|
-
}
|
|
3630
|
-
async function runHostChecks(input) {
|
|
3631
|
-
const { config, job, jobDir, deps } = input;
|
|
3632
|
-
const { runtime, log } = deps;
|
|
3633
|
-
const tag = `${itemTag(job)}`;
|
|
3634
|
-
const now = deps.now ?? (() => /* @__PURE__ */ new Date());
|
|
3635
|
-
const repository = job.repository ?? job.container?.repository;
|
|
3636
|
-
if (repository === void 0) return;
|
|
3637
|
-
const jobFilePath = join(jobDir, "job.json");
|
|
3638
|
-
const tokens = runtime.credentials.get(jobDir);
|
|
3639
|
-
let readToken;
|
|
3640
|
-
try {
|
|
3641
|
-
readToken = await tokens?.token("read");
|
|
3642
|
-
} catch (err) {
|
|
3643
|
-
log.error(`${tag} no read token for the checks: ${describeCallFailure(err)}`);
|
|
3644
|
-
}
|
|
3645
|
-
const scratch = join(jobDir, "checks", "definitions");
|
|
3646
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
3647
|
-
mkdirSync(scratch, { recursive: true });
|
|
3648
|
-
let definitions;
|
|
3649
|
-
try {
|
|
3650
|
-
definitions = await readDefinitions(input, repository, scratch, readToken, tokens);
|
|
3651
|
-
} catch (err) {
|
|
3652
|
-
definitions = { ok: false, reason: describeCallFailure(err) };
|
|
3653
|
-
} finally {
|
|
3654
|
-
removeStagedContext(scratch);
|
|
3655
|
-
}
|
|
3656
|
-
if (!definitions.ok) {
|
|
3657
|
-
log.line(`${tag} checks not run: the definitions could not be read: ${definitions.reason}`);
|
|
3658
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
3659
|
-
return;
|
|
3660
|
-
}
|
|
3661
|
-
const checks = definitions.checks.filter((check) => !isJudgeCheck(check));
|
|
3662
|
-
if (checks.length === 0) {
|
|
3663
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
3664
|
-
return;
|
|
3665
|
-
}
|
|
3666
|
-
log.line(`${tag} checks started (${checks.map((check) => check.name).join(", ")})`);
|
|
3667
|
-
const startedMs = now().getTime();
|
|
3668
|
-
const recorder = makeRecorder({ config, job, jobDir, deps });
|
|
3669
|
-
const record = (check, result, sentence, keyTail = "", items) => recorder(check.name, result, sentence, { keyTail, items });
|
|
3670
|
-
const failAll = async (reason, keyTail = "") => {
|
|
3671
|
-
log.error(`${tag} checks failed: ${reason}${keyTail}`);
|
|
3672
|
-
for (const check of checks) await record(check, "fail", reason, keyTail);
|
|
3673
|
-
};
|
|
3674
|
-
try {
|
|
3675
|
-
if (job.container === void 0 && job.microvm === void 0) return await failAll(noCleanEnvironment(job));
|
|
3676
|
-
if (tokens === void 0) return await failAll("the build's repository tokens are no longer held, so the branch cannot be fetched");
|
|
3677
|
-
if (job.branch === void 0) return await failAll("the build pushed no branch: the claim names none");
|
|
3678
|
-
const claimed = repositoriesOf({ repositories: job.repositories, repository, branch: job.branch });
|
|
3679
|
-
if (claimed.length === 0) return await failAll("the claim names no repository to check out");
|
|
3680
|
-
const remoteOf = (name) => runtime.remoteFor?.(name) ?? `${runtime.github.gitUrl}/${name}.git`;
|
|
3681
|
-
const remote = remoteOf(repository);
|
|
3682
|
-
const pushed = /* @__PURE__ */ new Map();
|
|
3683
|
-
const unpushed = [];
|
|
3684
|
-
let primaryHead;
|
|
3685
|
-
for (const entry of claimed) {
|
|
3686
|
-
let entryToken = readToken;
|
|
3687
|
-
if (!entry.primary) {
|
|
3688
|
-
try {
|
|
3689
|
-
entryToken = await tokens.token("read", entry.repository);
|
|
3690
|
-
} catch (err) {
|
|
3691
|
-
return await failAll(`no read token for ${entry.repository}: ${describeCallFailure(err)}`);
|
|
3692
|
-
}
|
|
3693
|
-
}
|
|
3694
|
-
const read = await (deps.readBranch ?? readBranchHead)({ remote: remoteOf(entry.repository), dir: scratch, branch: entry.branch, token: entryToken });
|
|
3695
|
-
if (read.kind === "unreadable") return await failAll(`the branch ${entry.branch} could not be read${entry.primary ? "" : ` on ${entry.repository}`}: ${read.reason}`);
|
|
3696
|
-
if (read.kind === "absent") {
|
|
3697
|
-
unpushed.push(entry);
|
|
3698
|
-
continue;
|
|
3699
|
-
}
|
|
3700
|
-
pushed.set(entry.repository, entry.branch);
|
|
3701
|
-
if (entry.primary) primaryHead = read.sha;
|
|
3702
|
-
}
|
|
3703
|
-
if (pushed.size === 0) {
|
|
3704
|
-
return await failAll(`the build pushed no branch: ${unpushed.map((entry) => `${entry.branch} does not exist on ${entry.repository}`).join("; ")}`);
|
|
3705
|
-
}
|
|
3706
|
-
const repositoriesToCheckOut = claimed;
|
|
3707
|
-
const limitMinutes = config.checks?.timeLimitMinutes ?? DEFAULT_CHECK_TIME_LIMIT_MINUTES;
|
|
3708
|
-
const minutes = `${limitMinutes} ${limitMinutes === 1 ? "minute" : "minutes"}`;
|
|
3709
|
-
const environmentLog = join(jobDir, "checks", "environment.log");
|
|
3710
|
-
let transcripts = { ok: false, reason: "the runner holds no way to read the build's sessions" };
|
|
3711
|
-
if (deps.collectSessions !== void 0) {
|
|
3712
|
-
try {
|
|
3713
|
-
transcripts = await deps.collectSessions();
|
|
3714
|
-
} catch (err) {
|
|
3715
|
-
transcripts = { ok: false, reason: describeCallFailure(err) };
|
|
3716
|
-
}
|
|
3717
|
-
}
|
|
3718
|
-
if (job.microvm !== void 0) {
|
|
3719
|
-
if (deps.endBuildMicroVm === void 0) return await failAll("the runner has no way to stop the build's microVM, so no check environment was made");
|
|
3720
|
-
try {
|
|
3721
|
-
await deps.endBuildMicroVm();
|
|
3722
|
-
} catch (err) {
|
|
3723
|
-
return await failAll(`the build's microVM could not be stopped, so no check environment was made: ${describeCallFailure(err)}`);
|
|
3724
|
-
}
|
|
3725
|
-
}
|
|
3726
|
-
let abandoned = false;
|
|
3727
|
-
let variables;
|
|
3728
|
-
const setup = async () => {
|
|
3729
|
-
const started = job.microvm !== void 0 ? await startCheckMicroVm({
|
|
3730
|
-
config: claimCalls(config, job),
|
|
3731
|
-
runtime,
|
|
3732
|
-
job,
|
|
3733
|
-
jobDir,
|
|
3734
|
-
tokens,
|
|
3735
|
-
log,
|
|
3736
|
-
aborted: () => abandoned,
|
|
3737
|
-
save: (state) => {
|
|
3738
|
-
if (abandoned) return;
|
|
3739
|
-
job.checkMicrovm = state;
|
|
3740
|
-
writeJobFile(jobFilePath, job);
|
|
3741
|
-
}
|
|
3742
|
-
}) : await startCheckContainer({
|
|
3743
|
-
config,
|
|
3744
|
-
runtime,
|
|
3745
|
-
job,
|
|
3746
|
-
jobDir,
|
|
3747
|
-
tokens,
|
|
3748
|
-
log,
|
|
3749
|
-
save: (state) => {
|
|
3750
|
-
job.checkContainer = state;
|
|
3751
|
-
writeJobFile(jobFilePath, job);
|
|
3752
|
-
}
|
|
3753
|
-
});
|
|
3754
|
-
if (!started.ok) return `the check environment could not be made: ${started.reason}`;
|
|
3755
|
-
const made = "container" in started ? started.container : started.environment;
|
|
3756
|
-
const checkBuildId = made.state.buildId;
|
|
3757
|
-
const steps = [
|
|
3758
|
-
{
|
|
3759
|
-
argv: ["git", "config", "--global", `url.http://127.0.0.1:${RELAY_PORT}/github.com/.insteadOf`, "https://github.com/"],
|
|
3760
|
-
workdir: "/build/work",
|
|
3761
|
-
failed: (status) => `configuring git failed (exit ${status})`
|
|
3762
|
-
},
|
|
3763
|
-
...repositoriesToCheckOut.flatMap((entry) => {
|
|
3764
|
-
const branch = pushed.get(entry.repository);
|
|
3765
|
-
const path = entry.primary ? CHECKOUT_DIR : repositoryPath(entry.repository);
|
|
3766
|
-
const clone = {
|
|
3767
|
-
argv: ["git", "clone", "--filter=blob:none", "--single-branch", ...branch === void 0 ? [] : ["--branch", branch], `https://github.com/${entry.repository}.git`, path],
|
|
3768
|
-
workdir: "/build/work",
|
|
3769
|
-
failed: (status) => `the clone of ${branch ?? `the default branch of ${entry.repository}`} failed (exit ${status})`
|
|
3770
|
-
};
|
|
3771
|
-
return entry.primary && repositoriesToCheckOut.length > 1 ? [
|
|
3772
|
-
clone,
|
|
3773
|
-
{
|
|
3774
|
-
argv: ["sh", "-c", 'mkdir -p "$(dirname "$1")" && ln -s "$2" "$1"', "link", repositoryPath(entry.repository), CHECKOUT_DIR],
|
|
3775
|
-
workdir: "/build/work",
|
|
3776
|
-
failed: (status) => `linking ${entry.repository} at ${repositoryPath(entry.repository)} failed (exit ${status})`
|
|
3777
|
-
}
|
|
3778
|
-
] : [clone];
|
|
3779
|
-
})
|
|
3780
|
-
];
|
|
3781
|
-
for (const step of steps) {
|
|
3782
|
-
if (abandoned) return "the check environment setup ran out of time";
|
|
3783
|
-
const ran = await made.exec(step.argv, step.workdir, { logPath: environmentLog });
|
|
3784
|
-
if (ran.status !== 0) return `the check environment could not be prepared: ${step.failed(ran.status)}`;
|
|
3785
|
-
}
|
|
3786
|
-
if (ranRuntimeStep(job.container ?? job.microvm)) {
|
|
3787
|
-
const chosen = await provisionRuntimes({
|
|
3788
|
-
repositories: repositoriesToCheckOut,
|
|
3789
|
-
exec: (argv, workdir, opts) => made.exec(argv, workdir, opts?.quiet === true ? {} : { logPath: environmentLog }),
|
|
3790
|
-
log,
|
|
3791
|
-
tag: `${tag} checks`,
|
|
3792
|
-
refusals: () => runtime.tunnels.get(checkBuildId)?.record().refusals ?? []
|
|
3793
|
-
});
|
|
3794
|
-
if (!chosen.ok) return `the check environment could not be prepared: ${chosen.reason}`;
|
|
3795
|
-
variables = chosen.variables;
|
|
3796
|
-
}
|
|
3797
|
-
const lifecycleDir = lifecycleWorkdir(repositoriesToCheckOut, (job.container ?? job.microvm)?.governing?.repository);
|
|
3798
|
-
for (const step of lifecycleSteps((job.container ?? job.microvm)?.lifecycle ?? {})) {
|
|
3799
|
-
log.line(`${tag} checks lifecycle ${step.name} (${step.source})`);
|
|
3800
|
-
const ran = await made.exec(step.argv, lifecycleDir, { logPath: environmentLog, variables });
|
|
3801
|
-
if (ran.status !== 0) return `the check environment could not be prepared: ${lifecycleFailureReason(step, ran.status)}`;
|
|
3802
|
-
}
|
|
3803
|
-
return made;
|
|
3804
|
-
};
|
|
3805
|
-
const setupTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
|
|
3806
|
-
let settled;
|
|
3807
|
-
try {
|
|
3808
|
-
const outcome = await Promise.race([
|
|
3809
|
-
setup().catch((err) => `the check environment could not be made: ${describeCallFailure(err)}`),
|
|
3810
|
-
setupTimer.done
|
|
3811
|
-
]);
|
|
3812
|
-
if (outcome === "timeout") {
|
|
3813
|
-
abandoned = true;
|
|
3814
|
-
return await failAll(`the check environment setup ran out of time after ${minutes}`, LIMIT_KEY_TAIL2);
|
|
3815
|
-
}
|
|
3816
|
-
settled = outcome;
|
|
3817
|
-
} finally {
|
|
3818
|
-
setupTimer.cancel();
|
|
3819
|
-
}
|
|
3820
|
-
if (typeof settled === "string") return await failAll(settled);
|
|
3821
|
-
const ready = settled;
|
|
3822
|
-
const setupMinutes = config.build?.setupMinutes ?? DEFAULT_SETUP_MINUTES;
|
|
3823
|
-
const prepared = await runSetupScripts({
|
|
3824
|
-
repositories: [...repositoriesToCheckOut.filter((entry) => entry.primary), ...repositoriesToCheckOut.filter((entry) => !entry.primary)],
|
|
3825
|
-
exec: (argv, workdir, opts) => ready.exec(argv, workdir, { logPath: opts?.quiet === true ? void 0 : environmentLog, timeoutMs: opts?.timeoutMs, variables }),
|
|
3826
|
-
log,
|
|
3827
|
-
tag: `${tag} checks`,
|
|
3828
|
-
timeoutMs: setupMinutes * 6e4,
|
|
3829
|
-
limitMinutes: setupMinutes,
|
|
3830
|
-
aborted: () => false
|
|
3831
|
-
});
|
|
3832
|
-
if (!prepared.ok) return await failAll(`the check environment could not be prepared: ${prepared.reason}`);
|
|
3833
|
-
try {
|
|
3834
|
-
await writeEvalInput({
|
|
3835
|
-
job,
|
|
3836
|
-
dir: ready.evalInput.dir,
|
|
3837
|
-
scratch,
|
|
3838
|
-
remote,
|
|
3839
|
-
readToken,
|
|
3840
|
-
branchHead: primaryHead ?? (job.container ?? job.microvm)?.startCommit ?? "",
|
|
3841
|
-
transcripts,
|
|
3842
|
-
scrub: transcriptScrub(config, [config.apiKey, job.itemKey, readToken], jobDir),
|
|
3843
|
-
readWorkItem: () => readWorkItem(itemCalls(config, job), job.item, deps.fetchImpl),
|
|
3844
|
-
fetchDiff: deps.fetchDiff ?? ((input2) => diffBetween(input2)),
|
|
3845
|
-
log
|
|
3846
|
-
});
|
|
3847
|
-
} catch (err) {
|
|
3848
|
-
return await failAll(`the checks' input could not be written: ${describeCallFailure(err)}`);
|
|
3849
|
-
}
|
|
3850
|
-
try {
|
|
3851
|
-
await ready.evalInput.deliver();
|
|
3852
|
-
} catch (err) {
|
|
3853
|
-
return await failAll(err instanceof DeliveryFailure ? err.message : `the checks' input could not be written: ${describeCallFailure(err)}`);
|
|
3854
|
-
}
|
|
3855
|
-
let usable = true;
|
|
3856
|
-
for (const check of checks) {
|
|
3857
|
-
if (!usable) {
|
|
3858
|
-
await record(check, "fail", "the check environment could not be restarted after an earlier check ran out of time");
|
|
3859
|
-
continue;
|
|
3860
|
-
}
|
|
3861
|
-
const checkStartedMs = now().getTime();
|
|
3862
|
-
const logPath = join(jobDir, "checks", `${check.name}.log`);
|
|
3863
|
-
await ready.exec(["rm", "-f", CHECK_ITEMS_FILE], CHECKOUT_DIR, { logPath });
|
|
3864
|
-
const running = ready.exec(["/bin/sh", "-c", check.command], CHECKOUT_DIR, { logPath, variables });
|
|
3865
|
-
running.catch(() => void 0);
|
|
3866
|
-
const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
|
|
3867
|
-
let outcome;
|
|
3868
|
-
try {
|
|
3869
|
-
outcome = await Promise.race([running, timer.done]);
|
|
3870
|
-
} catch (err) {
|
|
3871
|
-
timer.cancel();
|
|
3872
|
-
await record(check, "fail", `the command could not be run: ${describeCallFailure(err)}`);
|
|
3873
|
-
continue;
|
|
3874
|
-
}
|
|
3875
|
-
timer.cancel();
|
|
3876
|
-
if (outcome === "timeout") {
|
|
3877
|
-
await record(check, "fail", `ran out of time after ${minutes}`, LIMIT_KEY_TAIL2);
|
|
3878
|
-
const restarting = ready.restart().catch(() => false);
|
|
3879
|
-
const restartTimer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
|
|
3880
|
-
usable = await Promise.race([restarting, restartTimer.done]) === true;
|
|
3881
|
-
restartTimer.cancel();
|
|
3882
|
-
continue;
|
|
3883
|
-
}
|
|
3884
|
-
const read = await ready.exec(["head", "-c", String(MAX_CHECK_ITEMS_BYTES + 1), CHECK_ITEMS_FILE], CHECKOUT_DIR);
|
|
3885
|
-
const items = commandItems(read, (text2) => scrubReason(text2, config, void 0, jobDir) ?? text2);
|
|
3886
|
-
if (!items.ok) {
|
|
3887
|
-
await record(check, "fail", items.reason);
|
|
3888
|
-
continue;
|
|
3889
|
-
}
|
|
3890
|
-
const failing = items.items === void 0 ? void 0 : itemsSentence(items.items, { one: "check item", many: "check items" });
|
|
3891
|
-
if (outcome.status === 0 && failing === void 0) {
|
|
3892
|
-
log.line(`${tag} check ${check.name} ran (${formatDuration(now().getTime() - checkStartedMs)})`);
|
|
3893
|
-
await record(check, "pass", void 0, "", items.items);
|
|
3894
|
-
continue;
|
|
3895
|
-
}
|
|
3896
|
-
if (outcome.status === 0) {
|
|
3897
|
-
await record(check, "fail", failing, "", items.items);
|
|
3898
|
-
continue;
|
|
3899
|
-
}
|
|
3900
|
-
const last = lastLine(outcome.stderr) ?? lastLine(outcome.stdout);
|
|
3901
|
-
const scrubbed = last === void 0 ? void 0 : scrubReason(last, config, void 0, jobDir);
|
|
3902
|
-
await record(check, "fail", `exited ${outcome.status}${scrubbed === void 0 ? " with no output" : `: ${cutReasonLine(scrubbed, 140)}`}`, "", items.items);
|
|
3903
|
-
}
|
|
3904
|
-
} catch (err) {
|
|
3905
|
-
log.error(`${tag} checks stopped: ${describeCallFailure(err)}`);
|
|
3906
|
-
} finally {
|
|
3907
|
-
if (job.checkContainer !== void 0) {
|
|
3908
|
-
await cleanupContainer(runtime, job.checkContainer).catch((err) => log.error(`${tag} removing the check environment failed: ${describeCallFailure(err)}`));
|
|
3909
|
-
delete job.checkContainer;
|
|
3910
|
-
writeJobFile(jobFilePath, job);
|
|
3911
|
-
}
|
|
3912
|
-
if (job.checkMicrovm !== void 0) {
|
|
3913
|
-
await cleanupMicroVm(runtime, job.checkMicrovm).catch((err) => log.error(`${tag} removing the check environment failed: ${describeCallFailure(err)}`));
|
|
3914
|
-
delete job.checkMicrovm;
|
|
3915
|
-
writeJobFile(jobFilePath, job);
|
|
3916
|
-
}
|
|
3917
|
-
rmSync(scratch, { recursive: true, force: true });
|
|
3918
|
-
rmSync(join(jobDir, "checks", EVAL_INPUT_NAME), { recursive: true, force: true });
|
|
3919
|
-
if (existsSync(scratch)) log.error(`${tag} the checks' scratch directory could not be removed`);
|
|
3920
|
-
log.line(`${tag} checks done (${formatDuration(now().getTime() - startedMs)})`);
|
|
3921
|
-
}
|
|
3922
|
-
}
|
|
3923
|
-
|
|
3924
|
-
export { ALREADY_RELEASED, AttemptLogBook, EXIT_CODE_FILE, GUEST_USAGE_INTERVAL_MS, MAX_CHECK_ITEMS_BYTES, MICROVM_COMMAND_DIR, RENEW_INTERVAL_SECONDS, appendCommandLog, checksIn, commandItems, commandLogOffset, deliverCommitHooks, describeHookEnd, finishJob, foldSample, hookLimitsFor, inTeardown, isContainerTracked, isGroupAlive, isIdle, isJudgeCheck, isMicroVmTracked, isReadable, isTimedOut, isTracked, isUnfinished, jobEnv, judgeProbe, listAdoptableJobs, listTrackedJobs, listUnreadableJobs, makeRecorder, microVmCommandLine, microVmOomKills, needsRenew, outcomeFromExitCode, probeCommand, probeMicroVmCommand, provisionMicroVm, provisionProcess, readCpuUsec, readDefinitions, readProvisionOut, recoverLostJobs, releaseLease, renewLease, restartSender, restoreSessionIntoMicroVm, retryFailedReleases, runCommand, runHook, runHostChecks, sampleGuest, scrubReason, serveLease, setStage, startMicroVmCommand, startSender, stopOrphanedMicroVms, stripTerminalControls, testProofIn, timeoutAfter, unfinishedJobs, vmAlive, workdirProblem, writeGuestGhConfig };
|