@outerlayer/cli 0.34.0-next.20261028183303 → 0.34.0-next.20261029100612
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/agent-setup-UPIHHKI2.js +32 -0
- package/dist/{artifact-link-cmd-CSVFEM5O.js → artifact-link-cmd-6LFCTDBD.js} +5 -5
- package/dist/{bridge-4GRZFXJU.js → bridge-XM4QLKIX.js} +2 -2
- package/dist/{build-SEJJWGTO.js → build-UBZNVGY2.js} +10 -10
- package/dist/{build-disk-EYRR3PBT.js → build-disk-Y3MMWFYE.js} +7 -7
- package/dist/build-info.json +1 -1
- package/dist/{builder-T7B46P7T.js → builder-SX4W5VF3.js} +6 -6
- package/dist/caller-IIA2TPAA.js +18 -0
- package/dist/{check-VKSOYRNI.js → check-PVB3GPFQ.js} +88 -70
- package/dist/{chunk-ANPNT6PM.js → chunk-2AM6Z5BP.js} +1 -1
- package/dist/{chunk-3UIXIGYT.js → chunk-2B6EKUBD.js} +4 -4
- package/dist/{chunk-QGDGGLSD.js → chunk-2F43WNKI.js} +3 -3
- package/dist/{chunk-UZEIBWTP.js → chunk-2OE4BZRF.js} +8 -8
- package/dist/{chunk-VNVZDWO3.js → chunk-352DHUTP.js} +4 -1
- package/dist/{chunk-W7E6IMEW.js → chunk-3EEJGQEJ.js} +22 -21
- package/dist/{chunk-5W2YSWIW.js → chunk-4PH4DVV4.js} +1 -1
- package/dist/{chunk-WH3FPBPO.js → chunk-4PWOZU7R.js} +82 -33
- package/dist/{chunk-ZJSNLVCE.js → chunk-545P7JY7.js} +1 -1
- package/dist/{chunk-RRXS4CWE.js → chunk-5C7QZBLI.js} +9 -7
- package/dist/{chunk-2EEW6FJN.js → chunk-5FWBAM6O.js} +1 -1
- package/dist/{chunk-T2Q7ISPE.js → chunk-5WYT3PPE.js} +1 -1
- package/dist/{chunk-ARL6X6Y4.js → chunk-62TMOXQN.js} +2 -2
- package/dist/{chunk-EA2PQRCU.js → chunk-7CIU64OH.js} +6 -6
- package/dist/{chunk-AF4E4I4T.js → chunk-7G35LX5D.js} +11 -11
- package/dist/{chunk-KH4774QT.js → chunk-7R655ODT.js} +1 -1
- package/dist/{chunk-G2JWZDFA.js → chunk-A2LHHKTT.js} +3 -3
- package/dist/{chunk-ESVW4MJY.js → chunk-AAGU5FJF.js} +1 -1
- package/dist/{chunk-QGEGLZSY.js → chunk-ADI2CCDX.js} +47 -7
- package/dist/{chunk-I2MFIT6X.js → chunk-AGMR4IQE.js} +29 -28
- package/dist/{chunk-SAAVZ2BX.js → chunk-AUVJMR3P.js} +1 -1
- package/dist/{chunk-HFOZTCTN.js → chunk-AVHPSB65.js} +7 -7
- package/dist/{chunk-KQDK4Q6A.js → chunk-CDVG2LXT.js} +10 -10
- package/dist/{chunk-TUSYOL2O.js → chunk-CJPTUU2E.js} +1 -1
- package/dist/{chunk-OYQVLQP3.js → chunk-EQ5Y2OMK.js} +16 -14
- package/dist/{chunk-MH6QLXUO.js → chunk-FJ7KTNMD.js} +1 -1
- package/dist/{chunk-KPEMNHVF.js → chunk-FNKCMOQY.js} +3 -3
- package/dist/{chunk-2O5KA67H.js → chunk-FUJUFLDC.js} +2 -2
- package/dist/{chunk-CRB3POP4.js → chunk-FZKNTIOJ.js} +1 -1
- package/dist/{chunk-YLPJOUVF.js → chunk-GANAOCHR.js} +6 -1
- package/dist/{chunk-3KGXXTMN.js → chunk-GBMTKV5B.js} +5 -5
- package/dist/{chunk-POND2SMT.js → chunk-H2GESWLW.js} +3 -3
- package/dist/{chunk-DCBFUUDL.js → chunk-HERRKKES.js} +7 -7
- package/dist/{chunk-V2S2WRGD.js → chunk-HS5QEIJT.js} +11 -11
- package/dist/{chunk-PJDQPF5N.js → chunk-HWVMRETT.js} +1 -1
- package/dist/{chunk-H7MGR2GN.js → chunk-IUPYSGCP.js} +6 -6
- package/dist/{chunk-7JSWRF5O.js → chunk-JFALOM5G.js} +10 -10
- package/dist/{chunk-XDAMLX2L.js → chunk-JQOET7TF.js} +9 -9
- package/dist/{chunk-RQ3GCCHH.js → chunk-JXXMOAJG.js} +2 -2
- package/dist/{chunk-URIZRCRY.js → chunk-KYDEBGJG.js} +1 -1
- package/dist/{chunk-GG5OE6MY.js → chunk-KZUHEN26.js} +2 -2
- package/dist/{chunk-2EHDLYSL.js → chunk-LFXLFDEA.js} +30 -22
- package/dist/chunk-LW2A24IA.js +31 -0
- package/dist/{chunk-BBJ6BM5U.js → chunk-MKSZHT6S.js} +1 -1
- package/dist/{chunk-VX7WZK7J.js → chunk-MOG4XCYS.js} +189 -121
- package/dist/{chunk-P2PNBDUT.js → chunk-NM3L4JBB.js} +4 -4
- package/dist/{chunk-TDCHUEKY.js → chunk-O7CGRDRQ.js} +5 -5
- package/dist/{chunk-KT4CN6OU.js → chunk-OHQMGLJP.js} +3 -3
- package/dist/{chunk-5BHIBJDA.js → chunk-OLM2TIOW.js} +1 -1
- package/dist/{chunk-VEVNGDTP.js → chunk-PDK35AR4.js} +2 -2
- package/dist/{chunk-RPST2GNG.js → chunk-PQ5YDDDE.js} +3 -3
- package/dist/{chunk-OKJXZXYH.js → chunk-PUSCE544.js} +6 -6
- package/dist/{chunk-ZEFNJCN3.js → chunk-QM3HMXMU.js} +2 -2
- package/dist/chunk-RW3YMQVQ.js +39 -0
- package/dist/{chunk-ZCZY3Q55.js → chunk-SHPCVBT3.js} +1 -1
- package/dist/{chunk-UXC3RQLF.js → chunk-SQGD4LYW.js} +1 -1
- package/dist/{chunk-LJEZGHCH.js → chunk-SYCZTZXD.js} +647 -211
- package/dist/{chunk-4YD7N6V3.js → chunk-TKTVKEES.js} +4 -4
- package/dist/{chunk-GH2HEEOH.js → chunk-TV2OQUWE.js} +1 -1
- package/dist/{chunk-MRLJK4GZ.js → chunk-UGPJIKN4.js} +6 -6
- package/dist/{chunk-HH3BQG65.js → chunk-UPRZREBO.js} +1 -1
- package/dist/{chunk-ROQPDCRU.js → chunk-UZGQT6OF.js} +1 -1
- package/dist/{chunk-NCOL5RYI.js → chunk-VNHBDHX7.js} +1 -1
- package/dist/{chunk-64O2NUVA.js → chunk-VO46IHB4.js} +8 -7
- package/dist/{chunk-MCPCOKHN.js → chunk-W74HQLLZ.js} +9 -9
- package/dist/{chunk-ZCA34T7R.js → chunk-WDDVPDLQ.js} +8 -8
- package/dist/{chunk-YF7W7B7Y.js → chunk-WDIZT56W.js} +38 -11
- package/dist/{chunk-CUAB7FAZ.js → chunk-WG3VIMD2.js} +8 -8
- package/dist/{chunk-IAVT7R6O.js → chunk-WJTIU2A5.js} +6 -6
- package/dist/{chunk-NXUZFHMX.js → chunk-WK4OTIAN.js} +3 -3
- package/dist/{chunk-T3XPLCJG.js → chunk-XBECMHJW.js} +1 -1
- package/dist/{chunk-54LIUIZL.js → chunk-XGSJNXH2.js} +4 -4
- package/dist/{chunk-5TXQFZHY.js → chunk-XU5BNWX7.js} +1 -1
- package/dist/{chunk-I7RZHMXW.js → chunk-YBAVN6WS.js} +3 -3
- package/dist/{chunk-ZTOJIRJH.js → chunk-YBP22LDO.js} +8 -8
- package/dist/{chunk-S6UM3N4R.js → chunk-YQHUCOUL.js} +37 -12
- package/dist/{chunk-IXRUVR25.js → chunk-YTTESG7K.js} +46 -7
- package/dist/{chunk-CKDSPW35.js → chunk-ZI2E4YMY.js} +6 -6
- package/dist/{chunk-MEIDQLXQ.js → chunk-ZI7JENTB.js} +1 -1
- package/dist/{claude-launcher-7IC6QWR5.js → claude-launcher-MMCKOBND.js} +6 -6
- package/dist/{cli-HBS2PXSS.js → cli-W4B35KDN.js} +165 -155
- package/dist/{cli-update-notice-4DLZ3655.js → cli-update-notice-T3CXRWNB.js} +3 -3
- package/dist/{command-XA6DOFL4.js → command-SKJHW4C4.js} +43 -42
- package/dist/{config-RSOCVUKC.js → config-ZDENMTEW.js} +9 -9
- package/dist/connect-cmd-APZN6F6N.js +27 -0
- package/dist/{container-start-PYW7W2N2.js → container-start-PAUJUXJX.js} +57 -56
- package/dist/{context-cleanup-CEZSKNVL.js → context-cleanup-PASDUBGW.js} +5 -5
- package/dist/{context-show-cmd-XWLTP6R4.js → context-show-cmd-OWREGUJL.js} +19 -19
- package/dist/{control-plane-launch-3MBWPYWZ.js → control-plane-launch-FV5Q6GVJ.js} +16 -16
- package/dist/{demo-record-cmd-SIKNTSGI.js → demo-record-cmd-KXLLIMOP.js} +5 -5
- package/dist/{demo-voice-cmd-SAB3QU4F.js → demo-voice-cmd-TY2OX2CZ.js} +30 -30
- package/dist/{dist-VPXROEZO.js → dist-Z6WP743R.js} +2 -2
- package/dist/{docker-YMRPMDOW.js → docker-SKD6VTBD.js} +6 -6
- package/dist/{doctor-NVHAE4LS.js → doctor-DX6N3L5T.js} +53 -42
- package/dist/emit-artifact-cmd-NFAASCXW.js +43 -0
- package/dist/emit-cmd-UOPHEUW5.js +16 -0
- package/dist/emit-criteria-cmd-TX4LKO5X.js +33 -0
- package/dist/emit-demo-cmd-FRJFQKED.js +47 -0
- package/dist/emit-finding-cmd-LO2AKL4P.js +34 -0
- package/dist/{emit-result-cmd-VCKNZ4RZ.js → emit-result-cmd-7PL4HNXL.js} +28 -28
- package/dist/emit-step-cmd-RZT6VK5U.js +33 -0
- package/dist/{exec-client-U4544U2S.js → exec-client-UHL7WPDO.js} +3 -3
- package/dist/find-unit-IBL736HQ.js +19 -0
- package/dist/{findings-cmd-PLHBAQW4.js → findings-cmd-MEU7GDRC.js} +23 -23
- package/dist/fleet-readiness-4RNICYSP.js +28 -0
- package/dist/governing-source-6DH2NLPE.js +38 -0
- package/dist/{guest-init-MI4SEME4.js → guest-init-2TTQKFFZ.js} +4 -4
- package/dist/{hook-fast-GOH4T6SN.js → hook-fast-5PNO5HRO.js} +6 -6
- package/dist/{hook-wrap-fast-7VYYAAH4.js → hook-wrap-fast-545QJSBJ.js} +3 -3
- package/dist/host-checks-FWPDGTWF.js +78 -0
- package/dist/{host-key-OMLDWLNG.js → host-key-Q5DEAP3Q.js} +3 -3
- package/dist/{image-tag-YJJONB64.js → image-tag-JWATDMIZ.js} +3 -3
- package/dist/index.js +6 -6
- package/dist/init-DWOODP2Q.js +57 -0
- package/dist/{init-W3ATJYLX.js → init-GF43ZLZI.js} +39 -38
- package/dist/{init-cmd-XJOEU2PB.js → init-cmd-LW4TECQH.js} +33 -33
- package/dist/{init-update-KX3FFQ5H.js → init-update-5PTU5IE6.js} +30 -30
- package/dist/{install-cmd-QTIIMO6B.js → install-cmd-ZEFBJH24.js} +5 -5
- package/dist/{layout-7UOBWNYO.js → layout-ZE6F2Z7W.js} +4 -4
- package/dist/login-browser-YB7AMKHR.js +13 -0
- package/dist/{logout-cmd-5WE2N5B5.js → logout-cmd-Z2PYV5LF.js} +5 -5
- package/dist/{logs-YVXVAYTD.js → logs-3YRC7HRA.js} +23 -3
- package/dist/loop-UBAR3KBZ.js +89 -0
- package/dist/machine-H4Y5X424.js +57 -0
- package/dist/{machine-skills-KYN7XJEJ.js → machine-skills-DMHU7OLO.js} +5 -5
- package/dist/{mcp-install-cmd-LD7KYAOP.js → mcp-install-cmd-23ZY4IBK.js} +5 -5
- package/dist/{mcp-serve-cmd-ECTAPDB3.js → mcp-serve-cmd-PKXIHUOX.js} +9 -9
- package/dist/{paths-M66LPPUF.js → paths-7PC3K7C3.js} +2 -2
- package/dist/{pending-7LC6XDRO.js → pending-2VIIFAP4.js} +3 -3
- package/dist/{pidfile-DSQOOAZV.js → pidfile-KQ2EJABZ.js} +3 -3
- package/dist/{record-3KKIJ3XL.js → record-IHEF6C33.js} +3 -3
- package/dist/{runner-env-EX7YUIUH.js → runner-env-KCZBSWOB.js} +11 -11
- package/dist/{runner-identity-LJWBXNM5.js → runner-identity-D5HQ22D3.js} +6 -6
- package/dist/{runner-key-CAUO4RKG.js → runner-key-N5YEMI3S.js} +38 -37
- package/dist/session-sender-GJEVFQVN.js +43 -0
- package/dist/{settings-3DBYKYQD.js → settings-XB2HYBKO.js} +2 -2
- package/dist/{spec-cmd-SAWQYHKM.js → spec-cmd-BRTO5J2C.js} +23 -23
- package/dist/{starter-pack-KB3YH4KX.js → starter-pack-WUNNS6G7.js} +9 -9
- package/dist/state-FJPSFOID.js +9 -0
- package/dist/{status-VMFS7N6A.js → status-UPEZZYCT.js} +78 -56
- package/dist/{statusline-fast-YVHG6ZLX.js → statusline-fast-E7RWVREA.js} +4 -4
- package/dist/sync-cmd-EO4TP2K7.js +42 -0
- package/dist/update-steps-46FCWHJZ.js +79 -0
- package/dist/{watch-6NRBX6VE.js → watch-EG6MYI3O.js} +28 -28
- package/dist/{work-claim-cmd-SLVX2N4L.js → work-claim-cmd-MPH5HFLT.js} +26 -26
- package/dist/work-cmd-PKEAVXJC.js +31 -0
- package/dist/work-comment-cmd-7IEOLOOK.js +32 -0
- package/dist/{work-demo-cmd-SZJIKB5J.js → work-demo-cmd-GKMBBMMM.js} +20 -20
- package/dist/{work-launch-LEQ2MPM3.js → work-launch-TTGSHLY4.js} +3 -3
- package/dist/{work-logs-cmd-MRFSBMJM.js → work-logs-cmd-44ZOKO5E.js} +20 -20
- package/dist/{work-open-pr-cmd-N7GK7FUQ.js → work-open-pr-cmd-YYQLQ7DN.js} +22 -22
- package/dist/work-pr-cmd-ILCIO22W.js +32 -0
- package/dist/work-proofs-cmd-S7HZNVFC.js +35 -0
- package/dist/{work-proofs-record-JD4DNZUW.js → work-proofs-record-RJMR37XG.js} +34 -34
- package/dist/{work-stop-cmd-M6RIGF7S.js → work-stop-cmd-5VPW7HOZ.js} +20 -20
- package/package.json +1 -1
- package/dist/agent-setup-IVXRBFVV.js +0 -32
- package/dist/connect-cmd-JFGDHFLA.js +0 -27
- package/dist/emit-artifact-cmd-TZTVQCFT.js +0 -43
- package/dist/emit-cmd-4G3443OJ.js +0 -16
- package/dist/emit-criteria-cmd-WH7JB4RQ.js +0 -33
- package/dist/emit-demo-cmd-DNUVG5N7.js +0 -47
- package/dist/emit-finding-cmd-X5CBH4ZA.js +0 -34
- package/dist/emit-step-cmd-IUXZZYKO.js +0 -33
- package/dist/find-unit-QYD2FMCP.js +0 -19
- package/dist/fleet-readiness-F65CKZY5.js +0 -28
- package/dist/governing-source-JFCTQXYK.js +0 -38
- package/dist/host-checks-HC4XXGAD.js +0 -77
- package/dist/init-JVKJTJNM.js +0 -56
- package/dist/login-browser-Q25ZNSQ3.js +0 -13
- package/dist/loop-GBTANKUW.js +0 -87
- package/dist/machine-OQOHEMJB.js +0 -56
- package/dist/session-sender-U2CTD7EX.js +0 -43
- package/dist/sync-cmd-EBJQ5LMH.js +0 -42
- package/dist/update-steps-HMSVBDKH.js +0 -78
- package/dist/work-cmd-VSHZCXHQ.js +0 -31
- package/dist/work-comment-cmd-3DCL7VHN.js +0 -32
- package/dist/work-pr-cmd-HD5OIHUU.js +0 -32
- package/dist/work-proofs-cmd-GFGENBGN.js +0 -35
|
@@ -1,38 +1,334 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
|
+
import { readHousekeepingRecord, writeHousekeepingRecord } from './chunk-ZI7JENTB.js';
|
|
3
4
|
import { decideInstalledSwitch } from './chunk-3JVYMD4G.js';
|
|
4
|
-
import { reportFailedStart, fitImageBuildMemory } from './chunk-64O2NUVA.js';
|
|
5
5
|
import { fetchVersionMeta, npmVersion, npmSupportsProvenance, fetchTarballDigest, fetchProvenance, installVersion, auditSignatures } from './chunk-E7R2GIAW.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
6
|
+
import { reportFailedStart, fitImageBuildMemory } from './chunk-VO46IHB4.js';
|
|
7
|
+
import { renderUpdateSteps } from './chunk-WK4OTIAN.js';
|
|
8
|
+
import { releaseLease, isUnfinished, setStage, jobEnv, runHook, describeHookEnd, provisionMicroVm, deliverCommitHooks, provisionProcess, readProvisionOut, workdirProblem, MICROVM_COMMAND_DIR, commandLogOffset, startMicroVmCommand, microVmCommandLine, runCommand, startSender, finishJob, sampleGuest, foldSample, isContainerTracked, isMicroVmTracked, probeMicroVmCommand, probeCommand, appendCommandLog, judgeProbe, EXIT_CODE_FILE, isTracked, isGroupAlive, outcomeFromExitCode, isTimedOut, readCpuUsec, isIdle, stopOrphanedMicroVms, unfinishedJobs, listUnreadableJobs, listAdoptableJobs, retryFailedReleases, recoverLostJobs, GUEST_USAGE_INTERVAL_MS, hookLimitsFor, AttemptLogBook, microVmOomKills, needsRenew, renewLease, RENEW_INTERVAL_SECONDS, listTrackedJobs, restartSender, serveLease, vmAlive, writeGuestGhConfig, ALREADY_RELEASED, inTeardown } from './chunk-MOG4XCYS.js';
|
|
9
|
+
import { RUNNER_PROTOCOL } from './chunk-FJ7KTNMD.js';
|
|
10
|
+
import { writeFleetState } from './chunk-LW2A24IA.js';
|
|
11
|
+
import { writePendingUpdate } from './chunk-SHPCVBT3.js';
|
|
11
12
|
import { JOB_STATE_FORMAT, readsJobState, jobStateFormatOf } from './chunk-ZVB57WI6.js';
|
|
12
|
-
import { realBlockedProbes, checkBlockedWithSlots, largeBuildMemoryBytes, LARGE_BUILD_DISK_BYTES, checkManagedRoom,
|
|
13
|
-
import { writeMarker, readMarker, currentVersion, removeMarker, isInLayout, startedThroughCurrent, entryIn, versionDir, switchCurrent, cleanVersions, stagingDir } from './chunk-
|
|
13
|
+
import { fleetConfigProblem, realBlockedProbes, sliceMemoryOf, checkBlockedWithSlots, largeBuildMemoryBytes, LARGE_BUILD_DISK_BYTES, checkManagedRoom, createHousekeepingDocker, housekeepingDockerIfReachable, KIND_LABEL, RECIPE_KEY_LABEL, askFleetMachine } from './chunk-ADI2CCDX.js';
|
|
14
|
+
import { writeMarker, readMarker, currentVersion, removeMarker, isInLayout, startedThroughCurrent, entryIn, versionDir, switchCurrent, cleanVersions, stagingDir } from './chunk-7R655ODT.js';
|
|
14
15
|
import { RepoFilter } from './chunk-TBT347UY.js';
|
|
15
|
-
import { leaseTimeLeftMs, createContainerRuntime, withTunnelLog, LEASE_SECONDS, attemptNote, fetchManagedClaudeKey, holdManagedClaudeKey, fetchManagedBuildSettings, holdManagedBuildSettings, needsBuildSkill, provisionContainer, leaseExpiresAtMs, deliverContainerHooks, provisionEnvironment, resolveHostBuild, listenOnBroker, hostEnvironment, deliverExecMaven, runWithSpawn, hostCommandEnvironment, lifecycleSteps, execArgv, lifecycleFailureReason, decideHostBuildSkill, prepareBranch, currentSkillCommand, formerSkillProblem, containerCommandExec, execCommandLine, startHostServices, HELPER_ENV, setUpRunnerCgroup, sweepImageBuilders, stoppedHelperReason, measureDisk, parseCpuList, ensureTunnel, keepGhConfigCurrent, filesAreCopied, savedStatesDir, readRecords, removeOldestUntil, ensureNetworkReady, managedBuildSettingsFor, ADOPTED_FETCH_RETRY_WINDOW_MS, gitRemoteFor, startServicesFor, createLeaseChannel, selectStaleIncomplete, selectExpired, selectToFit, removeSavedState } from './chunk-
|
|
16
|
-
import {
|
|
17
|
-
import { imageTag, writeUpdateStatus, pullInstruction, tagToPull } from './chunk-UXC3RQLF.js';
|
|
18
|
-
import { isVersion, compareVersions, satisfiesRange } from './chunk-TQMI335S.js';
|
|
19
|
-
import { isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, keepsHostBuilder, DEFAULT_SAVED_STATE_MAX_BYTES, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-YF7W7B7Y.js';
|
|
20
|
-
import { formatBytes, allocatedBytes, freeBytes } from './chunk-RRXS4CWE.js';
|
|
21
|
-
import { writeJobFile, ensureRunnerIdentity, identityDockerFor, readJobFile, exceedsDisk, builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, spawnRun, readJobFileChecked } from './chunk-P2PNBDUT.js';
|
|
22
|
-
import { claimPidFile, releasePidFile, writePidFile, readPidFile } from './chunk-SAAVZ2BX.js';
|
|
16
|
+
import { leaseTimeLeftMs, createContainerRuntime, withTunnelLog, LEASE_SECONDS, itemTag, itemScope, attemptNote, fetchManagedClaudeKey, holdManagedClaudeKey, fetchManagedBuildSettings, holdManagedBuildSettings, needsBuildSkill, provisionContainer, leaseExpiresAtMs, deliverContainerHooks, provisionEnvironment, resolveHostBuild, listenOnBroker, hostEnvironment, deliverExecMaven, runWithSpawn, hostCommandEnvironment, lifecycleSteps, execArgv, lifecycleFailureReason, decideHostBuildSkill, prepareBranch, currentSkillCommand, formerSkillProblem, containerCommandExec, execCommandLine, startHostServices, HELPER_ENV, setUpRunnerCgroup, sweepImageBuilders, usableCpus, stoppedHelperReason, measureDisk, parseCpuList, ensureTunnel, keepGhConfigCurrent, filesAreCopied, savedStatesDir, readRecords, removeOldestUntil, ensureNetworkReady, managedBuildSettingsFor, ADOPTED_FETCH_RETRY_WINDOW_MS, gitRemoteFor, startServicesFor, createLeaseChannel, selectStaleIncomplete, selectExpired, selectToFit, removeSavedState } from './chunk-4PWOZU7R.js';
|
|
17
|
+
import { claimPidFile, releasePidFile, writePidFile, readPidFile } from './chunk-AUVJMR3P.js';
|
|
23
18
|
import { startTimeOf, isSameProcess } from './chunk-JBJTC3UU.js';
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
19
|
+
import { describeCallFailure, retrySend, shortClaim, relativeToRunner, formatDuration, resolveGoverningSource, BuildTokens, classifyRefusal, refusalCode, createRunnerLog, formatClock } from './chunk-HERRKKES.js';
|
|
20
|
+
import { imageTag, writeUpdateStatus, pullInstruction, tagToPull } from './chunk-SQGD4LYW.js';
|
|
21
|
+
import { isVersion, compareVersions, satisfiesRange } from './chunk-TQMI335S.js';
|
|
22
|
+
import { RUNNER_RESERVE_BYTES } from './chunk-TKTVKEES.js';
|
|
23
|
+
import { claimCalls, claimItem, spoolHomeFor, leftByRunnerKey, runnerCalls } from './chunk-RW3YMQVQ.js';
|
|
24
|
+
import { keyRunnerFactory, isBuiltinContainer, isBuiltinVm, isBuiltinProcess, loadRunnerConfig, checkHooks, builtinKind, keepsHostBuilder, DEFAULT_SAVED_STATE_MAX_BYTES, DEFAULT_UPDATE_MAX_WAIT_HOURS } from './chunk-WDIZT56W.js';
|
|
25
|
+
import { allocatedBytes, freeBytes, formatBytes } from './chunk-5C7QZBLI.js';
|
|
26
|
+
import { writeJobFile, ensureRunnerIdentity, identityDockerFor, readJobFile, exceedsDisk, builderName, runnerLabelValue, ensureFailureCount, builderSpec, ensureBuilder, spawnRun, readJobFileChecked } from './chunk-NM3L4JBB.js';
|
|
27
|
+
import { isSignatureRefusal, WorkCommandError, getJson, postJson } from './chunk-JQOET7TF.js';
|
|
28
|
+
import { loadHostKey, ensureHostKey, signerFor } from './chunk-XU5BNWX7.js';
|
|
26
29
|
import { resolveVersion } from './chunk-RA3O54FC.js';
|
|
27
|
-
import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, RUNNER_FILE_MODE, jobsDir, pidFilePath,
|
|
28
|
-
import { parseClaimIdentity, parseClaimedRepositories, parseUnreachable, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, takeClaudeCredential, heldClaudeCredential, currentClaudeCredential, fdName } from './chunk-
|
|
30
|
+
import { RUNNER_DIR_MODE, runnerDir, buildDir, attemptStamp, jobDir, RUNNER_FILE_MODE, jobsDir, pidFilePath, storeScopes, imagesDir, spoolDir, orgStoreIn, builderCacheDiskPath, rootfsDir } from './chunk-YQHUCOUL.js';
|
|
31
|
+
import { parseClaimIdentity, parseClaimedRepositories, parseUnreachable, repositoriesOf, primaryOf, sharedUserEnvironment, JOB_MOUNT, takeClaudeCredential, heldClaudeCredential, currentClaudeCredential, fdName } from './chunk-GANAOCHR.js';
|
|
29
32
|
import { COMMAND_LOST_REASON, CONTAINER_STOPPED_REASON } from './chunk-MESOOLF4.js';
|
|
30
|
-
import { randomBytes, createHash } from 'crypto';
|
|
31
33
|
import { mkdirSync, existsSync, chmodSync, writeFileSync, renameSync, statSync, readdirSync, readFileSync, rmSync, closeSync, lstatSync } from 'fs';
|
|
32
|
-
import { totalmem, userInfo, constants } from 'os';
|
|
33
34
|
import { join, dirname } from 'path';
|
|
35
|
+
import { randomBytes, createHash } from 'crypto';
|
|
36
|
+
import { totalmem, userInfo, constants } from 'os';
|
|
34
37
|
|
|
35
38
|
createRequire(import.meta.url);
|
|
39
|
+
var GIB = 1024 ** 3;
|
|
40
|
+
var GB = 1e9;
|
|
41
|
+
var BUILD_SIZES = {
|
|
42
|
+
small: { memoryGiB: 8, vcpus: 2, diskGB: 20 },
|
|
43
|
+
large: { memoryGiB: 16, vcpus: 4, diskGB: 30 }
|
|
44
|
+
};
|
|
45
|
+
var UNLIMITED_DISK_GB = 1e9;
|
|
46
|
+
var NONE = { memoryGiB: 0, vcpus: 0, diskGB: 0 };
|
|
47
|
+
function fleetRoom(facts, jobs, held, draining) {
|
|
48
|
+
if (draining || held) return { total: { ...NONE }, free: { ...NONE } };
|
|
49
|
+
const availableBytes = facts.sliceBytes ?? Math.max(0, facts.totalMemoryBytes - RUNNER_RESERVE_BYTES);
|
|
50
|
+
const total = {
|
|
51
|
+
memoryGiB: Math.floor(availableBytes / GIB),
|
|
52
|
+
vcpus: facts.cpus,
|
|
53
|
+
diskGB: facts.freeDiskBytes === void 0 ? UNLIMITED_DISK_GB : Math.floor((facts.freeDiskBytes + facts.heldDiskBytes) / GB)
|
|
54
|
+
};
|
|
55
|
+
const sizes = jobs.map((job) => BUILD_SIZES[job.size ?? "large"]);
|
|
56
|
+
const sum = (key) => sizes.reduce((all, size) => all + size[key], 0);
|
|
57
|
+
return {
|
|
58
|
+
total,
|
|
59
|
+
free: {
|
|
60
|
+
memoryGiB: Math.max(0, total.memoryGiB - sum("memoryGiB")),
|
|
61
|
+
vcpus: Math.max(0, total.vcpus - sum("vcpus")),
|
|
62
|
+
diskGB: Math.max(0, total.diskGB - sum("diskGB"))
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
var MAX_IMAGES = 500;
|
|
67
|
+
function names(path) {
|
|
68
|
+
try {
|
|
69
|
+
return readdirSync(path);
|
|
70
|
+
} catch {
|
|
71
|
+
return [];
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function heldImages(configPath) {
|
|
75
|
+
const held = /* @__PURE__ */ new Set();
|
|
76
|
+
for (const scope of storeScopes(configPath)) {
|
|
77
|
+
if (scope === void 0) continue;
|
|
78
|
+
const root = imagesDir(configPath, scope);
|
|
79
|
+
for (const owner of names(root)) {
|
|
80
|
+
if (owner.startsWith("_")) continue;
|
|
81
|
+
for (const name of names(join(root, owner))) {
|
|
82
|
+
const hasRecord = names(join(root, owner, name)).some((entry) => existsSync(join(root, owner, name, entry, "image.json")));
|
|
83
|
+
if (hasRecord) held.add(`${scope}/${owner}/${name}`.toLowerCase());
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return [...held].sort().slice(0, MAX_IMAGES);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// src/runner/fleet/loop.ts
|
|
91
|
+
var FLEET_CALL_TIMEOUT_MS = 2e4;
|
|
92
|
+
var MAX_CLAIMS_PER_POLL = 8;
|
|
93
|
+
var MAX_LOGGED_REASONS = 1e3;
|
|
94
|
+
var MACHINE_REVOKED = "machine_revoked";
|
|
95
|
+
var NO_ROOM = { memoryGiB: 0, vcpus: 0, diskGB: 0 };
|
|
96
|
+
function newFleetMemory() {
|
|
97
|
+
return { keyDoubted: false, bound: false, loggedReasons: /* @__PURE__ */ new Set(), calling: false, held: false };
|
|
98
|
+
}
|
|
99
|
+
function record(ports, memory) {
|
|
100
|
+
const state = {
|
|
101
|
+
calledAt: memory.last?.calledAt ?? null,
|
|
102
|
+
claimId: memory.last?.claimId ?? null,
|
|
103
|
+
reservation: memory.last?.reservation ?? null,
|
|
104
|
+
stops: memory.last?.stops ?? 0,
|
|
105
|
+
...memory.last?.reason === void 0 ? {} : { reason: memory.last.reason },
|
|
106
|
+
...memory.check === void 0 ? {} : { check: memory.check }
|
|
107
|
+
};
|
|
108
|
+
try {
|
|
109
|
+
writeFleetState(ports.config().configPath, state);
|
|
110
|
+
} catch {
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async function checkFleet(ports, memory) {
|
|
114
|
+
const config = ports.config();
|
|
115
|
+
const at = ports.now().toISOString();
|
|
116
|
+
const problem = fleetConfigProblem(config);
|
|
117
|
+
if (problem !== void 0) {
|
|
118
|
+
ports.repeating("the fleet check", `${problem} \xB7 the runner calls no route that hands out work until this is fixed`);
|
|
119
|
+
memory.check = { at, ok: false, message: problem };
|
|
120
|
+
record(ports, memory);
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
if (!memory.keyDoubted) {
|
|
124
|
+
ports.repeating("the fleet check", void 0);
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
const machine = await askFleetMachine(config, ports.fetchImpl);
|
|
128
|
+
if (!machine.ok && machine.refused && machine.code === MACHINE_REVOKED) ports.revoked();
|
|
129
|
+
if (!machine.ok) {
|
|
130
|
+
ports.repeating("the fleet check", `${machine.message} \xB7 the runner calls no route that hands out work until this is fixed`);
|
|
131
|
+
memory.check = { at, ok: false, message: machine.message };
|
|
132
|
+
record(ports, memory);
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
ports.repeating("the fleet check", void 0);
|
|
136
|
+
const recovered = memory.check?.ok === false;
|
|
137
|
+
memory.keyDoubted = false;
|
|
138
|
+
memory.check = { at, ok: true, machine: machine.name };
|
|
139
|
+
if (recovered) ports.log.line(`fleet machine ${machine.name} accepted the key`);
|
|
140
|
+
record(ports, memory);
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
function parseClaim(value) {
|
|
144
|
+
if (value === null || value === void 0) return void 0;
|
|
145
|
+
if (typeof value !== "object" || Array.isArray(value)) return { malformed: true };
|
|
146
|
+
const data = value;
|
|
147
|
+
const { id, kind, workItemId, number, appId, organizationId, size } = data;
|
|
148
|
+
if (typeof id !== "string" || id === "" || kind !== "implement" && kind !== "amend" || typeof workItemId !== "string" || workItemId === "" || typeof number !== "number" || !Number.isInteger(number) || typeof appId !== "string" || appId === "" || typeof organizationId !== "string" || organizationId === "" || size !== "small" && size !== "large") {
|
|
149
|
+
return {
|
|
150
|
+
malformed: true,
|
|
151
|
+
...typeof workItemId === "string" && workItemId !== "" ? { workItemId } : {},
|
|
152
|
+
...typeof appId === "string" && appId !== "" ? { appId } : {}
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
return { data, id, kind, workItemId, number, appId, organizationId, size };
|
|
156
|
+
}
|
|
157
|
+
function parseStops(value) {
|
|
158
|
+
if (!Array.isArray(value)) return [];
|
|
159
|
+
const stops = [];
|
|
160
|
+
for (const entry of value) {
|
|
161
|
+
const { claimId, requestedByName, note } = Object(entry);
|
|
162
|
+
if (typeof claimId !== "string" || claimId === "") continue;
|
|
163
|
+
stops.push({ claimId, requestedByName: typeof requestedByName === "string" ? requestedByName : "", note: typeof note === "string" ? note : null });
|
|
164
|
+
}
|
|
165
|
+
return stops;
|
|
166
|
+
}
|
|
167
|
+
function parseAnswer(data) {
|
|
168
|
+
const reservation = Object(data.reservation);
|
|
169
|
+
return {
|
|
170
|
+
claim: parseClaim(data.claim),
|
|
171
|
+
stops: parseStops(data.stops),
|
|
172
|
+
reservation: typeof reservation.workItemId === "string" && (reservation.size === "small" || reservation.size === "large") ? { workItemId: reservation.workItemId, size: reservation.size } : null,
|
|
173
|
+
...typeof data.reason === "string" ? { reason: data.reason } : {},
|
|
174
|
+
...typeof data.reasonWorkItemId === "string" ? { reasonWorkItemId: data.reasonWorkItemId } : {},
|
|
175
|
+
...typeof data.runnerVersion === "string" ? { runnerVersion: data.runnerVersion } : {}
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
async function callFleetRoute(ports, memory, room, images) {
|
|
179
|
+
const config = ports.config();
|
|
180
|
+
const hostKey = await ensureHostKey(config.configPath);
|
|
181
|
+
const body = {
|
|
182
|
+
total: room.total,
|
|
183
|
+
free: room.free,
|
|
184
|
+
images,
|
|
185
|
+
cliVersion: ports.cliVersion,
|
|
186
|
+
runnerProtocol: RUNNER_PROTOCOL,
|
|
187
|
+
// The first accepted call binds the machine to this host's key; the calls after it are signed and need not repeat it.
|
|
188
|
+
...memory.bound ? {} : { hostKey: hostKey.hostKey }
|
|
189
|
+
};
|
|
190
|
+
memory.calling = true;
|
|
191
|
+
try {
|
|
192
|
+
const { data } = await postJson(config.url, "/v1/fleet/next", config.apiKey, "", body, ports.fetchImpl, FLEET_CALL_TIMEOUT_MS, hostKey.signer);
|
|
193
|
+
memory.bound = true;
|
|
194
|
+
memory.lastAnsweredMs = ports.now().getTime();
|
|
195
|
+
ports.repeating("the fleet route", void 0);
|
|
196
|
+
const answer = parseAnswer(data);
|
|
197
|
+
ports.considerVersion(answer.runnerVersion);
|
|
198
|
+
return answer;
|
|
199
|
+
} catch (err) {
|
|
200
|
+
const refused = err instanceof WorkCommandError && !err.retryable;
|
|
201
|
+
const code = err instanceof WorkCommandError ? err.code : void 0;
|
|
202
|
+
if (code === MACHINE_REVOKED) ports.revoked();
|
|
203
|
+
ports.repeating(
|
|
204
|
+
"the fleet route",
|
|
205
|
+
refused ? `the gateway refused the call to the fleet route${code === void 0 ? "" : ` (${code})`}: ${describeCallFailure(err)} \xB7 testing the key again next poll` : `the call to the fleet route failed: ${describeCallFailure(err)} \xB7 trying again next poll`
|
|
206
|
+
);
|
|
207
|
+
if (refused) memory.keyDoubted = true;
|
|
208
|
+
memory.last = { calledAt: ports.now().toISOString(), claimId: null, reservation: null, stops: 0, reason: code ?? "failed" };
|
|
209
|
+
record(ports, memory);
|
|
210
|
+
return void 0;
|
|
211
|
+
} finally {
|
|
212
|
+
memory.calling = false;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function noteReason(ports, memory, answer) {
|
|
216
|
+
if (answer.reason === void 0) return;
|
|
217
|
+
const key = `${answer.reasonWorkItemId ?? "unknown"}:${answer.reason}`;
|
|
218
|
+
if (memory.loggedReasons.has(key)) return;
|
|
219
|
+
if (memory.loggedReasons.size >= MAX_LOGGED_REASONS) memory.loggedReasons.clear();
|
|
220
|
+
memory.loggedReasons.add(key);
|
|
221
|
+
ports.log.error(`the gateway did not claim work item ${answer.reasonWorkItemId ?? "(not named)"} for this machine: ${answer.reason}`);
|
|
222
|
+
}
|
|
223
|
+
async function runFleetTick(ports, memory) {
|
|
224
|
+
ports.reload();
|
|
225
|
+
if (!await checkFleet(ports, memory)) return;
|
|
226
|
+
const held = await ports.held();
|
|
227
|
+
memory.held = held;
|
|
228
|
+
const images = ports.images();
|
|
229
|
+
for (let called = 0; called < MAX_CLAIMS_PER_POLL; called++) {
|
|
230
|
+
const room = fleetRoom(ports.facts(), ports.jobs(), held, ports.draining());
|
|
231
|
+
const answer = await callFleetRoute(ports, memory, room, images);
|
|
232
|
+
if (answer === void 0) return;
|
|
233
|
+
memory.last = {
|
|
234
|
+
calledAt: ports.now().toISOString(),
|
|
235
|
+
claimId: answer.claim !== void 0 && "id" in answer.claim ? answer.claim.id : null,
|
|
236
|
+
reservation: answer.reservation,
|
|
237
|
+
stops: answer.stops.length,
|
|
238
|
+
...answer.reason === void 0 ? {} : { reason: answer.reason }
|
|
239
|
+
};
|
|
240
|
+
record(ports, memory);
|
|
241
|
+
if (answer.stops.length > 0) ports.applyStops(answer.stops);
|
|
242
|
+
noteReason(ports, memory, answer);
|
|
243
|
+
if (answer.claim === void 0) return;
|
|
244
|
+
if ("malformed" in answer.claim) {
|
|
245
|
+
const { workItemId, appId } = answer.claim;
|
|
246
|
+
const released = workItemId !== void 0 && appId !== void 0 ? await ports.release({ workItemId, appId }) : false;
|
|
247
|
+
ports.log.error(
|
|
248
|
+
"the gateway's claim lacked what a fleet runner needs (its id, kind, work item, number, factory, organization and size); update the runner \xB7 " + (released ? "the claim was given back" : "the claim could not be given back, so it stays held until its lease lapses")
|
|
249
|
+
);
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
if (ports.draining()) {
|
|
253
|
+
const released = await ports.release({ workItemId: answer.claim.workItemId, appId: answer.claim.appId });
|
|
254
|
+
ports.log.line(
|
|
255
|
+
`the runner is draining; the claim on work item ${answer.claim.workItemId} was ${released ? "given back" : "not given back, so it stays held until its lease lapses"}`
|
|
256
|
+
);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
const started = await ports.startBuild(answer.claim);
|
|
260
|
+
if (!started || held || ports.draining()) return;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
async function runFleetKeepalive(ports, memory, minGapMs) {
|
|
264
|
+
if (memory.calling) return;
|
|
265
|
+
if (memory.lastAnsweredMs !== void 0 && ports.now().getTime() - memory.lastAnsweredMs < minGapMs) return;
|
|
266
|
+
if (memory.keyDoubted || fleetConfigProblem(ports.config()) !== void 0) return;
|
|
267
|
+
const room = fleetRoom(ports.facts(), ports.jobs(), memory.held, ports.draining());
|
|
268
|
+
const answer = await callFleetRoute(ports, memory, { total: room.total, free: NO_ROOM }, ports.images());
|
|
269
|
+
if (answer === void 0) return;
|
|
270
|
+
memory.last = {
|
|
271
|
+
calledAt: ports.now().toISOString(),
|
|
272
|
+
claimId: null,
|
|
273
|
+
reservation: answer.reservation,
|
|
274
|
+
stops: answer.stops.length,
|
|
275
|
+
...answer.reason === void 0 ? {} : { reason: answer.reason }
|
|
276
|
+
};
|
|
277
|
+
record(ports, memory);
|
|
278
|
+
if (answer.stops.length > 0) ports.applyStops(answer.stops);
|
|
279
|
+
noteReason(ports, memory, answer);
|
|
280
|
+
if (answer.claim === void 0) return;
|
|
281
|
+
const { workItemId, appId } = answer.claim;
|
|
282
|
+
const released = workItemId !== void 0 && appId !== void 0 ? await ports.release({ workItemId, appId }) : false;
|
|
283
|
+
ports.log.error(
|
|
284
|
+
`the gateway handed out work item ${workItemId ?? "(not named)"} to a call that reported no free room \xB7 ` + (released ? "the claim was given back" : "the claim could not be given back, so it stays held until its lease lapses")
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
function coalesced(run) {
|
|
288
|
+
let running;
|
|
289
|
+
let again = false;
|
|
290
|
+
const call = async () => {
|
|
291
|
+
if (running !== void 0) {
|
|
292
|
+
again = true;
|
|
293
|
+
return running;
|
|
294
|
+
}
|
|
295
|
+
running = (async () => {
|
|
296
|
+
try {
|
|
297
|
+
do {
|
|
298
|
+
again = false;
|
|
299
|
+
await run();
|
|
300
|
+
} while (again);
|
|
301
|
+
} finally {
|
|
302
|
+
running = void 0;
|
|
303
|
+
}
|
|
304
|
+
})();
|
|
305
|
+
return running;
|
|
306
|
+
};
|
|
307
|
+
return Object.assign(call, { running: () => running !== void 0 });
|
|
308
|
+
}
|
|
309
|
+
function createFleetPoller(opts) {
|
|
310
|
+
const set = opts.setInterval ?? setInterval;
|
|
311
|
+
const clear = opts.clearInterval ?? clearInterval;
|
|
312
|
+
let timer;
|
|
313
|
+
let stopped = false;
|
|
314
|
+
return {
|
|
315
|
+
start() {
|
|
316
|
+
if (timer !== void 0 || stopped) return;
|
|
317
|
+
timer = set(() => {
|
|
318
|
+
void opts.keepalive?.();
|
|
319
|
+
void opts.run();
|
|
320
|
+
}, opts.everyMs);
|
|
321
|
+
},
|
|
322
|
+
wake() {
|
|
323
|
+
if (!stopped) void opts.run();
|
|
324
|
+
},
|
|
325
|
+
stop() {
|
|
326
|
+
stopped = true;
|
|
327
|
+
if (timer !== void 0) clear(timer);
|
|
328
|
+
timer = void 0;
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
}
|
|
36
332
|
|
|
37
333
|
// src/runner/heartbeat.ts
|
|
38
334
|
var HEARTBEAT_TIMEOUT_MS = 1e4;
|
|
@@ -339,10 +635,10 @@ var RESTART_EXIT_STATUS = 75;
|
|
|
339
635
|
var MAX_MARKER_STARTS = 3;
|
|
340
636
|
var RETRY_FIRST_WAIT_MS = 5 * 6e4;
|
|
341
637
|
var RETRY_MAX_WAIT_MS = 6 * 60 * 6e4;
|
|
342
|
-
function retryAfterMs(
|
|
343
|
-
const at =
|
|
638
|
+
function retryAfterMs(record2) {
|
|
639
|
+
const at = record2.at === void 0 ? Number.NaN : Date.parse(record2.at);
|
|
344
640
|
if (!Number.isFinite(at)) return 0;
|
|
345
|
-
const failures = Math.max(1,
|
|
641
|
+
const failures = Math.max(1, record2.attempts ?? 1);
|
|
346
642
|
return at + Math.min(RETRY_FIRST_WAIT_MS * 2 ** (failures - 1), RETRY_MAX_WAIT_MS);
|
|
347
643
|
}
|
|
348
644
|
function decide(input) {
|
|
@@ -350,10 +646,10 @@ function decide(input) {
|
|
|
350
646
|
if (order === void 0 || order <= 0) return "none";
|
|
351
647
|
if (!input.autoUpdate) return "none";
|
|
352
648
|
if (!input.fromCurrent) return "cannot_self_update";
|
|
353
|
-
const { record } = input;
|
|
354
|
-
if (
|
|
355
|
-
if (
|
|
356
|
-
if (
|
|
649
|
+
const { record: record2 } = input;
|
|
650
|
+
if (record2 !== void 0 && record2.to === input.named) {
|
|
651
|
+
if (record2.outcome === "verification_failed" || record2.outcome === "rolled_back") return "skip_known_bad";
|
|
652
|
+
if (record2.outcome === "retrying" && input.nowMs < retryAfterMs(record2)) return "retry_later";
|
|
357
653
|
}
|
|
358
654
|
return "update";
|
|
359
655
|
}
|
|
@@ -648,20 +944,22 @@ var UNRECORDED_IMAGE_GRACE_MS = 60 * 60 * 1e3;
|
|
|
648
944
|
function recordedImages(configPath) {
|
|
649
945
|
const imageIds = /* @__PURE__ */ new Set();
|
|
650
946
|
const recipeKeys = /* @__PURE__ */ new Set();
|
|
651
|
-
const
|
|
652
|
-
|
|
653
|
-
for (const
|
|
654
|
-
for (const
|
|
655
|
-
const
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
947
|
+
for (const scope of storeScopes(configPath)) {
|
|
948
|
+
const root = imagesDir(configPath, scope);
|
|
949
|
+
for (const owner of listDir(root)) {
|
|
950
|
+
for (const repo of listDir(join(root, owner))) {
|
|
951
|
+
for (const entry of listDir(join(root, owner, repo))) {
|
|
952
|
+
const file = join(root, owner, repo, entry, "image.json");
|
|
953
|
+
if (!existsSync(file)) continue;
|
|
954
|
+
let record2;
|
|
955
|
+
try {
|
|
956
|
+
record2 = JSON.parse(readFileSync(file, "utf8"));
|
|
957
|
+
} catch {
|
|
958
|
+
return void 0;
|
|
959
|
+
}
|
|
960
|
+
if (typeof record2.imageId === "string") imageIds.add(record2.imageId);
|
|
961
|
+
if (typeof record2.recipeKey === "string") recipeKeys.add(record2.recipeKey);
|
|
662
962
|
}
|
|
663
|
-
if (typeof record.imageId === "string") imageIds.add(record.imageId);
|
|
664
|
-
if (typeof record.recipeKey === "string") recipeKeys.add(record.recipeKey);
|
|
665
963
|
}
|
|
666
964
|
}
|
|
667
965
|
}
|
|
@@ -777,20 +1075,22 @@ async function runHousekeeping(options) {
|
|
|
777
1075
|
}
|
|
778
1076
|
freed += await removeSavedStates(options, nowMs, savedStatesDir(root));
|
|
779
1077
|
freed += removeStaleBuilderCaches(options, nowMs);
|
|
780
|
-
const
|
|
781
|
-
|
|
782
|
-
const
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
const
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
1078
|
+
for (const scope of storeScopes(options.configPath)) {
|
|
1079
|
+
const spoolPath = spoolDir(orgStoreIn(root, scope));
|
|
1080
|
+
const spoolFiles = listDir(spoolPath).flatMap((name) => {
|
|
1081
|
+
const mtimeMs = mtimeOf(join(spoolPath, name));
|
|
1082
|
+
return mtimeMs === void 0 ? [] : [{ name, mtimeMs }];
|
|
1083
|
+
});
|
|
1084
|
+
for (const file of selectSpool(spoolFiles, nowMs, jobs.unfinished.length > 0)) {
|
|
1085
|
+
const path = join(spoolPath, file.name);
|
|
1086
|
+
const size = sizeOf(path);
|
|
1087
|
+
try {
|
|
1088
|
+
rmSync(path, { recursive: true, force: true });
|
|
1089
|
+
freed += size;
|
|
1090
|
+
log.line(`housekeeping: removed spool file ${file.name} (${formatBytes(size)}), last changed ${formatDuration(nowMs - file.mtimeMs)} ago and no build is running`);
|
|
1091
|
+
} catch (err) {
|
|
1092
|
+
log.error(`housekeeping: could not remove spool file ${file.name}: ${describeCallFailure(err)}`);
|
|
1093
|
+
}
|
|
794
1094
|
}
|
|
795
1095
|
}
|
|
796
1096
|
log.line(`housekeeping: pass finished, freed ${formatBytes(freed)}`);
|
|
@@ -799,36 +1099,41 @@ async function runHousekeeping(options) {
|
|
|
799
1099
|
async function removeSavedStates(options, nowMs, dir) {
|
|
800
1100
|
const records = readRecords(dir);
|
|
801
1101
|
const reasons = /* @__PURE__ */ new Map();
|
|
802
|
-
for (const
|
|
803
|
-
for (const
|
|
1102
|
+
for (const record2 of selectStaleIncomplete(records, nowMs)) reasons.set(record2.id, "its copy never finished");
|
|
1103
|
+
for (const record2 of selectExpired(records.filter((r) => r.complete), nowMs)) reasons.set(record2.id, "it is at least 7 days old");
|
|
804
1104
|
if (options.savedStateMax !== void 0) {
|
|
805
|
-
const kept = records.filter((
|
|
1105
|
+
const kept = records.filter((record2) => record2.complete && !reasons.has(record2.id));
|
|
806
1106
|
const max = options.savedStateMax === "off" ? 0 : options.savedStateMax;
|
|
807
|
-
for (const
|
|
1107
|
+
for (const record2 of selectToFit(kept, max)) reasons.set(record2.id, "the saved states pass runner.housekeeping.savedStateMax");
|
|
808
1108
|
}
|
|
809
1109
|
let freed = 0;
|
|
810
1110
|
const docker = options.docker;
|
|
811
|
-
for (const
|
|
1111
|
+
for (const record2 of records.filter((r) => reasons.has(r.id))) {
|
|
812
1112
|
try {
|
|
813
|
-
await removeSavedState(
|
|
1113
|
+
await removeSavedState(record2, {
|
|
814
1114
|
dir,
|
|
815
1115
|
removeVolume: async (name) => {
|
|
816
1116
|
if (docker === void 0) throw new Error("Docker is not available");
|
|
817
1117
|
await docker.removeVolume(name);
|
|
818
1118
|
}
|
|
819
1119
|
});
|
|
820
|
-
freed +=
|
|
821
|
-
options.log.line(`housekeeping: removed saved state ${
|
|
1120
|
+
freed += record2.bytes;
|
|
1121
|
+
options.log.line(`housekeeping: removed saved state ${record2.id} (${formatBytes(record2.bytes)}), ${reasons.get(record2.id)}`);
|
|
822
1122
|
} catch (err) {
|
|
823
|
-
options.log.error(`housekeeping: could not remove saved state ${
|
|
1123
|
+
options.log.error(`housekeeping: could not remove saved state ${record2.id} (${describeSaved(record2)}): ${describeCallFailure(err)}`);
|
|
824
1124
|
}
|
|
825
1125
|
}
|
|
826
1126
|
return freed;
|
|
827
1127
|
}
|
|
828
|
-
function describeSaved(
|
|
829
|
-
return
|
|
1128
|
+
function describeSaved(record2) {
|
|
1129
|
+
return record2.volume ?? record2.path ?? record2.kind;
|
|
830
1130
|
}
|
|
831
1131
|
async function removeStaleRootDisks(root, unfinished, options, docker) {
|
|
1132
|
+
let freed = 0;
|
|
1133
|
+
for (const scope of storeScopes(options.configPath)) freed += await removeStaleRootDisksIn(orgStoreIn(root, scope), unfinished, options, docker);
|
|
1134
|
+
return freed;
|
|
1135
|
+
}
|
|
1136
|
+
async function removeStaleRootDisksIn(root, unfinished, options, docker) {
|
|
832
1137
|
const dir = rootfsDir(root);
|
|
833
1138
|
const inUse = new Set(unfinished.flatMap((job) => job.microvm?.rootfs === void 0 ? [] : [job.microvm.rootfs]));
|
|
834
1139
|
let freed = 0;
|
|
@@ -851,18 +1156,21 @@ async function removeStaleRootDisks(root, unfinished, options, docker) {
|
|
|
851
1156
|
}
|
|
852
1157
|
function removeStaleBuilderCaches(options, nowMs) {
|
|
853
1158
|
let freed = 0;
|
|
854
|
-
for (const
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
const
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1159
|
+
for (const scope of storeScopes(options.configPath)) {
|
|
1160
|
+
const where = scope === void 0 ? "" : `${scope}/`;
|
|
1161
|
+
for (const owner of listDir(imagesDir(options.configPath, scope))) {
|
|
1162
|
+
for (const name of listDir(join(imagesDir(options.configPath, scope), owner))) {
|
|
1163
|
+
const path = builderCacheDiskPath(options.configPath, owner, name, scope);
|
|
1164
|
+
const mtimeMs = mtimeOf(path);
|
|
1165
|
+
if (mtimeMs === void 0 || nowMs - mtimeMs < options.keepDays * 864e5) continue;
|
|
1166
|
+
const size = sizeOf(path);
|
|
1167
|
+
try {
|
|
1168
|
+
rmSync(path, { force: true });
|
|
1169
|
+
freed += size;
|
|
1170
|
+
options.log.line(`housekeeping: removed the builder cache disk of ${where}${owner}/${name} (${formatBytes(size)}), last used ${formatDuration(nowMs - mtimeMs)} ago`);
|
|
1171
|
+
} catch (err) {
|
|
1172
|
+
options.log.error(`housekeeping: could not remove the builder cache disk of ${where}${owner}/${name}: ${describeCallFailure(err)}`);
|
|
1173
|
+
}
|
|
866
1174
|
}
|
|
867
1175
|
}
|
|
868
1176
|
}
|
|
@@ -977,11 +1285,11 @@ async function judgeMicroVmAdoption(input) {
|
|
|
977
1285
|
return probe.unreachable === true ? "unasked" : "unservable";
|
|
978
1286
|
}
|
|
979
1287
|
const inherited = runtime.vm.fdStore.inherited();
|
|
980
|
-
const
|
|
981
|
-
return
|
|
1288
|
+
const names2 = state.fdStore ?? [];
|
|
1289
|
+
return names2.length > 0 && names2.every((name) => inherited.has(name)) ? "adopt" : "unservable";
|
|
982
1290
|
}
|
|
983
1291
|
function linuxSignalName(number) {
|
|
984
|
-
const
|
|
1292
|
+
const names2 = {
|
|
985
1293
|
1: "SIGHUP",
|
|
986
1294
|
2: "SIGINT",
|
|
987
1295
|
3: "SIGQUIT",
|
|
@@ -1014,7 +1322,7 @@ function linuxSignalName(number) {
|
|
|
1014
1322
|
30: "SIGPWR",
|
|
1015
1323
|
31: "SIGSYS"
|
|
1016
1324
|
};
|
|
1017
|
-
return
|
|
1325
|
+
return names2[number];
|
|
1018
1326
|
}
|
|
1019
1327
|
function hostSignalName(number) {
|
|
1020
1328
|
return Object.entries(constants.signals).find(([, value]) => value === number)?.[0];
|
|
@@ -1149,7 +1457,7 @@ async function listStartable(config, queue, fetchImpl, cursor = null) {
|
|
|
1149
1457
|
config.url,
|
|
1150
1458
|
`/v1/work-items?${query}${cursor === null ? "" : `&cursor=${encodeURIComponent(cursor)}`}`,
|
|
1151
1459
|
config.apiKey,
|
|
1152
|
-
config
|
|
1460
|
+
keyRunnerFactory(config),
|
|
1153
1461
|
fetchImpl,
|
|
1154
1462
|
await signerFor(config.configPath)
|
|
1155
1463
|
);
|
|
@@ -1187,6 +1495,7 @@ function fitForImage(deps, config) {
|
|
|
1187
1495
|
return fitImageBuildMemory(config, (deps.blocked?.totalMemoryBytes ?? totalmem)());
|
|
1188
1496
|
}
|
|
1189
1497
|
function describeConfig(config) {
|
|
1498
|
+
if (config.fleet === true) return `fleet \xB7 poll ${config.pollSeconds}s`;
|
|
1190
1499
|
return `queues ${config.queues.join(",")} \xB7 concurrency ${config.concurrency} \xB7 poll ${config.pollSeconds}s`;
|
|
1191
1500
|
}
|
|
1192
1501
|
function slotsTotalOf(state) {
|
|
@@ -1210,7 +1519,7 @@ async function sendTickHeartbeat(state, workRequest) {
|
|
|
1210
1519
|
configPath: state.config.configPath,
|
|
1211
1520
|
url: state.config.url,
|
|
1212
1521
|
apiKey: state.config.apiKey,
|
|
1213
|
-
appId: state.config
|
|
1522
|
+
appId: keyRunnerFactory(state.config),
|
|
1214
1523
|
host: state.config.host,
|
|
1215
1524
|
cliVersion: state.deps.cliVersion ?? resolveVersion(),
|
|
1216
1525
|
pollSeconds: state.pollTimerSeconds ?? state.config.pollSeconds,
|
|
@@ -1256,16 +1565,16 @@ function applyStops(state, stops) {
|
|
|
1256
1565
|
reportRepeating(
|
|
1257
1566
|
state,
|
|
1258
1567
|
`stop of claim ${stop.claimId}`,
|
|
1259
|
-
held === void 0 ? `ignoring a stop for claim ${stop.claimId}: this runner holds no build under that claim` :
|
|
1568
|
+
held === void 0 ? `ignoring a stop for claim ${stop.claimId}: this runner holds no build under that claim` : `${itemTag(held.job)} a stop was asked for; the build is already ending`,
|
|
1260
1569
|
"line"
|
|
1261
1570
|
);
|
|
1262
1571
|
continue;
|
|
1263
1572
|
}
|
|
1264
1573
|
const who = `stopped by ${stop.requestedByName === "" ? "someone" : stop.requestedByName} through OuterLayer`;
|
|
1265
1574
|
const reason = stop.note === null ? who : `${who}: ${stop.note}`;
|
|
1266
|
-
logOf(state).line(
|
|
1575
|
+
logOf(state).line(`${itemTag(active.job)} ${reason} \xB7 ending the build`);
|
|
1267
1576
|
void endJob(state, active, "stopped", { reason }).catch((err) => {
|
|
1268
|
-
logOf(state).error(
|
|
1577
|
+
logOf(state).error(`${itemTag(active.job)} could not be stopped: ${describeCallFailure(err)}`);
|
|
1269
1578
|
});
|
|
1270
1579
|
}
|
|
1271
1580
|
}
|
|
@@ -1357,16 +1666,16 @@ async function readOwnContainerOnce(state) {
|
|
|
1357
1666
|
function recordImageStatus(state) {
|
|
1358
1667
|
if (state.deps.image === void 0) return;
|
|
1359
1668
|
const named = state.imageNamed;
|
|
1360
|
-
const
|
|
1669
|
+
const record2 = {
|
|
1361
1670
|
own: ownVersionOf(state.deps),
|
|
1362
1671
|
...named === void 0 ? {} : { named, tag: imageTag(named) },
|
|
1363
1672
|
upgradeRequired: state.upgradeRefused === true,
|
|
1364
1673
|
at: nowOf(state).toISOString()
|
|
1365
1674
|
};
|
|
1366
|
-
const key = JSON.stringify({ ...
|
|
1675
|
+
const key = JSON.stringify({ ...record2, at: void 0 });
|
|
1367
1676
|
if (state.imageRecord === key) return;
|
|
1368
1677
|
try {
|
|
1369
|
-
writeUpdateStatus(runnerDir(state.config.configPath),
|
|
1678
|
+
writeUpdateStatus(runnerDir(state.config.configPath), record2);
|
|
1370
1679
|
state.imageRecord = key;
|
|
1371
1680
|
reportRepeating(state, "the update-status.json record", void 0);
|
|
1372
1681
|
} catch (err) {
|
|
@@ -1451,15 +1760,15 @@ function holdWithoutRoomForLarge(state) {
|
|
|
1451
1760
|
}
|
|
1452
1761
|
async function freeDiskFromSavedStates(state, root, readFree) {
|
|
1453
1762
|
const dir = savedStatesDir(root);
|
|
1454
|
-
const records = readRecords(dir).filter((
|
|
1763
|
+
const records = readRecords(dir).filter((record2) => record2.complete);
|
|
1455
1764
|
if (records.length === 0) return readFree();
|
|
1456
1765
|
const log = logOf(state);
|
|
1457
1766
|
const minimum = state.config.housekeeping?.minFreeDisk ?? 0;
|
|
1458
1767
|
try {
|
|
1459
1768
|
state.deps.housekeeping ??= {};
|
|
1460
|
-
const
|
|
1769
|
+
const names2 = { run: state.deps.housekeeping.run, runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) };
|
|
1461
1770
|
const hostBuilder = keepsHostBuilder(state.config, state.deps.blocked?.builderDiskPinned);
|
|
1462
|
-
const docker = state.deps.housekeeping.docker ?? (hostBuilder ? createHousekeepingDocker(
|
|
1771
|
+
const docker = state.deps.housekeeping.docker ?? (hostBuilder ? createHousekeepingDocker(names2) : await housekeepingDockerIfReachable(names2));
|
|
1463
1772
|
await removeOldestUntil({
|
|
1464
1773
|
records,
|
|
1465
1774
|
deps: {
|
|
@@ -1477,11 +1786,11 @@ async function freeDiskFromSavedStates(state, root, readFree) {
|
|
|
1477
1786
|
}
|
|
1478
1787
|
return readFree();
|
|
1479
1788
|
}
|
|
1480
|
-
function recordPendingUpdate(state,
|
|
1481
|
-
const text =
|
|
1789
|
+
function recordPendingUpdate(state, record2) {
|
|
1790
|
+
const text = record2 === void 0 ? "none" : JSON.stringify(record2);
|
|
1482
1791
|
if (state.updateRecord === text) return;
|
|
1483
1792
|
try {
|
|
1484
|
-
writePendingUpdate(runnerDir(state.config.configPath),
|
|
1793
|
+
writePendingUpdate(runnerDir(state.config.configPath), record2);
|
|
1485
1794
|
state.updateRecord = text;
|
|
1486
1795
|
reportRepeating(state, "the update-pending.json record", void 0);
|
|
1487
1796
|
} catch (err) {
|
|
@@ -1512,10 +1821,10 @@ function holdForUpdate(state, version, mayHold = true, waits = []) {
|
|
|
1512
1821
|
}
|
|
1513
1822
|
const maxWaitMs = (state.config.update?.maxWaitHours ?? DEFAULT_UPDATE_MAX_WAIT_HOURS) * 60 * 6e4;
|
|
1514
1823
|
const holding = mayHold && (state.upgradeRefused === true || nowMs - wait.sinceMs >= maxWaitMs);
|
|
1515
|
-
const
|
|
1824
|
+
const record2 = { version, since: new Date(wait.sinceMs).toISOString() };
|
|
1516
1825
|
if (waits.length > 0) {
|
|
1517
|
-
|
|
1518
|
-
|
|
1826
|
+
record2.via = "self-update";
|
|
1827
|
+
record2.waits = recordedWaits(waits);
|
|
1519
1828
|
}
|
|
1520
1829
|
noteWaits(state, version, waits);
|
|
1521
1830
|
if (holding) {
|
|
@@ -1526,12 +1835,12 @@ function holdForUpdate(state, version, mayHold = true, waits = []) {
|
|
|
1526
1835
|
logged.hold = key;
|
|
1527
1836
|
log.line(`claiming nothing: update to ${version} waits for ${buildsPhrase(builds)} to finish`);
|
|
1528
1837
|
}
|
|
1529
|
-
|
|
1838
|
+
record2.holding = { builds, since: new Date(wait.holdSinceMs).toISOString() };
|
|
1530
1839
|
} else {
|
|
1531
1840
|
wait.holdSinceMs = void 0;
|
|
1532
1841
|
logged.hold = void 0;
|
|
1533
1842
|
}
|
|
1534
|
-
recordPendingUpdate(state,
|
|
1843
|
+
recordPendingUpdate(state, record2);
|
|
1535
1844
|
return holding;
|
|
1536
1845
|
}
|
|
1537
1846
|
function recordedWaits(waits) {
|
|
@@ -1678,9 +1987,9 @@ async function updateWhenIdle(state) {
|
|
|
1678
1987
|
return false;
|
|
1679
1988
|
}
|
|
1680
1989
|
if (state.jobs.size > 0) {
|
|
1681
|
-
const
|
|
1682
|
-
if (
|
|
1683
|
-
const failedAtMs = new Date(
|
|
1990
|
+
const record2 = readLastUpdate(state.config.configPath);
|
|
1991
|
+
if (record2?.to === version && record2.outcome === "retrying") {
|
|
1992
|
+
const failedAtMs = new Date(record2.at ?? Number.NaN).getTime();
|
|
1684
1993
|
if (!(nowOf(state).getTime() >= failedAtMs + RETRY_MAX_WAIT_MS)) {
|
|
1685
1994
|
holdForUpdate(state, version, false);
|
|
1686
1995
|
return false;
|
|
@@ -1713,6 +2022,10 @@ async function updateWhenIdle(state) {
|
|
|
1713
2022
|
return false;
|
|
1714
2023
|
}
|
|
1715
2024
|
async function tick(state) {
|
|
2025
|
+
if (state.config.fleet === true) {
|
|
2026
|
+
await fleetTickOf(state)();
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
1716
2029
|
if (state.ticking) {
|
|
1717
2030
|
await sendTickHeartbeat(state, void 0);
|
|
1718
2031
|
return;
|
|
@@ -1728,6 +2041,108 @@ async function tick(state) {
|
|
|
1728
2041
|
}
|
|
1729
2042
|
await sendTickHeartbeat(state, state.lastWorkRequest);
|
|
1730
2043
|
}
|
|
2044
|
+
function fleetTickOf(state) {
|
|
2045
|
+
state.fleet ??= newFleetMemory();
|
|
2046
|
+
state.fleetTick ??= coalesced(async () => {
|
|
2047
|
+
try {
|
|
2048
|
+
await runFleetTick(fleetPortsOf(state), state.fleet);
|
|
2049
|
+
} catch (err) {
|
|
2050
|
+
logOf(state).error(`unexpected error: ${describeCallFailure(err)}`);
|
|
2051
|
+
}
|
|
2052
|
+
});
|
|
2053
|
+
return state.fleetTick;
|
|
2054
|
+
}
|
|
2055
|
+
async function fleetKeepalive(state) {
|
|
2056
|
+
if (state.fleetTick?.running() !== true || state.fleet === void 0) return;
|
|
2057
|
+
try {
|
|
2058
|
+
await runFleetKeepalive(fleetPortsOf(state), state.fleet, (state.pollTimerSeconds ?? state.config.pollSeconds) * 1e3 / 2);
|
|
2059
|
+
} catch (err) {
|
|
2060
|
+
logOf(state).error(`unexpected error: ${describeCallFailure(err)}`);
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
function endRevokedBuilds(state) {
|
|
2064
|
+
for (const active of state.jobs.values()) {
|
|
2065
|
+
if (active.ending) continue;
|
|
2066
|
+
const reason = "the fleet machine was revoked, and its builds went back to the queue";
|
|
2067
|
+
logOf(state).error(`${itemTag(active.job)} ${reason} \xB7 ending the build`);
|
|
2068
|
+
void endJob(state, active, "lease_lost", { reason }).catch((err) => {
|
|
2069
|
+
logOf(state).error(`${itemTag(active.job)} could not be ended: ${describeCallFailure(err)}`);
|
|
2070
|
+
});
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
async function fleetHeld(state) {
|
|
2074
|
+
await readOwnContainerOnce(state);
|
|
2075
|
+
state.restarting = false;
|
|
2076
|
+
state.holdingForUpdate = false;
|
|
2077
|
+
if (state.draining === true) return false;
|
|
2078
|
+
if (await restartOntoInstalled(state)) {
|
|
2079
|
+
state.restarting = true;
|
|
2080
|
+
return true;
|
|
2081
|
+
}
|
|
2082
|
+
if (await updateWhenIdle(state)) {
|
|
2083
|
+
state.holdingForUpdate = true;
|
|
2084
|
+
return true;
|
|
2085
|
+
}
|
|
2086
|
+
const jailer = state.config.managed === true && state.deps.blocked?.jailer === void 0 ? await jailerState(state) : void 0;
|
|
2087
|
+
if (holdWhenBlocked(state, jailer)) return true;
|
|
2088
|
+
return holdOffForDisk(state);
|
|
2089
|
+
}
|
|
2090
|
+
function fleetFactsOf(state) {
|
|
2091
|
+
const probes = {
|
|
2092
|
+
...realBlockedProbes(),
|
|
2093
|
+
...state.deps.container?.hostEnv === void 0 ? {} : { env: state.deps.container.hostEnv },
|
|
2094
|
+
...state.deps.blocked
|
|
2095
|
+
};
|
|
2096
|
+
let heldDiskBytes = 0;
|
|
2097
|
+
for (const active of state.jobs.values()) {
|
|
2098
|
+
const buildDisk = active.job?.microvm?.buildDisk;
|
|
2099
|
+
if (buildDisk !== void 0) heldDiskBytes += (state.deps.allocatedBytes ?? allocatedBytes)(buildDisk);
|
|
2100
|
+
}
|
|
2101
|
+
const sliceBytes = sliceMemoryOf(probes.env);
|
|
2102
|
+
return {
|
|
2103
|
+
totalMemoryBytes: probes.totalMemoryBytes(),
|
|
2104
|
+
...sliceBytes === void 0 ? {} : { sliceBytes },
|
|
2105
|
+
cpus: (state.deps.fleet?.cpus ?? (() => usableCpus().length))(),
|
|
2106
|
+
freeDiskBytes: (state.deps.freeBytes ?? freeBytes)(runnerDir(state.config.configPath)),
|
|
2107
|
+
heldDiskBytes
|
|
2108
|
+
};
|
|
2109
|
+
}
|
|
2110
|
+
function fleetPortsOf(state) {
|
|
2111
|
+
return {
|
|
2112
|
+
config: () => state.config,
|
|
2113
|
+
fetchImpl: state.deps.fetchImpl,
|
|
2114
|
+
now: () => nowOf(state),
|
|
2115
|
+
log: logOf(state),
|
|
2116
|
+
cliVersion: state.deps.cliVersion ?? resolveVersion(),
|
|
2117
|
+
reload: () => reloadConfig(state),
|
|
2118
|
+
held: () => fleetHeld(state),
|
|
2119
|
+
draining: () => state.draining === true,
|
|
2120
|
+
jobs: () => [...state.jobs.values()].map((active) => active.job.size === void 0 ? {} : { size: active.job.size }),
|
|
2121
|
+
facts: () => fleetFactsOf(state),
|
|
2122
|
+
images: () => heldImages(state.config.configPath),
|
|
2123
|
+
applyStops: (stops) => applyStops(state, stops),
|
|
2124
|
+
considerVersion: (named) => considerNamedVersion(state, named),
|
|
2125
|
+
revoked: () => endRevokedBuilds(state),
|
|
2126
|
+
startBuild: async (claim) => {
|
|
2127
|
+
const item = { id: claim.workItemId, number: claim.number, trackerKey: `organization ${claim.organizationId.slice(0, 8)}`, startable: true };
|
|
2128
|
+
const attempt = await startClaimedBuild(state, item, claim.kind, claim.data, {
|
|
2129
|
+
workItemId: claim.workItemId,
|
|
2130
|
+
appId: claim.appId,
|
|
2131
|
+
organizationId: claim.organizationId,
|
|
2132
|
+
size: claim.size
|
|
2133
|
+
});
|
|
2134
|
+
return attempt.kind === "started" && attempt.provisionFailed !== true;
|
|
2135
|
+
},
|
|
2136
|
+
release: async (claim) => {
|
|
2137
|
+
const released = await retrySend(() => releaseLease(claimCalls(state.config, { appId: claim.appId }), claim.workItemId, void 0, {}, state.deps.fetchImpl), {
|
|
2138
|
+
now: state.deps.now,
|
|
2139
|
+
sleep: state.deps.sleep
|
|
2140
|
+
});
|
|
2141
|
+
return released.ok;
|
|
2142
|
+
},
|
|
2143
|
+
repeating: (key, message) => reportRepeating(state, key, message)
|
|
2144
|
+
};
|
|
2145
|
+
}
|
|
1731
2146
|
function imageBuilderState(state) {
|
|
1732
2147
|
const kind = builtinKind(state.config.hooks.provision);
|
|
1733
2148
|
if (kind !== "container" && kind !== "vm") return "usable";
|
|
@@ -1825,8 +2240,8 @@ async function housekeepingPass(state) {
|
|
|
1825
2240
|
try {
|
|
1826
2241
|
state.deps.housekeeping ??= {};
|
|
1827
2242
|
const hostBuilder = keepsHostBuilder(state.config, state.deps.blocked?.builderDiskPinned);
|
|
1828
|
-
const
|
|
1829
|
-
const docker = state.deps.housekeeping.docker ?? (hostBuilder ? createHousekeepingDocker(
|
|
2243
|
+
const names2 = { run: state.deps.housekeeping.run, runnerId: runnerLabelValue(state.config.configPath), builder: builderName(state.config.configPath) };
|
|
2244
|
+
const docker = state.deps.housekeeping.docker ?? (hostBuilder ? createHousekeepingDocker(names2) : await housekeepingDockerIfReachable(names2));
|
|
1830
2245
|
if (hostBuilder) state.deps.housekeeping.docker ??= docker;
|
|
1831
2246
|
const result = await runHousekeeping({
|
|
1832
2247
|
configPath: state.config.configPath,
|
|
@@ -1922,7 +2337,7 @@ var CLAIM_REFUSAL_CODES = new Set(
|
|
|
1922
2337
|
"pull_request_from_fork pull_request_on_default_branch pull_request_ambiguous pull_request_repository_not_connected repository_not_reachable branch_claimed_elsewhere".split(" ")
|
|
1923
2338
|
);
|
|
1924
2339
|
async function tryClaimAndStart(state, item, queue) {
|
|
1925
|
-
const { postJson: postJson2 } = await import('./work-cmd-
|
|
2340
|
+
const { postJson: postJson2 } = await import('./work-cmd-PKEAVXJC.js');
|
|
1926
2341
|
const log = logOf(state);
|
|
1927
2342
|
const hostKey = await loadHostKey(state.config.configPath);
|
|
1928
2343
|
const claim = await retrySend(
|
|
@@ -1930,7 +2345,7 @@ async function tryClaimAndStart(state, item, queue) {
|
|
|
1930
2345
|
state.config.url,
|
|
1931
2346
|
`/v1/work-items/${item.number}/claim`,
|
|
1932
2347
|
state.config.apiKey,
|
|
1933
|
-
state.config
|
|
2348
|
+
keyRunnerFactory(state.config),
|
|
1934
2349
|
{
|
|
1935
2350
|
host: state.config.host,
|
|
1936
2351
|
kind: queue,
|
|
@@ -1997,45 +2412,49 @@ async function tryClaimAndStart(state, item, queue) {
|
|
|
1997
2412
|
state.versionRefusalLogged = void 0;
|
|
1998
2413
|
recordImageStatus(state);
|
|
1999
2414
|
reportRepeating(state, "the request signature", void 0);
|
|
2000
|
-
|
|
2415
|
+
return startClaimedBuild(state, item, queue, claim.value.data);
|
|
2416
|
+
}
|
|
2417
|
+
function releaseUntried(state, item, fleet) {
|
|
2418
|
+
return retrySend(
|
|
2419
|
+
() => fleet === void 0 ? releaseLease(runnerCalls(state.config), item.number, void 0, {}, state.deps.fetchImpl) : releaseLease(claimCalls(state.config, { appId: fleet.appId }), fleet.workItemId, void 0, {}, state.deps.fetchImpl),
|
|
2420
|
+
{ now: state.deps.now, sleep: state.deps.sleep }
|
|
2421
|
+
);
|
|
2422
|
+
}
|
|
2423
|
+
async function startClaimedBuild(state, item, queue, data, fleet) {
|
|
2424
|
+
const log = logOf(state);
|
|
2425
|
+
const tag = itemTag({ item: item.number, organizationId: fleet?.organizationId, claimId: data.id === void 0 || data.id === null ? void 0 : String(data.id) });
|
|
2001
2426
|
const claimId = data.id === void 0 || data.id === null ? "" : String(data.id);
|
|
2002
2427
|
if (claimId === "") {
|
|
2003
2428
|
return { kind: "error", reason: "the claim response carried no claim id" };
|
|
2004
2429
|
}
|
|
2005
2430
|
if (wasExtended(data)) {
|
|
2006
2431
|
log.line(
|
|
2007
|
-
|
|
2432
|
+
`${tag} skipped: the claim extended a live lease under the runner name "${state.config.host}" \xB7 set a distinct runner.name per config file`
|
|
2008
2433
|
);
|
|
2009
2434
|
return { kind: "held" };
|
|
2010
2435
|
}
|
|
2011
2436
|
if (claimIsAlreadyRunning(state, claimId)) {
|
|
2012
|
-
log.line(
|
|
2437
|
+
log.line(`${tag} skipped: claim ${shortClaim(claimId)} is already running here`);
|
|
2013
2438
|
return { kind: "held" };
|
|
2014
2439
|
}
|
|
2015
2440
|
const itemKey = typeof data.itemKey === "string" && data.itemKey !== "" ? data.itemKey : void 0;
|
|
2016
2441
|
if (itemKey === void 0) {
|
|
2017
|
-
const released = await
|
|
2018
|
-
() => releaseLease(state.config, item.number, void 0, {}, state.deps.fetchImpl),
|
|
2019
|
-
{ now: state.deps.now, sleep: state.deps.sleep }
|
|
2020
|
-
);
|
|
2442
|
+
const released = await releaseUntried(state, item, fleet);
|
|
2021
2443
|
reportRepeating(
|
|
2022
2444
|
state,
|
|
2023
2445
|
"the item key",
|
|
2024
|
-
`the gateway returned no item key for
|
|
2446
|
+
`the gateway returned no item key for ${tag}; upgrade the gateway. ` + (released.ok ? "The claim was released and no build started" : `No build started, but releasing the claim failed (${released.reason}), so it stays held until its lease lapses`)
|
|
2025
2447
|
);
|
|
2026
2448
|
return { kind: "no_item_key" };
|
|
2027
2449
|
}
|
|
2028
2450
|
reportRepeating(state, "the item key", void 0);
|
|
2029
2451
|
const branch = typeof data.branch === "string" && data.branch !== "" ? data.branch : void 0;
|
|
2030
2452
|
if (branch === void 0) {
|
|
2031
|
-
const released = await
|
|
2032
|
-
() => releaseLease(state.config, item.number, void 0, {}, state.deps.fetchImpl),
|
|
2033
|
-
{ now: state.deps.now, sleep: state.deps.sleep }
|
|
2034
|
-
);
|
|
2453
|
+
const released = await releaseUntried(state, item, fleet);
|
|
2035
2454
|
reportRepeating(
|
|
2036
2455
|
state,
|
|
2037
2456
|
"the branch",
|
|
2038
|
-
`the gateway named no branch for
|
|
2457
|
+
`the gateway named no branch for ${tag}; upgrade the gateway. ` + (released.ok ? "The claim was released and no build started" : `No build started, but releasing the claim failed (${released.reason}), so it stays held until its lease lapses`)
|
|
2039
2458
|
);
|
|
2040
2459
|
return { kind: "no_branch" };
|
|
2041
2460
|
}
|
|
@@ -2043,31 +2462,30 @@ async function tryClaimAndStart(state, item, queue) {
|
|
|
2043
2462
|
const identity = parseClaimIdentity(data);
|
|
2044
2463
|
const repositories = parseClaimedRepositories(data.repositories);
|
|
2045
2464
|
if (repositories === null) {
|
|
2046
|
-
const released = await
|
|
2047
|
-
() => releaseLease(state.config, item.number, void 0, {}, state.deps.fetchImpl),
|
|
2048
|
-
{ now: state.deps.now, sleep: state.deps.sleep }
|
|
2049
|
-
);
|
|
2465
|
+
const released = await releaseUntried(state, item, fleet);
|
|
2050
2466
|
reportRepeating(
|
|
2051
2467
|
state,
|
|
2052
2468
|
"the repositories",
|
|
2053
|
-
`the gateway named repositories for
|
|
2469
|
+
`the gateway named repositories for ${tag} that this runner cannot read; upgrade the runner or the gateway. ` + (released.ok ? "The claim was released and no build started" : `No build started, but releasing the claim failed (${released.reason}), so it stays held until its lease lapses`)
|
|
2054
2470
|
);
|
|
2055
2471
|
return { kind: "unreadable_repositories" };
|
|
2056
2472
|
}
|
|
2057
2473
|
reportRepeating(state, "the repositories", void 0);
|
|
2058
2474
|
for (const entry of parseUnreachable(data.unreachable)) {
|
|
2059
|
-
log.line(
|
|
2475
|
+
log.line(`${tag}: ${entry.repository} is left out of this build because its GitHub App installation is suspended`);
|
|
2060
2476
|
}
|
|
2061
2477
|
try {
|
|
2062
|
-
await startJob(state, item, queue, claimId, itemKey, branch, identity, repositories);
|
|
2478
|
+
await startJob(state, item, queue, claimId, itemKey, branch, identity, repositories, fleet);
|
|
2063
2479
|
} catch (err) {
|
|
2064
2480
|
const stranded = [...state.jobs.values()].find((active) => active.job.claimId === claimId);
|
|
2065
2481
|
if (stranded === void 0) throw err;
|
|
2066
2482
|
const reason = `provision failed: ${describeCallFailure(err)}`;
|
|
2067
|
-
logOf(state).error(
|
|
2483
|
+
logOf(state).error(`${tag} ${reason} \xB7 ending job as failed`);
|
|
2068
2484
|
await endJob(state, stranded, "failed", { stage: "provision", reason });
|
|
2485
|
+
return { kind: "started", provisionFailed: true };
|
|
2069
2486
|
}
|
|
2070
|
-
|
|
2487
|
+
const running = [...state.jobs.values()].some((active) => active.job.claimId === claimId);
|
|
2488
|
+
return running ? { kind: "started" } : { kind: "started", provisionFailed: true };
|
|
2071
2489
|
}
|
|
2072
2490
|
function claimIsAlreadyRunning(state, claimId) {
|
|
2073
2491
|
for (const active of state.jobs.values()) {
|
|
@@ -2084,7 +2502,7 @@ function wasExtended(claim) {
|
|
|
2084
2502
|
if (!Number.isFinite(createdAtMs) || !Number.isFinite(expiresAtMs)) return false;
|
|
2085
2503
|
return expiresAtMs - createdAtMs > LEASE_SECONDS * 1e3 + 2e3;
|
|
2086
2504
|
}
|
|
2087
|
-
async function startJob(state, item, queue, claimId, itemKey, branch, identity, claimed) {
|
|
2505
|
+
async function startJob(state, item, queue, claimId, itemKey, branch, identity, claimed, fleet) {
|
|
2088
2506
|
const log = logOf(state);
|
|
2089
2507
|
const startedAt = nowOf(state);
|
|
2090
2508
|
const repositories = repositoriesOf({ repositories: claimed, repository: item.specRepository, branch });
|
|
@@ -2101,6 +2519,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2101
2519
|
itemKey,
|
|
2102
2520
|
branch,
|
|
2103
2521
|
commitIdentity: identity,
|
|
2522
|
+
...fleet === void 0 ? {} : { workItemId: fleet.workItemId, appId: fleet.appId, organizationId: fleet.organizationId, size: fleet.size },
|
|
2104
2523
|
...primary === void 0 ? {} : { repository: primary.repository, repositories },
|
|
2105
2524
|
startedAt: startedAt.toISOString()
|
|
2106
2525
|
};
|
|
@@ -2114,12 +2533,12 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2114
2533
|
ending: false
|
|
2115
2534
|
};
|
|
2116
2535
|
state.jobs.set(dirPath, active);
|
|
2117
|
-
attemptLogsOf(state).open(item.number, dirPath);
|
|
2118
|
-
const claimLine = `claimed ${queue}
|
|
2536
|
+
attemptLogsOf(state).open(item.number, dirPath, itemScope(job));
|
|
2537
|
+
const claimLine = `claimed ${queue} ${itemTag(job)} ${item.trackerKey} (claim ${shortClaim(claimId)}) \u2192 ${relativeToRunner(runnerRoot, dirPath)}`;
|
|
2119
2538
|
log.line(claimLine);
|
|
2120
|
-
attemptLogsOf(state).note(item.number, claimLine);
|
|
2539
|
+
attemptLogsOf(state).note(item.number, claimLine, itemScope(job));
|
|
2121
2540
|
const hookLimits = () => hookLimitsFor(state.config, startedAt.getTime(), state.deps.now);
|
|
2122
|
-
const tag =
|
|
2541
|
+
const tag = itemTag(job);
|
|
2123
2542
|
const noteOnce = attemptNote(log, tag);
|
|
2124
2543
|
const containerMode = isBuiltinContainer(state.config.hooks.provision);
|
|
2125
2544
|
const vmMode = isBuiltinVm(state.config.hooks.provision);
|
|
@@ -2127,7 +2546,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2127
2546
|
if (containerMode || vmMode) {
|
|
2128
2547
|
active.job.environment = { isolation: vmMode ? "vm" : "container", cliVersion: state.deps.cliVersion ?? resolveVersion() };
|
|
2129
2548
|
writeJobFile(jobFilePath, active.job);
|
|
2130
|
-
tokens = buildTokensFor(state, active
|
|
2549
|
+
tokens = buildTokensFor(state, active);
|
|
2131
2550
|
runtimeOf(state).credentials.set(dirPath, tokens);
|
|
2132
2551
|
let asking;
|
|
2133
2552
|
try {
|
|
@@ -2143,14 +2562,14 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2143
2562
|
}
|
|
2144
2563
|
if (active.ending) return;
|
|
2145
2564
|
if (state.config.managed === true) {
|
|
2146
|
-
const fetched = await fetchManagedClaudeKey(state.config,
|
|
2565
|
+
const fetched = await fetchManagedClaudeKey(claimCalls(state.config, active.job), claimItem(state.config, active.job), { fetchImpl: state.deps.fetchImpl, now: state.deps.now, sleep: state.deps.sleep });
|
|
2147
2566
|
if (active.ending) return;
|
|
2148
2567
|
if (!fetched.ok) {
|
|
2149
2568
|
await endOnManagedRefusal(state, active, fetched, { what: "Anthropic key", was: "was" });
|
|
2150
2569
|
return;
|
|
2151
2570
|
}
|
|
2152
2571
|
holdManagedClaudeKey(dirPath, fetched.apiKey);
|
|
2153
|
-
const settings = await fetchManagedBuildSettings(state.config,
|
|
2572
|
+
const settings = await fetchManagedBuildSettings(claimCalls(state.config, active.job), claimItem(state.config, active.job), { fetchImpl: state.deps.fetchImpl, now: state.deps.now, sleep: state.deps.sleep });
|
|
2154
2573
|
if (active.ending) return;
|
|
2155
2574
|
if (!settings.ok) {
|
|
2156
2575
|
await endOnManagedRefusal(state, active, settings, { what: "settings", was: "were" });
|
|
@@ -2164,7 +2583,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2164
2583
|
setStage(job, dirPath, "report");
|
|
2165
2584
|
log.line(`${tag} report started \xB7 phase start`);
|
|
2166
2585
|
const reportStartedMs = nowOf(state).getTime();
|
|
2167
|
-
const startEnv = { ...jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), OUTERLAYER_PHASE: "start" };
|
|
2586
|
+
const startEnv = { ...jobEnv(claimCalls(state.config, active.job), item.number, itemKey, dirPath, process.env, branch, identity?.author, spoolHomeFor(state.config, active.job)), OUTERLAYER_PHASE: "start" };
|
|
2168
2587
|
const report = await runHook(
|
|
2169
2588
|
state.config.hooks.report,
|
|
2170
2589
|
startEnv,
|
|
@@ -2195,7 +2614,8 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2195
2614
|
let cgroup;
|
|
2196
2615
|
if (containerMode) {
|
|
2197
2616
|
const provisioned = await provisionContainer({
|
|
2198
|
-
config: state.config,
|
|
2617
|
+
config: claimCalls(state.config, active.job),
|
|
2618
|
+
...active.job.organizationId === void 0 ? {} : { organizationId: active.job.organizationId, claimId: active.job.claimId },
|
|
2199
2619
|
runtime: runtimeOf(state),
|
|
2200
2620
|
item,
|
|
2201
2621
|
itemKey,
|
|
@@ -2235,7 +2655,8 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2235
2655
|
}
|
|
2236
2656
|
} else if (vmMode) {
|
|
2237
2657
|
const provisioned = await provisionMicroVm({
|
|
2238
|
-
config: state.config,
|
|
2658
|
+
config: claimCalls(state.config, active.job),
|
|
2659
|
+
...active.job.organizationId === void 0 ? {} : { organizationId: active.job.organizationId, claimId: active.job.claimId },
|
|
2239
2660
|
runtime: runtimeOf(state),
|
|
2240
2661
|
item,
|
|
2241
2662
|
itemKey,
|
|
@@ -2311,7 +2732,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2311
2732
|
} else {
|
|
2312
2733
|
const runtime = runtimeOf(state);
|
|
2313
2734
|
const build = await startHostBuild(state, active, primary?.repository);
|
|
2314
|
-
const provisionEnv = provisionEnvironment(jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), {
|
|
2735
|
+
const provisionEnv = provisionEnvironment(jobEnv(claimCalls(state.config, active.job), item.number, itemKey, dirPath, process.env, branch, identity?.author, spoolHomeFor(state.config, active.job)), {
|
|
2315
2736
|
buildDir: build.state.buildDir,
|
|
2316
2737
|
repository: primary?.repository ?? null,
|
|
2317
2738
|
recipeCommit: build.recipeCommit,
|
|
@@ -2403,11 +2824,11 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2403
2824
|
}
|
|
2404
2825
|
active.job.environment = hostEnvironment(named, cliVersion);
|
|
2405
2826
|
writeJobFile(jobFilePath, active.job);
|
|
2406
|
-
await deliverExecMaven(build?.state, provisioned.workdir, jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), runWithSpawn(state.deps.spawnFn), noteOnce);
|
|
2827
|
+
await deliverExecMaven(build?.state, provisioned.workdir, jobEnv(claimCalls(state.config, active.job), item.number, itemKey, dirPath, process.env, branch, identity?.author, spoolHomeFor(state.config, active.job)), runWithSpawn(state.deps.spawnFn), noteOnce);
|
|
2407
2828
|
if (active.ending) return;
|
|
2408
2829
|
writeJobFile(jobFilePath, active.job);
|
|
2409
2830
|
if (named.exec !== void 0 && named.recipe !== void 0) {
|
|
2410
|
-
const lifecycleEnv = hostCommandEnvironment(jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), build.state, state.config, void 0, noteOnce);
|
|
2831
|
+
const lifecycleEnv = hostCommandEnvironment(jobEnv(claimCalls(state.config, active.job), item.number, itemKey, dirPath, process.env, branch, identity?.author, spoolHomeFor(state.config, active.job)), build.state, state.config, void 0, noteOnce);
|
|
2411
2832
|
for (const step of lifecycleSteps(named.recipe.apply.lifecycle)) {
|
|
2412
2833
|
if (active.ending) return;
|
|
2413
2834
|
log.line(`${tag} lifecycle ${step.name} (${step.source})`);
|
|
@@ -2446,7 +2867,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2446
2867
|
vmMode,
|
|
2447
2868
|
workdir,
|
|
2448
2869
|
repository: primary?.repository,
|
|
2449
|
-
lookup: (repository) => resolveGoverningSource({ config: state.config, repository, fetchImpl: state.deps.fetchImpl }).catch(() => void 0)
|
|
2870
|
+
lookup: (repository) => resolveGoverningSource({ config: claimCalls(state.config, active.job), repository, fetchImpl: state.deps.fetchImpl }).catch(() => void 0)
|
|
2450
2871
|
});
|
|
2451
2872
|
if (skillDecision.kind === "refuse") {
|
|
2452
2873
|
log.error(`${tag} ${skillDecision.reason} \xB7 ending job as failed`);
|
|
@@ -2503,7 +2924,7 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2503
2924
|
}
|
|
2504
2925
|
} else {
|
|
2505
2926
|
const build = job.build;
|
|
2506
|
-
const commandEnv = hostCommandEnvironment(jobEnv(state.config, item.number, itemKey, dirPath, process.env, branch, identity?.author), build, state.config, identity?.hooksPath, noteOnce);
|
|
2927
|
+
const commandEnv = hostCommandEnvironment(jobEnv(claimCalls(state.config, active.job), item.number, itemKey, dirPath, process.env, branch, identity?.author, spoolHomeFor(state.config, active.job)), build, state.config, identity?.hooksPath, noteOnce);
|
|
2507
2928
|
running = build.exec === void 0 ? runCommand(cmdline, commandEnv, workdir, join(dirPath, "command.log"), state.deps.spawnFn) : runCommand(execCommandLine(build.exec, workdir, cmdline), commandEnv, dirPath, join(dirPath, "command.log"), state.deps.spawnFn);
|
|
2508
2929
|
}
|
|
2509
2930
|
} catch (err) {
|
|
@@ -2555,10 +2976,10 @@ async function startJob(state, item, queue, claimId, itemKey, branch, identity,
|
|
|
2555
2976
|
void endJob(state, active, code === 0 ? "ok" : "failed");
|
|
2556
2977
|
});
|
|
2557
2978
|
}
|
|
2558
|
-
function buildTokensFor(state, active
|
|
2979
|
+
function buildTokensFor(state, active) {
|
|
2559
2980
|
return new BuildTokens({
|
|
2560
|
-
config: state.config,
|
|
2561
|
-
item,
|
|
2981
|
+
config: claimCalls(state.config, active.job),
|
|
2982
|
+
item: claimItem(state.config, active.job),
|
|
2562
2983
|
primary: primaryOf(repositoriesOf({ repositories: active.job.repositories, repository: active.job.repository, branch: active.job.branch }))?.repository,
|
|
2563
2984
|
fetchImpl: state.deps.fetchImpl,
|
|
2564
2985
|
now: state.deps.now,
|
|
@@ -2573,7 +2994,7 @@ function buildTokensFor(state, active, item) {
|
|
|
2573
2994
|
}
|
|
2574
2995
|
async function endOnManagedRefusal(state, active, refused, subject) {
|
|
2575
2996
|
const log = logOf(state);
|
|
2576
|
-
const tag =
|
|
2997
|
+
const tag = `${itemTag(active.job)}`;
|
|
2577
2998
|
const code = refused.code ?? "unavailable";
|
|
2578
2999
|
if (refused.code === "claim_expired" || refused.code === "claim_not_held") {
|
|
2579
3000
|
log.error(`${tag} the build's ${subject.what} ${subject.was} refused: ${code} \xB7 ending job as lease_lost`);
|
|
@@ -2588,7 +3009,7 @@ async function endOnManagedRefusal(state, active, refused, subject) {
|
|
|
2588
3009
|
}
|
|
2589
3010
|
async function endOnTokenRefusal(state, active, err, repository) {
|
|
2590
3011
|
const log = logOf(state);
|
|
2591
|
-
const tag =
|
|
3012
|
+
const tag = `${itemTag(active.job)}`;
|
|
2592
3013
|
const refusal = classifyRefusal(err);
|
|
2593
3014
|
const which = repository === void 0 ? "the build's read token" : `the build's read token for ${repository}`;
|
|
2594
3015
|
if (refusal.kind === "retry") {
|
|
@@ -2605,7 +3026,7 @@ async function endOnTokenRefusal(state, active, err, repository) {
|
|
|
2605
3026
|
}
|
|
2606
3027
|
async function endOnProvisionFailure(state, active, tokens, took, reason) {
|
|
2607
3028
|
const log = logOf(state);
|
|
2608
|
-
const tag =
|
|
3029
|
+
const tag = `${itemTag(active.job)}`;
|
|
2609
3030
|
if (tokens.refusal()?.kind === "lease_lost") {
|
|
2610
3031
|
log.error(`${tag} provision failed (${took}): a token request was refused as a lost lease \xB7 ending job as lease_lost`);
|
|
2611
3032
|
await endJob(state, active, "lease_lost");
|
|
@@ -2643,10 +3064,10 @@ async function startHostBuild(state, active, repository) {
|
|
|
2643
3064
|
const runtime = runtimeOf(state);
|
|
2644
3065
|
let failed2;
|
|
2645
3066
|
try {
|
|
2646
|
-
await startHostServices({ runtime, config: state.config, build, jobDir: active.dirPath, itemKey: active.job.itemKey, process: false, routeLog: (line) => logOf(state).line(
|
|
3067
|
+
await startHostServices({ runtime, config: state.config, build, jobDir: active.dirPath, itemKey: active.job.itemKey, process: false, routeLog: (line) => logOf(state).line(`${itemTag(active.job)} ${line}`) });
|
|
2647
3068
|
} catch (err) {
|
|
2648
3069
|
failed2 = describeCallFailure(err);
|
|
2649
|
-
logOf(state).error(
|
|
3070
|
+
logOf(state).error(`${itemTag(active.job)} the tunnel and destination servers could not start: ${failed2} \xB7 continuing`);
|
|
2650
3071
|
}
|
|
2651
3072
|
writeJobFile(join(active.dirPath, "job.json"), active.job);
|
|
2652
3073
|
let recipeCommit;
|
|
@@ -2671,16 +3092,17 @@ async function endJob(state, active, outcome, ending) {
|
|
|
2671
3092
|
const finished = await finishJob(active.job, active.dirPath, outcome, state.config, finishDepsOf(state), ending);
|
|
2672
3093
|
active.job = finished;
|
|
2673
3094
|
} catch (err) {
|
|
2674
|
-
logOf(state).error(
|
|
3095
|
+
logOf(state).error(`${itemTag(active.job)} could not be ended cleanly: ${describeCallFailure(err)}`);
|
|
2675
3096
|
} finally {
|
|
2676
3097
|
state.jobs.delete(active.dirPath);
|
|
3098
|
+
if (!(outcome === "failed" && ending?.stage === "provision")) state.afterBuildEnd?.();
|
|
2677
3099
|
}
|
|
2678
3100
|
}
|
|
2679
3101
|
async function renewLeaseIfDue(state, active, nowMs) {
|
|
2680
3102
|
const log = logOf(state);
|
|
2681
|
-
const tag =
|
|
3103
|
+
const tag = `${itemTag(active.job)}`;
|
|
2682
3104
|
if (!needsRenew(active.lastRenewAtMs, nowMs)) return true;
|
|
2683
|
-
const renew = await retrySend(() => renewLease(state.config, active.job.
|
|
3105
|
+
const renew = await retrySend(() => renewLease(claimCalls(state.config, active.job), claimItem(state.config, active.job), state.deps.fetchImpl), {
|
|
2684
3106
|
now: state.deps.now,
|
|
2685
3107
|
sleep: state.deps.sleep
|
|
2686
3108
|
});
|
|
@@ -2709,7 +3131,7 @@ async function renewLeaseIfDue(state, active, nowMs) {
|
|
|
2709
3131
|
var COMMAND_START_GRACE_MS = 3e4;
|
|
2710
3132
|
async function endStoppedBuild(state, active, where) {
|
|
2711
3133
|
const log = logOf(state);
|
|
2712
|
-
const tag =
|
|
3134
|
+
const tag = `${itemTag(active.job)}`;
|
|
2713
3135
|
const read = where === "stopped" ? await readStoppedBuild(state, active.dirPath, active.job) : { code: void 0, readable: true };
|
|
2714
3136
|
if (active.ending) return;
|
|
2715
3137
|
active.job.container.stopped = true;
|
|
@@ -2762,7 +3184,7 @@ async function checkDisk(state, active, container, tag) {
|
|
|
2762
3184
|
}
|
|
2763
3185
|
async function endOrWaitForMicroVm(state, active) {
|
|
2764
3186
|
const log = logOf(state);
|
|
2765
|
-
const tag =
|
|
3187
|
+
const tag = `${itemTag(active.job)}`;
|
|
2766
3188
|
const microvm = active.job.microvm;
|
|
2767
3189
|
const runtime = runtimeOf(state);
|
|
2768
3190
|
const helper = await runtime.vm.helper.probe();
|
|
@@ -2787,7 +3209,7 @@ async function sampleUsage(state, active) {
|
|
|
2787
3209
|
active.job.guestUsage = foldSample(active.job.guestUsage, sample);
|
|
2788
3210
|
writeJobFile(join(active.dirPath, "job.json"), active.job);
|
|
2789
3211
|
} catch (err) {
|
|
2790
|
-
logOf(state).error(
|
|
3212
|
+
logOf(state).error(`${itemTag(active.job)} reading the build's guest for its usage failed: ${describeCallFailure(err)}`);
|
|
2791
3213
|
} finally {
|
|
2792
3214
|
active.sampling = false;
|
|
2793
3215
|
}
|
|
@@ -2798,7 +3220,7 @@ async function probeBuild(state, active) {
|
|
|
2798
3220
|
if (tracked === void 0 || active.ending || active.probing) return;
|
|
2799
3221
|
active.probing = true;
|
|
2800
3222
|
const log = logOf(state);
|
|
2801
|
-
const tag =
|
|
3223
|
+
const tag = `${itemTag(active.job)}`;
|
|
2802
3224
|
try {
|
|
2803
3225
|
const runtime = runtimeOf(state);
|
|
2804
3226
|
for (; ; ) {
|
|
@@ -2841,7 +3263,7 @@ async function probeBuild(state, active) {
|
|
|
2841
3263
|
}
|
|
2842
3264
|
function recordCommandExit(state, active, code) {
|
|
2843
3265
|
const log = logOf(state);
|
|
2844
|
-
const tag =
|
|
3266
|
+
const tag = `${itemTag(active.job)}`;
|
|
2845
3267
|
const file = join(active.dirPath, EXIT_CODE_FILE);
|
|
2846
3268
|
writeFileSync(`${file}.partial`, `${code}
|
|
2847
3269
|
`, { mode: RUNNER_FILE_MODE });
|
|
@@ -2858,7 +3280,7 @@ async function watchTick(state, active, nowMs, logMtimeMs) {
|
|
|
2858
3280
|
return;
|
|
2859
3281
|
}
|
|
2860
3282
|
const log = logOf(state);
|
|
2861
|
-
const tag =
|
|
3283
|
+
const tag = `${itemTag(active.job)}`;
|
|
2862
3284
|
const pid = active.job.pid;
|
|
2863
3285
|
const started = pid !== void 0 || isTracked(active.job);
|
|
2864
3286
|
if (pid !== void 0 && !isGroupAlive(pid)) {
|
|
@@ -2963,24 +3385,24 @@ async function rejoinContainerBuild(state, adopted) {
|
|
|
2963
3385
|
const container = job.container;
|
|
2964
3386
|
const cores = container.cpuset === void 0 ? void 0 : parseCpuList(container.cpuset);
|
|
2965
3387
|
if (cores !== void 0) runtimeOf(state).cpus.hold(container.buildId, cores);
|
|
2966
|
-
const tokens = buildTokensFor(state, adopted
|
|
3388
|
+
const tokens = buildTokensFor(state, adopted);
|
|
2967
3389
|
runtimeOf(state).credentials.set(dirPath, tokens);
|
|
2968
|
-
await ensureTunnel(runtimeOf(state), container, dirPath, config, job.itemKey, job.branch, job.repositories, (line) => logOf(state).line(
|
|
3390
|
+
await ensureTunnel(runtimeOf(state), container, dirPath, config, job.itemKey, job.branch, job.repositories, (line) => logOf(state).line(`${itemTag(job)} ${line}`), () => leaseExpiresAtMs(adopted.job));
|
|
2969
3391
|
try {
|
|
2970
3392
|
await tokens.token("read");
|
|
2971
3393
|
keepGhConfigCurrent(
|
|
2972
3394
|
tokens,
|
|
2973
3395
|
container.buildDir,
|
|
2974
|
-
(err) => logOf(state).error(
|
|
3396
|
+
(err) => logOf(state).error(`${itemTag(job)} the read token could not be written for gh: ${describeCallFailure(err)}`),
|
|
2975
3397
|
filesAreCopied(container) ? {
|
|
2976
3398
|
runtime: runtimeOf(state),
|
|
2977
3399
|
state: container,
|
|
2978
3400
|
secrets: () => tokens.issuedTokens(),
|
|
2979
|
-
onFailure: (reason) => logOf(state).error(
|
|
3401
|
+
onFailure: (reason) => logOf(state).error(`${itemTag(job)} ${reason}`)
|
|
2980
3402
|
} : void 0
|
|
2981
3403
|
);
|
|
2982
3404
|
} catch (err) {
|
|
2983
|
-
logOf(state).error(
|
|
3405
|
+
logOf(state).error(`${itemTag(job)} could not give the adopted build a read token: ${describeCallFailure(err)}`);
|
|
2984
3406
|
}
|
|
2985
3407
|
}
|
|
2986
3408
|
async function adoptTrackedBuilds(state, jobsDirPath) {
|
|
@@ -2989,6 +3411,7 @@ async function adoptTrackedBuilds(state, jobsDirPath) {
|
|
|
2989
3411
|
const runtime = runtimeOf(state);
|
|
2990
3412
|
for (const { dirPath, job, known } of listTrackedJobs(jobsDirPath)) {
|
|
2991
3413
|
if (!known || job.container === void 0) continue;
|
|
3414
|
+
if (leftByRunnerKey(state.config, job)) continue;
|
|
2992
3415
|
const container = job.container;
|
|
2993
3416
|
const where = await runtime.host.state(container.container).catch(() => "unknown");
|
|
2994
3417
|
if (where === "stopped" || where === "missing") {
|
|
@@ -2997,15 +3420,15 @@ async function adoptTrackedBuilds(state, jobsDirPath) {
|
|
|
2997
3420
|
}
|
|
2998
3421
|
const adopted = { job, dirPath, lastRenewAtMs: 0, cmd: null, cpuSample: null, ending: false };
|
|
2999
3422
|
state.jobs.set(dirPath, adopted);
|
|
3000
|
-
attemptLogs.open(job.item, dirPath);
|
|
3423
|
+
attemptLogs.open(job.item, dirPath, itemScope(job));
|
|
3001
3424
|
await rejoinContainerBuild(state, adopted);
|
|
3002
3425
|
if (!await renewLeaseIfDue(state, adopted, nowOf(state).getTime())) continue;
|
|
3003
3426
|
const sender = await restartSender({ config: state.config, job: adopted.job, jobDir: dirPath, deps: { spawnFn: state.deps.spawnFn, runtime, log, sleep: state.deps.sleep } });
|
|
3004
3427
|
adopted.job = sender === void 0 ? { ...adopted.job, sender: void 0 } : { ...adopted.job, sender };
|
|
3005
3428
|
writeJobFile(join(dirPath, "job.json"), adopted.job);
|
|
3006
|
-
const line = `adopted job ${job.queue}
|
|
3429
|
+
const line = `adopted job ${job.queue} ${itemTag(job)} (claim ${shortClaim(job.claimId)}) \xB7 command tracked in container ${container.container}`;
|
|
3007
3430
|
log.line(line);
|
|
3008
|
-
attemptLogs.note(job.item, line);
|
|
3431
|
+
attemptLogs.note(job.item, line, itemScope(job));
|
|
3009
3432
|
void probeBuild(state, adopted);
|
|
3010
3433
|
}
|
|
3011
3434
|
}
|
|
@@ -3013,6 +3436,7 @@ async function planMicroVmAdoption(state, jobsDirPath) {
|
|
|
3013
3436
|
const plan = { adopt: [], away: [], unservable: [], unasked: [] };
|
|
3014
3437
|
for (const { dirPath, job, known } of listTrackedJobs(jobsDirPath)) {
|
|
3015
3438
|
if (!known || !isMicroVmTracked(job)) continue;
|
|
3439
|
+
if (leftByRunnerKey(state.config, job)) continue;
|
|
3016
3440
|
const verdict = await judgeMicroVmAdoption({ job, runtime: runtimeOf(state), onError: (line) => logOf(state).error(line) });
|
|
3017
3441
|
plan[verdict === "adopt" ? "adopt" : verdict === "stopped" ? "away" : verdict === "unasked" ? "unasked" : "unservable"].push({ dirPath, job });
|
|
3018
3442
|
}
|
|
@@ -3023,7 +3447,7 @@ async function rejoinMicroVmBuild(state, adopted) {
|
|
|
3023
3447
|
const config = state.config;
|
|
3024
3448
|
const microvm = job.microvm;
|
|
3025
3449
|
const runtime = runtimeOf(state);
|
|
3026
|
-
const tag =
|
|
3450
|
+
const tag = `${itemTag(job)}`;
|
|
3027
3451
|
const log = logOf(state);
|
|
3028
3452
|
const inherited = runtime.vm.fdStore.inherited();
|
|
3029
3453
|
const inherit = (socket) => {
|
|
@@ -3031,7 +3455,7 @@ async function rejoinMicroVmBuild(state, adopted) {
|
|
|
3031
3455
|
return fd === void 0 ? void 0 : (server) => runtime.vm.fdStore.listenOn(server, fd);
|
|
3032
3456
|
};
|
|
3033
3457
|
if (config.managed === true) {
|
|
3034
|
-
const fetched = await fetchManagedClaudeKey(config, job
|
|
3458
|
+
const fetched = await fetchManagedClaudeKey(claimCalls(config, job), claimItem(config, job), {
|
|
3035
3459
|
fetchImpl: state.deps.fetchImpl,
|
|
3036
3460
|
now: state.deps.now,
|
|
3037
3461
|
sleep: state.deps.sleep,
|
|
@@ -3044,7 +3468,7 @@ async function rejoinMicroVmBuild(state, adopted) {
|
|
|
3044
3468
|
return { reason: `the adopted build's Anthropic key could not be had after the runner restarted: ${cause}`, why: "the gateway did not give its Anthropic key again" };
|
|
3045
3469
|
}
|
|
3046
3470
|
holdManagedClaudeKey(dirPath, fetched.apiKey);
|
|
3047
|
-
const settings = await fetchManagedBuildSettings(config, job
|
|
3471
|
+
const settings = await fetchManagedBuildSettings(claimCalls(config, job), claimItem(config, job), {
|
|
3048
3472
|
fetchImpl: state.deps.fetchImpl,
|
|
3049
3473
|
now: state.deps.now,
|
|
3050
3474
|
sleep: state.deps.sleep,
|
|
@@ -3060,13 +3484,13 @@ async function rejoinMicroVmBuild(state, adopted) {
|
|
|
3060
3484
|
job.managedVariables = [.../* @__PURE__ */ new Set([...job.managedVariables ?? [], ...settings.settings.variables.map(({ name }) => name)])];
|
|
3061
3485
|
writeJobFile(join(dirPath, "job.json"), job);
|
|
3062
3486
|
}
|
|
3063
|
-
const tokens = buildTokensFor(state, adopted
|
|
3487
|
+
const tokens = buildTokensFor(state, adopted);
|
|
3064
3488
|
runtime.credentials.set(dirPath, tokens);
|
|
3065
3489
|
let servicesServed = false;
|
|
3066
3490
|
let leaseServed = false;
|
|
3067
3491
|
try {
|
|
3068
3492
|
const repositories = repositoriesOf({ repositories: job.repositories, repository: microvm.repository, branch: job.branch });
|
|
3069
|
-
const gitRemote = repositories.length === 0 ? void 0 : gitRemoteFor(runtime, { repositories, tokens, buildDir: microvm.buildDir, log: (line) => log.line(`${tag} ${line}`) });
|
|
3493
|
+
const gitRemote = repositories.length === 0 ? void 0 : gitRemoteFor(runtime, { repositories, tokens, buildDir: microvm.buildDir, organizationId: job.organizationId, log: (line) => log.line(`${tag} ${line}`) });
|
|
3070
3494
|
runtime.tunnels.set(microvm.buildId, await startServicesFor(runtime, config, microvm.buildDir, dirPath, job.itemKey, microvm.builtIns === true, gitRemote, "socket", void 0, inherit));
|
|
3071
3495
|
servicesServed = true;
|
|
3072
3496
|
} catch (err) {
|
|
@@ -3104,7 +3528,7 @@ async function adoptTrackedMicroVms(state, plan) {
|
|
|
3104
3528
|
for (const { dirPath, job } of plan.adopt) {
|
|
3105
3529
|
const adopted = { job, dirPath, lastRenewAtMs: 0, cmd: null, cpuSample: null, ending: false };
|
|
3106
3530
|
state.jobs.set(dirPath, adopted);
|
|
3107
|
-
attemptLogs.open(job.item, dirPath);
|
|
3531
|
+
attemptLogs.open(job.item, dirPath, itemScope(job));
|
|
3108
3532
|
const rejoined = await rejoinMicroVmBuild(state, adopted);
|
|
3109
3533
|
if (rejoined !== true) {
|
|
3110
3534
|
state.jobs.delete(dirPath);
|
|
@@ -3115,9 +3539,9 @@ async function adoptTrackedMicroVms(state, plan) {
|
|
|
3115
3539
|
const sender = await restartSender({ config: state.config, job: adopted.job, jobDir: dirPath, deps: { spawnFn: state.deps.spawnFn, runtime, log, sleep: state.deps.sleep } });
|
|
3116
3540
|
adopted.job = sender === void 0 ? { ...adopted.job, sender: void 0 } : { ...adopted.job, sender };
|
|
3117
3541
|
writeJobFile(join(dirPath, "job.json"), adopted.job);
|
|
3118
|
-
const line = `adopted job ${job.queue}
|
|
3542
|
+
const line = `adopted job ${job.queue} ${itemTag(job)} (claim ${shortClaim(job.claimId)}) \xB7 command tracked in microVM build ${job.microvm.buildId}`;
|
|
3119
3543
|
log.line(line);
|
|
3120
|
-
attemptLogs.note(job.item, line);
|
|
3544
|
+
attemptLogs.note(job.item, line, itemScope(job));
|
|
3121
3545
|
void probeBuild(state, adopted);
|
|
3122
3546
|
}
|
|
3123
3547
|
await removeUnadoptedSockets(state, new Set(plan.adopt.map(({ job }) => job.microvm.buildId)));
|
|
@@ -3137,9 +3561,9 @@ async function removeUnadoptedSockets(state, adopted) {
|
|
|
3137
3561
|
async function finishMicroVmWhileAway(state, dirPath, job, reason, why) {
|
|
3138
3562
|
const log = logOf(state);
|
|
3139
3563
|
if ((job.managedVariables?.length ?? 0) > 0 && managedBuildSettingsFor(dirPath) === void 0) {
|
|
3140
|
-
const settings = await fetchManagedBuildSettings(state.config, job.
|
|
3564
|
+
const settings = await fetchManagedBuildSettings(claimCalls(state.config, job), claimItem(state.config, job), { fetchImpl: state.deps.fetchImpl, now: state.deps.now, sleep: state.deps.sleep });
|
|
3141
3565
|
if (settings.ok) holdManagedBuildSettings(dirPath, settings.settings);
|
|
3142
|
-
else log.error(
|
|
3566
|
+
else log.error(`${itemTag(job)} the build's settings could not be had again: ${settings.code ?? "unavailable"}`);
|
|
3143
3567
|
}
|
|
3144
3568
|
const recorded = outcomeFromExitCode(dirPath);
|
|
3145
3569
|
let outcome = "interrupted";
|
|
@@ -3152,7 +3576,7 @@ async function finishMicroVmWhileAway(state, dirPath, job, reason, why) {
|
|
|
3152
3576
|
}
|
|
3153
3577
|
const finished = await finishJob(job, dirPath, outcome, state.config, finishDepsOf(state), ending);
|
|
3154
3578
|
log.line(
|
|
3155
|
-
`recovered job ${finished.queue}
|
|
3579
|
+
`recovered job ${finished.queue} ${itemTag(finished)} (claim ${shortClaim(finished.claimId)}) as ${outcome}: ${recorded === void 0 ? `${why}, and no exit code had been read` : `its command recorded exit ${recorded.code}`}`
|
|
3156
3580
|
);
|
|
3157
3581
|
}
|
|
3158
3582
|
async function readStoppedBuild(state, dirPath, job) {
|
|
@@ -3174,10 +3598,10 @@ async function readStoppedBuild(state, dirPath, job) {
|
|
|
3174
3598
|
}
|
|
3175
3599
|
if (attempt < STOPPED_READ_ATTEMPTS) await (state.deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(STOPPED_READ_RETRY_MS);
|
|
3176
3600
|
}
|
|
3177
|
-
logOf(state).error(
|
|
3601
|
+
logOf(state).error(`${itemTag(job)} the stopped build's volume could not be read after ${STOPPED_READ_ATTEMPTS} tries${network === void 0 ? "" : `: ${network}`}`);
|
|
3178
3602
|
return { code: void 0, readable: false };
|
|
3179
3603
|
} catch (err) {
|
|
3180
|
-
logOf(state).error(
|
|
3604
|
+
logOf(state).error(`${itemTag(job)} the stopped build's volume could not be read: ${describeCallFailure(err)}`);
|
|
3181
3605
|
return { code: void 0, readable: false };
|
|
3182
3606
|
}
|
|
3183
3607
|
}
|
|
@@ -3213,7 +3637,7 @@ async function finishTrackedWhileAway(state, dirPath, job, where) {
|
|
|
3213
3637
|
}
|
|
3214
3638
|
const finished = await finishJob(job, dirPath, outcome, state.config, finishDepsOf(state), ending);
|
|
3215
3639
|
log.line(
|
|
3216
|
-
`recovered job ${finished.queue}
|
|
3640
|
+
`recovered job ${finished.queue} ${itemTag(finished)} (claim ${shortClaim(finished.claimId)}) as ${outcome}: ${code === void 0 ? `its container ${where === "missing" ? "is gone" : "stopped"} and its volume ${read.readable ? "holds no exit code" : "could not be read"}` : `its command recorded exit ${code}`}`
|
|
3217
3641
|
);
|
|
3218
3642
|
}
|
|
3219
3643
|
async function canLeaveBuilds(state) {
|
|
@@ -3255,7 +3679,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3255
3679
|
check: update.check ? () => update.check(configPath) : async () => {
|
|
3256
3680
|
const held = heldClaudeCredential();
|
|
3257
3681
|
const env = held === void 0 ? process.env : { ...process.env, [held.variable]: held.value };
|
|
3258
|
-
return (await import('./check-
|
|
3682
|
+
return (await import('./check-PVB3GPFQ.js')).runRunnerCheck(configPath, { fetchImpl: deps.fetchImpl, env });
|
|
3259
3683
|
},
|
|
3260
3684
|
now: () => (deps.now ?? (() => /* @__PURE__ */ new Date()))().getTime(),
|
|
3261
3685
|
log
|
|
@@ -3268,10 +3692,12 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3268
3692
|
const loaded = loadRunnerConfig(configPath);
|
|
3269
3693
|
let check = "hooks";
|
|
3270
3694
|
let probeRefusal;
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3695
|
+
if (loaded.fleet !== true) {
|
|
3696
|
+
startup.report = (err) => reportFailedStart(
|
|
3697
|
+
{ config: loaded, cliVersion: deps.cliVersion ?? resolveVersion(), check, error: err, refusal: probeRefusal },
|
|
3698
|
+
deps.fetchImpl
|
|
3699
|
+
);
|
|
3700
|
+
}
|
|
3275
3701
|
const hookProblem = checkHooks(loaded);
|
|
3276
3702
|
if (hookProblem) throw new RunnerStartError(hookProblem);
|
|
3277
3703
|
check = "image";
|
|
@@ -3285,13 +3711,15 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3285
3711
|
log: (line) => log.error(line)
|
|
3286
3712
|
}).catch((err) => log.error(`the runner's Docker identity could not be stored: ${describeCallFailure(err)} \xB7 using the names its folder's path gives`));
|
|
3287
3713
|
}
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3714
|
+
if (config.fleet !== true) {
|
|
3715
|
+
check = "credentials";
|
|
3716
|
+
const probe = await probeCredentials(config, deps.fetchImpl, deps.image === void 0 ? "config" : "variable");
|
|
3717
|
+
if (!probe.ok && probe.fatal) {
|
|
3718
|
+
probeRefusal = { status: probe.status, code: probe.code };
|
|
3719
|
+
throw new RunnerStartError(probe.reason);
|
|
3720
|
+
}
|
|
3721
|
+
if (!probe.ok) log.error(`the gateway did not answer at start: ${probe.reason} \xB7 starting anyway`);
|
|
3293
3722
|
}
|
|
3294
|
-
if (!probe.ok) log.error(`the gateway did not answer at start: ${probe.reason} \xB7 starting anyway`);
|
|
3295
3723
|
check = "pid-file";
|
|
3296
3724
|
mkdirSync(jobsDir(configPath), { recursive: true, mode: RUNNER_DIR_MODE });
|
|
3297
3725
|
const self = { pid: process.pid, start: startTimeOf(process.pid) ?? "" };
|
|
@@ -3355,21 +3783,22 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3355
3783
|
for (const buildId of sweptBuilders) log.line(`removed builder microVM ${buildId}, left by an earlier runner`);
|
|
3356
3784
|
const attemptLogs = attemptLogsOf(state);
|
|
3357
3785
|
for (const { job, format } of listUnreadableJobs(jobsDir(configPath))) {
|
|
3358
|
-
log.error(`leaving job ${job.queue}
|
|
3786
|
+
log.error(`leaving job ${job.queue} ${itemTag(job)} alone: its job-state format ${format} is not one this version reads`);
|
|
3359
3787
|
}
|
|
3360
3788
|
for (const { dirPath, job } of listAdoptableJobs(jobsDir(configPath))) {
|
|
3789
|
+
if (leftByRunnerKey(config, job)) continue;
|
|
3361
3790
|
const adopted = { job, dirPath, lastRenewAtMs: nowMs(), cmd: null, cpuSample: null, ending: false };
|
|
3362
3791
|
state.jobs.set(dirPath, adopted);
|
|
3363
|
-
attemptLogs.open(job.item, dirPath);
|
|
3792
|
+
attemptLogs.open(job.item, dirPath, itemScope(job));
|
|
3364
3793
|
if (job.container !== void 0) await rejoinContainerBuild(state, adopted);
|
|
3365
3794
|
if (job.build !== void 0 && !runtimeOf(state).tunnels.has(job.build.buildId)) {
|
|
3366
3795
|
await startHostServices({ runtime: runtimeOf(state), config, build: job.build, jobDir: dirPath, itemKey: job.itemKey, process: job.build.workDir !== void 0 }).catch((err) => {
|
|
3367
|
-
logOf(state).error(`the tunnel of
|
|
3796
|
+
logOf(state).error(`the tunnel of ${itemTag(job)} could not start again: ${describeCallFailure(err)}`);
|
|
3368
3797
|
});
|
|
3369
3798
|
}
|
|
3370
|
-
const adoptedLine = `adopted job ${job.queue}
|
|
3799
|
+
const adoptedLine = `adopted job ${job.queue} ${itemTag(job)} (claim ${shortClaim(job.claimId)}) \xB7 pid ${job.pid} still running`;
|
|
3371
3800
|
log.line(adoptedLine);
|
|
3372
|
-
attemptLogs.note(job.item, adoptedLine);
|
|
3801
|
+
attemptLogs.note(job.item, adoptedLine, itemScope(job));
|
|
3373
3802
|
}
|
|
3374
3803
|
await adoptTrackedBuilds(state, jobsDir(configPath));
|
|
3375
3804
|
await adoptTrackedMicroVms(state, microVmPlan);
|
|
@@ -3379,13 +3808,13 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3379
3808
|
log
|
|
3380
3809
|
})) {
|
|
3381
3810
|
log.line(
|
|
3382
|
-
released ? `released the lease on
|
|
3811
|
+
released ? `released the lease on ${itemTag(job)} that an earlier attempt could not \xB7 outcome ${job.outcome ?? "none"}` : `the lease on ${itemTag(job)} still cannot be released: ${job.releaseFailed ?? "no reason recorded"}`
|
|
3383
3812
|
);
|
|
3384
3813
|
}
|
|
3385
3814
|
for (const recovered of await recoverLostJobs(jobsDir(configPath), config, finishDepsOf(state))) {
|
|
3386
3815
|
const why = recovered.exitCode === void 0 ? `pid ${recovered.pid ?? "unknown"} is gone, no exit code recorded` : `its command recorded exit ${recovered.exitCode}`;
|
|
3387
3816
|
log.line(
|
|
3388
|
-
`recovered job ${recovered.queue}
|
|
3817
|
+
`recovered job ${recovered.queue} ${itemTag(recovered)} (claim ${shortClaim(recovered.claimId)}) as ${recovered.outcome ?? "interrupted"}: ${why}`
|
|
3389
3818
|
);
|
|
3390
3819
|
}
|
|
3391
3820
|
startup.report = void 0;
|
|
@@ -3395,7 +3824,10 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3395
3824
|
let exited = false;
|
|
3396
3825
|
let interrupts = 0;
|
|
3397
3826
|
state.pollTimerSeconds = state.config.pollSeconds;
|
|
3398
|
-
|
|
3827
|
+
const fleetPoller = config.fleet === true ? createFleetPoller({ everyMs: state.pollTimerSeconds * 1e3, run: () => tick(state), keepalive: () => fleetKeepalive(state) }) : void 0;
|
|
3828
|
+
if (fleetPoller !== void 0) state.afterBuildEnd = () => fleetPoller.wake();
|
|
3829
|
+
let pollTimer = fleetPoller === void 0 ? setInterval(() => void tick(state), state.pollTimerSeconds * 1e3) : void 0;
|
|
3830
|
+
fleetPoller?.start();
|
|
3399
3831
|
const hooksKind = builtinKind(config.hooks.provision);
|
|
3400
3832
|
const housekeepingTimer = hooksKind === "container" || hooksKind === "vm" ? (() => {
|
|
3401
3833
|
const everyMs = deps.housekeeping?.intervalMs ?? HOUSEKEEPING_INTERVAL_MS;
|
|
@@ -3411,7 +3843,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3411
3843
|
} catch {
|
|
3412
3844
|
}
|
|
3413
3845
|
void watchTick(state, active, now, mtime).catch((err) => {
|
|
3414
|
-
logOf(state).error(
|
|
3846
|
+
logOf(state).error(`${itemTag(active.job)} health check failed: ${describeCallFailure(err)}`);
|
|
3415
3847
|
});
|
|
3416
3848
|
}
|
|
3417
3849
|
}, deps.watchIntervalMs ?? 3e4);
|
|
@@ -3448,7 +3880,9 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3448
3880
|
if (intakeClosed) return;
|
|
3449
3881
|
intakeClosed = true;
|
|
3450
3882
|
clearInterval(pollTimer);
|
|
3451
|
-
|
|
3883
|
+
if (fleetPoller === void 0) {
|
|
3884
|
+
pollTimer = setInterval(() => void sendTickHeartbeat(state, void 0), (state.pollTimerSeconds ?? state.config.pollSeconds) * 1e3);
|
|
3885
|
+
}
|
|
3452
3886
|
clearInterval(housekeepingTimer);
|
|
3453
3887
|
};
|
|
3454
3888
|
const close = () => {
|
|
@@ -3457,6 +3891,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3457
3891
|
if (closed) return;
|
|
3458
3892
|
closed = true;
|
|
3459
3893
|
clearInterval(pollTimer);
|
|
3894
|
+
fleetPoller?.stop();
|
|
3460
3895
|
clearInterval(watchTimer);
|
|
3461
3896
|
clearInterval(probeTimer);
|
|
3462
3897
|
clearInterval(usageTimer);
|
|
@@ -3478,7 +3913,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3478
3913
|
while (state.jobs.size > 0 && !exited) {
|
|
3479
3914
|
onWaiting?.([...state.jobs.keys()]);
|
|
3480
3915
|
for (const active of state.jobs.values()) {
|
|
3481
|
-
attemptLogs.noteChanged(active.job.item, "drain", `stopping: draining, waiting on
|
|
3916
|
+
attemptLogs.noteChanged(active.job.item, "drain", `stopping: draining, waiting on ${itemTag(active.job)} (${active.job.stage ?? "starting"})`, itemScope(active.job));
|
|
3482
3917
|
}
|
|
3483
3918
|
if (passes % DRAIN_LOG_EVERY_PASSES === 0) {
|
|
3484
3919
|
log.line(
|
|
@@ -3508,6 +3943,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3508
3943
|
stopping = true;
|
|
3509
3944
|
stopIntake();
|
|
3510
3945
|
clearInterval(pollTimer);
|
|
3946
|
+
fleetPoller?.stop();
|
|
3511
3947
|
clearInterval(watchTimer);
|
|
3512
3948
|
clearInterval(probeTimer);
|
|
3513
3949
|
clearInterval(usageTimer);
|
|
@@ -3536,7 +3972,7 @@ async function startRunner(configPath, deps, startup) {
|
|
|
3536
3972
|
}
|
|
3537
3973
|
function describeWaiting(active, nowMs) {
|
|
3538
3974
|
const elapsed = formatDuration(nowMs - new Date(active.job.startedAt).getTime());
|
|
3539
|
-
return
|
|
3975
|
+
return `${itemTag(active.job)} (${active.job.stage ?? "starting"}, ${elapsed})`;
|
|
3540
3976
|
}
|
|
3541
3977
|
var RunnerStartError = class extends Error {
|
|
3542
3978
|
};
|
|
@@ -3585,7 +4021,7 @@ The container stays up. Run \`outerlayer runner status\` to see what is left, th
|
|
|
3585
4021
|
const now = /* @__PURE__ */ new Date();
|
|
3586
4022
|
for (const { job } of listJobFiles(configPath).filter(({ job: job2 }) => isUnfinished(job2))) {
|
|
3587
4023
|
const elapsed = formatDuration(now.getTime() - new Date(job.startedAt).getTime());
|
|
3588
|
-
process.stdout.write(`waiting on
|
|
4024
|
+
process.stdout.write(`waiting on ${itemTag(job)} (${job.stage ?? "starting"}, ${elapsed})
|
|
3589
4025
|
`);
|
|
3590
4026
|
}
|
|
3591
4027
|
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
|
@@ -3609,4 +4045,4 @@ function runnerProcessState(configPath) {
|
|
|
3609
4045
|
return isSameProcess(entry.pid, entry.start) ? { state: "running", pid: entry.pid } : { state: "stale", pid: entry.pid };
|
|
3610
4046
|
}
|
|
3611
4047
|
|
|
3612
|
-
export { RunnerStartError, leaseTimeLeftFor, listJobFiles, logUnusableHelper, probeBuild, probeCredentials, requestStop, runnerProcessState, runtimeOf, sampleUsage, start, tick, watchTick };
|
|
4048
|
+
export { RunnerStartError, fleetKeepalive, leaseTimeLeftFor, listJobFiles, logUnusableHelper, probeBuild, probeCredentials, requestStop, runnerProcessState, runtimeOf, sampleUsage, start, tick, watchTick };
|