mirab 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +223 -3
- package/package.json +56 -7
- package/src/agent/approved.ts +261 -0
- package/src/agent/define.ts +235 -0
- package/src/agent/guest.ts +138 -0
- package/src/agent/init.ts +150 -0
- package/src/agent/manifest.ts +141 -0
- package/src/agent/modules.ts +23 -0
- package/src/agent/runtime.ts +239 -0
- package/src/agent-profiles.ts +327 -0
- package/src/api/bootstrap-contract.ts +95 -0
- package/src/api/channel-contract.ts +19 -0
- package/src/api/client.ts +100 -0
- package/src/api/contract.ts +2454 -0
- package/src/api/events-contract.ts +103 -0
- package/src/api/import-contract.ts +22 -0
- package/src/api/orchestrator-contract.ts +73 -0
- package/src/api/runner-contract.ts +25 -0
- package/src/api/scopes.ts +41 -0
- package/src/api/sessions-contract.ts +71 -0
- package/src/api/settings-contract.ts +50 -0
- package/src/channel/bridge.ts +97 -0
- package/src/channel/local.ts +154 -0
- package/src/channel/state.ts +11 -0
- package/src/channel/wire.ts +104 -0
- package/src/dev-sync.ts +206 -0
- package/src/dev.ts +453 -0
- package/src/events-tail.ts +75 -0
- package/src/images.ts +498 -0
- package/src/index.ts +21 -0
- package/src/invocation.ts +34 -0
- package/src/isolation/anchored-fs.ts +284 -0
- package/src/isolation/colima.ts +159 -0
- package/src/isolation/container.ts +189 -0
- package/src/isolation/firecracker.ts +414 -0
- package/src/isolation/guest-protocol.json +1 -0
- package/src/isolation/host-user.ts +247 -0
- package/src/isolation/image.ts +336 -0
- package/src/isolation/index.ts +477 -0
- package/src/isolation/multipass.ts +541 -0
- package/src/isolation/network.ts +167 -0
- package/src/isolation/policy.ts +86 -0
- package/src/isolation/session.ts +385 -0
- package/src/isolation/sync.ts +329 -0
- package/src/isolation/vz-network-setup.ts +247 -0
- package/src/isolation/vz.ts +312 -0
- package/src/mirab-main.ts +536 -0
- package/src/mirab.ts +13 -0
- package/src/runner/define.ts +187 -0
- package/src/runner/image.ts +239 -0
- package/src/runner/index.ts +37 -0
- package/src/runner/serve.ts +491 -0
- package/src/runner/toolsets.ts +205 -0
- package/src/spawn-adapters/claude.ts +112 -0
- package/src/spawn-adapters/codex-logs.ts +130 -0
- package/src/spawn-adapters/codex-notify.ts +73 -0
- package/src/spawn-adapters/codex.ts +89 -0
- package/src/spawn-adapters/index.ts +40 -0
- package/src/spawn-adapters/types.ts +55 -0
- package/src/spawn-attach.ts +243 -0
- package/src/spawn-binary.ts +90 -0
- package/src/spawn-bridge.ts +504 -0
- package/src/spawn-bundle.ts +16 -0
- package/src/spawn-detach.ts +301 -0
- package/src/spawn-events.ts +35 -0
- package/src/spawn-hook.ts +86 -0
- package/src/spawn-input.ts +107 -0
- package/src/spawn-native.ts +49 -0
- package/src/spawn-nudge-worker.ts +61 -0
- package/src/spawn-nudge.ts +446 -0
- package/src/spawn-output.ts +45 -0
- package/src/spawn-queue.ts +135 -0
- package/src/spawn-quiet.ts +69 -0
- package/src/spawn-report.ts +114 -0
- package/src/spawn-sessions.ts +166 -0
- package/src/spawn-vm.ts +356 -0
- package/src/spawn-worker.ts +766 -0
- package/src/spawn.ts +468 -0
- package/src/station/config.ts +89 -0
- package/src/station/gateway.ts +94 -0
- package/src/station/vault.ts +96 -0
- package/src/test-cli.ts +90 -0
- package/src/tools.ts +3 -0
- package/types/agent/define.d.ts +76 -0
- package/types/agent/manifest.d.ts +53 -0
- package/types/agent/modules.d.ts +1 -0
- package/types/images.d.ts +124 -0
- package/types/index.d.ts +6 -0
- package/types/isolation/colima.d.ts +37 -0
- package/types/isolation/container.d.ts +25 -0
- package/types/isolation/firecracker.d.ts +55 -0
- package/types/isolation/host-user.d.ts +47 -0
- package/types/isolation/image.d.ts +158 -0
- package/types/isolation/index.d.ts +128 -0
- package/types/isolation/network.d.ts +33 -0
- package/types/isolation/policy.d.ts +29 -0
- package/types/isolation/session.d.ts +99 -0
- package/types/isolation/vz.d.ts +63 -0
- package/types/runner/define.d.ts +101 -0
- package/types/runner/image.d.ts +52 -0
- package/types/runner/index.d.ts +13 -0
- package/types/runner/serve.d.ts +69 -0
- package/types/runner/toolsets.d.ts +35 -0
- package/types/tools.d.ts +2 -0
- package/index.js +0 -1
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
// serveRunners: keeps each definition's slot filled with a fresh VM that takes
|
|
2
|
+
// exactly one job, then throws the VM away. Outbound only: the forge's runner
|
|
3
|
+
// in the VM polls the forge; nothing reaches this host.
|
|
4
|
+
//
|
|
5
|
+
// A slot: image (cached) → VM from a clone of it, filtered internet, nothing
|
|
6
|
+
// of the host's inside, no guest-to-host port → ticket (minted only once the
|
|
7
|
+
// guest answers, so a failed boot leaves no registration) → files, steps as
|
|
8
|
+
// the runner user → the job timer starts at the runner's "job started" line →
|
|
9
|
+
// VM closed, disk discarded → next slot.
|
|
10
|
+
import { randomBytes } from 'node:crypto'
|
|
11
|
+
import {
|
|
12
|
+
appendFile,
|
|
13
|
+
mkdir,
|
|
14
|
+
rename,
|
|
15
|
+
rm,
|
|
16
|
+
statfs,
|
|
17
|
+
writeFile,
|
|
18
|
+
} from 'node:fs/promises'
|
|
19
|
+
import { hostname, totalmem } from 'node:os'
|
|
20
|
+
import { join } from 'node:path'
|
|
21
|
+
import { startVm, vmBackend, type VmConfig } from '../isolation'
|
|
22
|
+
import { networkState } from '../isolation/network'
|
|
23
|
+
import type { Isolation } from '../isolation/session'
|
|
24
|
+
import { vzNetworkState } from '../isolation/vz'
|
|
25
|
+
import {
|
|
26
|
+
RUNNER_SETPRIV,
|
|
27
|
+
RUNNER_USER,
|
|
28
|
+
hostRunnerPlatform,
|
|
29
|
+
platformLabel,
|
|
30
|
+
type RunnerDefinition,
|
|
31
|
+
type RunnerPlatform,
|
|
32
|
+
type RunnerStep,
|
|
33
|
+
} from './define'
|
|
34
|
+
import {
|
|
35
|
+
runnerImage,
|
|
36
|
+
runnerRecipe,
|
|
37
|
+
type RunnerImage,
|
|
38
|
+
type RunnerRecipe,
|
|
39
|
+
} from './image'
|
|
40
|
+
|
|
41
|
+
export const runnersDir = (configDir: string) => join(configDir, 'runners')
|
|
42
|
+
|
|
43
|
+
export type SlotPhase = 'image' | 'booting' | 'waiting' | 'busy' | 'stopping'
|
|
44
|
+
|
|
45
|
+
export interface SlotState {
|
|
46
|
+
vmName: string
|
|
47
|
+
phase: SlotPhase
|
|
48
|
+
since: string
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface RunnerState {
|
|
52
|
+
name: string
|
|
53
|
+
host: string
|
|
54
|
+
forge: string
|
|
55
|
+
image?: string
|
|
56
|
+
slots: SlotState[]
|
|
57
|
+
jobs: number
|
|
58
|
+
lastError?: { at: string; message: string }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Things serveRunners does to the world; tests pass their own. */
|
|
62
|
+
export interface ServeDeps {
|
|
63
|
+
startVm: (
|
|
64
|
+
vm: VmConfig,
|
|
65
|
+
vmDir: string,
|
|
66
|
+
log: (line: string) => void,
|
|
67
|
+
) => Promise<Isolation>
|
|
68
|
+
image: (def: RunnerDefinition, recipe: RunnerRecipe) => Promise<RunnerImage>
|
|
69
|
+
recipe: (def: RunnerDefinition) => Promise<RunnerRecipe>
|
|
70
|
+
sleep: (ms: number, signal: AbortSignal) => Promise<void>
|
|
71
|
+
now: () => Date
|
|
72
|
+
/** How long a minute is (the job and idle timers); tests shorten it. */
|
|
73
|
+
minuteMs: number
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface ServeOptions {
|
|
77
|
+
configDir: string
|
|
78
|
+
vm: VmConfig
|
|
79
|
+
/** Names registrations `<host>-<name>-<random>`. */
|
|
80
|
+
host?: string
|
|
81
|
+
/** RAM all slots together may use (default: host RAM − 3 GiB). */
|
|
82
|
+
budgetMiB?: number
|
|
83
|
+
signal?: AbortSignal
|
|
84
|
+
log?: (line: string) => void
|
|
85
|
+
deps?: Partial<ServeDeps>
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const RECIPE_RECHECK_MS = 24 * 3600_000
|
|
89
|
+
const BACKOFF_MS = [10_000, 30_000, 60_000, 120_000, 300_000]
|
|
90
|
+
const SHUTDOWN_GRACE_MS = 10 * 60_000
|
|
91
|
+
|
|
92
|
+
const sleep = (ms: number, signal: AbortSignal) =>
|
|
93
|
+
new Promise<void>((resolve) => {
|
|
94
|
+
if (signal.aborted) return resolve()
|
|
95
|
+
const t = setTimeout(done, ms)
|
|
96
|
+
function done() {
|
|
97
|
+
clearTimeout(t)
|
|
98
|
+
signal.removeEventListener('abort', done)
|
|
99
|
+
resolve()
|
|
100
|
+
}
|
|
101
|
+
signal.addEventListener('abort', done)
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
/** A runner's VM: nothing of the host's inside (no work share or drive) and
|
|
105
|
+
* no guest-to-host port. */
|
|
106
|
+
export function startRunnerVm(
|
|
107
|
+
vm: VmConfig,
|
|
108
|
+
vmDir: string,
|
|
109
|
+
log: (line: string) => void,
|
|
110
|
+
): Promise<Isolation> {
|
|
111
|
+
return startVm(vm, null, vmDir, log, undefined, { listen: [] })
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** A host name usable in a registration name. */
|
|
115
|
+
export function hostLabel(name: string = hostname()): string {
|
|
116
|
+
return (
|
|
117
|
+
name
|
|
118
|
+
.split('.')[0]!
|
|
119
|
+
.toLowerCase()
|
|
120
|
+
.replace(/[^a-z0-9-]/g, '-')
|
|
121
|
+
.replace(/^-+|-+$/g, '')
|
|
122
|
+
.slice(0, 24) || 'host'
|
|
123
|
+
)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Replaces every secret value in `line` (the ticket's, while it lives). */
|
|
127
|
+
export function redact(line: string, secrets: Iterable<string>): string {
|
|
128
|
+
let out = line
|
|
129
|
+
for (const s of secrets)
|
|
130
|
+
if (s.length >= 8) out = out.split(s).join('[redacted]')
|
|
131
|
+
return out
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The env every job step gets: base, then the image's toolsets, then the
|
|
135
|
+
* forge's job env, then the step's own (later wins). PATH is composed. */
|
|
136
|
+
export function jobEnv(
|
|
137
|
+
def: RunnerDefinition,
|
|
138
|
+
image: Pick<RunnerImage['meta'], 'env' | 'path'>,
|
|
139
|
+
job: Record<string, string>,
|
|
140
|
+
step: Record<string, string> = {},
|
|
141
|
+
): Record<string, string> {
|
|
142
|
+
const home = RUNNER_USER.home
|
|
143
|
+
const basePath =
|
|
144
|
+
'/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
|
145
|
+
return {
|
|
146
|
+
HOME: home,
|
|
147
|
+
USER: RUNNER_USER.name,
|
|
148
|
+
LOGNAME: RUNNER_USER.name,
|
|
149
|
+
LANG: 'C.UTF-8',
|
|
150
|
+
// /tmp is a RAM disk in the guest; temp files go on the job's disk.
|
|
151
|
+
TMPDIR: `${home}/tmp`,
|
|
152
|
+
CARGO_BUILD_JOBS: String(def.vm.vcpus),
|
|
153
|
+
...image.env,
|
|
154
|
+
...job,
|
|
155
|
+
...step,
|
|
156
|
+
PATH: [...image.path, step.PATH ?? job.PATH ?? basePath].join(':'),
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The argv mirab really execs for a step: as the runner user unless asRoot,
|
|
161
|
+
* output lines cut to 4000 characters (the guest channel refuses a line over
|
|
162
|
+
* 32 KiB, which would end the job), the step's exit status kept. */
|
|
163
|
+
export function stepArgv(step: RunnerStep): string[] {
|
|
164
|
+
const argv = step.asRoot ? step.argv : [...RUNNER_SETPRIV, ...step.argv]
|
|
165
|
+
return [
|
|
166
|
+
'bash',
|
|
167
|
+
'-o',
|
|
168
|
+
'pipefail',
|
|
169
|
+
'-c',
|
|
170
|
+
'"$@" 2>&1 | cut -c1-4000',
|
|
171
|
+
'mirab-step',
|
|
172
|
+
...argv,
|
|
173
|
+
]
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Refuses a host that can't run these definitions safely. */
|
|
177
|
+
export async function admission(
|
|
178
|
+
defs: RunnerDefinition[],
|
|
179
|
+
o: {
|
|
180
|
+
configDir: string
|
|
181
|
+
budgetMiB: number
|
|
182
|
+
backend: 'vz' | 'firecracker'
|
|
183
|
+
filtered: boolean
|
|
184
|
+
freeBytes: number
|
|
185
|
+
},
|
|
186
|
+
): Promise<void> {
|
|
187
|
+
const names = new Set<string>()
|
|
188
|
+
for (const d of defs) {
|
|
189
|
+
if (names.has(d.name)) throw new Error(`two runners are named ${d.name}`)
|
|
190
|
+
names.add(d.name)
|
|
191
|
+
}
|
|
192
|
+
const mem = defs.reduce((s, d) => s + d.pool.max * d.vm.memoryMiB, 0)
|
|
193
|
+
if (mem > o.budgetMiB)
|
|
194
|
+
throw new Error(
|
|
195
|
+
`runners need ${mem} MiB of VM memory at once; this host's budget is ${o.budgetMiB} MiB (lower vm.memoryMiB or pool.max, or pass budgetMiB)`,
|
|
196
|
+
)
|
|
197
|
+
const disk =
|
|
198
|
+
defs.reduce((s, d) => s + d.pool.max * d.vm.diskGiB, 0) * 1024 ** 3
|
|
199
|
+
const reserve = 20 * 1024 ** 3
|
|
200
|
+
if (disk + reserve > o.freeBytes)
|
|
201
|
+
throw new Error(
|
|
202
|
+
`runners may fill ${disk / 1024 ** 3} GiB of disk at once; ${Math.floor(o.freeBytes / 1024 ** 3)} GiB is free and 20 GiB stays in reserve`,
|
|
203
|
+
)
|
|
204
|
+
if (!o.filtered)
|
|
205
|
+
throw new Error(
|
|
206
|
+
o.backend === 'vz'
|
|
207
|
+
? 'runner VMs need the vz network filter (internet yes, LAN and tailnet no): sudo ~/.mirab/bin/mirab vm network setup'
|
|
208
|
+
: 'runner VMs need the filtered mirabbr0 bridge (internet yes, LAN and tailnet no): see docs/VM.md',
|
|
209
|
+
)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export async function serveRunners(
|
|
213
|
+
defs: RunnerDefinition[],
|
|
214
|
+
o: ServeOptions,
|
|
215
|
+
): Promise<void> {
|
|
216
|
+
const log = o.log ?? ((l: string) => console.log(l))
|
|
217
|
+
const host = hostLabel(o.host)
|
|
218
|
+
const signal = o.signal ?? new AbortController().signal
|
|
219
|
+
const deps: ServeDeps = {
|
|
220
|
+
startVm: startRunnerVm,
|
|
221
|
+
recipe: (def) => runnerRecipe(def),
|
|
222
|
+
image: (_def, recipe) =>
|
|
223
|
+
runnerImage({ configDir: o.configDir, vm: o.vm, recipe, log }),
|
|
224
|
+
sleep,
|
|
225
|
+
now: () => new Date(),
|
|
226
|
+
minuteMs: 60_000,
|
|
227
|
+
...o.deps,
|
|
228
|
+
}
|
|
229
|
+
if (!o.deps?.startVm) {
|
|
230
|
+
const backend = vmBackend()
|
|
231
|
+
const fs = await statfs(o.configDir).catch(() => undefined)
|
|
232
|
+
await admission(defs, {
|
|
233
|
+
configDir: o.configDir,
|
|
234
|
+
budgetMiB: o.budgetMiB ?? Math.floor(totalmem() / 1024 ** 2) - 3072,
|
|
235
|
+
backend,
|
|
236
|
+
filtered:
|
|
237
|
+
backend === 'vz' ? vzNetworkState().filtered : networkState().ready,
|
|
238
|
+
freeBytes: fs ? fs.bavail * fs.bsize : 0,
|
|
239
|
+
})
|
|
240
|
+
}
|
|
241
|
+
await Promise.all(
|
|
242
|
+
defs.map((def) => serveOne(def, host, deps, o, log, signal)),
|
|
243
|
+
)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async function serveOne(
|
|
247
|
+
def: RunnerDefinition,
|
|
248
|
+
host: string,
|
|
249
|
+
deps: ServeDeps,
|
|
250
|
+
o: ServeOptions,
|
|
251
|
+
hostLog: (line: string) => void,
|
|
252
|
+
signal: AbortSignal,
|
|
253
|
+
) {
|
|
254
|
+
const dir = join(runnersDir(o.configDir), def.name)
|
|
255
|
+
const vmsDir = join(dir, 'vms')
|
|
256
|
+
const prefix = `${host}-${def.name}-`
|
|
257
|
+
await mkdir(dir, { recursive: true, mode: 0o700 })
|
|
258
|
+
const logFile = join(dir, 'serve.log')
|
|
259
|
+
const secrets = new Set<string>()
|
|
260
|
+
const log = (line: string) => {
|
|
261
|
+
const clean = redact(line, secrets)
|
|
262
|
+
hostLog(`[${def.name}] ${clean}`)
|
|
263
|
+
void appendFile(logFile, `${deps.now().toISOString()} ${clean}\n`).catch(
|
|
264
|
+
() => {},
|
|
265
|
+
)
|
|
266
|
+
}
|
|
267
|
+
const state: RunnerState = {
|
|
268
|
+
name: def.name,
|
|
269
|
+
host,
|
|
270
|
+
forge: def.forge.name,
|
|
271
|
+
slots: [],
|
|
272
|
+
jobs: 0,
|
|
273
|
+
}
|
|
274
|
+
// One write at a time, each whole (temp file + rename): status readers and
|
|
275
|
+
// a crash never see half a file.
|
|
276
|
+
let saving: Promise<void> = Promise.resolve()
|
|
277
|
+
const save = () =>
|
|
278
|
+
(saving = saving.then(async () => {
|
|
279
|
+
const file = join(dir, 'state.json')
|
|
280
|
+
const tmp = `${file}.${process.pid}.tmp`
|
|
281
|
+
try {
|
|
282
|
+
await writeFile(tmp, JSON.stringify(state, null, 2) + '\n', {
|
|
283
|
+
mode: 0o600,
|
|
284
|
+
})
|
|
285
|
+
await rename(tmp, file)
|
|
286
|
+
} catch {}
|
|
287
|
+
}))
|
|
288
|
+
const fail = (message: string) => {
|
|
289
|
+
state.lastError = {
|
|
290
|
+
at: deps.now().toISOString(),
|
|
291
|
+
message: redact(message, secrets),
|
|
292
|
+
}
|
|
293
|
+
log(`error: ${message}`)
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// A crash or reboot left VM directories behind: nothing in them is wanted.
|
|
297
|
+
await rm(vmsDir, { recursive: true, force: true })
|
|
298
|
+
await mkdir(vmsDir, { recursive: true, mode: 0o700 })
|
|
299
|
+
await def.forge
|
|
300
|
+
.reconcile(prefix, [])
|
|
301
|
+
.catch((e) => fail(`reconcile: ${e.message}`))
|
|
302
|
+
await save()
|
|
303
|
+
|
|
304
|
+
let recipe: RunnerRecipe | undefined
|
|
305
|
+
let recipeAt = 0
|
|
306
|
+
let failures = 0
|
|
307
|
+
const backoff = async () => {
|
|
308
|
+
const ms = BACKOFF_MS[Math.min(failures, BACKOFF_MS.length - 1)]!
|
|
309
|
+
failures++
|
|
310
|
+
log(`retrying in ${ms / 1000} s`)
|
|
311
|
+
await deps.sleep(ms, signal)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
while (!signal.aborted) {
|
|
315
|
+
const vmName = prefix + randomBytes(3).toString('hex')
|
|
316
|
+
const slot: SlotState = {
|
|
317
|
+
vmName,
|
|
318
|
+
phase: 'image',
|
|
319
|
+
since: deps.now().toISOString(),
|
|
320
|
+
}
|
|
321
|
+
const phase = (p: SlotPhase) => {
|
|
322
|
+
slot.phase = p
|
|
323
|
+
slot.since = deps.now().toISOString()
|
|
324
|
+
void save()
|
|
325
|
+
}
|
|
326
|
+
state.slots = [slot]
|
|
327
|
+
let iso: Isolation | undefined
|
|
328
|
+
let ticketed = false
|
|
329
|
+
try {
|
|
330
|
+
// The recipe names the forge's runner release: look again daily, so a
|
|
331
|
+
// new release becomes a new image between jobs.
|
|
332
|
+
if (!recipe || deps.now().getTime() - recipeAt > RECIPE_RECHECK_MS) {
|
|
333
|
+
recipe = await deps.recipe(def)
|
|
334
|
+
recipeAt = deps.now().getTime()
|
|
335
|
+
}
|
|
336
|
+
phase('image')
|
|
337
|
+
const image = await deps.image(def, recipe)
|
|
338
|
+
state.image = image.meta.key
|
|
339
|
+
if (signal.aborted) break
|
|
340
|
+
phase('booting')
|
|
341
|
+
iso = await deps.startVm(
|
|
342
|
+
{
|
|
343
|
+
...o.vm,
|
|
344
|
+
rootfs: image.rootfs,
|
|
345
|
+
vcpus: def.vm.vcpus,
|
|
346
|
+
memoryMiB: def.vm.memoryMiB,
|
|
347
|
+
egress: 'allow',
|
|
348
|
+
},
|
|
349
|
+
join(vmsDir, vmName),
|
|
350
|
+
log,
|
|
351
|
+
)
|
|
352
|
+
// Booted and answering: only now is a registration worth making.
|
|
353
|
+
const ticket = await def.forge.ticket({
|
|
354
|
+
name: vmName,
|
|
355
|
+
labels: [...new Set([platformLabel(recipe.platform), ...def.labels])],
|
|
356
|
+
platform: recipe.platform,
|
|
357
|
+
})
|
|
358
|
+
ticketed = true
|
|
359
|
+
for (const v of Object.values(ticket.secret)) secrets.add(v)
|
|
360
|
+
const job = def.forge.guest.job(ticket)
|
|
361
|
+
for (const f of job.files) {
|
|
362
|
+
if (f.secret) secrets.add(f.content)
|
|
363
|
+
await iso.put(f.path, f.content, f.mode)
|
|
364
|
+
}
|
|
365
|
+
phase('waiting')
|
|
366
|
+
const outcome = await runJob(
|
|
367
|
+
def,
|
|
368
|
+
iso,
|
|
369
|
+
image,
|
|
370
|
+
job,
|
|
371
|
+
signal,
|
|
372
|
+
log,
|
|
373
|
+
deps.minuteMs,
|
|
374
|
+
() => {
|
|
375
|
+
phase('busy')
|
|
376
|
+
state.jobs++
|
|
377
|
+
},
|
|
378
|
+
)
|
|
379
|
+
log(`${vmName}: ${outcome.why}`)
|
|
380
|
+
if (outcome.tookJob || outcome.expected) failures = 0
|
|
381
|
+
else {
|
|
382
|
+
// The runner gave up before any job (a refused or spent registration,
|
|
383
|
+
// a broken image): without this, a new VM would boot every few seconds.
|
|
384
|
+
fail(
|
|
385
|
+
`${vmName}: the runner exited before taking a job (${outcome.why})`,
|
|
386
|
+
)
|
|
387
|
+
if (!signal.aborted) await backoff()
|
|
388
|
+
}
|
|
389
|
+
} catch (e) {
|
|
390
|
+
fail(`${vmName}: ${(e as Error).message}`)
|
|
391
|
+
if (!signal.aborted) await backoff()
|
|
392
|
+
} finally {
|
|
393
|
+
phase('stopping')
|
|
394
|
+
await iso?.close().catch((e) => fail(`close ${vmName}: ${e.message}`))
|
|
395
|
+
if (ticketed)
|
|
396
|
+
// The runner deregisters itself after its job; anything left (an idle
|
|
397
|
+
// slot ended, a killed job) is removed here.
|
|
398
|
+
await def.forge
|
|
399
|
+
.reconcile(prefix, [])
|
|
400
|
+
.catch((e) => fail(`reconcile: ${e.message}`))
|
|
401
|
+
secrets.clear()
|
|
402
|
+
state.slots = []
|
|
403
|
+
await save()
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Runs the job's steps; returns how the slot ended. */
|
|
409
|
+
async function runJob(
|
|
410
|
+
def: RunnerDefinition,
|
|
411
|
+
iso: Isolation,
|
|
412
|
+
image: RunnerImage,
|
|
413
|
+
job: { steps: RunnerStep[]; env: Record<string, string> },
|
|
414
|
+
stop: AbortSignal,
|
|
415
|
+
log: (line: string) => void,
|
|
416
|
+
minuteMs: number,
|
|
417
|
+
onStarted: () => void,
|
|
418
|
+
): Promise<{ why: string; tookJob: boolean; expected: boolean }> {
|
|
419
|
+
const ctl = new AbortController()
|
|
420
|
+
let started = false
|
|
421
|
+
let why = 'finished'
|
|
422
|
+
/** Ended on purpose (idle lifetime, shutdown), not by the runner giving up. */
|
|
423
|
+
let expected = false
|
|
424
|
+
const idleMs = def.pool.idleHours * 60 * minuteMs
|
|
425
|
+
const jobMs = def.timeoutMinutes * minuteMs
|
|
426
|
+
const idleTimer = setTimeout(() => {
|
|
427
|
+
if (!started) {
|
|
428
|
+
why = `no job in ${def.pool.idleHours} h, ended`
|
|
429
|
+
expected = true
|
|
430
|
+
ctl.abort()
|
|
431
|
+
}
|
|
432
|
+
}, idleMs)
|
|
433
|
+
let jobTimer: ReturnType<typeof setTimeout> | undefined
|
|
434
|
+
let graceTimer: ReturnType<typeof setTimeout> | undefined
|
|
435
|
+
const onStop = () => {
|
|
436
|
+
if (!started) {
|
|
437
|
+
why = 'stopped while waiting'
|
|
438
|
+
expected = true
|
|
439
|
+
ctl.abort()
|
|
440
|
+
} else
|
|
441
|
+
graceTimer = setTimeout(
|
|
442
|
+
() => {
|
|
443
|
+
why = 'stopped: job killed after the shutdown grace period'
|
|
444
|
+
ctl.abort()
|
|
445
|
+
},
|
|
446
|
+
(SHUTDOWN_GRACE_MS / 60_000) * minuteMs,
|
|
447
|
+
)
|
|
448
|
+
}
|
|
449
|
+
stop.addEventListener('abort', onStop)
|
|
450
|
+
if (stop.aborted) onStop()
|
|
451
|
+
try {
|
|
452
|
+
for (const step of job.steps) {
|
|
453
|
+
if (ctl.signal.aborted) break
|
|
454
|
+
const code = await iso.exec(stepArgv(step), {
|
|
455
|
+
cwd: step.asRoot ? '/' : RUNNER_USER.home,
|
|
456
|
+
env: jobEnv(def, image.meta, job.env, step.env),
|
|
457
|
+
// A backstop: the timers above end it first.
|
|
458
|
+
timeoutMs: idleMs + jobMs + 10 * 60_000,
|
|
459
|
+
signal: ctl.signal,
|
|
460
|
+
onLine: async (line) => {
|
|
461
|
+
log(line)
|
|
462
|
+
if (!started && def.forge.guest.started(line)) {
|
|
463
|
+
started = true
|
|
464
|
+
clearTimeout(idleTimer)
|
|
465
|
+
onStarted()
|
|
466
|
+
jobTimer = setTimeout(() => {
|
|
467
|
+
why = `job killed at its ${def.timeoutMinutes} min limit`
|
|
468
|
+
ctl.abort()
|
|
469
|
+
}, jobMs)
|
|
470
|
+
}
|
|
471
|
+
},
|
|
472
|
+
})
|
|
473
|
+
if (ctl.signal.aborted) break
|
|
474
|
+
if (code !== 0)
|
|
475
|
+
return {
|
|
476
|
+
why: `step ${step.argv[0]} exited ${code}`,
|
|
477
|
+
tookJob: started,
|
|
478
|
+
expected,
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return { why, tookJob: started, expected }
|
|
482
|
+
} finally {
|
|
483
|
+
clearTimeout(idleTimer)
|
|
484
|
+
clearTimeout(jobTimer)
|
|
485
|
+
clearTimeout(graceTimer)
|
|
486
|
+
stop.removeEventListener('abort', onStop)
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
export { hostRunnerPlatform }
|
|
491
|
+
export type { RunnerPlatform }
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
// Toolsets baked into a runner image: versions resolved on the host (so the
|
|
2
|
+
// image key names exact versions), archives checked in the build VM against
|
|
3
|
+
// the upstream checksum. Laid out the way GitHub's hosted images do
|
|
4
|
+
// (<tool cache>/<tool>/<version>/<arch>/ plus a sibling <arch>.complete), so
|
|
5
|
+
// actions/setup-node and setup-go find them and skip their download.
|
|
6
|
+
import type { RunnerPlatform, RunnerStep } from './define'
|
|
7
|
+
|
|
8
|
+
export const TOOL_CACHE = '/opt/hostedtoolcache'
|
|
9
|
+
export const RUST_HOME = '/opt/rust'
|
|
10
|
+
|
|
11
|
+
export interface ResolvedTool {
|
|
12
|
+
tool: 'rust' | 'node' | 'go'
|
|
13
|
+
version: string
|
|
14
|
+
url: string
|
|
15
|
+
sha256: string
|
|
16
|
+
/** Hosts the build VM fetches from. */
|
|
17
|
+
hosts: string[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type Fetcher = (url: string) => Promise<string>
|
|
21
|
+
|
|
22
|
+
const defaultFetch: Fetcher = async (url) => {
|
|
23
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(30_000) })
|
|
24
|
+
if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`)
|
|
25
|
+
return res.text()
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const SHA256 = /^[0-9a-f]{64}$/
|
|
29
|
+
|
|
30
|
+
/** `24` matches 24.x.y, `24.21` matches 24.21.y, `24.21.0` only itself. */
|
|
31
|
+
export function versionMatches(spec: string, version: string): boolean {
|
|
32
|
+
return version === spec || version.startsWith(spec + '.')
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const nodeArch = (p: RunnerPlatform) => (p === 'linux-arm64' ? 'arm64' : 'x64')
|
|
36
|
+
const goArch = (p: RunnerPlatform) => (p === 'linux-arm64' ? 'arm64' : 'amd64')
|
|
37
|
+
const rustTriple = (p: RunnerPlatform) =>
|
|
38
|
+
p === 'linux-arm64' ? 'aarch64-unknown-linux-gnu' : 'x86_64-unknown-linux-gnu'
|
|
39
|
+
/** The tool cache's arch directory (setup-node and setup-go use these). */
|
|
40
|
+
export const cacheArch = nodeArch
|
|
41
|
+
|
|
42
|
+
export async function resolveNode(
|
|
43
|
+
spec: string,
|
|
44
|
+
platform: RunnerPlatform,
|
|
45
|
+
get: Fetcher = defaultFetch,
|
|
46
|
+
): Promise<ResolvedTool> {
|
|
47
|
+
const index = JSON.parse(await get('https://nodejs.org/dist/index.json')) as {
|
|
48
|
+
version: string
|
|
49
|
+
}[]
|
|
50
|
+
const hit = index.find((r) =>
|
|
51
|
+
versionMatches(spec, r.version.replace(/^v/, '')),
|
|
52
|
+
)
|
|
53
|
+
if (!hit) throw new Error(`node ${spec}: no such release`)
|
|
54
|
+
const version = hit.version.replace(/^v/, '')
|
|
55
|
+
const file = `node-v${version}-linux-${nodeArch(platform)}.tar.xz`
|
|
56
|
+
const sums = await get(`https://nodejs.org/dist/v${version}/SHASUMS256.txt`)
|
|
57
|
+
const sha256 = sums
|
|
58
|
+
.split('\n')
|
|
59
|
+
.map((l) => l.trim().split(/\s+/))
|
|
60
|
+
.find(([, f]) => f === file)?.[0]
|
|
61
|
+
if (!sha256 || !SHA256.test(sha256))
|
|
62
|
+
throw new Error(`node ${version}: no checksum for ${file}`)
|
|
63
|
+
return {
|
|
64
|
+
tool: 'node',
|
|
65
|
+
version,
|
|
66
|
+
url: `https://nodejs.org/dist/v${version}/${file}`,
|
|
67
|
+
sha256,
|
|
68
|
+
hosts: ['nodejs.org'],
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function resolveGo(
|
|
73
|
+
spec: string,
|
|
74
|
+
platform: RunnerPlatform,
|
|
75
|
+
get: Fetcher = defaultFetch,
|
|
76
|
+
): Promise<ResolvedTool> {
|
|
77
|
+
const releases = JSON.parse(
|
|
78
|
+
await get('https://go.dev/dl/?mode=json&include=all'),
|
|
79
|
+
) as {
|
|
80
|
+
version: string
|
|
81
|
+
stable: boolean
|
|
82
|
+
files: {
|
|
83
|
+
filename: string
|
|
84
|
+
os: string
|
|
85
|
+
arch: string
|
|
86
|
+
sha256: string
|
|
87
|
+
kind: string
|
|
88
|
+
}[]
|
|
89
|
+
}[]
|
|
90
|
+
const hit = releases.find(
|
|
91
|
+
(r) => r.stable && versionMatches(spec, r.version.replace(/^go/, '')),
|
|
92
|
+
)
|
|
93
|
+
if (!hit) throw new Error(`go ${spec}: no such stable release`)
|
|
94
|
+
const file = hit.files.find(
|
|
95
|
+
(f) =>
|
|
96
|
+
f.os === 'linux' && f.arch === goArch(platform) && f.kind === 'archive',
|
|
97
|
+
)
|
|
98
|
+
if (!file || !SHA256.test(file.sha256))
|
|
99
|
+
throw new Error(`go ${hit.version}: no linux-${goArch(platform)} archive`)
|
|
100
|
+
return {
|
|
101
|
+
tool: 'go',
|
|
102
|
+
version: hit.version.replace(/^go/, ''),
|
|
103
|
+
url: `https://dl.google.com/go/${file.filename}`,
|
|
104
|
+
sha256: file.sha256,
|
|
105
|
+
hosts: ['dl.google.com'],
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Rust: rustup-init (checked by its published sha256) installs the toolchain
|
|
110
|
+
* `spec`; rustup checks every component it downloads itself. */
|
|
111
|
+
export async function resolveRust(
|
|
112
|
+
spec: string,
|
|
113
|
+
platform: RunnerPlatform,
|
|
114
|
+
get: Fetcher = defaultFetch,
|
|
115
|
+
): Promise<ResolvedTool> {
|
|
116
|
+
const url = `https://static.rust-lang.org/rustup/dist/${rustTriple(platform)}/rustup-init`
|
|
117
|
+
const sha256 = (await get(`${url}.sha256`)).trim().split(/\s+/)[0]
|
|
118
|
+
if (!sha256 || !SHA256.test(sha256))
|
|
119
|
+
throw new Error(`rustup-init: no checksum at ${url}.sha256`)
|
|
120
|
+
return {
|
|
121
|
+
tool: 'rust',
|
|
122
|
+
version: spec,
|
|
123
|
+
url,
|
|
124
|
+
sha256,
|
|
125
|
+
hosts: ['static.rust-lang.org'],
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export async function resolveTools(
|
|
130
|
+
tools: { rust?: string; node?: string; go?: string },
|
|
131
|
+
platform: RunnerPlatform,
|
|
132
|
+
get: Fetcher = defaultFetch,
|
|
133
|
+
): Promise<ResolvedTool[]> {
|
|
134
|
+
const out: ResolvedTool[] = []
|
|
135
|
+
if (tools.rust) out.push(await resolveRust(tools.rust, platform, get))
|
|
136
|
+
if (tools.node) out.push(await resolveNode(tools.node, platform, get))
|
|
137
|
+
if (tools.go) out.push(await resolveGo(tools.go, platform, get))
|
|
138
|
+
return out
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const q = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`
|
|
142
|
+
|
|
143
|
+
/** Download to the root disk (not tmpfs /tmp) and check the checksum first. */
|
|
144
|
+
const fetchChecked = (t: ResolvedTool, to: string) =>
|
|
145
|
+
`curl -fsSL -o ${q(to)} ${q(t.url)} && echo ${q(`${t.sha256} ${to}`)} | sha256sum -c -`
|
|
146
|
+
|
|
147
|
+
/** The build steps (root, in the build VM) for the resolved toolsets. */
|
|
148
|
+
export function toolSteps(
|
|
149
|
+
tools: ResolvedTool[],
|
|
150
|
+
platform: RunnerPlatform,
|
|
151
|
+
): RunnerStep[] {
|
|
152
|
+
const arch = cacheArch(platform)
|
|
153
|
+
return tools.map((t) => {
|
|
154
|
+
if (t.tool === 'rust')
|
|
155
|
+
return {
|
|
156
|
+
argv: [
|
|
157
|
+
'sh',
|
|
158
|
+
'-euc',
|
|
159
|
+
[
|
|
160
|
+
`mkdir -p ${RUST_HOME}/dl`,
|
|
161
|
+
fetchChecked(t, `${RUST_HOME}/dl/rustup-init`),
|
|
162
|
+
`chmod 755 ${RUST_HOME}/dl/rustup-init`,
|
|
163
|
+
`RUSTUP_HOME=${RUST_HOME}/rustup CARGO_HOME=${RUST_HOME}/cargo ${RUST_HOME}/dl/rustup-init -y --no-modify-path --profile minimal --default-toolchain ${q(t.version)} -c clippy -c rustfmt`,
|
|
164
|
+
`rm -rf ${RUST_HOME}/dl`,
|
|
165
|
+
].join(' && '),
|
|
166
|
+
],
|
|
167
|
+
}
|
|
168
|
+
const dir = `${TOOL_CACHE}/${t.tool}/${t.version}/${arch}`
|
|
169
|
+
const archive = `${TOOL_CACHE}/.dl-${t.tool}`
|
|
170
|
+
return {
|
|
171
|
+
argv: [
|
|
172
|
+
'sh',
|
|
173
|
+
'-euc',
|
|
174
|
+
[
|
|
175
|
+
`mkdir -p ${q(dir)}`,
|
|
176
|
+
fetchChecked(t, archive),
|
|
177
|
+
`tar -xf ${q(archive)} -C ${q(dir)} --strip-components=1`,
|
|
178
|
+
`rm -f ${q(archive)}`,
|
|
179
|
+
`touch ${q(`${dir}.complete`)}`,
|
|
180
|
+
].join(' && '),
|
|
181
|
+
],
|
|
182
|
+
}
|
|
183
|
+
})
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** What a job's env needs to find the toolsets (merged into the base env). */
|
|
187
|
+
export function toolEnv(
|
|
188
|
+
tools: ResolvedTool[],
|
|
189
|
+
platform: RunnerPlatform,
|
|
190
|
+
): { env: Record<string, string>; path: string[] } {
|
|
191
|
+
const arch = cacheArch(platform)
|
|
192
|
+
const env: Record<string, string> = {
|
|
193
|
+
RUNNER_TOOL_CACHE: TOOL_CACHE,
|
|
194
|
+
AGENT_TOOLSDIRECTORY: TOOL_CACHE,
|
|
195
|
+
}
|
|
196
|
+
const path: string[] = []
|
|
197
|
+
for (const t of tools) {
|
|
198
|
+
if (t.tool === 'rust') {
|
|
199
|
+
env.RUSTUP_HOME = `${RUST_HOME}/rustup`
|
|
200
|
+
env.CARGO_HOME = `${RUST_HOME}/cargo`
|
|
201
|
+
path.push(`${RUST_HOME}/cargo/bin`)
|
|
202
|
+
} else path.push(`${TOOL_CACHE}/${t.tool}/${t.version}/${arch}/bin`)
|
|
203
|
+
}
|
|
204
|
+
return { env, path }
|
|
205
|
+
}
|