mirab 0.0.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +223 -3
  3. package/package.json +56 -7
  4. package/src/agent/approved.ts +261 -0
  5. package/src/agent/define.ts +235 -0
  6. package/src/agent/guest.ts +138 -0
  7. package/src/agent/init.ts +150 -0
  8. package/src/agent/manifest.ts +141 -0
  9. package/src/agent/modules.ts +23 -0
  10. package/src/agent/runtime.ts +239 -0
  11. package/src/agent-profiles.ts +327 -0
  12. package/src/api/bootstrap-contract.ts +95 -0
  13. package/src/api/channel-contract.ts +19 -0
  14. package/src/api/client.ts +100 -0
  15. package/src/api/contract.ts +2454 -0
  16. package/src/api/events-contract.ts +103 -0
  17. package/src/api/import-contract.ts +22 -0
  18. package/src/api/orchestrator-contract.ts +73 -0
  19. package/src/api/runner-contract.ts +25 -0
  20. package/src/api/scopes.ts +41 -0
  21. package/src/api/sessions-contract.ts +71 -0
  22. package/src/api/settings-contract.ts +50 -0
  23. package/src/channel/bridge.ts +97 -0
  24. package/src/channel/local.ts +154 -0
  25. package/src/channel/state.ts +11 -0
  26. package/src/channel/wire.ts +104 -0
  27. package/src/dev-sync.ts +206 -0
  28. package/src/dev.ts +453 -0
  29. package/src/events-tail.ts +75 -0
  30. package/src/images.ts +498 -0
  31. package/src/index.ts +21 -0
  32. package/src/invocation.ts +34 -0
  33. package/src/isolation/anchored-fs.ts +284 -0
  34. package/src/isolation/colima.ts +159 -0
  35. package/src/isolation/container.ts +189 -0
  36. package/src/isolation/firecracker.ts +414 -0
  37. package/src/isolation/guest-protocol.json +1 -0
  38. package/src/isolation/host-user.ts +247 -0
  39. package/src/isolation/image.ts +336 -0
  40. package/src/isolation/index.ts +477 -0
  41. package/src/isolation/multipass.ts +541 -0
  42. package/src/isolation/network.ts +167 -0
  43. package/src/isolation/policy.ts +86 -0
  44. package/src/isolation/session.ts +385 -0
  45. package/src/isolation/sync.ts +329 -0
  46. package/src/isolation/vz-network-setup.ts +247 -0
  47. package/src/isolation/vz.ts +312 -0
  48. package/src/mirab-main.ts +536 -0
  49. package/src/mirab.ts +13 -0
  50. package/src/runner/define.ts +187 -0
  51. package/src/runner/image.ts +239 -0
  52. package/src/runner/index.ts +37 -0
  53. package/src/runner/serve.ts +491 -0
  54. package/src/runner/toolsets.ts +205 -0
  55. package/src/spawn-adapters/claude.ts +112 -0
  56. package/src/spawn-adapters/codex-logs.ts +130 -0
  57. package/src/spawn-adapters/codex-notify.ts +73 -0
  58. package/src/spawn-adapters/codex.ts +89 -0
  59. package/src/spawn-adapters/index.ts +40 -0
  60. package/src/spawn-adapters/types.ts +55 -0
  61. package/src/spawn-attach.ts +243 -0
  62. package/src/spawn-binary.ts +90 -0
  63. package/src/spawn-bridge.ts +504 -0
  64. package/src/spawn-bundle.ts +16 -0
  65. package/src/spawn-detach.ts +301 -0
  66. package/src/spawn-events.ts +35 -0
  67. package/src/spawn-hook.ts +86 -0
  68. package/src/spawn-input.ts +107 -0
  69. package/src/spawn-native.ts +49 -0
  70. package/src/spawn-nudge-worker.ts +61 -0
  71. package/src/spawn-nudge.ts +446 -0
  72. package/src/spawn-output.ts +45 -0
  73. package/src/spawn-queue.ts +135 -0
  74. package/src/spawn-quiet.ts +69 -0
  75. package/src/spawn-report.ts +114 -0
  76. package/src/spawn-sessions.ts +166 -0
  77. package/src/spawn-vm.ts +356 -0
  78. package/src/spawn-worker.ts +766 -0
  79. package/src/spawn.ts +468 -0
  80. package/src/station/config.ts +89 -0
  81. package/src/station/gateway.ts +94 -0
  82. package/src/station/vault.ts +96 -0
  83. package/src/test-cli.ts +90 -0
  84. package/src/tools.ts +3 -0
  85. package/types/agent/define.d.ts +76 -0
  86. package/types/agent/manifest.d.ts +53 -0
  87. package/types/agent/modules.d.ts +1 -0
  88. package/types/images.d.ts +124 -0
  89. package/types/index.d.ts +6 -0
  90. package/types/isolation/colima.d.ts +37 -0
  91. package/types/isolation/container.d.ts +25 -0
  92. package/types/isolation/firecracker.d.ts +55 -0
  93. package/types/isolation/host-user.d.ts +47 -0
  94. package/types/isolation/image.d.ts +158 -0
  95. package/types/isolation/index.d.ts +128 -0
  96. package/types/isolation/network.d.ts +33 -0
  97. package/types/isolation/policy.d.ts +29 -0
  98. package/types/isolation/session.d.ts +99 -0
  99. package/types/isolation/vz.d.ts +63 -0
  100. package/types/runner/define.d.ts +101 -0
  101. package/types/runner/image.d.ts +52 -0
  102. package/types/runner/index.d.ts +13 -0
  103. package/types/runner/serve.d.ts +69 -0
  104. package/types/runner/toolsets.d.ts +35 -0
  105. package/types/tools.d.ts +2 -0
  106. package/index.js +0 -1
@@ -0,0 +1,491 @@
1
+ // serveRunners: keeps each definition's slot filled with a fresh VM that takes
2
+ // exactly one job, then throws the VM away. Outbound only: the forge's runner
3
+ // in the VM polls the forge; nothing reaches this host.
4
+ //
5
+ // A slot: image (cached) → VM from a clone of it, filtered internet, nothing
6
+ // of the host's inside, no guest-to-host port → ticket (minted only once the
7
+ // guest answers, so a failed boot leaves no registration) → files, steps as
8
+ // the runner user → the job timer starts at the runner's "job started" line →
9
+ // VM closed, disk discarded → next slot.
10
+ import { randomBytes } from 'node:crypto'
11
+ import {
12
+ appendFile,
13
+ mkdir,
14
+ rename,
15
+ rm,
16
+ statfs,
17
+ writeFile,
18
+ } from 'node:fs/promises'
19
+ import { hostname, totalmem } from 'node:os'
20
+ import { join } from 'node:path'
21
+ import { startVm, vmBackend, type VmConfig } from '../isolation'
22
+ import { networkState } from '../isolation/network'
23
+ import type { Isolation } from '../isolation/session'
24
+ import { vzNetworkState } from '../isolation/vz'
25
+ import {
26
+ RUNNER_SETPRIV,
27
+ RUNNER_USER,
28
+ hostRunnerPlatform,
29
+ platformLabel,
30
+ type RunnerDefinition,
31
+ type RunnerPlatform,
32
+ type RunnerStep,
33
+ } from './define'
34
+ import {
35
+ runnerImage,
36
+ runnerRecipe,
37
+ type RunnerImage,
38
+ type RunnerRecipe,
39
+ } from './image'
40
+
41
+ export const runnersDir = (configDir: string) => join(configDir, 'runners')
42
+
43
+ export type SlotPhase = 'image' | 'booting' | 'waiting' | 'busy' | 'stopping'
44
+
45
+ export interface SlotState {
46
+ vmName: string
47
+ phase: SlotPhase
48
+ since: string
49
+ }
50
+
51
+ export interface RunnerState {
52
+ name: string
53
+ host: string
54
+ forge: string
55
+ image?: string
56
+ slots: SlotState[]
57
+ jobs: number
58
+ lastError?: { at: string; message: string }
59
+ }
60
+
61
+ /** Things serveRunners does to the world; tests pass their own. */
62
+ export interface ServeDeps {
63
+ startVm: (
64
+ vm: VmConfig,
65
+ vmDir: string,
66
+ log: (line: string) => void,
67
+ ) => Promise<Isolation>
68
+ image: (def: RunnerDefinition, recipe: RunnerRecipe) => Promise<RunnerImage>
69
+ recipe: (def: RunnerDefinition) => Promise<RunnerRecipe>
70
+ sleep: (ms: number, signal: AbortSignal) => Promise<void>
71
+ now: () => Date
72
+ /** How long a minute is (the job and idle timers); tests shorten it. */
73
+ minuteMs: number
74
+ }
75
+
76
+ export interface ServeOptions {
77
+ configDir: string
78
+ vm: VmConfig
79
+ /** Names registrations `<host>-<name>-<random>`. */
80
+ host?: string
81
+ /** RAM all slots together may use (default: host RAM − 3 GiB). */
82
+ budgetMiB?: number
83
+ signal?: AbortSignal
84
+ log?: (line: string) => void
85
+ deps?: Partial<ServeDeps>
86
+ }
87
+
88
+ const RECIPE_RECHECK_MS = 24 * 3600_000
89
+ const BACKOFF_MS = [10_000, 30_000, 60_000, 120_000, 300_000]
90
+ const SHUTDOWN_GRACE_MS = 10 * 60_000
91
+
92
+ const sleep = (ms: number, signal: AbortSignal) =>
93
+ new Promise<void>((resolve) => {
94
+ if (signal.aborted) return resolve()
95
+ const t = setTimeout(done, ms)
96
+ function done() {
97
+ clearTimeout(t)
98
+ signal.removeEventListener('abort', done)
99
+ resolve()
100
+ }
101
+ signal.addEventListener('abort', done)
102
+ })
103
+
104
+ /** A runner's VM: nothing of the host's inside (no work share or drive) and
105
+ * no guest-to-host port. */
106
+ export function startRunnerVm(
107
+ vm: VmConfig,
108
+ vmDir: string,
109
+ log: (line: string) => void,
110
+ ): Promise<Isolation> {
111
+ return startVm(vm, null, vmDir, log, undefined, { listen: [] })
112
+ }
113
+
114
+ /** A host name usable in a registration name. */
115
+ export function hostLabel(name: string = hostname()): string {
116
+ return (
117
+ name
118
+ .split('.')[0]!
119
+ .toLowerCase()
120
+ .replace(/[^a-z0-9-]/g, '-')
121
+ .replace(/^-+|-+$/g, '')
122
+ .slice(0, 24) || 'host'
123
+ )
124
+ }
125
+
126
+ /** Replaces every secret value in `line` (the ticket's, while it lives). */
127
+ export function redact(line: string, secrets: Iterable<string>): string {
128
+ let out = line
129
+ for (const s of secrets)
130
+ if (s.length >= 8) out = out.split(s).join('[redacted]')
131
+ return out
132
+ }
133
+
134
+ /** The env every job step gets: base, then the image's toolsets, then the
135
+ * forge's job env, then the step's own (later wins). PATH is composed. */
136
+ export function jobEnv(
137
+ def: RunnerDefinition,
138
+ image: Pick<RunnerImage['meta'], 'env' | 'path'>,
139
+ job: Record<string, string>,
140
+ step: Record<string, string> = {},
141
+ ): Record<string, string> {
142
+ const home = RUNNER_USER.home
143
+ const basePath =
144
+ '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
145
+ return {
146
+ HOME: home,
147
+ USER: RUNNER_USER.name,
148
+ LOGNAME: RUNNER_USER.name,
149
+ LANG: 'C.UTF-8',
150
+ // /tmp is a RAM disk in the guest; temp files go on the job's disk.
151
+ TMPDIR: `${home}/tmp`,
152
+ CARGO_BUILD_JOBS: String(def.vm.vcpus),
153
+ ...image.env,
154
+ ...job,
155
+ ...step,
156
+ PATH: [...image.path, step.PATH ?? job.PATH ?? basePath].join(':'),
157
+ }
158
+ }
159
+
160
+ /** The argv mirab really execs for a step: as the runner user unless asRoot,
161
+ * output lines cut to 4000 characters (the guest channel refuses a line over
162
+ * 32 KiB, which would end the job), the step's exit status kept. */
163
+ export function stepArgv(step: RunnerStep): string[] {
164
+ const argv = step.asRoot ? step.argv : [...RUNNER_SETPRIV, ...step.argv]
165
+ return [
166
+ 'bash',
167
+ '-o',
168
+ 'pipefail',
169
+ '-c',
170
+ '"$@" 2>&1 | cut -c1-4000',
171
+ 'mirab-step',
172
+ ...argv,
173
+ ]
174
+ }
175
+
176
+ /** Refuses a host that can't run these definitions safely. */
177
+ export async function admission(
178
+ defs: RunnerDefinition[],
179
+ o: {
180
+ configDir: string
181
+ budgetMiB: number
182
+ backend: 'vz' | 'firecracker'
183
+ filtered: boolean
184
+ freeBytes: number
185
+ },
186
+ ): Promise<void> {
187
+ const names = new Set<string>()
188
+ for (const d of defs) {
189
+ if (names.has(d.name)) throw new Error(`two runners are named ${d.name}`)
190
+ names.add(d.name)
191
+ }
192
+ const mem = defs.reduce((s, d) => s + d.pool.max * d.vm.memoryMiB, 0)
193
+ if (mem > o.budgetMiB)
194
+ throw new Error(
195
+ `runners need ${mem} MiB of VM memory at once; this host's budget is ${o.budgetMiB} MiB (lower vm.memoryMiB or pool.max, or pass budgetMiB)`,
196
+ )
197
+ const disk =
198
+ defs.reduce((s, d) => s + d.pool.max * d.vm.diskGiB, 0) * 1024 ** 3
199
+ const reserve = 20 * 1024 ** 3
200
+ if (disk + reserve > o.freeBytes)
201
+ throw new Error(
202
+ `runners may fill ${disk / 1024 ** 3} GiB of disk at once; ${Math.floor(o.freeBytes / 1024 ** 3)} GiB is free and 20 GiB stays in reserve`,
203
+ )
204
+ if (!o.filtered)
205
+ throw new Error(
206
+ o.backend === 'vz'
207
+ ? 'runner VMs need the vz network filter (internet yes, LAN and tailnet no): sudo ~/.mirab/bin/mirab vm network setup'
208
+ : 'runner VMs need the filtered mirabbr0 bridge (internet yes, LAN and tailnet no): see docs/VM.md',
209
+ )
210
+ }
211
+
212
+ export async function serveRunners(
213
+ defs: RunnerDefinition[],
214
+ o: ServeOptions,
215
+ ): Promise<void> {
216
+ const log = o.log ?? ((l: string) => console.log(l))
217
+ const host = hostLabel(o.host)
218
+ const signal = o.signal ?? new AbortController().signal
219
+ const deps: ServeDeps = {
220
+ startVm: startRunnerVm,
221
+ recipe: (def) => runnerRecipe(def),
222
+ image: (_def, recipe) =>
223
+ runnerImage({ configDir: o.configDir, vm: o.vm, recipe, log }),
224
+ sleep,
225
+ now: () => new Date(),
226
+ minuteMs: 60_000,
227
+ ...o.deps,
228
+ }
229
+ if (!o.deps?.startVm) {
230
+ const backend = vmBackend()
231
+ const fs = await statfs(o.configDir).catch(() => undefined)
232
+ await admission(defs, {
233
+ configDir: o.configDir,
234
+ budgetMiB: o.budgetMiB ?? Math.floor(totalmem() / 1024 ** 2) - 3072,
235
+ backend,
236
+ filtered:
237
+ backend === 'vz' ? vzNetworkState().filtered : networkState().ready,
238
+ freeBytes: fs ? fs.bavail * fs.bsize : 0,
239
+ })
240
+ }
241
+ await Promise.all(
242
+ defs.map((def) => serveOne(def, host, deps, o, log, signal)),
243
+ )
244
+ }
245
+
246
+ async function serveOne(
247
+ def: RunnerDefinition,
248
+ host: string,
249
+ deps: ServeDeps,
250
+ o: ServeOptions,
251
+ hostLog: (line: string) => void,
252
+ signal: AbortSignal,
253
+ ) {
254
+ const dir = join(runnersDir(o.configDir), def.name)
255
+ const vmsDir = join(dir, 'vms')
256
+ const prefix = `${host}-${def.name}-`
257
+ await mkdir(dir, { recursive: true, mode: 0o700 })
258
+ const logFile = join(dir, 'serve.log')
259
+ const secrets = new Set<string>()
260
+ const log = (line: string) => {
261
+ const clean = redact(line, secrets)
262
+ hostLog(`[${def.name}] ${clean}`)
263
+ void appendFile(logFile, `${deps.now().toISOString()} ${clean}\n`).catch(
264
+ () => {},
265
+ )
266
+ }
267
+ const state: RunnerState = {
268
+ name: def.name,
269
+ host,
270
+ forge: def.forge.name,
271
+ slots: [],
272
+ jobs: 0,
273
+ }
274
+ // One write at a time, each whole (temp file + rename): status readers and
275
+ // a crash never see half a file.
276
+ let saving: Promise<void> = Promise.resolve()
277
+ const save = () =>
278
+ (saving = saving.then(async () => {
279
+ const file = join(dir, 'state.json')
280
+ const tmp = `${file}.${process.pid}.tmp`
281
+ try {
282
+ await writeFile(tmp, JSON.stringify(state, null, 2) + '\n', {
283
+ mode: 0o600,
284
+ })
285
+ await rename(tmp, file)
286
+ } catch {}
287
+ }))
288
+ const fail = (message: string) => {
289
+ state.lastError = {
290
+ at: deps.now().toISOString(),
291
+ message: redact(message, secrets),
292
+ }
293
+ log(`error: ${message}`)
294
+ }
295
+
296
+ // A crash or reboot left VM directories behind: nothing in them is wanted.
297
+ await rm(vmsDir, { recursive: true, force: true })
298
+ await mkdir(vmsDir, { recursive: true, mode: 0o700 })
299
+ await def.forge
300
+ .reconcile(prefix, [])
301
+ .catch((e) => fail(`reconcile: ${e.message}`))
302
+ await save()
303
+
304
+ let recipe: RunnerRecipe | undefined
305
+ let recipeAt = 0
306
+ let failures = 0
307
+ const backoff = async () => {
308
+ const ms = BACKOFF_MS[Math.min(failures, BACKOFF_MS.length - 1)]!
309
+ failures++
310
+ log(`retrying in ${ms / 1000} s`)
311
+ await deps.sleep(ms, signal)
312
+ }
313
+
314
+ while (!signal.aborted) {
315
+ const vmName = prefix + randomBytes(3).toString('hex')
316
+ const slot: SlotState = {
317
+ vmName,
318
+ phase: 'image',
319
+ since: deps.now().toISOString(),
320
+ }
321
+ const phase = (p: SlotPhase) => {
322
+ slot.phase = p
323
+ slot.since = deps.now().toISOString()
324
+ void save()
325
+ }
326
+ state.slots = [slot]
327
+ let iso: Isolation | undefined
328
+ let ticketed = false
329
+ try {
330
+ // The recipe names the forge's runner release: look again daily, so a
331
+ // new release becomes a new image between jobs.
332
+ if (!recipe || deps.now().getTime() - recipeAt > RECIPE_RECHECK_MS) {
333
+ recipe = await deps.recipe(def)
334
+ recipeAt = deps.now().getTime()
335
+ }
336
+ phase('image')
337
+ const image = await deps.image(def, recipe)
338
+ state.image = image.meta.key
339
+ if (signal.aborted) break
340
+ phase('booting')
341
+ iso = await deps.startVm(
342
+ {
343
+ ...o.vm,
344
+ rootfs: image.rootfs,
345
+ vcpus: def.vm.vcpus,
346
+ memoryMiB: def.vm.memoryMiB,
347
+ egress: 'allow',
348
+ },
349
+ join(vmsDir, vmName),
350
+ log,
351
+ )
352
+ // Booted and answering: only now is a registration worth making.
353
+ const ticket = await def.forge.ticket({
354
+ name: vmName,
355
+ labels: [...new Set([platformLabel(recipe.platform), ...def.labels])],
356
+ platform: recipe.platform,
357
+ })
358
+ ticketed = true
359
+ for (const v of Object.values(ticket.secret)) secrets.add(v)
360
+ const job = def.forge.guest.job(ticket)
361
+ for (const f of job.files) {
362
+ if (f.secret) secrets.add(f.content)
363
+ await iso.put(f.path, f.content, f.mode)
364
+ }
365
+ phase('waiting')
366
+ const outcome = await runJob(
367
+ def,
368
+ iso,
369
+ image,
370
+ job,
371
+ signal,
372
+ log,
373
+ deps.minuteMs,
374
+ () => {
375
+ phase('busy')
376
+ state.jobs++
377
+ },
378
+ )
379
+ log(`${vmName}: ${outcome.why}`)
380
+ if (outcome.tookJob || outcome.expected) failures = 0
381
+ else {
382
+ // The runner gave up before any job (a refused or spent registration,
383
+ // a broken image): without this, a new VM would boot every few seconds.
384
+ fail(
385
+ `${vmName}: the runner exited before taking a job (${outcome.why})`,
386
+ )
387
+ if (!signal.aborted) await backoff()
388
+ }
389
+ } catch (e) {
390
+ fail(`${vmName}: ${(e as Error).message}`)
391
+ if (!signal.aborted) await backoff()
392
+ } finally {
393
+ phase('stopping')
394
+ await iso?.close().catch((e) => fail(`close ${vmName}: ${e.message}`))
395
+ if (ticketed)
396
+ // The runner deregisters itself after its job; anything left (an idle
397
+ // slot ended, a killed job) is removed here.
398
+ await def.forge
399
+ .reconcile(prefix, [])
400
+ .catch((e) => fail(`reconcile: ${e.message}`))
401
+ secrets.clear()
402
+ state.slots = []
403
+ await save()
404
+ }
405
+ }
406
+ }
407
+
408
+ /** Runs the job's steps; returns how the slot ended. */
409
+ async function runJob(
410
+ def: RunnerDefinition,
411
+ iso: Isolation,
412
+ image: RunnerImage,
413
+ job: { steps: RunnerStep[]; env: Record<string, string> },
414
+ stop: AbortSignal,
415
+ log: (line: string) => void,
416
+ minuteMs: number,
417
+ onStarted: () => void,
418
+ ): Promise<{ why: string; tookJob: boolean; expected: boolean }> {
419
+ const ctl = new AbortController()
420
+ let started = false
421
+ let why = 'finished'
422
+ /** Ended on purpose (idle lifetime, shutdown), not by the runner giving up. */
423
+ let expected = false
424
+ const idleMs = def.pool.idleHours * 60 * minuteMs
425
+ const jobMs = def.timeoutMinutes * minuteMs
426
+ const idleTimer = setTimeout(() => {
427
+ if (!started) {
428
+ why = `no job in ${def.pool.idleHours} h, ended`
429
+ expected = true
430
+ ctl.abort()
431
+ }
432
+ }, idleMs)
433
+ let jobTimer: ReturnType<typeof setTimeout> | undefined
434
+ let graceTimer: ReturnType<typeof setTimeout> | undefined
435
+ const onStop = () => {
436
+ if (!started) {
437
+ why = 'stopped while waiting'
438
+ expected = true
439
+ ctl.abort()
440
+ } else
441
+ graceTimer = setTimeout(
442
+ () => {
443
+ why = 'stopped: job killed after the shutdown grace period'
444
+ ctl.abort()
445
+ },
446
+ (SHUTDOWN_GRACE_MS / 60_000) * minuteMs,
447
+ )
448
+ }
449
+ stop.addEventListener('abort', onStop)
450
+ if (stop.aborted) onStop()
451
+ try {
452
+ for (const step of job.steps) {
453
+ if (ctl.signal.aborted) break
454
+ const code = await iso.exec(stepArgv(step), {
455
+ cwd: step.asRoot ? '/' : RUNNER_USER.home,
456
+ env: jobEnv(def, image.meta, job.env, step.env),
457
+ // A backstop: the timers above end it first.
458
+ timeoutMs: idleMs + jobMs + 10 * 60_000,
459
+ signal: ctl.signal,
460
+ onLine: async (line) => {
461
+ log(line)
462
+ if (!started && def.forge.guest.started(line)) {
463
+ started = true
464
+ clearTimeout(idleTimer)
465
+ onStarted()
466
+ jobTimer = setTimeout(() => {
467
+ why = `job killed at its ${def.timeoutMinutes} min limit`
468
+ ctl.abort()
469
+ }, jobMs)
470
+ }
471
+ },
472
+ })
473
+ if (ctl.signal.aborted) break
474
+ if (code !== 0)
475
+ return {
476
+ why: `step ${step.argv[0]} exited ${code}`,
477
+ tookJob: started,
478
+ expected,
479
+ }
480
+ }
481
+ return { why, tookJob: started, expected }
482
+ } finally {
483
+ clearTimeout(idleTimer)
484
+ clearTimeout(jobTimer)
485
+ clearTimeout(graceTimer)
486
+ stop.removeEventListener('abort', onStop)
487
+ }
488
+ }
489
+
490
+ export { hostRunnerPlatform }
491
+ export type { RunnerPlatform }
@@ -0,0 +1,205 @@
1
+ // Toolsets baked into a runner image: versions resolved on the host (so the
2
+ // image key names exact versions), archives checked in the build VM against
3
+ // the upstream checksum. Laid out the way GitHub's hosted images do
4
+ // (<tool cache>/<tool>/<version>/<arch>/ plus a sibling <arch>.complete), so
5
+ // actions/setup-node and setup-go find them and skip their download.
6
+ import type { RunnerPlatform, RunnerStep } from './define'
7
+
8
+ export const TOOL_CACHE = '/opt/hostedtoolcache'
9
+ export const RUST_HOME = '/opt/rust'
10
+
11
+ export interface ResolvedTool {
12
+ tool: 'rust' | 'node' | 'go'
13
+ version: string
14
+ url: string
15
+ sha256: string
16
+ /** Hosts the build VM fetches from. */
17
+ hosts: string[]
18
+ }
19
+
20
+ export type Fetcher = (url: string) => Promise<string>
21
+
22
+ const defaultFetch: Fetcher = async (url) => {
23
+ const res = await fetch(url, { signal: AbortSignal.timeout(30_000) })
24
+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`)
25
+ return res.text()
26
+ }
27
+
28
+ const SHA256 = /^[0-9a-f]{64}$/
29
+
30
+ /** `24` matches 24.x.y, `24.21` matches 24.21.y, `24.21.0` only itself. */
31
+ export function versionMatches(spec: string, version: string): boolean {
32
+ return version === spec || version.startsWith(spec + '.')
33
+ }
34
+
35
+ const nodeArch = (p: RunnerPlatform) => (p === 'linux-arm64' ? 'arm64' : 'x64')
36
+ const goArch = (p: RunnerPlatform) => (p === 'linux-arm64' ? 'arm64' : 'amd64')
37
+ const rustTriple = (p: RunnerPlatform) =>
38
+ p === 'linux-arm64' ? 'aarch64-unknown-linux-gnu' : 'x86_64-unknown-linux-gnu'
39
+ /** The tool cache's arch directory (setup-node and setup-go use these). */
40
+ export const cacheArch = nodeArch
41
+
42
+ export async function resolveNode(
43
+ spec: string,
44
+ platform: RunnerPlatform,
45
+ get: Fetcher = defaultFetch,
46
+ ): Promise<ResolvedTool> {
47
+ const index = JSON.parse(await get('https://nodejs.org/dist/index.json')) as {
48
+ version: string
49
+ }[]
50
+ const hit = index.find((r) =>
51
+ versionMatches(spec, r.version.replace(/^v/, '')),
52
+ )
53
+ if (!hit) throw new Error(`node ${spec}: no such release`)
54
+ const version = hit.version.replace(/^v/, '')
55
+ const file = `node-v${version}-linux-${nodeArch(platform)}.tar.xz`
56
+ const sums = await get(`https://nodejs.org/dist/v${version}/SHASUMS256.txt`)
57
+ const sha256 = sums
58
+ .split('\n')
59
+ .map((l) => l.trim().split(/\s+/))
60
+ .find(([, f]) => f === file)?.[0]
61
+ if (!sha256 || !SHA256.test(sha256))
62
+ throw new Error(`node ${version}: no checksum for ${file}`)
63
+ return {
64
+ tool: 'node',
65
+ version,
66
+ url: `https://nodejs.org/dist/v${version}/${file}`,
67
+ sha256,
68
+ hosts: ['nodejs.org'],
69
+ }
70
+ }
71
+
72
+ export async function resolveGo(
73
+ spec: string,
74
+ platform: RunnerPlatform,
75
+ get: Fetcher = defaultFetch,
76
+ ): Promise<ResolvedTool> {
77
+ const releases = JSON.parse(
78
+ await get('https://go.dev/dl/?mode=json&include=all'),
79
+ ) as {
80
+ version: string
81
+ stable: boolean
82
+ files: {
83
+ filename: string
84
+ os: string
85
+ arch: string
86
+ sha256: string
87
+ kind: string
88
+ }[]
89
+ }[]
90
+ const hit = releases.find(
91
+ (r) => r.stable && versionMatches(spec, r.version.replace(/^go/, '')),
92
+ )
93
+ if (!hit) throw new Error(`go ${spec}: no such stable release`)
94
+ const file = hit.files.find(
95
+ (f) =>
96
+ f.os === 'linux' && f.arch === goArch(platform) && f.kind === 'archive',
97
+ )
98
+ if (!file || !SHA256.test(file.sha256))
99
+ throw new Error(`go ${hit.version}: no linux-${goArch(platform)} archive`)
100
+ return {
101
+ tool: 'go',
102
+ version: hit.version.replace(/^go/, ''),
103
+ url: `https://dl.google.com/go/${file.filename}`,
104
+ sha256: file.sha256,
105
+ hosts: ['dl.google.com'],
106
+ }
107
+ }
108
+
109
+ /** Rust: rustup-init (checked by its published sha256) installs the toolchain
110
+ * `spec`; rustup checks every component it downloads itself. */
111
+ export async function resolveRust(
112
+ spec: string,
113
+ platform: RunnerPlatform,
114
+ get: Fetcher = defaultFetch,
115
+ ): Promise<ResolvedTool> {
116
+ const url = `https://static.rust-lang.org/rustup/dist/${rustTriple(platform)}/rustup-init`
117
+ const sha256 = (await get(`${url}.sha256`)).trim().split(/\s+/)[0]
118
+ if (!sha256 || !SHA256.test(sha256))
119
+ throw new Error(`rustup-init: no checksum at ${url}.sha256`)
120
+ return {
121
+ tool: 'rust',
122
+ version: spec,
123
+ url,
124
+ sha256,
125
+ hosts: ['static.rust-lang.org'],
126
+ }
127
+ }
128
+
129
+ export async function resolveTools(
130
+ tools: { rust?: string; node?: string; go?: string },
131
+ platform: RunnerPlatform,
132
+ get: Fetcher = defaultFetch,
133
+ ): Promise<ResolvedTool[]> {
134
+ const out: ResolvedTool[] = []
135
+ if (tools.rust) out.push(await resolveRust(tools.rust, platform, get))
136
+ if (tools.node) out.push(await resolveNode(tools.node, platform, get))
137
+ if (tools.go) out.push(await resolveGo(tools.go, platform, get))
138
+ return out
139
+ }
140
+
141
+ const q = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`
142
+
143
+ /** Download to the root disk (not tmpfs /tmp) and check the checksum first. */
144
+ const fetchChecked = (t: ResolvedTool, to: string) =>
145
+ `curl -fsSL -o ${q(to)} ${q(t.url)} && echo ${q(`${t.sha256} ${to}`)} | sha256sum -c -`
146
+
147
+ /** The build steps (root, in the build VM) for the resolved toolsets. */
148
+ export function toolSteps(
149
+ tools: ResolvedTool[],
150
+ platform: RunnerPlatform,
151
+ ): RunnerStep[] {
152
+ const arch = cacheArch(platform)
153
+ return tools.map((t) => {
154
+ if (t.tool === 'rust')
155
+ return {
156
+ argv: [
157
+ 'sh',
158
+ '-euc',
159
+ [
160
+ `mkdir -p ${RUST_HOME}/dl`,
161
+ fetchChecked(t, `${RUST_HOME}/dl/rustup-init`),
162
+ `chmod 755 ${RUST_HOME}/dl/rustup-init`,
163
+ `RUSTUP_HOME=${RUST_HOME}/rustup CARGO_HOME=${RUST_HOME}/cargo ${RUST_HOME}/dl/rustup-init -y --no-modify-path --profile minimal --default-toolchain ${q(t.version)} -c clippy -c rustfmt`,
164
+ `rm -rf ${RUST_HOME}/dl`,
165
+ ].join(' && '),
166
+ ],
167
+ }
168
+ const dir = `${TOOL_CACHE}/${t.tool}/${t.version}/${arch}`
169
+ const archive = `${TOOL_CACHE}/.dl-${t.tool}`
170
+ return {
171
+ argv: [
172
+ 'sh',
173
+ '-euc',
174
+ [
175
+ `mkdir -p ${q(dir)}`,
176
+ fetchChecked(t, archive),
177
+ `tar -xf ${q(archive)} -C ${q(dir)} --strip-components=1`,
178
+ `rm -f ${q(archive)}`,
179
+ `touch ${q(`${dir}.complete`)}`,
180
+ ].join(' && '),
181
+ ],
182
+ }
183
+ })
184
+ }
185
+
186
+ /** What a job's env needs to find the toolsets (merged into the base env). */
187
+ export function toolEnv(
188
+ tools: ResolvedTool[],
189
+ platform: RunnerPlatform,
190
+ ): { env: Record<string, string>; path: string[] } {
191
+ const arch = cacheArch(platform)
192
+ const env: Record<string, string> = {
193
+ RUNNER_TOOL_CACHE: TOOL_CACHE,
194
+ AGENT_TOOLSDIRECTORY: TOOL_CACHE,
195
+ }
196
+ const path: string[] = []
197
+ for (const t of tools) {
198
+ if (t.tool === 'rust') {
199
+ env.RUSTUP_HOME = `${RUST_HOME}/rustup`
200
+ env.CARGO_HOME = `${RUST_HOME}/cargo`
201
+ path.push(`${RUST_HOME}/cargo/bin`)
202
+ } else path.push(`${TOOL_CACHE}/${t.tool}/${t.version}/${arch}/bin`)
203
+ }
204
+ return { env, path }
205
+ }