@vzn/vx-dte 0.0.0 → 0.0.650
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +923 -1
- package/index.ts +4 -0
- package/package.json +40 -4
- package/src/cpu-account.ts +134 -0
- package/src/dispatch.ts +268 -0
- package/src/docker.ts +156 -0
- package/src/forecast.ts +68 -0
- package/src/github.ts +523 -0
- package/src/gitlab.ts +130 -0
- package/src/index.ts +12 -0
- package/src/init.ts +522 -0
- package/src/local.ts +256 -0
- package/src/messages.ts +219 -0
- package/src/noise.ts +248 -0
- package/src/orchestrator.ts +2359 -0
- package/src/outputs.ts +117 -0
- package/src/planner.ts +988 -0
- package/src/plugin.ts +1018 -0
- package/src/printable.ts +11 -0
- package/src/seal.ts +509 -0
- package/src/strategy.ts +50 -0
- package/src/trace.ts +288 -0
- package/src/tree.ts +78 -0
- package/src/usage.ts +135 -0
- package/src/warm-ups.ts +85 -0
- package/src/wire.ts +1060 -0
- package/src/worker.ts +1038 -0
package/src/github.ts
ADDED
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
// `github()`: workers as jobs of one workflow the run dispatches per wave
|
|
2
|
+
// (design dte-2026-10.md, "Adapters"): `workflow_dispatch` with the run,
|
|
3
|
+
// the relay, the commit, a size and the workers of that size, which the
|
|
4
|
+
// workflow runs as a matrix on the runners it names for the size. Workers
|
|
5
|
+
// prove themselves by their OIDC identity: the run makes no secret.
|
|
6
|
+
|
|
7
|
+
import { readFileSync } from 'node:fs'
|
|
8
|
+
import { appendFile } from 'node:fs/promises'
|
|
9
|
+
import path from 'node:path'
|
|
10
|
+
import { refuseUnknownOptions, secretNamed, UserError, ui, type PluginOptionKinds } from '@vzn/vx'
|
|
11
|
+
import type { WorkerSpawn, DteAdapter } from './plugin.js'
|
|
12
|
+
import { printable } from './printable.js'
|
|
13
|
+
|
|
14
|
+
export interface GithubOptions {
|
|
15
|
+
/** The workers' workflow file in `.github/workflows` (default `vx-worker.yml`). */
|
|
16
|
+
readonly workflow?: string
|
|
17
|
+
/** The runner label of each size, by size name, which `vx dte init` writes; absent, its platform's GitHub-hosted runner. */
|
|
18
|
+
readonly runsOn?: Readonly<Record<string, string>>
|
|
19
|
+
/**
|
|
20
|
+
* Variables the run hands its workers, as the workflow's `env` input (a
|
|
21
|
+
* JSON object): a value the run's job decides, such as a server's URL
|
|
22
|
+
* for this commit. An undefined one is left out.
|
|
23
|
+
*/
|
|
24
|
+
readonly env?: Readonly<Record<string, string | undefined>>
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface Job {
|
|
28
|
+
readonly name: string
|
|
29
|
+
readonly status: string
|
|
30
|
+
readonly conclusion: string | null
|
|
31
|
+
readonly steps?: readonly { name: string; conclusion: string | null }[]
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const KEYS: PluginOptionKinds<GithubOptions> = {
|
|
35
|
+
workflow: 'string',
|
|
36
|
+
runsOn: 'object',
|
|
37
|
+
env: 'object',
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** In `RUNNER_TEMP`: one dispatched workflow run's API path per line. */
|
|
41
|
+
const RUNS_FILE = 'vx-dte-runs'
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* In `RUNNER_TEMP`: the worker runs a step dispatched before the run began,
|
|
45
|
+
* one per line, `<API path> <worker>,<worker>`: they start while the job
|
|
46
|
+
* checks out and installs (`adopt`).
|
|
47
|
+
*/
|
|
48
|
+
const STARTED_FILE = 'vx-dte-started'
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* A dispatched run's jobs are first read this soon, then twice as late each
|
|
52
|
+
* time, up to `WATCH_MAX_MS`. Every job of a repository shares
|
|
53
|
+
* GITHUB_TOKEN's API limit (1000 an hour): at 10 s, a dozen runs at once
|
|
54
|
+
* spent it, and dispatches and cancels were refused (2026-10-28).
|
|
55
|
+
*/
|
|
56
|
+
const WATCH_FIRST_MS = 2_000
|
|
57
|
+
const WATCH_MAX_MS = 30_000
|
|
58
|
+
|
|
59
|
+
/** A dispatch the API limit refused waits for its reset only this long: the worker is `starting` meanwhile. */
|
|
60
|
+
const DISPATCH_WAIT_MS = 5 * 60_000
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A cancel the API limit refused waits for its reset only this long: a
|
|
64
|
+
* worker run left queued starts for no one, then reads its run is over
|
|
65
|
+
* and cancels itself (the workflow's first step, `parent`).
|
|
66
|
+
*/
|
|
67
|
+
const CANCEL_WAIT_MS = 60_000
|
|
68
|
+
|
|
69
|
+
/** When a refused answer's API limit resets (epoch ms), or undefined for a refusal that is no limit. */
|
|
70
|
+
function limitReset(res: Response): number | undefined {
|
|
71
|
+
if (res.status !== 403 && res.status !== 429) return undefined
|
|
72
|
+
const after = res.headers.get('retry-after')
|
|
73
|
+
if (after !== null && Number.isFinite(Number(after))) return Date.now() + Number(after) * 1000
|
|
74
|
+
const reset = res.headers.get('x-ratelimit-reset')
|
|
75
|
+
if (
|
|
76
|
+
res.headers.get('x-ratelimit-remaining') === '0' &&
|
|
77
|
+
reset !== null &&
|
|
78
|
+
Number.isFinite(Number(reset))
|
|
79
|
+
)
|
|
80
|
+
return Number(reset) * 1000
|
|
81
|
+
return undefined
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const resetsAt = (at: number): string =>
|
|
85
|
+
`; the repository's API limit resets at ${new Date(at).toISOString()}`
|
|
86
|
+
|
|
87
|
+
/** The jobs a matrix makes in one workflow run at most: one dispatch's workers. */
|
|
88
|
+
const MATRIX_MAX = 256
|
|
89
|
+
|
|
90
|
+
export function github(options: GithubOptions = {}): DteAdapter {
|
|
91
|
+
refuseUnknownOptions('github()', options, KEYS)
|
|
92
|
+
const workflow = options.workflow ?? 'vx-worker.yml'
|
|
93
|
+
// GitHub reads no workflow in a subdirectory, and `vx dte init` writes
|
|
94
|
+
// the run's own ci.yml beside it: the same name wrote one over the other.
|
|
95
|
+
if (workflow === '' || workflow.includes('/') || workflow === 'ci.yml')
|
|
96
|
+
throw new UserError(
|
|
97
|
+
`github() option "workflow" must be a file name in .github/workflows other than ci.yml, such as vx-worker.yml, got ${JSON.stringify(workflow)}`,
|
|
98
|
+
)
|
|
99
|
+
const handed = handedEnv(options.env ?? {})
|
|
100
|
+
// The workers one demand starts, by size: one dispatch each.
|
|
101
|
+
const waves = new Map<string, WorkerSpawn[]>()
|
|
102
|
+
// The workflow runs dispatched, and the dispatches still in flight: a
|
|
103
|
+
// run that ends first cancels those still queued, or they hold runners
|
|
104
|
+
// (macOS: five a plan) that the next run's workers wait for.
|
|
105
|
+
const runs: string[] = []
|
|
106
|
+
const pending = new Set<Promise<void>>()
|
|
107
|
+
// A job that fails before its worker joins (checkout, install) told the
|
|
108
|
+
// run nothing: its worker stayed `starting`, held its size's `max`, and a
|
|
109
|
+
// run with no other worker waited for the job's timeout (2026-10-22).
|
|
110
|
+
// Each job is named for its worker; one that ends badly says so.
|
|
111
|
+
const watched = new Map<string, Map<string, WorkerSpawn>>()
|
|
112
|
+
let watch: ReturnType<typeof setTimeout> | undefined
|
|
113
|
+
let closed = false
|
|
114
|
+
// Each run's jobs answer's ETag: asked with it, an unchanged answer is a
|
|
115
|
+
// 304, which the API limit does not count.
|
|
116
|
+
const etags = new Map<string, string>()
|
|
117
|
+
// No jobs are asked before this (epoch ms): the API limit is spent.
|
|
118
|
+
let held = 0
|
|
119
|
+
// The workflow runs whose jobs are this run's workers, by id: a run's id
|
|
120
|
+
// is a dispatch input anyone who reads the repository's Actions sees, so
|
|
121
|
+
// a job of any other run is no worker of it.
|
|
122
|
+
const workerRuns = new Set<string>()
|
|
123
|
+
// Workers the run no longer needs: a run of only these is cancelled.
|
|
124
|
+
const cancelled = new Set<string>()
|
|
125
|
+
// A poll or cancel that failed is said, each cause once (2026-10-23: no
|
|
126
|
+
// fault is silent): a job that ended badly went unseen, a run held runners.
|
|
127
|
+
let warn = (m: string): void => ui.say('warn', m)
|
|
128
|
+
const said = new Set<string>()
|
|
129
|
+
const runOf = (run: string): string => `workflow run ${run.slice(run.lastIndexOf('/') + 1)}`
|
|
130
|
+
const failed = (what: string, why: string): void => {
|
|
131
|
+
const m = `[vx] github(): ${what} failed: ${why}`
|
|
132
|
+
if (said.has(m)) return
|
|
133
|
+
said.add(m)
|
|
134
|
+
warn(m)
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* A cancel's answer: 202, or 409 for a run already finished; anything
|
|
138
|
+
* else is said. One the spent API limit refused is asked again at its
|
|
139
|
+
* reset, when that comes within `CANCEL_WAIT_MS`.
|
|
140
|
+
*/
|
|
141
|
+
const cancelOf = async (run: string): Promise<void> => {
|
|
142
|
+
const end = Date.now() + CANCEL_WAIT_MS
|
|
143
|
+
for (;;) {
|
|
144
|
+
const res = await call(`${run}/cancel`, { method: 'POST' }).catch((err: unknown) => {
|
|
145
|
+
failed(`cancelling ${runOf(run)}`, err instanceof Error ? err.message : String(err))
|
|
146
|
+
return undefined
|
|
147
|
+
})
|
|
148
|
+
if (res === undefined || res.ok || res.status === 409) return
|
|
149
|
+
// quiet: the status is said below either way; the body only adds to it
|
|
150
|
+
const text = printable((await res.text().catch(() => '')).slice(0, 200))
|
|
151
|
+
const reset = limitReset(res)
|
|
152
|
+
if (reset !== undefined && reset <= end) {
|
|
153
|
+
await Bun.sleep(Math.max(0, reset - Date.now()))
|
|
154
|
+
continue
|
|
155
|
+
}
|
|
156
|
+
failed(
|
|
157
|
+
`cancelling ${runOf(run)}`,
|
|
158
|
+
`${res.status} ${text}${reset === undefined ? '' : resetsAt(reset)}`,
|
|
159
|
+
)
|
|
160
|
+
return
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// A worker `cancel` was asked to stop, until it is known whether it was:
|
|
164
|
+
// true when it was left running (`cancel`'s answer).
|
|
165
|
+
const deciding = new Map<string, (left: boolean) => void>()
|
|
166
|
+
// Workers whose dispatch has not answered yet.
|
|
167
|
+
const flying = new Set<string>()
|
|
168
|
+
// Of `cancelled`, those given up as late: stuck or queued, stopped whatever their job holds.
|
|
169
|
+
const late = new Set<string>()
|
|
170
|
+
const decide = (id: string, left: boolean): void => {
|
|
171
|
+
deciding.get(id)?.(left)
|
|
172
|
+
deciding.delete(id)
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Whether a job of `run` has a runner: its worker's setup is paid, and
|
|
176
|
+
* a cancel only races its join. Read fresh, past the poll's ETag, which
|
|
177
|
+
* it leaves alone; undefined when GitHub does not answer.
|
|
178
|
+
*/
|
|
179
|
+
const hasRunner = async (run: string): Promise<boolean | undefined> => {
|
|
180
|
+
const res = await call(`${run}/jobs?per_page=100`).catch((err: unknown) => {
|
|
181
|
+
failed(`reading ${runOf(run)}'s jobs`, err instanceof Error ? err.message : String(err))
|
|
182
|
+
return undefined
|
|
183
|
+
})
|
|
184
|
+
if (res === undefined) return undefined
|
|
185
|
+
if (!res.ok) {
|
|
186
|
+
failed(
|
|
187
|
+
`reading ${runOf(run)}'s jobs`,
|
|
188
|
+
`${res.status} ${printable((await res.text().catch(() => '')).slice(0, 200))}`,
|
|
189
|
+
)
|
|
190
|
+
return undefined
|
|
191
|
+
}
|
|
192
|
+
// quiet: an answer that is no JSON leaves the run to be cancelled, as before it was read
|
|
193
|
+
const body = (await res.json().catch(() => undefined)) as { jobs?: Job[] } | null | undefined
|
|
194
|
+
if (!Array.isArray(body?.jobs)) return undefined
|
|
195
|
+
return body.jobs.some((j) => j.status === 'in_progress')
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Cancels `run` once none of its workers is still needed and none of its
|
|
199
|
+
* jobs has a runner. Otherwise it goes on, and each worker let go in it
|
|
200
|
+
* is left running: it joins, and the run takes it (2026-10-29: small#3,
|
|
201
|
+
* let go with its job running, joined to be told to exit while two tasks
|
|
202
|
+
* waited for a core).
|
|
203
|
+
*/
|
|
204
|
+
const letGo = (run: string, workers: Map<string, WorkerSpawn>): void => {
|
|
205
|
+
const ids = [...workers.keys()]
|
|
206
|
+
if (!ids.every((id) => cancelled.has(id))) {
|
|
207
|
+
for (const id of ids)
|
|
208
|
+
if (late.has(id)) decide(id, false)
|
|
209
|
+
else if (cancelled.delete(id)) decide(id, true)
|
|
210
|
+
return
|
|
211
|
+
}
|
|
212
|
+
watched.delete(run)
|
|
213
|
+
// Let go already (a cancel while its dispatch wrote RUNNER_TEMP).
|
|
214
|
+
const i = runs.indexOf(run)
|
|
215
|
+
if (i < 0) return
|
|
216
|
+
runs.splice(i, 1)
|
|
217
|
+
const p = (ids.some((id) => late.has(id)) ? Promise.resolve(false) : hasRunner(run))
|
|
218
|
+
.then((held) => {
|
|
219
|
+
if (held === true && !closed) {
|
|
220
|
+
runs.push(run)
|
|
221
|
+
watched.set(run, workers)
|
|
222
|
+
if (watch === undefined) look(WATCH_FIRST_MS)
|
|
223
|
+
for (const id of ids) {
|
|
224
|
+
cancelled.delete(id)
|
|
225
|
+
decide(id, true)
|
|
226
|
+
}
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
for (const id of ids) decide(id, false)
|
|
230
|
+
return cancelOf(run)
|
|
231
|
+
})
|
|
232
|
+
.finally(() => pending.delete(p))
|
|
233
|
+
pending.add(p)
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Every page of `run`'s jobs: a wave of more than 100 workers is a run of
|
|
237
|
+
* more than 100 jobs. Undefined when unread, or when a one-page run has
|
|
238
|
+
* not changed since the last read (a 304 costs no API limit).
|
|
239
|
+
*/
|
|
240
|
+
const jobsOf = async (run: string): Promise<Job[] | undefined> => {
|
|
241
|
+
const all: Job[] = []
|
|
242
|
+
for (let page = 1; ; page++) {
|
|
243
|
+
const tag = page === 1 ? etags.get(run) : undefined
|
|
244
|
+
const res = await call(
|
|
245
|
+
`${run}/jobs?per_page=100&page=${page}`,
|
|
246
|
+
tag === undefined ? {} : { headers: { 'if-none-match': tag } },
|
|
247
|
+
).catch((err: unknown) => {
|
|
248
|
+
failed(`reading ${runOf(run)}'s jobs`, err instanceof Error ? err.message : String(err))
|
|
249
|
+
return undefined
|
|
250
|
+
})
|
|
251
|
+
if (res === undefined || res.status === 304) return undefined
|
|
252
|
+
if (!res.ok) {
|
|
253
|
+
const reset = limitReset(res)
|
|
254
|
+
if (reset !== undefined) held = Math.max(held, reset)
|
|
255
|
+
failed(
|
|
256
|
+
`reading ${runOf(run)}'s jobs`,
|
|
257
|
+
`${res.status} ${printable((await res.text().catch(() => '')).slice(0, 200))}${reset === undefined ? '' : resetsAt(reset)}`,
|
|
258
|
+
)
|
|
259
|
+
return undefined
|
|
260
|
+
}
|
|
261
|
+
const body = (await res.json().catch((err: unknown) => {
|
|
262
|
+
failed(
|
|
263
|
+
`reading ${runOf(run)}'s jobs`,
|
|
264
|
+
`no JSON: ${err instanceof Error ? err.message : String(err)}`,
|
|
265
|
+
)
|
|
266
|
+
return undefined
|
|
267
|
+
})) as { jobs?: Job[] } | null | undefined
|
|
268
|
+
if (body === undefined) return undefined
|
|
269
|
+
const jobs = body?.jobs ?? []
|
|
270
|
+
all.push(...jobs)
|
|
271
|
+
// A first page's tag says nothing of the next: only a run of one page is asked by it.
|
|
272
|
+
const etag = res.headers.get('etag')
|
|
273
|
+
if (page === 1 && etag !== null && jobs.length < 100) etags.set(run, etag)
|
|
274
|
+
else etags.delete(run)
|
|
275
|
+
if (jobs.length < 100) return all
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
const look = (ms: number): void => {
|
|
279
|
+
watch = setTimeout(() => {
|
|
280
|
+
void Promise.all(
|
|
281
|
+
[...watched].map(async ([run, workers]) => {
|
|
282
|
+
const jobs = await jobsOf(run)
|
|
283
|
+
if (jobs === undefined) return
|
|
284
|
+
for (const j of jobs) {
|
|
285
|
+
const a = workers.get(j.name)
|
|
286
|
+
if (a === undefined || j.status !== 'completed') continue
|
|
287
|
+
workers.delete(j.name)
|
|
288
|
+
if (closed || cancelled.has(j.name)) continue
|
|
289
|
+
// A success too: a worker that found no live run left at once,
|
|
290
|
+
// and the run must not wait on it (2026-10-23).
|
|
291
|
+
const step = j.steps?.find((x) => x.conclusion === 'failure')?.name
|
|
292
|
+
a.exited(`job ${j.conclusion}${step === undefined ? '' : ` at ${step}`}`)
|
|
293
|
+
}
|
|
294
|
+
if (workers.size === 0) watched.delete(run)
|
|
295
|
+
}),
|
|
296
|
+
).then(() => {
|
|
297
|
+
const next = Math.min(2 * ms, WATCH_MAX_MS)
|
|
298
|
+
if (!closed && watched.size > 0) look(Math.max(next, held - Date.now()))
|
|
299
|
+
else watch = undefined
|
|
300
|
+
})
|
|
301
|
+
}, ms)
|
|
302
|
+
watch.unref()
|
|
303
|
+
}
|
|
304
|
+
const call = (path: string, init: RequestInit = {}): Promise<Response> => {
|
|
305
|
+
const { GITHUB_API_URL: api, GITHUB_TOKEN: token } = process.env
|
|
306
|
+
return fetch(`${api}${path}`, {
|
|
307
|
+
...init,
|
|
308
|
+
headers: {
|
|
309
|
+
...(init.headers as Record<string, string> | undefined),
|
|
310
|
+
authorization: `Bearer ${token}`,
|
|
311
|
+
accept: 'application/vnd.github+json',
|
|
312
|
+
'content-type': 'application/json',
|
|
313
|
+
// The version whose dispatch answers with the workflow run's id.
|
|
314
|
+
'x-github-api-version': '2026-03-10',
|
|
315
|
+
},
|
|
316
|
+
signal: AbortSignal.timeout(10_000),
|
|
317
|
+
})
|
|
318
|
+
}
|
|
319
|
+
const dispatch = async (size: string, workers: WorkerSpawn[]): Promise<void> => {
|
|
320
|
+
const env = process.env
|
|
321
|
+
const { GITHUB_API_URL: api, GITHUB_REPOSITORY: repo, GITHUB_TOKEN: token } = env
|
|
322
|
+
if (!api || !repo || !token)
|
|
323
|
+
throw new Error(
|
|
324
|
+
'github(): dispatching workers needs GITHUB_TOKEN (actions: write) in a GitHub Actions job',
|
|
325
|
+
)
|
|
326
|
+
const sha = env['GITHUB_SHA']
|
|
327
|
+
const first = workers[0]!
|
|
328
|
+
// The branch's workflow file, or, once a merged PR's branch is deleted
|
|
329
|
+
// mid-run (GitHub answers 422 "No ref found"), its base's: the workers
|
|
330
|
+
// check out `sha` either way.
|
|
331
|
+
const refs = [env['GITHUB_HEAD_REF'] || env['GITHUB_REF_NAME'], env['GITHUB_BASE_REF']]
|
|
332
|
+
let res: Response
|
|
333
|
+
const end = Date.now() + DISPATCH_WAIT_MS
|
|
334
|
+
for (let i = 0; ;) {
|
|
335
|
+
res = await call(`/repos/${repo}/actions/workflows/${workflow}/dispatches`, {
|
|
336
|
+
method: 'POST',
|
|
337
|
+
body: JSON.stringify({
|
|
338
|
+
ref: refs[i],
|
|
339
|
+
inputs: {
|
|
340
|
+
run: first.run,
|
|
341
|
+
relay: first.relay,
|
|
342
|
+
sha,
|
|
343
|
+
size,
|
|
344
|
+
workers: JSON.stringify(workers.map((a) => a.id)),
|
|
345
|
+
// This run's attempt: a worker that starts after it ended
|
|
346
|
+
// cancels its own workflow run instead of working for no one.
|
|
347
|
+
...(env['GITHUB_RUN_ID'] === undefined
|
|
348
|
+
? {}
|
|
349
|
+
: { parent: `${env['GITHUB_RUN_ID']}/attempts/${env['GITHUB_RUN_ATTEMPT'] ?? '1'}` }),
|
|
350
|
+
...(options.env === undefined ? {} : { env: handed }),
|
|
351
|
+
},
|
|
352
|
+
}),
|
|
353
|
+
})
|
|
354
|
+
if (res.ok) break
|
|
355
|
+
const text = printable((await res.text()).slice(0, 200))
|
|
356
|
+
const reset = limitReset(res)
|
|
357
|
+
if (reset !== undefined) {
|
|
358
|
+
if (reset > end || closed)
|
|
359
|
+
throw new Error(
|
|
360
|
+
`github(): dispatching ${workflow}: ${res.status} ${text}${resetsAt(reset)}`,
|
|
361
|
+
)
|
|
362
|
+
await Bun.sleep(Math.max(0, reset - Date.now()))
|
|
363
|
+
continue
|
|
364
|
+
}
|
|
365
|
+
if (res.status !== 422 || !text.includes('No ref found') || !refs[++i]) {
|
|
366
|
+
// GitHub's own words named no fix for the two a first run meets (2026-10-26).
|
|
367
|
+
const fix =
|
|
368
|
+
res.status === 404
|
|
369
|
+
? `: .github/workflows/${workflow} is not on ${refs[i]} (vx dte init writes it; commit and push it)`
|
|
370
|
+
: res.status === 403
|
|
371
|
+
? ': the job needs `permissions: actions: write` to start workers'
|
|
372
|
+
: ''
|
|
373
|
+
throw new Error(`github(): dispatching ${workflow}: ${res.status} ${text}${fix}`)
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
const id =
|
|
377
|
+
res.status === 200
|
|
378
|
+
? ((await res.json()) as { workflow_run_id?: number }).workflow_run_id
|
|
379
|
+
: undefined
|
|
380
|
+
// An API naming no run (a 204) is refused, not trusted by repository
|
|
381
|
+
// alone: any job of it with id-token: write would join as a worker.
|
|
382
|
+
if (id === undefined) {
|
|
383
|
+
// No run to cancel: each let go meanwhile is settled before the refusal.
|
|
384
|
+
for (const a of workers) if (cancelled.delete(a.id)) decide(a.id, true)
|
|
385
|
+
throw new Error(
|
|
386
|
+
`github(): dispatching ${workflow} named no workflow run (${res.status}): its workers cannot be told from any job of the repository`,
|
|
387
|
+
)
|
|
388
|
+
}
|
|
389
|
+
workerRuns.add(String(id))
|
|
390
|
+
const run = `/repos/${repo}/actions/runs/${id}`
|
|
391
|
+
runs.push(run)
|
|
392
|
+
const mine = new Map(workers.map((a) => [a.id, a]))
|
|
393
|
+
watched.set(run, mine)
|
|
394
|
+
if (watch === undefined && !closed) look(WATCH_FIRST_MS)
|
|
395
|
+
// For the job's last step, which cancels them when this process
|
|
396
|
+
// dies before close() (a kill, an OOM, the job's timeout).
|
|
397
|
+
if (env['RUNNER_TEMP']) await appendFile(path.join(env['RUNNER_TEMP'], RUNS_FILE), `${run}\n`)
|
|
398
|
+
// Let go while this dispatch was in flight: cancel() knew no run.
|
|
399
|
+
if (!closed) letGo(run, mine)
|
|
400
|
+
}
|
|
401
|
+
return {
|
|
402
|
+
// Loaded only by `vx dte init`: a run never needs it.
|
|
403
|
+
init: async (setup) => (await import('./init.js')).githubInit(options, setup),
|
|
404
|
+
keyless: true,
|
|
405
|
+
workerRuns: () => workerRuns,
|
|
406
|
+
report(w) {
|
|
407
|
+
warn = w
|
|
408
|
+
},
|
|
409
|
+
adopt(worker) {
|
|
410
|
+
const temp = process.env['RUNNER_TEMP']
|
|
411
|
+
if (!temp) return
|
|
412
|
+
let text: string
|
|
413
|
+
try {
|
|
414
|
+
text = readFileSync(path.join(temp, STARTED_FILE), 'utf8')
|
|
415
|
+
// quiet: no list: this job started no worker
|
|
416
|
+
} catch {
|
|
417
|
+
return
|
|
418
|
+
}
|
|
419
|
+
for (const line of text.split('\n')) {
|
|
420
|
+
const [run, ids] = line.trim().split(/\s+/)
|
|
421
|
+
if (!run || !ids) continue
|
|
422
|
+
const workers = new Map<string, WorkerSpawn>()
|
|
423
|
+
for (const id of ids.split(','))
|
|
424
|
+
workers.set(id, worker(id, id.slice(0, id.lastIndexOf('#'))))
|
|
425
|
+
runs.push(run)
|
|
426
|
+
watched.set(run, workers)
|
|
427
|
+
workerRuns.add(run.slice(run.lastIndexOf('/') + 1))
|
|
428
|
+
}
|
|
429
|
+
if (watched.size > 0 && watch === undefined) look(WATCH_FIRST_MS)
|
|
430
|
+
},
|
|
431
|
+
spawn(a) {
|
|
432
|
+
flying.add(a.id)
|
|
433
|
+
const wave = waves.get(a.size)
|
|
434
|
+
// A full wave dispatches as it is; the next worker starts another.
|
|
435
|
+
if (wave !== undefined && wave.length < MATRIX_MAX) {
|
|
436
|
+
wave.push(a)
|
|
437
|
+
return
|
|
438
|
+
}
|
|
439
|
+
const workers = [a]
|
|
440
|
+
waves.set(a.size, workers)
|
|
441
|
+
queueMicrotask(() => {
|
|
442
|
+
if (waves.get(a.size) === workers) waves.delete(a.size)
|
|
443
|
+
const p = dispatch(a.size, workers)
|
|
444
|
+
.catch((err: unknown) => {
|
|
445
|
+
const why = err instanceof Error ? err.message : String(err)
|
|
446
|
+
for (const x of workers) {
|
|
447
|
+
x.exited(why)
|
|
448
|
+
decide(x.id, false)
|
|
449
|
+
}
|
|
450
|
+
})
|
|
451
|
+
.finally(() => {
|
|
452
|
+
for (const x of workers) flying.delete(x.id)
|
|
453
|
+
pending.delete(p)
|
|
454
|
+
})
|
|
455
|
+
pending.add(p)
|
|
456
|
+
})
|
|
457
|
+
},
|
|
458
|
+
// Before a worker's setup when its job has yet to run: its runner is
|
|
459
|
+
// free for others (2026-10-23). A run with a worker still needed, or
|
|
460
|
+
// a job with a runner, goes on; the workers let go in it are left
|
|
461
|
+
// running. One given up as late is stopped whatever its job holds.
|
|
462
|
+
cancel(ids, givenUp) {
|
|
463
|
+
if (givenUp === true) for (const id of ids) late.add(id)
|
|
464
|
+
const left = ids.map((id) => new Promise<boolean>((done) => deciding.set(id, done)))
|
|
465
|
+
// Neither watched nor in flight (its job ended, its dispatch failed): nothing to stop.
|
|
466
|
+
const gone = ids.filter(
|
|
467
|
+
(id) => !flying.has(id) && ![...watched.values()].some((w) => w.has(id)),
|
|
468
|
+
)
|
|
469
|
+
for (const id of ids) cancelled.add(id)
|
|
470
|
+
for (const [run, workers] of [...watched])
|
|
471
|
+
if (ids.some((id) => workers.has(id))) letGo(run, workers)
|
|
472
|
+
for (const id of gone) decide(id, false)
|
|
473
|
+
return Promise.all(left).then((l) => ids.filter((_, i) => l[i]))
|
|
474
|
+
},
|
|
475
|
+
// A finished run answers 409; one still queued or running is the
|
|
476
|
+
// run's no more: its results are in, or the run failed without them.
|
|
477
|
+
async close() {
|
|
478
|
+
closed = true
|
|
479
|
+
clearTimeout(watch)
|
|
480
|
+
await Promise.resolve() // a wave spawned this tick dispatches first
|
|
481
|
+
await Promise.all(pending)
|
|
482
|
+
for (const id of [...deciding.keys()]) decide(id, false)
|
|
483
|
+
// A cancel that fails leaves the run to its timeout, and says so.
|
|
484
|
+
await Promise.all(runs.splice(0).map(cancelOf))
|
|
485
|
+
},
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/** The relay token on GitHub Actions: an OIDC token for the relay's URL as audience. */
|
|
490
|
+
export async function githubRelayToken(relay: string): Promise<string | undefined> {
|
|
491
|
+
const { ACTIONS_ID_TOKEN_REQUEST_URL: url, ACTIONS_ID_TOKEN_REQUEST_TOKEN: bearer } = process.env
|
|
492
|
+
if (!url || !bearer) return undefined
|
|
493
|
+
// Loaded when asked: a run that is not distributed loads none of it.
|
|
494
|
+
const { oidcToken } = await import('./seal.js')
|
|
495
|
+
return oidcToken(url, bearer, relay).catch((err: unknown) => {
|
|
496
|
+
throw new UserError(
|
|
497
|
+
`dte(): ${err instanceof Error ? err.message : String(err)} (for the relay)`,
|
|
498
|
+
)
|
|
499
|
+
})
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* `env` as the workflow's input: each name a variable's, each value one
|
|
504
|
+
* line, since the workflow writes each as a line of `GITHUB_ENV`. No
|
|
505
|
+
* secret: a dispatch's inputs are shown with its run, and the worker
|
|
506
|
+
* job's log prints each step's env unmasked.
|
|
507
|
+
*/
|
|
508
|
+
function handedEnv(env: Readonly<Record<string, string | undefined>>): string {
|
|
509
|
+
const out: Record<string, string> = {}
|
|
510
|
+
for (const [name, value] of Object.entries(env)) {
|
|
511
|
+
if (value === undefined) continue
|
|
512
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== 'string' || /[\r\n]/.test(value))
|
|
513
|
+
throw new UserError(
|
|
514
|
+
`github(): env ${JSON.stringify(name)}: a variable's name, and a string of one line`,
|
|
515
|
+
)
|
|
516
|
+
if (secretNamed(name))
|
|
517
|
+
throw new UserError(
|
|
518
|
+
`github(): env ${JSON.stringify(name)} is a secret, which the worker job's log would show: give it to that job from secrets in its workflow`,
|
|
519
|
+
)
|
|
520
|
+
out[name] = value
|
|
521
|
+
}
|
|
522
|
+
return JSON.stringify(out)
|
|
523
|
+
}
|
package/src/gitlab.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// `gitlab()`: workers as pipelines of the project's own `.gitlab-ci.yml`
|
|
2
|
+
// that the run triggers once it is on the relay (design dte-2026-10.md,
|
|
3
|
+
// "Adapters"): a worker never arrives before its run. The trigger takes
|
|
4
|
+
// the job's `CI_JOB_TOKEN`, so no token is stored. One pipeline per size
|
|
5
|
+
// for each wave the run asks: its `vx-worker <size>` job is a matrix of
|
|
6
|
+
// `max` slots (`VX_DTE_SLOT`), of which the rules run those `VX_DTE_PICK`
|
|
7
|
+
// names, at the run's commit (`VX_DTE_SHA`). Each proves itself by the
|
|
8
|
+
// run's secret.
|
|
9
|
+
|
|
10
|
+
import { refuseUnknownOptions, type PluginOptionKinds } from '@vzn/vx'
|
|
11
|
+
import type { DteAdapter, WorkerSpawn } from './plugin.js'
|
|
12
|
+
import { printable } from './printable.js'
|
|
13
|
+
|
|
14
|
+
export interface GitlabOptions {
|
|
15
|
+
/** Runner tags by worker size: the runners each size's jobs go to. */
|
|
16
|
+
readonly tags?: Readonly<Record<string, readonly string[]>>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const KEYS: PluginOptionKinds<GitlabOptions> = { tags: 'object' }
|
|
20
|
+
|
|
21
|
+
export function gitlab(options: GitlabOptions = {}): DteAdapter {
|
|
22
|
+
refuseUnknownOptions('gitlab()', options, KEYS)
|
|
23
|
+
// The workers asked in one tick, by size: the run asks a wave at once.
|
|
24
|
+
const waves = new Map<string, { workers: WorkerSpawn[]; sent: Promise<void> }>()
|
|
25
|
+
return {
|
|
26
|
+
init: async (setup) => (await import('./init.js')).gitlabInit(options, setup),
|
|
27
|
+
|
|
28
|
+
spawn(a) {
|
|
29
|
+
let wave = waves.get(a.size)
|
|
30
|
+
// Two workers in one slot would be one job: the second starts a wave of its own.
|
|
31
|
+
if (wave === undefined || wave.workers.some((w) => slotOf(w) === slotOf(a))) {
|
|
32
|
+
const workers: WorkerSpawn[] = []
|
|
33
|
+
const sent = Promise.resolve().then(() => {
|
|
34
|
+
if (waves.get(a.size) === mine) waves.delete(a.size)
|
|
35
|
+
return trigger(workers)
|
|
36
|
+
})
|
|
37
|
+
const mine = { workers, sent }
|
|
38
|
+
wave = mine
|
|
39
|
+
waves.set(a.size, wave)
|
|
40
|
+
}
|
|
41
|
+
wave.workers.push(a)
|
|
42
|
+
return wave.sent
|
|
43
|
+
},
|
|
44
|
+
|
|
45
|
+
workerId() {
|
|
46
|
+
const slot = process.env['VX_DTE_SLOT']
|
|
47
|
+
for (const pair of (process.env['VX_DTE_IDS'] ?? '').split(' ')) {
|
|
48
|
+
const at = pair.indexOf('=')
|
|
49
|
+
if (at > 0 && pair.slice(0, at) === slot) return pair.slice(at + 1)
|
|
50
|
+
}
|
|
51
|
+
return undefined
|
|
52
|
+
},
|
|
53
|
+
|
|
54
|
+
// Random: a pipeline id is guessable.
|
|
55
|
+
runId() {
|
|
56
|
+
return process.env['GITLAB_CI'] === 'true' && process.env['CI_JOB_TOKEN']
|
|
57
|
+
? crypto.randomUUID()
|
|
58
|
+
: undefined
|
|
59
|
+
},
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** One pipeline for a wave of one size's workers. */
|
|
64
|
+
async function trigger(workers: readonly WorkerSpawn[]): Promise<void> {
|
|
65
|
+
const {
|
|
66
|
+
CI_API_V4_URL: base,
|
|
67
|
+
CI_PROJECT_ID: project,
|
|
68
|
+
CI_JOB_TOKEN: token,
|
|
69
|
+
CI_COMMIT_SHA: sha,
|
|
70
|
+
} = process.env
|
|
71
|
+
if (!base || !project || !token || !sha) {
|
|
72
|
+
const missing = Object.entries({
|
|
73
|
+
CI_API_V4_URL: base,
|
|
74
|
+
CI_JOB_TOKEN: token,
|
|
75
|
+
CI_PROJECT_ID: project,
|
|
76
|
+
CI_COMMIT_SHA: sha,
|
|
77
|
+
}).flatMap(([k, v]) => (v ? [] : [k]))
|
|
78
|
+
throw new Error(`gitlab(): not in a GitLab job (${missing.join(', ')})`)
|
|
79
|
+
}
|
|
80
|
+
const first = workers[0]!
|
|
81
|
+
const slots = workers.map((w) => [slotOf(w), w.id] as const)
|
|
82
|
+
// A merge request's pipeline is of its source branch; any other, of its
|
|
83
|
+
// ref. Once a merged request's source branch is deleted mid-run
|
|
84
|
+
// (GitLab's default), GitLab answers 400 "Reference not found": its
|
|
85
|
+
// target's file then, as github() does; the workers check out `sha` either way.
|
|
86
|
+
const refs = [
|
|
87
|
+
process.env['CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'] || process.env['CI_COMMIT_REF_NAME']!,
|
|
88
|
+
process.env['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'],
|
|
89
|
+
]
|
|
90
|
+
const { VX_DTE_WORKER: _, ...env } = first.env
|
|
91
|
+
const vars: Record<string, string> = {
|
|
92
|
+
...env,
|
|
93
|
+
VX_DTE_SIZE: first.size,
|
|
94
|
+
VX_DTE_SHA: sha,
|
|
95
|
+
VX_DTE_PICK: `/^(${slots.map(([s]) => s).join('|')})$/`,
|
|
96
|
+
VX_DTE_IDS: slots.map(([s, id]) => `${s}=${id}`).join(' '),
|
|
97
|
+
}
|
|
98
|
+
for (let i = 0; ; i++) {
|
|
99
|
+
const form = new FormData()
|
|
100
|
+
form.set('token', token)
|
|
101
|
+
form.set('ref', refs[i]!)
|
|
102
|
+
for (const [k, v] of Object.entries(vars)) form.set(`variables[${k}]`, v)
|
|
103
|
+
const res = await fetch(`${base}/projects/${project}/trigger/pipeline`, {
|
|
104
|
+
method: 'POST',
|
|
105
|
+
body: form,
|
|
106
|
+
signal: AbortSignal.timeout(10_000),
|
|
107
|
+
})
|
|
108
|
+
if (res.ok) return
|
|
109
|
+
const text = printable((await res.text()).slice(0, 200))
|
|
110
|
+
if (res.status === 400 && text.includes('Reference not found') && refs[i + 1]) continue
|
|
111
|
+
// GitLab.com's new projects let no one set pipeline variables (17.7),
|
|
112
|
+
// and GitLab's words name no setting.
|
|
113
|
+
const fix = text.includes('Insufficient permissions to set pipeline variables')
|
|
114
|
+
? ": the workers take the run's variables; in the project's Settings > CI/CD > Variables, let the vx job's role use pipeline variables"
|
|
115
|
+
: ''
|
|
116
|
+
throw new Error(
|
|
117
|
+
`gitlab(): triggering ${workers.map((w) => w.id).join(', ')}: ${res.status} ${text}${fix}`,
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* A worker's matrix slot: its number within its size, wrapped at `max`.
|
|
124
|
+
* A lost worker's number is never reused, so numbers pass `max`; a wave's
|
|
125
|
+
* are consecutive and at most `max`, so its slots differ.
|
|
126
|
+
*/
|
|
127
|
+
function slotOf(w: WorkerSpawn): number {
|
|
128
|
+
const n = Number(w.id.slice(w.id.lastIndexOf('#') + 1))
|
|
129
|
+
return ((n - 1) % w.max) + 1
|
|
130
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export {
|
|
2
|
+
dte,
|
|
3
|
+
type WorkerSize,
|
|
4
|
+
type WorkerSpawn,
|
|
5
|
+
type DteAdapter,
|
|
6
|
+
type DteOptions,
|
|
7
|
+
} from './plugin.js'
|
|
8
|
+
export type { DteInit, DteSetup, PackageManager } from './init.js'
|
|
9
|
+
export { local, type LocalOptions } from './local.js'
|
|
10
|
+
export { docker, type DockerOptions } from './docker.js'
|
|
11
|
+
export { gitlab, type GitlabOptions } from './gitlab.js'
|
|
12
|
+
export { github, type GithubOptions } from './github.js'
|