@vzn/vx-dte 0.0.0 → 0.0.649

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/github.ts ADDED
@@ -0,0 +1,523 @@
1
+ // `github()`: workers as jobs of one workflow the run dispatches per wave
2
+ // (design dte-2026-10.md, "Adapters"): `workflow_dispatch` with the run,
3
+ // the relay, the commit, a size and the workers of that size, which the
4
+ // workflow runs as a matrix on the runners it names for the size. Workers
5
+ // prove themselves by their OIDC identity: the run makes no secret.
6
+
7
+ import { readFileSync } from 'node:fs'
8
+ import { appendFile } from 'node:fs/promises'
9
+ import path from 'node:path'
10
+ import { refuseUnknownOptions, secretNamed, UserError, ui, type PluginOptionKinds } from '@vzn/vx'
11
+ import type { WorkerSpawn, DteAdapter } from './plugin.js'
12
+ import { printable } from './printable.js'
13
+
14
+ export interface GithubOptions {
15
+ /** The workers' workflow file in `.github/workflows` (default `vx-worker.yml`). */
16
+ readonly workflow?: string
17
+ /** The runner label of each size, by size name, which `vx dte init` writes; absent, its platform's GitHub-hosted runner. */
18
+ readonly runsOn?: Readonly<Record<string, string>>
19
+ /**
20
+ * Variables the run hands its workers, as the workflow's `env` input (a
21
+ * JSON object): a value the run's job decides, such as a server's URL
22
+ * for this commit. An undefined one is left out.
23
+ */
24
+ readonly env?: Readonly<Record<string, string | undefined>>
25
+ }
26
+
27
+ interface Job {
28
+ readonly name: string
29
+ readonly status: string
30
+ readonly conclusion: string | null
31
+ readonly steps?: readonly { name: string; conclusion: string | null }[]
32
+ }
33
+
34
+ const KEYS: PluginOptionKinds<GithubOptions> = {
35
+ workflow: 'string',
36
+ runsOn: 'object',
37
+ env: 'object',
38
+ }
39
+
40
+ /** In `RUNNER_TEMP`: one dispatched workflow run's API path per line. */
41
+ const RUNS_FILE = 'vx-dte-runs'
42
+
43
+ /**
44
+ * In `RUNNER_TEMP`: the worker runs a step dispatched before the run began,
45
+ * one per line, `<API path> <worker>,<worker>`: they start while the job
46
+ * checks out and installs (`adopt`).
47
+ */
48
+ const STARTED_FILE = 'vx-dte-started'
49
+
50
+ /**
51
+ * A dispatched run's jobs are first read this soon, then twice as late each
52
+ * time, up to `WATCH_MAX_MS`. Every job of a repository shares
53
+ * GITHUB_TOKEN's API limit (1000 an hour): at 10 s, a dozen runs at once
54
+ * spent it, and dispatches and cancels were refused (2026-10-28).
55
+ */
56
+ const WATCH_FIRST_MS = 2_000
57
+ const WATCH_MAX_MS = 30_000
58
+
59
+ /** A dispatch the API limit refused waits for its reset only this long: the worker is `starting` meanwhile. */
60
+ const DISPATCH_WAIT_MS = 5 * 60_000
61
+
62
+ /**
63
+ * A cancel the API limit refused waits for its reset only this long: a
64
+ * worker run left queued starts for no one, then reads its run is over
65
+ * and cancels itself (the workflow's first step, `parent`).
66
+ */
67
+ const CANCEL_WAIT_MS = 60_000
68
+
69
+ /** When a refused answer's API limit resets (epoch ms), or undefined for a refusal that is no limit. */
70
+ function limitReset(res: Response): number | undefined {
71
+ if (res.status !== 403 && res.status !== 429) return undefined
72
+ const after = res.headers.get('retry-after')
73
+ if (after !== null && Number.isFinite(Number(after))) return Date.now() + Number(after) * 1000
74
+ const reset = res.headers.get('x-ratelimit-reset')
75
+ if (
76
+ res.headers.get('x-ratelimit-remaining') === '0' &&
77
+ reset !== null &&
78
+ Number.isFinite(Number(reset))
79
+ )
80
+ return Number(reset) * 1000
81
+ return undefined
82
+ }
83
+
84
+ const resetsAt = (at: number): string =>
85
+ `; the repository's API limit resets at ${new Date(at).toISOString()}`
86
+
87
+ /** The jobs a matrix makes in one workflow run at most: one dispatch's workers. */
88
+ const MATRIX_MAX = 256
89
+
90
+ export function github(options: GithubOptions = {}): DteAdapter {
91
+ refuseUnknownOptions('github()', options, KEYS)
92
+ const workflow = options.workflow ?? 'vx-worker.yml'
93
+ // GitHub reads no workflow in a subdirectory, and `vx dte init` writes
94
+ // the run's own ci.yml beside it: the same name wrote one over the other.
95
+ if (workflow === '' || workflow.includes('/') || workflow === 'ci.yml')
96
+ throw new UserError(
97
+ `github() option "workflow" must be a file name in .github/workflows other than ci.yml, such as vx-worker.yml, got ${JSON.stringify(workflow)}`,
98
+ )
99
+ const handed = handedEnv(options.env ?? {})
100
+ // The workers one demand starts, by size: one dispatch each.
101
+ const waves = new Map<string, WorkerSpawn[]>()
102
+ // The workflow runs dispatched, and the dispatches still in flight: a
103
+ // run that ends first cancels those still queued, or they hold runners
104
+ // (macOS: five a plan) that the next run's workers wait for.
105
+ const runs: string[] = []
106
+ const pending = new Set<Promise<void>>()
107
+ // A job that fails before its worker joins (checkout, install) told the
108
+ // run nothing: its worker stayed `starting`, held its size's `max`, and a
109
+ // run with no other worker waited for the job's timeout (2026-10-22).
110
+ // Each job is named for its worker; one that ends badly says so.
111
+ const watched = new Map<string, Map<string, WorkerSpawn>>()
112
+ let watch: ReturnType<typeof setTimeout> | undefined
113
+ let closed = false
114
+ // Each run's jobs answer's ETag: asked with it, an unchanged answer is a
115
+ // 304, which the API limit does not count.
116
+ const etags = new Map<string, string>()
117
+ // No jobs are asked before this (epoch ms): the API limit is spent.
118
+ let held = 0
119
+ // The workflow runs whose jobs are this run's workers, by id: a run's id
120
+ // is a dispatch input anyone who reads the repository's Actions sees, so
121
+ // a job of any other run is no worker of it.
122
+ const workerRuns = new Set<string>()
123
+ // Workers the run no longer needs: a run of only these is cancelled.
124
+ const cancelled = new Set<string>()
125
+ // A poll or cancel that failed is said, each cause once (2026-10-23: no
126
+ // fault is silent): a job that ended badly went unseen, a run held runners.
127
+ let warn = (m: string): void => ui.say('warn', m)
128
+ const said = new Set<string>()
129
+ const runOf = (run: string): string => `workflow run ${run.slice(run.lastIndexOf('/') + 1)}`
130
+ const failed = (what: string, why: string): void => {
131
+ const m = `[vx] github(): ${what} failed: ${why}`
132
+ if (said.has(m)) return
133
+ said.add(m)
134
+ warn(m)
135
+ }
136
+ /**
137
+ * A cancel's answer: 202, or 409 for a run already finished; anything
138
+ * else is said. One the spent API limit refused is asked again at its
139
+ * reset, when that comes within `CANCEL_WAIT_MS`.
140
+ */
141
+ const cancelOf = async (run: string): Promise<void> => {
142
+ const end = Date.now() + CANCEL_WAIT_MS
143
+ for (;;) {
144
+ const res = await call(`${run}/cancel`, { method: 'POST' }).catch((err: unknown) => {
145
+ failed(`cancelling ${runOf(run)}`, err instanceof Error ? err.message : String(err))
146
+ return undefined
147
+ })
148
+ if (res === undefined || res.ok || res.status === 409) return
149
+ // quiet: the status is said below either way; the body only adds to it
150
+ const text = printable((await res.text().catch(() => '')).slice(0, 200))
151
+ const reset = limitReset(res)
152
+ if (reset !== undefined && reset <= end) {
153
+ await Bun.sleep(Math.max(0, reset - Date.now()))
154
+ continue
155
+ }
156
+ failed(
157
+ `cancelling ${runOf(run)}`,
158
+ `${res.status} ${text}${reset === undefined ? '' : resetsAt(reset)}`,
159
+ )
160
+ return
161
+ }
162
+ }
163
+ // A worker `cancel` was asked to stop, until it is known whether it was:
164
+ // true when it was left running (`cancel`'s answer).
165
+ const deciding = new Map<string, (left: boolean) => void>()
166
+ // Workers whose dispatch has not answered yet.
167
+ const flying = new Set<string>()
168
+ // Of `cancelled`, those given up as late: stuck or queued, stopped whatever their job holds.
169
+ const late = new Set<string>()
170
+ const decide = (id: string, left: boolean): void => {
171
+ deciding.get(id)?.(left)
172
+ deciding.delete(id)
173
+ }
174
+ /**
175
+ * Whether a job of `run` has a runner: its worker's setup is paid, and
176
+ * a cancel only races its join. Read fresh, past the poll's ETag, which
177
+ * it leaves alone; undefined when GitHub does not answer.
178
+ */
179
+ const hasRunner = async (run: string): Promise<boolean | undefined> => {
180
+ const res = await call(`${run}/jobs?per_page=100`).catch((err: unknown) => {
181
+ failed(`reading ${runOf(run)}'s jobs`, err instanceof Error ? err.message : String(err))
182
+ return undefined
183
+ })
184
+ if (res === undefined) return undefined
185
+ if (!res.ok) {
186
+ failed(
187
+ `reading ${runOf(run)}'s jobs`,
188
+ `${res.status} ${printable((await res.text().catch(() => '')).slice(0, 200))}`,
189
+ )
190
+ return undefined
191
+ }
192
+ // quiet: an answer that is no JSON leaves the run to be cancelled, as before it was read
193
+ const body = (await res.json().catch(() => undefined)) as { jobs?: Job[] } | null | undefined
194
+ if (!Array.isArray(body?.jobs)) return undefined
195
+ return body.jobs.some((j) => j.status === 'in_progress')
196
+ }
197
+ /**
198
+ * Cancels `run` once none of its workers is still needed and none of its
199
+ * jobs has a runner. Otherwise it goes on, and each worker let go in it
200
+ * is left running: it joins, and the run takes it (2026-10-29: small#3,
201
+ * let go with its job running, joined to be told to exit while two tasks
202
+ * waited for a core).
203
+ */
204
+ const letGo = (run: string, workers: Map<string, WorkerSpawn>): void => {
205
+ const ids = [...workers.keys()]
206
+ if (!ids.every((id) => cancelled.has(id))) {
207
+ for (const id of ids)
208
+ if (late.has(id)) decide(id, false)
209
+ else if (cancelled.delete(id)) decide(id, true)
210
+ return
211
+ }
212
+ watched.delete(run)
213
+ // Let go already (a cancel while its dispatch wrote RUNNER_TEMP).
214
+ const i = runs.indexOf(run)
215
+ if (i < 0) return
216
+ runs.splice(i, 1)
217
+ const p = (ids.some((id) => late.has(id)) ? Promise.resolve(false) : hasRunner(run))
218
+ .then((held) => {
219
+ if (held === true && !closed) {
220
+ runs.push(run)
221
+ watched.set(run, workers)
222
+ if (watch === undefined) look(WATCH_FIRST_MS)
223
+ for (const id of ids) {
224
+ cancelled.delete(id)
225
+ decide(id, true)
226
+ }
227
+ return
228
+ }
229
+ for (const id of ids) decide(id, false)
230
+ return cancelOf(run)
231
+ })
232
+ .finally(() => pending.delete(p))
233
+ pending.add(p)
234
+ }
235
+ /**
236
+ * Every page of `run`'s jobs: a wave of more than 100 workers is a run of
237
+ * more than 100 jobs. Undefined when unread, or when a one-page run has
238
+ * not changed since the last read (a 304 costs no API limit).
239
+ */
240
+ const jobsOf = async (run: string): Promise<Job[] | undefined> => {
241
+ const all: Job[] = []
242
+ for (let page = 1; ; page++) {
243
+ const tag = page === 1 ? etags.get(run) : undefined
244
+ const res = await call(
245
+ `${run}/jobs?per_page=100&page=${page}`,
246
+ tag === undefined ? {} : { headers: { 'if-none-match': tag } },
247
+ ).catch((err: unknown) => {
248
+ failed(`reading ${runOf(run)}'s jobs`, err instanceof Error ? err.message : String(err))
249
+ return undefined
250
+ })
251
+ if (res === undefined || res.status === 304) return undefined
252
+ if (!res.ok) {
253
+ const reset = limitReset(res)
254
+ if (reset !== undefined) held = Math.max(held, reset)
255
+ failed(
256
+ `reading ${runOf(run)}'s jobs`,
257
+ `${res.status} ${printable((await res.text().catch(() => '')).slice(0, 200))}${reset === undefined ? '' : resetsAt(reset)}`,
258
+ )
259
+ return undefined
260
+ }
261
+ const body = (await res.json().catch((err: unknown) => {
262
+ failed(
263
+ `reading ${runOf(run)}'s jobs`,
264
+ `no JSON: ${err instanceof Error ? err.message : String(err)}`,
265
+ )
266
+ return undefined
267
+ })) as { jobs?: Job[] } | null | undefined
268
+ if (body === undefined) return undefined
269
+ const jobs = body?.jobs ?? []
270
+ all.push(...jobs)
271
+ // A first page's tag says nothing of the next: only a run of one page is asked by it.
272
+ const etag = res.headers.get('etag')
273
+ if (page === 1 && etag !== null && jobs.length < 100) etags.set(run, etag)
274
+ else etags.delete(run)
275
+ if (jobs.length < 100) return all
276
+ }
277
+ }
278
+ const look = (ms: number): void => {
279
+ watch = setTimeout(() => {
280
+ void Promise.all(
281
+ [...watched].map(async ([run, workers]) => {
282
+ const jobs = await jobsOf(run)
283
+ if (jobs === undefined) return
284
+ for (const j of jobs) {
285
+ const a = workers.get(j.name)
286
+ if (a === undefined || j.status !== 'completed') continue
287
+ workers.delete(j.name)
288
+ if (closed || cancelled.has(j.name)) continue
289
+ // A success too: a worker that found no live run left at once,
290
+ // and the run must not wait on it (2026-10-23).
291
+ const step = j.steps?.find((x) => x.conclusion === 'failure')?.name
292
+ a.exited(`job ${j.conclusion}${step === undefined ? '' : ` at ${step}`}`)
293
+ }
294
+ if (workers.size === 0) watched.delete(run)
295
+ }),
296
+ ).then(() => {
297
+ const next = Math.min(2 * ms, WATCH_MAX_MS)
298
+ if (!closed && watched.size > 0) look(Math.max(next, held - Date.now()))
299
+ else watch = undefined
300
+ })
301
+ }, ms)
302
+ watch.unref()
303
+ }
304
+ const call = (path: string, init: RequestInit = {}): Promise<Response> => {
305
+ const { GITHUB_API_URL: api, GITHUB_TOKEN: token } = process.env
306
+ return fetch(`${api}${path}`, {
307
+ ...init,
308
+ headers: {
309
+ ...(init.headers as Record<string, string> | undefined),
310
+ authorization: `Bearer ${token}`,
311
+ accept: 'application/vnd.github+json',
312
+ 'content-type': 'application/json',
313
+ // The version whose dispatch answers with the workflow run's id.
314
+ 'x-github-api-version': '2026-03-10',
315
+ },
316
+ signal: AbortSignal.timeout(10_000),
317
+ })
318
+ }
319
+ const dispatch = async (size: string, workers: WorkerSpawn[]): Promise<void> => {
320
+ const env = process.env
321
+ const { GITHUB_API_URL: api, GITHUB_REPOSITORY: repo, GITHUB_TOKEN: token } = env
322
+ if (!api || !repo || !token)
323
+ throw new Error(
324
+ 'github(): dispatching workers needs GITHUB_TOKEN (actions: write) in a GitHub Actions job',
325
+ )
326
+ const sha = env['GITHUB_SHA']
327
+ const first = workers[0]!
328
+ // The branch's workflow file, or, once a merged PR's branch is deleted
329
+ // mid-run (GitHub answers 422 "No ref found"), its base's: the workers
330
+ // check out `sha` either way.
331
+ const refs = [env['GITHUB_HEAD_REF'] || env['GITHUB_REF_NAME'], env['GITHUB_BASE_REF']]
332
+ let res: Response
333
+ const end = Date.now() + DISPATCH_WAIT_MS
334
+ for (let i = 0; ;) {
335
+ res = await call(`/repos/${repo}/actions/workflows/${workflow}/dispatches`, {
336
+ method: 'POST',
337
+ body: JSON.stringify({
338
+ ref: refs[i],
339
+ inputs: {
340
+ run: first.run,
341
+ relay: first.relay,
342
+ sha,
343
+ size,
344
+ workers: JSON.stringify(workers.map((a) => a.id)),
345
+ // This run's attempt: a worker that starts after it ended
346
+ // cancels its own workflow run instead of working for no one.
347
+ ...(env['GITHUB_RUN_ID'] === undefined
348
+ ? {}
349
+ : { parent: `${env['GITHUB_RUN_ID']}/attempts/${env['GITHUB_RUN_ATTEMPT'] ?? '1'}` }),
350
+ ...(options.env === undefined ? {} : { env: handed }),
351
+ },
352
+ }),
353
+ })
354
+ if (res.ok) break
355
+ const text = printable((await res.text()).slice(0, 200))
356
+ const reset = limitReset(res)
357
+ if (reset !== undefined) {
358
+ if (reset > end || closed)
359
+ throw new Error(
360
+ `github(): dispatching ${workflow}: ${res.status} ${text}${resetsAt(reset)}`,
361
+ )
362
+ await Bun.sleep(Math.max(0, reset - Date.now()))
363
+ continue
364
+ }
365
+ if (res.status !== 422 || !text.includes('No ref found') || !refs[++i]) {
366
+ // GitHub's own words named no fix for the two a first run meets (2026-10-26).
367
+ const fix =
368
+ res.status === 404
369
+ ? `: .github/workflows/${workflow} is not on ${refs[i]} (vx dte init writes it; commit and push it)`
370
+ : res.status === 403
371
+ ? ': the job needs `permissions: actions: write` to start workers'
372
+ : ''
373
+ throw new Error(`github(): dispatching ${workflow}: ${res.status} ${text}${fix}`)
374
+ }
375
+ }
376
+ const id =
377
+ res.status === 200
378
+ ? ((await res.json()) as { workflow_run_id?: number }).workflow_run_id
379
+ : undefined
380
+ // An API naming no run (a 204) is refused, not trusted by repository
381
+ // alone: any job of it with id-token: write would join as a worker.
382
+ if (id === undefined) {
383
+ // No run to cancel: each let go meanwhile is settled before the refusal.
384
+ for (const a of workers) if (cancelled.delete(a.id)) decide(a.id, true)
385
+ throw new Error(
386
+ `github(): dispatching ${workflow} named no workflow run (${res.status}): its workers cannot be told from any job of the repository`,
387
+ )
388
+ }
389
+ workerRuns.add(String(id))
390
+ const run = `/repos/${repo}/actions/runs/${id}`
391
+ runs.push(run)
392
+ const mine = new Map(workers.map((a) => [a.id, a]))
393
+ watched.set(run, mine)
394
+ if (watch === undefined && !closed) look(WATCH_FIRST_MS)
395
+ // For the job's last step, which cancels them when this process
396
+ // dies before close() (a kill, an OOM, the job's timeout).
397
+ if (env['RUNNER_TEMP']) await appendFile(path.join(env['RUNNER_TEMP'], RUNS_FILE), `${run}\n`)
398
+ // Let go while this dispatch was in flight: cancel() knew no run.
399
+ if (!closed) letGo(run, mine)
400
+ }
401
+ return {
402
+ // Loaded only by `vx dte init`: a run never needs it.
403
+ init: async (setup) => (await import('./init.js')).githubInit(options, setup),
404
+ keyless: true,
405
+ workerRuns: () => workerRuns,
406
+ report(w) {
407
+ warn = w
408
+ },
409
+ adopt(worker) {
410
+ const temp = process.env['RUNNER_TEMP']
411
+ if (!temp) return
412
+ let text: string
413
+ try {
414
+ text = readFileSync(path.join(temp, STARTED_FILE), 'utf8')
415
+ // quiet: no list: this job started no worker
416
+ } catch {
417
+ return
418
+ }
419
+ for (const line of text.split('\n')) {
420
+ const [run, ids] = line.trim().split(/\s+/)
421
+ if (!run || !ids) continue
422
+ const workers = new Map<string, WorkerSpawn>()
423
+ for (const id of ids.split(','))
424
+ workers.set(id, worker(id, id.slice(0, id.lastIndexOf('#'))))
425
+ runs.push(run)
426
+ watched.set(run, workers)
427
+ workerRuns.add(run.slice(run.lastIndexOf('/') + 1))
428
+ }
429
+ if (watched.size > 0 && watch === undefined) look(WATCH_FIRST_MS)
430
+ },
431
+ spawn(a) {
432
+ flying.add(a.id)
433
+ const wave = waves.get(a.size)
434
+ // A full wave dispatches as it is; the next worker starts another.
435
+ if (wave !== undefined && wave.length < MATRIX_MAX) {
436
+ wave.push(a)
437
+ return
438
+ }
439
+ const workers = [a]
440
+ waves.set(a.size, workers)
441
+ queueMicrotask(() => {
442
+ if (waves.get(a.size) === workers) waves.delete(a.size)
443
+ const p = dispatch(a.size, workers)
444
+ .catch((err: unknown) => {
445
+ const why = err instanceof Error ? err.message : String(err)
446
+ for (const x of workers) {
447
+ x.exited(why)
448
+ decide(x.id, false)
449
+ }
450
+ })
451
+ .finally(() => {
452
+ for (const x of workers) flying.delete(x.id)
453
+ pending.delete(p)
454
+ })
455
+ pending.add(p)
456
+ })
457
+ },
458
+ // Before a worker's setup when its job has yet to run: its runner is
459
+ // free for others (2026-10-23). A run with a worker still needed, or
460
+ // a job with a runner, goes on; the workers let go in it are left
461
+ // running. One given up as late is stopped whatever its job holds.
462
+ cancel(ids, givenUp) {
463
+ if (givenUp === true) for (const id of ids) late.add(id)
464
+ const left = ids.map((id) => new Promise<boolean>((done) => deciding.set(id, done)))
465
+ // Neither watched nor in flight (its job ended, its dispatch failed): nothing to stop.
466
+ const gone = ids.filter(
467
+ (id) => !flying.has(id) && ![...watched.values()].some((w) => w.has(id)),
468
+ )
469
+ for (const id of ids) cancelled.add(id)
470
+ for (const [run, workers] of [...watched])
471
+ if (ids.some((id) => workers.has(id))) letGo(run, workers)
472
+ for (const id of gone) decide(id, false)
473
+ return Promise.all(left).then((l) => ids.filter((_, i) => l[i]))
474
+ },
475
+ // A finished run answers 409; one still queued or running is the
476
+ // run's no more: its results are in, or the run failed without them.
477
+ async close() {
478
+ closed = true
479
+ clearTimeout(watch)
480
+ await Promise.resolve() // a wave spawned this tick dispatches first
481
+ await Promise.all(pending)
482
+ for (const id of [...deciding.keys()]) decide(id, false)
483
+ // A cancel that fails leaves the run to its timeout, and says so.
484
+ await Promise.all(runs.splice(0).map(cancelOf))
485
+ },
486
+ }
487
+ }
488
+
489
+ /** The relay token on GitHub Actions: an OIDC token for the relay's URL as audience. */
490
+ export async function githubRelayToken(relay: string): Promise<string | undefined> {
491
+ const { ACTIONS_ID_TOKEN_REQUEST_URL: url, ACTIONS_ID_TOKEN_REQUEST_TOKEN: bearer } = process.env
492
+ if (!url || !bearer) return undefined
493
+ // Loaded when asked: a run that is not distributed loads none of it.
494
+ const { oidcToken } = await import('./seal.js')
495
+ return oidcToken(url, bearer, relay).catch((err: unknown) => {
496
+ throw new UserError(
497
+ `dte(): ${err instanceof Error ? err.message : String(err)} (for the relay)`,
498
+ )
499
+ })
500
+ }
501
+
502
+ /**
503
+ * `env` as the workflow's input: each name a variable's, each value one
504
+ * line, since the workflow writes each as a line of `GITHUB_ENV`. No
505
+ * secret: a dispatch's inputs are shown with its run, and the worker
506
+ * job's log prints each step's env unmasked.
507
+ */
508
+ function handedEnv(env: Readonly<Record<string, string | undefined>>): string {
509
+ const out: Record<string, string> = {}
510
+ for (const [name, value] of Object.entries(env)) {
511
+ if (value === undefined) continue
512
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || typeof value !== 'string' || /[\r\n]/.test(value))
513
+ throw new UserError(
514
+ `github(): env ${JSON.stringify(name)}: a variable's name, and a string of one line`,
515
+ )
516
+ if (secretNamed(name))
517
+ throw new UserError(
518
+ `github(): env ${JSON.stringify(name)} is a secret, which the worker job's log would show: give it to that job from secrets in its workflow`,
519
+ )
520
+ out[name] = value
521
+ }
522
+ return JSON.stringify(out)
523
+ }
package/src/gitlab.ts ADDED
@@ -0,0 +1,130 @@
1
+ // `gitlab()`: workers as pipelines of the project's own `.gitlab-ci.yml`
2
+ // that the run triggers once it is on the relay (design dte-2026-10.md,
3
+ // "Adapters"): a worker never arrives before its run. The trigger takes
4
+ // the job's `CI_JOB_TOKEN`, so no token is stored. One pipeline per size
5
+ // for each wave the run asks: its `vx-worker <size>` job is a matrix of
6
+ // `max` slots (`VX_DTE_SLOT`), of which the rules run those `VX_DTE_PICK`
7
+ // names, at the run's commit (`VX_DTE_SHA`). Each proves itself by the
8
+ // run's secret.
9
+
10
+ import { refuseUnknownOptions, type PluginOptionKinds } from '@vzn/vx'
11
+ import type { DteAdapter, WorkerSpawn } from './plugin.js'
12
+ import { printable } from './printable.js'
13
+
14
+ export interface GitlabOptions {
15
+ /** Runner tags by worker size: the runners each size's jobs go to. */
16
+ readonly tags?: Readonly<Record<string, readonly string[]>>
17
+ }
18
+
19
+ const KEYS: PluginOptionKinds<GitlabOptions> = { tags: 'object' }
20
+
21
+ export function gitlab(options: GitlabOptions = {}): DteAdapter {
22
+ refuseUnknownOptions('gitlab()', options, KEYS)
23
+ // The workers asked in one tick, by size: the run asks a wave at once.
24
+ const waves = new Map<string, { workers: WorkerSpawn[]; sent: Promise<void> }>()
25
+ return {
26
+ init: async (setup) => (await import('./init.js')).gitlabInit(options, setup),
27
+
28
+ spawn(a) {
29
+ let wave = waves.get(a.size)
30
+ // Two workers in one slot would be one job: the second starts a wave of its own.
31
+ if (wave === undefined || wave.workers.some((w) => slotOf(w) === slotOf(a))) {
32
+ const workers: WorkerSpawn[] = []
33
+ const sent = Promise.resolve().then(() => {
34
+ if (waves.get(a.size) === mine) waves.delete(a.size)
35
+ return trigger(workers)
36
+ })
37
+ const mine = { workers, sent }
38
+ wave = mine
39
+ waves.set(a.size, wave)
40
+ }
41
+ wave.workers.push(a)
42
+ return wave.sent
43
+ },
44
+
45
+ workerId() {
46
+ const slot = process.env['VX_DTE_SLOT']
47
+ for (const pair of (process.env['VX_DTE_IDS'] ?? '').split(' ')) {
48
+ const at = pair.indexOf('=')
49
+ if (at > 0 && pair.slice(0, at) === slot) return pair.slice(at + 1)
50
+ }
51
+ return undefined
52
+ },
53
+
54
+ // Random: a pipeline id is guessable.
55
+ runId() {
56
+ return process.env['GITLAB_CI'] === 'true' && process.env['CI_JOB_TOKEN']
57
+ ? crypto.randomUUID()
58
+ : undefined
59
+ },
60
+ }
61
+ }
62
+
63
+ /** One pipeline for a wave of one size's workers. */
64
+ async function trigger(workers: readonly WorkerSpawn[]): Promise<void> {
65
+ const {
66
+ CI_API_V4_URL: base,
67
+ CI_PROJECT_ID: project,
68
+ CI_JOB_TOKEN: token,
69
+ CI_COMMIT_SHA: sha,
70
+ } = process.env
71
+ if (!base || !project || !token || !sha) {
72
+ const missing = Object.entries({
73
+ CI_API_V4_URL: base,
74
+ CI_JOB_TOKEN: token,
75
+ CI_PROJECT_ID: project,
76
+ CI_COMMIT_SHA: sha,
77
+ }).flatMap(([k, v]) => (v ? [] : [k]))
78
+ throw new Error(`gitlab(): not in a GitLab job (${missing.join(', ')})`)
79
+ }
80
+ const first = workers[0]!
81
+ const slots = workers.map((w) => [slotOf(w), w.id] as const)
82
+ // A merge request's pipeline is of its source branch; any other, of its
83
+ // ref. Once a merged request's source branch is deleted mid-run
84
+ // (GitLab's default), GitLab answers 400 "Reference not found": its
85
+ // target's file then, as github() does; the workers check out `sha` either way.
86
+ const refs = [
87
+ process.env['CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'] || process.env['CI_COMMIT_REF_NAME']!,
88
+ process.env['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'],
89
+ ]
90
+ const { VX_DTE_WORKER: _, ...env } = first.env
91
+ const vars: Record<string, string> = {
92
+ ...env,
93
+ VX_DTE_SIZE: first.size,
94
+ VX_DTE_SHA: sha,
95
+ VX_DTE_PICK: `/^(${slots.map(([s]) => s).join('|')})$/`,
96
+ VX_DTE_IDS: slots.map(([s, id]) => `${s}=${id}`).join(' '),
97
+ }
98
+ for (let i = 0; ; i++) {
99
+ const form = new FormData()
100
+ form.set('token', token)
101
+ form.set('ref', refs[i]!)
102
+ for (const [k, v] of Object.entries(vars)) form.set(`variables[${k}]`, v)
103
+ const res = await fetch(`${base}/projects/${project}/trigger/pipeline`, {
104
+ method: 'POST',
105
+ body: form,
106
+ signal: AbortSignal.timeout(10_000),
107
+ })
108
+ if (res.ok) return
109
+ const text = printable((await res.text()).slice(0, 200))
110
+ if (res.status === 400 && text.includes('Reference not found') && refs[i + 1]) continue
111
+ // GitLab.com's new projects let no one set pipeline variables (17.7),
112
+ // and GitLab's words name no setting.
113
+ const fix = text.includes('Insufficient permissions to set pipeline variables')
114
+ ? ": the workers take the run's variables; in the project's Settings > CI/CD > Variables, let the vx job's role use pipeline variables"
115
+ : ''
116
+ throw new Error(
117
+ `gitlab(): triggering ${workers.map((w) => w.id).join(', ')}: ${res.status} ${text}${fix}`,
118
+ )
119
+ }
120
+ }
121
+
122
+ /**
123
+ * A worker's matrix slot: its number within its size, wrapped at `max`.
124
+ * A lost worker's number is never reused, so numbers pass `max`; a wave's
125
+ * are consecutive and at most `max`, so its slots differ.
126
+ */
127
+ function slotOf(w: WorkerSpawn): number {
128
+ const n = Number(w.id.slice(w.id.lastIndexOf('#') + 1))
129
+ return ((n - 1) % w.max) + 1
130
+ }
package/src/index.ts ADDED
@@ -0,0 +1,12 @@
1
+ export {
2
+ dte,
3
+ type WorkerSize,
4
+ type WorkerSpawn,
5
+ type DteAdapter,
6
+ type DteOptions,
7
+ } from './plugin.js'
8
+ export type { DteInit, DteSetup, PackageManager } from './init.js'
9
+ export { local, type LocalOptions } from './local.js'
10
+ export { docker, type DockerOptions } from './docker.js'
11
+ export { gitlab, type GitlabOptions } from './gitlab.js'
12
+ export { github, type GithubOptions } from './github.js'