@vzn/vx-migrate 0.0.595 → 0.0.596
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -18
- package/package.json +2 -2
- package/src/adopt.ts +1 -1
- package/src/migrate-turbo.ts +1 -30
- package/src/migrate-vite-task.ts +40 -0
- package/src/migrate.ts +69 -14
- package/src/nx/index.ts +4 -1
- package/src/nx/lerna.ts +60 -0
- package/src/nx/nx-inputs.ts +9 -5
- package/src/nx/nx-map.ts +16 -2
- package/src/nx/nx-outputs.ts +62 -12
- package/src/nx/nx-upstream.ts +4 -1
- package/src/nx-cache/index.ts +20 -14
- package/src/remote-breaker.ts +60 -0
- package/src/shared-outputs.ts +3 -1
- package/src/spelled-env.ts +36 -0
- package/src/turbo/turbo-map.ts +13 -2
- package/src/turbo-cache/index.ts +93 -26
- package/src/vite-task/vite-task-map.ts +558 -0
package/src/nx/nx-inputs.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// (`dependentTasksOutputFiles`) is nothing; `externalDependencies` is a todo saying
|
|
8
8
|
// so. Extracted from `buildTask` in item 606.
|
|
9
9
|
|
|
10
|
-
import { nxWorkspacePath, underProject } from './nx-outputs.js'
|
|
10
|
+
import { nxWorkspacePath, underProject, withProjectDir } from './nx-outputs.js'
|
|
11
11
|
import { minimatchToVx } from '../glob-grammar.js'
|
|
12
12
|
import { shellQuote } from '../nx-command.js'
|
|
13
13
|
|
|
@@ -70,15 +70,16 @@ export function expandNxInputs(
|
|
|
70
70
|
}
|
|
71
71
|
// Nx's file map skips gitignored files, so it hashes nothing here
|
|
72
72
|
// either: a changed `dist` file is a cache hit under Nx 23.3 (YL-1).
|
|
73
|
-
|
|
73
|
+
const lit = withProjectDir(p, at.rel, false)
|
|
74
|
+
if (neg === '' && at.outputs?.some((o) => lit === o || lit.startsWith(`${o}/`))) return
|
|
74
75
|
const g = minimatchToVx(p, neg !== '')
|
|
75
76
|
if (g === null) {
|
|
76
77
|
todos.push(`input ${JSON.stringify(entry)}: glob syntax vx cannot take — map manually`)
|
|
77
78
|
return
|
|
78
79
|
}
|
|
79
80
|
const own = underProject(g, at.rel)
|
|
80
|
-
if (own === null) into.wsFiles.push(neg + g)
|
|
81
|
-
else into.files.push(neg + own)
|
|
81
|
+
if (own === null) into.wsFiles.push(neg + withProjectDir(g, at.rel, true))
|
|
82
|
+
else into.files.push(neg + withProjectDir(own, at.rel, true))
|
|
82
83
|
return
|
|
83
84
|
}
|
|
84
85
|
// Bare string = named-input reference.
|
|
@@ -193,7 +194,10 @@ export function expandNxInputs(
|
|
|
193
194
|
}
|
|
194
195
|
// A negation filters only other includeIgnored matches, and a literal
|
|
195
196
|
// reads no others.
|
|
196
|
-
if (!neg)
|
|
197
|
+
if (!neg)
|
|
198
|
+
into.runtimeCmds.push(
|
|
199
|
+
`cat -- ${shellQuote(withProjectDir(p, at.rel, false))} 2>/dev/null; echo "$?"`,
|
|
200
|
+
)
|
|
197
201
|
}
|
|
198
202
|
for (const entry of entries) expand(entry, new Set())
|
|
199
203
|
// Nx matches a project fileset with no positive glob against every project
|
package/src/nx/nx-map.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
} from '../shared-outputs.js'
|
|
38
38
|
import type { TrackedKinds } from '../tracked-outputs.js'
|
|
39
39
|
import { packageScripts, relPosix } from '../paths.js'
|
|
40
|
+
import { withLernaOrder } from './lerna.js'
|
|
40
41
|
import { mapNxDeps, matchNxProjects, type TaskNameFor } from './nx-deps.js'
|
|
41
42
|
import {
|
|
42
43
|
dotenvCandidates,
|
|
@@ -304,7 +305,17 @@ export async function mapNxWorkspace(
|
|
|
304
305
|
graph: NxGraph,
|
|
305
306
|
opts: MapNxOptions,
|
|
306
307
|
): Promise<NxMapping> {
|
|
307
|
-
const
|
|
308
|
+
const byRel = new Map(metas.map((m) => [normRel(relPosix(root, m.dir)), m.packageJson]))
|
|
309
|
+
const nodeMap = await withLernaOrder(
|
|
310
|
+
root,
|
|
311
|
+
graph.nodes,
|
|
312
|
+
(await readNxJson(root).catch(() => null))?.json,
|
|
313
|
+
async (rel) =>
|
|
314
|
+
(byRel.get(normRel(rel)) as Record<string, unknown> | undefined) ??
|
|
315
|
+
(await Bun.file(path.join(root, rel, 'package.json'))
|
|
316
|
+
.json()
|
|
317
|
+
.catch(() => undefined)),
|
|
318
|
+
)
|
|
308
319
|
const g = graph
|
|
309
320
|
|
|
310
321
|
const { namedInputs, cacheable, globalSync } = await readNxJsonFacts(root)
|
|
@@ -923,7 +934,10 @@ function buildTask(
|
|
|
923
934
|
}
|
|
924
935
|
const exec: Record<string, unknown> = { command: mapped.command }
|
|
925
936
|
const env: Record<string, unknown> = {}
|
|
926
|
-
|
|
937
|
+
// A name that is no shell identifier never reached a task (`sh` drops it)
|
|
938
|
+
// and core refuses it in passThrough; the key still reads it.
|
|
939
|
+
const passThrough = inputs.envNames.filter((n) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(n))
|
|
940
|
+
if (passThrough.length > 0) env.passThrough = passThrough
|
|
927
941
|
// Nx hands every task its target (`getNxEnvVariablesForTask`), and
|
|
928
942
|
// `nx exec -- <cmd>`, a package script's way to run under Nx, reads it:
|
|
929
943
|
// unset, it booted Nx's own task runner, which ran the target and its
|
package/src/nx/nx-outputs.ts
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// workspace-root globs. Extracted from `buildTask` in item 606.
|
|
3
3
|
|
|
4
4
|
import path from 'node:path'
|
|
5
|
-
import {
|
|
5
|
+
import { isLiteralPattern } from '@vzn/vx'
|
|
6
|
+
import { literalGlob, minimatchToVx } from '../glob-grammar.js'
|
|
6
7
|
import { takingBack } from '../shared-outputs.js'
|
|
7
8
|
|
|
8
9
|
interface NxOutputs {
|
|
@@ -18,12 +19,13 @@ interface NxOutputs {
|
|
|
18
19
|
* became the literal glob `coverage/{projectRoot}/**`, saved nothing, and a
|
|
19
20
|
* hit restored nothing. Null when a token is left or the path leaves the
|
|
20
21
|
* workspace; a brace set is no token (item 914 — 912 read one as a gap).
|
|
22
|
+
* The project dir comes back masked: {@link withProjectDir} writes it in.
|
|
21
23
|
*/
|
|
22
24
|
export function nxWorkspacePath(s: string, projectRel: string, projectName: string): string | null {
|
|
23
25
|
const rel = projectRel === '.' ? '' : projectRel
|
|
24
26
|
const p = s
|
|
25
27
|
.replaceAll('{workspaceRoot}', '')
|
|
26
|
-
.replaceAll('{projectRoot}', rel)
|
|
28
|
+
.replaceAll('{projectRoot}', masked(rel))
|
|
27
29
|
.replaceAll('{projectName}', projectName)
|
|
28
30
|
// A token left over is a gap; a brace set (`*.{ts,tsx}`) is a glob.
|
|
29
31
|
if (/\{[\w.]+\}/.test(p)) return null
|
|
@@ -31,10 +33,38 @@ export function nxWorkspacePath(s: string, projectRel: string, projectName: stri
|
|
|
31
33
|
return n === '.' || n === '..' || n.startsWith('../') ? null : n
|
|
32
34
|
}
|
|
33
35
|
|
|
36
|
+
// A project dir is a path, never a glob: `packages/a{b}` read as a token
|
|
37
|
+
// dropped every `{projectRoot}` path, and `packages/[x]` translated as a
|
|
38
|
+
// class reached `packages/x`. Each dir segment stands in as `\uE000<i>\uE001`
|
|
39
|
+
// until the glob is vx's, so `..` still pops one.
|
|
40
|
+
const MASKED = /\uE000(\d+)\uE001/g
|
|
41
|
+
|
|
42
|
+
function masked(rel: string): string {
|
|
43
|
+
return rel === ''
|
|
44
|
+
? ''
|
|
45
|
+
: rel
|
|
46
|
+
.split('/')
|
|
47
|
+
.map((_, i) => `\uE000${i}\uE001`)
|
|
48
|
+
.join('/')
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** `p` with the masked project dir written back: as a path, or escaped for a glob. */
|
|
52
|
+
export function withProjectDir(p: string, projectRel: string, asGlob: boolean): string {
|
|
53
|
+
const segs = projectRel.split('/')
|
|
54
|
+
return p.replace(MASKED, (_, i: string, at: number) => {
|
|
55
|
+
const seg = segs[Number(i)]!
|
|
56
|
+
if (!asGlob) return seg
|
|
57
|
+
// Only a glob's head reads `!` as a negation.
|
|
58
|
+
return at === 0 ? literalGlob(seg) : literalGlob(`/${seg}`).slice(1)
|
|
59
|
+
})
|
|
60
|
+
}
|
|
61
|
+
|
|
34
62
|
/** A workspace path relative to the project dir, or null when it lies outside. */
|
|
35
63
|
export function underProject(p: string, projectRel: string): string | null {
|
|
36
64
|
if (projectRel === '.' || projectRel === '') return p
|
|
37
|
-
|
|
65
|
+
for (const dir of [masked(projectRel), projectRel])
|
|
66
|
+
if (p.startsWith(`${dir}/`)) return p.slice(dir.length + 1)
|
|
67
|
+
return null
|
|
38
68
|
}
|
|
39
69
|
|
|
40
70
|
/**
|
|
@@ -64,15 +94,16 @@ export function nxDefaultOutputs(
|
|
|
64
94
|
}
|
|
65
95
|
if (targetName !== 'build' && targetName !== 'prepare') return []
|
|
66
96
|
const root = projectRel === '.' ? '' : projectRel
|
|
67
|
-
|
|
97
|
+
// As tokens, so the dir stays a path whatever its name holds.
|
|
98
|
+
const at = (p: string): string => (root === '' ? p : `{projectRoot}/${p}`)
|
|
68
99
|
const held = ['build', 'public'].filter((d) => tops === undefined || tops.has(d))
|
|
69
100
|
if (held.length > 0)
|
|
70
101
|
todos.push(
|
|
71
|
-
`no outputs declared: Nx also caches ${held.map(
|
|
102
|
+
`no outputs declared: Nx also caches ${held.map((d) => path.posix.join(root, d)).join(' and ')} for this target — vx cleans an output before the run, so add ${held.length === 1 ? 'it' : 'them'} to the outputs by hand only if ${held.length === 1 ? 'it holds' : 'they hold'} nothing committed`,
|
|
72
103
|
)
|
|
73
104
|
// The root project's two are one path.
|
|
74
105
|
return [
|
|
75
|
-
...new Set([
|
|
106
|
+
...new Set([root === '' ? 'dist' : '{workspaceRoot}/dist/{projectRoot}', at('dist')]),
|
|
76
107
|
...['build', 'public'].filter((d) => !held.includes(d)).map(at),
|
|
77
108
|
]
|
|
78
109
|
}
|
|
@@ -145,7 +176,7 @@ export function mapNxOutputs(
|
|
|
145
176
|
const norm = path.posix.normalize(s.replace(/^\/+/, ''))
|
|
146
177
|
if (norm === '..' || norm.startsWith('../')) {
|
|
147
178
|
todos.push(
|
|
148
|
-
`output ${JSON.stringify(o)} resolves to ${JSON.stringify(norm)}, outside the workspace — ` +
|
|
179
|
+
`output ${JSON.stringify(o)} resolves to ${JSON.stringify(withProjectDir(norm, projectRel, false))}, outside the workspace — ` +
|
|
149
180
|
'vx caches only inside it; dropped',
|
|
150
181
|
)
|
|
151
182
|
continue
|
|
@@ -158,9 +189,18 @@ export function mapNxOutputs(
|
|
|
158
189
|
for (const g of globs) {
|
|
159
190
|
const gn = g.startsWith('!') ? '!' : ''
|
|
160
191
|
const gs = g.slice(gn.length)
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
192
|
+
const own = underProject(gs, projectRel)
|
|
193
|
+
if (own !== null) {
|
|
194
|
+
outFiles.push(gn + withProjectDir(own, projectRel, true))
|
|
195
|
+
continue
|
|
196
|
+
}
|
|
197
|
+
const m = path.posix.normalize(gs).replace(/^\.\//, '')
|
|
198
|
+
const w = withProjectDir(m, projectRel, true)
|
|
199
|
+
wsOutFiles.push(gn + w)
|
|
200
|
+
// An escaped brace makes a literal a glob, which core matches as the
|
|
201
|
+
// path alone (`asTrees` adds the tree only to a literal).
|
|
202
|
+
if (w !== withProjectDir(m, projectRel, false) && isLiteralPattern(m.replace(MASKED, '')))
|
|
203
|
+
wsOutFiles.push(`${gn}${w}/**`)
|
|
164
204
|
}
|
|
165
205
|
}
|
|
166
206
|
return { outFiles: takingBack(outFiles), wsOutFiles: takingBack(wsOutFiles) }
|
|
@@ -217,9 +257,19 @@ export function nxProjectOutputs(
|
|
|
217
257
|
projectName,
|
|
218
258
|
scratch,
|
|
219
259
|
)
|
|
220
|
-
|
|
260
|
+
const literal = (f: string): string | null => {
|
|
221
261
|
const lit = f.replace(/\/\*\*(\/\*)?$/, '')
|
|
222
|
-
|
|
262
|
+
return lit.startsWith('!') || /[*?[]|(?<!\\)[{}]/.test(lit)
|
|
263
|
+
? null
|
|
264
|
+
: lit.replace(/\\([{}!])/g, '$1')
|
|
265
|
+
}
|
|
266
|
+
for (const f of outFiles) {
|
|
267
|
+
const lit = literal(f)
|
|
268
|
+
if (lit !== null) out.add(path.posix.join(rel, lit))
|
|
269
|
+
}
|
|
270
|
+
for (const f of wsOutFiles) {
|
|
271
|
+
const lit = literal(f)
|
|
272
|
+
if (lit !== null) out.add(lit)
|
|
223
273
|
}
|
|
224
274
|
}
|
|
225
275
|
const r = [...out]
|
package/src/nx/nx-upstream.ts
CHANGED
|
@@ -20,6 +20,7 @@ import type { GeneratedTask, ProjectMeta } from '@vzn/vx'
|
|
|
20
20
|
import { emptyNxInputs, expandNxInputs, type NxInputs } from './nx-inputs.js'
|
|
21
21
|
import { nxProjectOutputs } from './nx-outputs.js'
|
|
22
22
|
import { matchNxProjects } from './nx-deps.js'
|
|
23
|
+
import { literalGlob } from '../glob-grammar.js'
|
|
23
24
|
|
|
24
25
|
interface GraphNode {
|
|
25
26
|
name?: string
|
|
@@ -192,7 +193,9 @@ export function planNxUpstream(
|
|
|
192
193
|
const inputs = emptyNxInputs()
|
|
193
194
|
for (const g of own.files) {
|
|
194
195
|
const neg = g.startsWith('!') ? '!' : ''
|
|
195
|
-
inputs.wsFiles.push(
|
|
196
|
+
inputs.wsFiles.push(
|
|
197
|
+
rel === '' ? g : `${neg}${literalGlob(rel)}/${neg === '' ? g : g.slice(1)}`,
|
|
198
|
+
)
|
|
196
199
|
}
|
|
197
200
|
inputs.wsFiles.push(...own.wsFiles)
|
|
198
201
|
inputs.envNames.push(...own.envNames)
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type PluginOptionKinds,
|
|
22
22
|
} from '@vzn/vx'
|
|
23
23
|
import { deadlineNamed } from '../remote-deadline.js'
|
|
24
|
+
import { OutageBreaker } from '../remote-breaker.js'
|
|
24
25
|
import { withRetry } from '../remote-retry.js'
|
|
25
26
|
import { headerValueFault } from '../remote-token.js'
|
|
26
27
|
|
|
@@ -102,6 +103,7 @@ export function resolveNxCacheConfig(
|
|
|
102
103
|
*/
|
|
103
104
|
export class NxRemoteCache implements RemoteCacheLayer {
|
|
104
105
|
private disabled = false
|
|
106
|
+
private readonly breaker = new OutageBreaker()
|
|
105
107
|
/**
|
|
106
108
|
* A `403` on a write is the spec's read-only token (a CI's pull-request
|
|
107
109
|
* token): uploads stop, reads go on. Read as a refused token, it turned
|
|
@@ -140,19 +142,23 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
140
142
|
// every hit read as a corrupt artifact (nx#33092).
|
|
141
143
|
headers['Accept'] = 'application/octet-stream'
|
|
142
144
|
}
|
|
143
|
-
const res = await
|
|
144
|
-
() =>
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
145
|
+
const res = await this.breaker
|
|
146
|
+
.send(() =>
|
|
147
|
+
withRetry(
|
|
148
|
+
() =>
|
|
149
|
+
this.fetchImpl(`${this.config.server}/v1/cache/${hash}`, {
|
|
150
|
+
method,
|
|
151
|
+
headers,
|
|
152
|
+
...(body === undefined ? {} : { body }),
|
|
153
|
+
signal: AbortSignal.timeout(this.config.timeoutMs),
|
|
154
|
+
}),
|
|
155
|
+
this.config.retries,
|
|
156
|
+
this.wait,
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
.catch((err: unknown) => {
|
|
160
|
+
throw deadlineNamed(err, this.config.timeoutMs)
|
|
161
|
+
})
|
|
156
162
|
if (method === 'PUT' && res.status === 403) {
|
|
157
163
|
const first = !this.writesRefused
|
|
158
164
|
this.writesRefused = true
|
|
@@ -166,7 +172,7 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
166
172
|
this.disabled = true
|
|
167
173
|
if (!first) return undefined
|
|
168
174
|
throw new Error(
|
|
169
|
-
`HTTP ${res.status}: ${res.status === 401 ? 'missing or invalid token' : 'access forbidden (
|
|
175
|
+
`HTTP ${res.status}: ${res.status === 401 ? 'missing or invalid token' : 'access forbidden (the token may not read)'}; remote cache off for this run`,
|
|
170
176
|
)
|
|
171
177
|
}
|
|
172
178
|
return res
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turbo's outage breaker (`turborepo-cache/src/outage_breaker.rs`): three
|
|
3
|
+
* outages in a row — no connection, no answer in time, or a 5xx once the
|
|
4
|
+
* retries are spent — open it for 30 s, and while open no request is sent.
|
|
5
|
+
* After the cooldown one request probes; its success closes the breaker,
|
|
6
|
+
* its outage opens it again. Without it a server that hung cost every
|
|
7
|
+
* task its full deadline. A result that lands after the breaker moved on
|
|
8
|
+
* (a request already in flight when it opened) changes nothing. Shared by
|
|
9
|
+
* `turboCache()` and `nxCache()`.
|
|
10
|
+
*/
|
|
11
|
+
export class OutageBreaker {
|
|
12
|
+
private failures = 0
|
|
13
|
+
private openUntil: number | undefined
|
|
14
|
+
private probing = false
|
|
15
|
+
private generation = 0
|
|
16
|
+
constructor(private readonly now: () => number = Date.now) {}
|
|
17
|
+
|
|
18
|
+
async send(request: () => Promise<Response>): Promise<Response> {
|
|
19
|
+
let probe = false
|
|
20
|
+
if (this.openUntil !== undefined) {
|
|
21
|
+
if (this.now() < this.openUntil || this.probing)
|
|
22
|
+
throw new Error(
|
|
23
|
+
`unavailable after ${FAILURE_THRESHOLD} failures in a row; not asked again for ${COOLDOWN_MS / 1000} s`,
|
|
24
|
+
)
|
|
25
|
+
this.probing = probe = true
|
|
26
|
+
}
|
|
27
|
+
const generation = this.generation
|
|
28
|
+
let res: Response
|
|
29
|
+
try {
|
|
30
|
+
res = await request()
|
|
31
|
+
} catch (err) {
|
|
32
|
+
this.finish(generation, probe, true)
|
|
33
|
+
throw err
|
|
34
|
+
}
|
|
35
|
+
this.finish(generation, probe, res.status >= 500)
|
|
36
|
+
return res
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
private finish(generation: number, probe: boolean, outage: boolean): void {
|
|
40
|
+
if (generation !== this.generation) return
|
|
41
|
+
if (outage) {
|
|
42
|
+
this.failures++
|
|
43
|
+
if (probe || this.failures >= FAILURE_THRESHOLD) {
|
|
44
|
+
this.openUntil = this.now() + COOLDOWN_MS
|
|
45
|
+
this.probing = false
|
|
46
|
+
this.generation++
|
|
47
|
+
}
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
this.failures = 0
|
|
51
|
+
if (probe) {
|
|
52
|
+
this.openUntil = undefined
|
|
53
|
+
this.probing = false
|
|
54
|
+
this.generation++
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const FAILURE_THRESHOLD = 3
|
|
60
|
+
const COOLDOWN_MS = 30_000
|
package/src/shared-outputs.ts
CHANGED
|
@@ -413,6 +413,8 @@ export function excludeWorkspaceOutputs(
|
|
|
413
413
|
readonly dir: string
|
|
414
414
|
readonly tasks: GeneratedTask[]
|
|
415
415
|
}[],
|
|
416
|
+
/** The globs an opaque entry stands for (`vx-migrate`'s `...globalInputs` spread). */
|
|
417
|
+
expand: (entry: unknown) => readonly string[] = () => [],
|
|
416
418
|
): void {
|
|
417
419
|
const positive = (v: unknown): string[] =>
|
|
418
420
|
Array.isArray(v)
|
|
@@ -444,7 +446,7 @@ export function excludeWorkspaceOutputs(
|
|
|
444
446
|
?.inputs
|
|
445
447
|
if (inputs === undefined || !Array.isArray(inputs.workspaceFiles)) continue
|
|
446
448
|
const files = inputs.workspaceFiles as string[]
|
|
447
|
-
const reads = positive(files)
|
|
449
|
+
const reads = positive(files.flatMap((f) => (typeof f === 'string' ? [f] : expand(f))))
|
|
448
450
|
if (reads.length === 0) continue
|
|
449
451
|
for (const i of candidates(reads)) {
|
|
450
452
|
const w = writers[i]!
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The env names a package's tracked source spells: what stands in, in a
|
|
2
|
+
// written config, for a `*` env entry Turbo or Vite Task matched against
|
|
3
|
+
// the environment a run sees.
|
|
4
|
+
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { relPosix } from './paths.js'
|
|
7
|
+
|
|
8
|
+
/** Source and env-example files a framework build reads its variables from. */
|
|
9
|
+
const SPELLS_ENV =
|
|
10
|
+
/\.(c|m)?(j|t)sx?$|\.(vue|svelte|astro|html)$|(^|\/)\.env\.(example|sample|template)$/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The upper-case names the tracked source under `dirs` spells (`NEXT_PUBLIC_API`
|
|
14
|
+
* in `process.env.NEXT_PUBLIC_API` or an `.env.example`), sorted. Without
|
|
15
|
+
* git, none: the note still names the framework's prefix.
|
|
16
|
+
*/
|
|
17
|
+
export async function spelledNames(
|
|
18
|
+
root: string,
|
|
19
|
+
dirs: readonly string[],
|
|
20
|
+
tracked: readonly string[] | null,
|
|
21
|
+
): Promise<string[]> {
|
|
22
|
+
if (tracked === null) return []
|
|
23
|
+
const prefixes = dirs.map((dir) => {
|
|
24
|
+
const rel = relPosix(root, dir)
|
|
25
|
+
return rel === '' || rel === '.' ? '' : `${rel}/`
|
|
26
|
+
})
|
|
27
|
+
const names = new Set<string>()
|
|
28
|
+
for (const f of tracked) {
|
|
29
|
+
if (!SPELLS_ENV.test(f) || !prefixes.some((p) => f.startsWith(p))) continue
|
|
30
|
+
const text = await Bun.file(path.join(root, f))
|
|
31
|
+
.text()
|
|
32
|
+
.catch(() => '')
|
|
33
|
+
for (const m of text.matchAll(/\b[A-Z][A-Z0-9_]*_[A-Z0-9_]+\b/g)) names.add(m[0])
|
|
34
|
+
}
|
|
35
|
+
return [...names].sort()
|
|
36
|
+
}
|
package/src/turbo/turbo-map.ts
CHANGED
|
@@ -65,6 +65,8 @@ interface TurboJson {
|
|
|
65
65
|
envMode?: unknown
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
+
const SHELL_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
69
|
+
|
|
68
70
|
/** A name core takes in `cache.inputs.env` and `exec.env.passThrough`. */
|
|
69
71
|
const keyable = (name: string): boolean =>
|
|
70
72
|
name.length > 0 && !name.startsWith('!') && !/[*?[\]{}=\0]/.test(name)
|
|
@@ -1488,7 +1490,14 @@ export async function mapTurboWorkspace(
|
|
|
1488
1490
|
)
|
|
1489
1491
|
literalEnvGapsOnce(projects, notes)
|
|
1490
1492
|
resolveSharedWorkspaceOutputs(root, projects)
|
|
1491
|
-
|
|
1493
|
+
// A written config spreads the preset (`...globalInputs`), which the
|
|
1494
|
+
// take-back pass must read as its globs: unread, opencode's configs
|
|
1495
|
+
// read `packages/plugin/**` with no `!packages/plugin/dist/**` and
|
|
1496
|
+
// core refused to load them.
|
|
1497
|
+
const opaque = new Map<string, readonly string[]>()
|
|
1498
|
+
for (const e of opts.splice('inputs', globals.inputs))
|
|
1499
|
+
if (typeof e !== 'string') opaque.set(JSON.stringify(e), globals.inputs)
|
|
1500
|
+
excludeWorkspaceOutputs(root, projects, (e) => opaque.get(JSON.stringify(e)) ?? [])
|
|
1492
1501
|
pruneUnreachedPersistentNotes(projects, metas, opts.persistentTodo)
|
|
1493
1502
|
pruneOrphanPersistentNotes(projects, opts.persistentTodo)
|
|
1494
1503
|
return { projects, notes, globals }
|
|
@@ -1882,10 +1891,12 @@ function buildTask(
|
|
|
1882
1891
|
spelled,
|
|
1883
1892
|
)
|
|
1884
1893
|
|
|
1894
|
+
// A name that is no shell identifier never reached a task (`sh` drops it)
|
|
1895
|
+
// and core refuses it in passThrough.
|
|
1885
1896
|
const passThrough = uniq(
|
|
1886
1897
|
[...global('env'), ...global('pass'), ...envNames, ...passNames],
|
|
1887
1898
|
hidden('env', 'pass'),
|
|
1888
|
-
)
|
|
1899
|
+
).filter((n) => typeof n !== 'string' || SHELL_NAME.test(n))
|
|
1889
1900
|
|
|
1890
1901
|
const exec: Record<string, unknown> = { command }
|
|
1891
1902
|
if (passThrough.length > 0) exec.env = { passThrough }
|
package/src/turbo-cache/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
type PluginOptionKinds,
|
|
26
26
|
} from '@vzn/vx'
|
|
27
27
|
import { deadlineNamed } from '../remote-deadline.js'
|
|
28
|
+
import { OutageBreaker } from '../remote-breaker.js'
|
|
28
29
|
import { withRetry } from '../remote-retry.js'
|
|
29
30
|
import { headerValueFault } from '../remote-token.js'
|
|
30
31
|
|
|
@@ -55,6 +56,12 @@ export interface TurboCacheOptions {
|
|
|
55
56
|
uploadTimeoutMs?: number
|
|
56
57
|
/** Resends of a request answered 429 / 5xx or never connected (default 1, Turbo's); 0 turns them off. */
|
|
57
58
|
retries?: number
|
|
59
|
+
/**
|
|
60
|
+
* Send Turbo's preflight (`OPTIONS`) before each artifact request and
|
|
61
|
+
* follow its `Location`, or `TURBO_PREFLIGHT` / turbo.json's
|
|
62
|
+
* `remoteCache.preflight`; default off.
|
|
63
|
+
*/
|
|
64
|
+
preflight?: boolean
|
|
58
65
|
}
|
|
59
66
|
|
|
60
67
|
/** turbo.json's `remoteCache` block: the source below Turbo's environment, as in Turbo. */
|
|
@@ -66,6 +73,7 @@ export interface TurboJsonRemoteCache {
|
|
|
66
73
|
enabled?: unknown
|
|
67
74
|
timeout?: unknown
|
|
68
75
|
uploadTimeout?: unknown
|
|
76
|
+
preflight?: unknown
|
|
69
77
|
}
|
|
70
78
|
|
|
71
79
|
/** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
|
|
@@ -85,6 +93,7 @@ export interface TurboCacheConfig {
|
|
|
85
93
|
timeoutMs: number
|
|
86
94
|
uploadTimeoutMs: number
|
|
87
95
|
retries: number
|
|
96
|
+
preflight: boolean
|
|
88
97
|
}
|
|
89
98
|
|
|
90
99
|
/** Turbo's signature message prefix (`crates/turborepo-cache/src/signature_authentication.rs`). */
|
|
@@ -250,18 +259,14 @@ export function resolveTurboCacheConfig(
|
|
|
250
259
|
fromFile(file.teamId)
|
|
251
260
|
const teamSlug =
|
|
252
261
|
options.teamSlug ?? env['TURBO_TEAM'] ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
|
|
262
|
+
const preflight =
|
|
263
|
+
options.preflight ??
|
|
264
|
+
truthy(env['TURBO_PREFLIGHT'], 'TURBO_PREFLIGHT') ??
|
|
265
|
+
file.preflight === true
|
|
253
266
|
// Turbo signs only under `remoteCache.signature: true`; the env key alone
|
|
254
267
|
// signs nothing there, and a short one here refused the whole cache.
|
|
255
268
|
// `TURBO_SIGNATURE` (1/true, 0/false) sits above turbo.json, as in Turbo.
|
|
256
|
-
const
|
|
257
|
-
let signing = file.signature === true
|
|
258
|
-
if (signEnv !== undefined && signEnv !== '') {
|
|
259
|
-
if (!/^(1|0|true|false)$/.test(signEnv))
|
|
260
|
-
throw new Error(
|
|
261
|
-
`vx/turbo-cache: TURBO_SIGNATURE should be 1 or 0, got ${JSON.stringify(signEnv)}`,
|
|
262
|
-
)
|
|
263
|
-
signing = signEnv === '1' || signEnv === 'true'
|
|
264
|
-
}
|
|
269
|
+
const signing = truthy(env['TURBO_SIGNATURE'], 'TURBO_SIGNATURE') ?? file.signature === true
|
|
265
270
|
const signatureKey =
|
|
266
271
|
options.signatureKey ?? (signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] : undefined)
|
|
267
272
|
if (signatureKey !== undefined) {
|
|
@@ -303,9 +308,18 @@ export function resolveTurboCacheConfig(
|
|
|
303
308
|
seconds(file.uploadTimeout, 'remoteCache.uploadTimeout') ??
|
|
304
309
|
60_000,
|
|
305
310
|
retries,
|
|
311
|
+
preflight,
|
|
306
312
|
}
|
|
307
313
|
}
|
|
308
314
|
|
|
315
|
+
/** Turbo's `truth_env_var`: 1/true, 0/false; unset or empty is `undefined`. */
|
|
316
|
+
function truthy(v: string | undefined, name: string): boolean | undefined {
|
|
317
|
+
if (v === undefined || v === '') return undefined
|
|
318
|
+
if (!/^(1|0|true|false)$/.test(v))
|
|
319
|
+
throw new Error(`vx/turbo-cache: ${name} should be 1 or 0, got ${JSON.stringify(v)}`)
|
|
320
|
+
return v === '1' || v === 'true'
|
|
321
|
+
}
|
|
322
|
+
|
|
309
323
|
/**
|
|
310
324
|
* Turbo's timeout, whole seconds, as ms; `undefined` when unset. A value
|
|
311
325
|
* Turbo would refuse is refused here rather than read as some other wait.
|
|
@@ -365,6 +379,7 @@ const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
|
365
379
|
*/
|
|
366
380
|
export class TurboRemoteCache implements RemoteCacheLayer {
|
|
367
381
|
private disabled = false
|
|
382
|
+
private readonly breaker = new OutageBreaker()
|
|
368
383
|
/**
|
|
369
384
|
* A `403` on an upload is a read-only token or server
|
|
370
385
|
* (turborepo-remote-cache's `READ_ONLY` and its JWTs without the write
|
|
@@ -396,14 +411,50 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
396
411
|
return u.toString()
|
|
397
412
|
}
|
|
398
413
|
|
|
399
|
-
private headers(extra: Record<string, string> = {}): Record<string, string> {
|
|
414
|
+
private headers(extra: Record<string, string> = {}, auth = true): Record<string, string> {
|
|
400
415
|
return {
|
|
401
|
-
Authorization: `Bearer ${this.config.token}
|
|
416
|
+
...(auth ? { Authorization: `Bearer ${this.config.token}` } : {}),
|
|
402
417
|
'x-artifact-client-interactive': process.stdout.isTTY ? '1' : '0',
|
|
403
418
|
...extra,
|
|
404
419
|
}
|
|
405
420
|
}
|
|
406
421
|
|
|
422
|
+
/**
|
|
423
|
+
* Turbo's preflight: an `OPTIONS` to the request's URL names the method
|
|
424
|
+
* and headers to come; the answer's `Location` (relative to `apiUrl`) is
|
|
425
|
+
* where the request goes, as given — a signed storage URL signs its own
|
|
426
|
+
* query, so nothing is appended — and the token goes along only when
|
|
427
|
+
* `Access-Control-Allow-Headers` admits `Authorization` (or `*`), so it
|
|
428
|
+
* never reaches a third-party store. The answer's status is not read, as
|
|
429
|
+
* in Turbo.
|
|
430
|
+
*/
|
|
431
|
+
private async preflight(
|
|
432
|
+
url: string,
|
|
433
|
+
method: string,
|
|
434
|
+
headers: Record<string, string>,
|
|
435
|
+
signal: () => AbortSignal | undefined,
|
|
436
|
+
): Promise<{ url: string; auth: boolean }> {
|
|
437
|
+
const s = signal()
|
|
438
|
+
const res = await this.fetchImpl(url, {
|
|
439
|
+
method: 'OPTIONS',
|
|
440
|
+
headers: {
|
|
441
|
+
Authorization: `Bearer ${this.config.token}`,
|
|
442
|
+
'Access-Control-Request-Method': method === 'HEAD' ? 'GET' : method,
|
|
443
|
+
'Access-Control-Request-Headers': ['Authorization', ...Object.keys(headers)].join(', '),
|
|
444
|
+
},
|
|
445
|
+
...(s === undefined ? {} : { signal: s }),
|
|
446
|
+
})
|
|
447
|
+
await res.body?.cancel()
|
|
448
|
+
const location = res.headers.get('location')
|
|
449
|
+
const allowed = res.headers.get('access-control-allow-headers') ?? ''
|
|
450
|
+
return {
|
|
451
|
+
url: location === null ? res.url || url : new URL(location, this.config.apiUrl).toString(),
|
|
452
|
+
auth:
|
|
453
|
+
allowed.trim() === '*' ||
|
|
454
|
+
allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization'),
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
407
458
|
/**
|
|
408
459
|
* `undefined` = the token was refused and the refusal is ALREADY
|
|
409
460
|
* reported, so the caller degrades to its miss value in silence. Only the
|
|
@@ -416,20 +467,33 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
416
467
|
init: { body?: Blob | string; headers?: Record<string, string>; timeoutMs?: number } = {},
|
|
417
468
|
): Promise<Response | undefined> {
|
|
418
469
|
const timeoutMs = init.timeoutMs ?? this.config.timeoutMs
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
470
|
+
// 0 is Turbo's "no deadline".
|
|
471
|
+
const signal = () => (timeoutMs === 0 ? undefined : AbortSignal.timeout(timeoutMs))
|
|
472
|
+
// The batch query is never preflighted, as in Turbo.
|
|
473
|
+
const preflight = this.config.preflight && method !== 'POST'
|
|
474
|
+
const res = await this.breaker
|
|
475
|
+
.send(() =>
|
|
476
|
+
withRetry(
|
|
477
|
+
async () => {
|
|
478
|
+
const extra = init.headers ?? {}
|
|
479
|
+
const to = preflight
|
|
480
|
+
? await this.preflight(this.url(pathname), method, extra, signal)
|
|
481
|
+
: { url: this.url(pathname), auth: true }
|
|
482
|
+
const s = signal()
|
|
483
|
+
return this.fetchImpl(to.url, {
|
|
484
|
+
method,
|
|
485
|
+
headers: this.headers(extra, to.auth),
|
|
486
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
487
|
+
...(s === undefined ? {} : { signal: s }),
|
|
488
|
+
})
|
|
489
|
+
},
|
|
490
|
+
this.config.retries,
|
|
491
|
+
this.wait,
|
|
492
|
+
),
|
|
493
|
+
)
|
|
494
|
+
.catch((err: unknown) => {
|
|
495
|
+
throw deadlineNamed(err, timeoutMs)
|
|
496
|
+
})
|
|
433
497
|
if (method === 'PUT' && res.status === 403) {
|
|
434
498
|
const first = !this.writesRefused
|
|
435
499
|
this.writesRefused = true
|
|
@@ -551,7 +615,9 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
551
615
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
552
616
|
})
|
|
553
617
|
if (res === undefined) return
|
|
554
|
-
|
|
618
|
+
// Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
|
|
619
|
+
// 204 stored the artifact and was counted a failed upload.
|
|
620
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
|
555
621
|
}
|
|
556
622
|
}
|
|
557
623
|
|
|
@@ -636,6 +702,7 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
636
702
|
timeoutMs: 'number',
|
|
637
703
|
uploadTimeoutMs: 'number',
|
|
638
704
|
retries: 'number',
|
|
705
|
+
preflight: 'boolean',
|
|
639
706
|
}
|
|
640
707
|
|
|
641
708
|
/**
|