@vzn/vx-migrate 0.0.594 → 0.0.596
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -21
- package/package.json +2 -2
- package/src/adopt.ts +1 -1
- package/src/glob-grammar.ts +20 -6
- package/src/migrate-nx.ts +37 -17
- package/src/migrate-turbo.ts +1 -30
- package/src/migrate-vite-task.ts +40 -0
- package/src/migrate.ts +179 -24
- package/src/nx/index.ts +4 -1
- package/src/nx/lerna.ts +60 -0
- package/src/nx/nx-inputs.ts +11 -11
- package/src/nx/nx-map.ts +16 -2
- package/src/nx/nx-outputs.ts +62 -12
- package/src/nx/nx-upstream.ts +4 -1
- package/src/nx-cache/index.ts +20 -14
- package/src/nx-env.cjs +4 -1
- package/src/nx-exec.cjs +4 -1
- package/src/remote-breaker.ts +60 -0
- package/src/shared-outputs.ts +3 -1
- package/src/spelled-env.ts +36 -0
- package/src/tracked-outputs.ts +36 -2
- package/src/turbo/turbo-map.ts +24 -20
- package/src/turbo-cache/index.ts +111 -28
- package/src/vite-task/vite-task-map.ts +558 -0
- package/src/workspace-plugins.ts +2 -0
package/src/nx/nx-upstream.ts
CHANGED
|
@@ -20,6 +20,7 @@ import type { GeneratedTask, ProjectMeta } from '@vzn/vx'
|
|
|
20
20
|
import { emptyNxInputs, expandNxInputs, type NxInputs } from './nx-inputs.js'
|
|
21
21
|
import { nxProjectOutputs } from './nx-outputs.js'
|
|
22
22
|
import { matchNxProjects } from './nx-deps.js'
|
|
23
|
+
import { literalGlob } from '../glob-grammar.js'
|
|
23
24
|
|
|
24
25
|
interface GraphNode {
|
|
25
26
|
name?: string
|
|
@@ -192,7 +193,9 @@ export function planNxUpstream(
|
|
|
192
193
|
const inputs = emptyNxInputs()
|
|
193
194
|
for (const g of own.files) {
|
|
194
195
|
const neg = g.startsWith('!') ? '!' : ''
|
|
195
|
-
inputs.wsFiles.push(
|
|
196
|
+
inputs.wsFiles.push(
|
|
197
|
+
rel === '' ? g : `${neg}${literalGlob(rel)}/${neg === '' ? g : g.slice(1)}`,
|
|
198
|
+
)
|
|
196
199
|
}
|
|
197
200
|
inputs.wsFiles.push(...own.wsFiles)
|
|
198
201
|
inputs.envNames.push(...own.envNames)
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type PluginOptionKinds,
|
|
22
22
|
} from '@vzn/vx'
|
|
23
23
|
import { deadlineNamed } from '../remote-deadline.js'
|
|
24
|
+
import { OutageBreaker } from '../remote-breaker.js'
|
|
24
25
|
import { withRetry } from '../remote-retry.js'
|
|
25
26
|
import { headerValueFault } from '../remote-token.js'
|
|
26
27
|
|
|
@@ -102,6 +103,7 @@ export function resolveNxCacheConfig(
|
|
|
102
103
|
*/
|
|
103
104
|
export class NxRemoteCache implements RemoteCacheLayer {
|
|
104
105
|
private disabled = false
|
|
106
|
+
private readonly breaker = new OutageBreaker()
|
|
105
107
|
/**
|
|
106
108
|
* A `403` on a write is the spec's read-only token (a CI's pull-request
|
|
107
109
|
* token): uploads stop, reads go on. Read as a refused token, it turned
|
|
@@ -140,19 +142,23 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
140
142
|
// every hit read as a corrupt artifact (nx#33092).
|
|
141
143
|
headers['Accept'] = 'application/octet-stream'
|
|
142
144
|
}
|
|
143
|
-
const res = await
|
|
144
|
-
() =>
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
145
|
+
const res = await this.breaker
|
|
146
|
+
.send(() =>
|
|
147
|
+
withRetry(
|
|
148
|
+
() =>
|
|
149
|
+
this.fetchImpl(`${this.config.server}/v1/cache/${hash}`, {
|
|
150
|
+
method,
|
|
151
|
+
headers,
|
|
152
|
+
...(body === undefined ? {} : { body }),
|
|
153
|
+
signal: AbortSignal.timeout(this.config.timeoutMs),
|
|
154
|
+
}),
|
|
155
|
+
this.config.retries,
|
|
156
|
+
this.wait,
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
.catch((err: unknown) => {
|
|
160
|
+
throw deadlineNamed(err, this.config.timeoutMs)
|
|
161
|
+
})
|
|
156
162
|
if (method === 'PUT' && res.status === 403) {
|
|
157
163
|
const first = !this.writesRefused
|
|
158
164
|
this.writesRefused = true
|
|
@@ -166,7 +172,7 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
166
172
|
this.disabled = true
|
|
167
173
|
if (!first) return undefined
|
|
168
174
|
throw new Error(
|
|
169
|
-
`HTTP ${res.status}: ${res.status === 401 ? 'missing or invalid token' : 'access forbidden (
|
|
175
|
+
`HTTP ${res.status}: ${res.status === 401 ? 'missing or invalid token' : 'access forbidden (the token may not read)'}; remote cache off for this run`,
|
|
170
176
|
)
|
|
171
177
|
}
|
|
172
178
|
return res
|
package/src/nx-env.cjs
CHANGED
|
@@ -25,7 +25,10 @@
|
|
|
25
25
|
const { spawn, spawnSync } = require('node:child_process')
|
|
26
26
|
const { createRequire } = require('node:module')
|
|
27
27
|
const path = require('node:path')
|
|
28
|
-
const {
|
|
28
|
+
const { realpathSync } = require('node:fs')
|
|
29
|
+
// Through npm's `.bin` symlink under `--preserve-symlinks-main`, Node keeps
|
|
30
|
+
// the link as the main module, so a relative require would look in `.bin`.
|
|
31
|
+
const { loadTaskEnv } = require(path.join(path.dirname(realpathSync(__filename)), 'nx-dotenv.cjs'))
|
|
29
32
|
|
|
30
33
|
// `--envFile`, Nx's own option name, and not `--env-file`: Node 22 takes
|
|
31
34
|
// `--env-file` from anywhere on its command line, the script's arguments
|
package/src/nx-exec.cjs
CHANGED
|
@@ -21,7 +21,10 @@
|
|
|
21
21
|
|
|
22
22
|
const { createRequire, enableCompileCache } = require('node:module')
|
|
23
23
|
const path = require('node:path')
|
|
24
|
-
const {
|
|
24
|
+
const { realpathSync } = require('node:fs')
|
|
25
|
+
// Through npm's `.bin` symlink under `--preserve-symlinks-main`, Node keeps
|
|
26
|
+
// the link as the main module, so a relative require would look in `.bin`.
|
|
27
|
+
const { loadTaskEnv } = require(path.join(path.dirname(realpathSync(__filename)), 'nx-dotenv.cjs'))
|
|
25
28
|
|
|
26
29
|
// Nx's module graph is ~150 ms of every executed task; Node's on-disk
|
|
27
30
|
// compile cache (22.1+, a no-op below) takes ~30 of them back on the
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turbo's outage breaker (`turborepo-cache/src/outage_breaker.rs`): three
|
|
3
|
+
* outages in a row — no connection, no answer in time, or a 5xx once the
|
|
4
|
+
* retries are spent — open it for 30 s, and while open no request is sent.
|
|
5
|
+
* After the cooldown one request probes; its success closes the breaker,
|
|
6
|
+
* its outage opens it again. Without it a server that hung cost every
|
|
7
|
+
* task its full deadline. A result that lands after the breaker moved on
|
|
8
|
+
* (a request already in flight when it opened) changes nothing. Shared by
|
|
9
|
+
* `turboCache()` and `nxCache()`.
|
|
10
|
+
*/
|
|
11
|
+
export class OutageBreaker {
|
|
12
|
+
private failures = 0
|
|
13
|
+
private openUntil: number | undefined
|
|
14
|
+
private probing = false
|
|
15
|
+
private generation = 0
|
|
16
|
+
constructor(private readonly now: () => number = Date.now) {}
|
|
17
|
+
|
|
18
|
+
async send(request: () => Promise<Response>): Promise<Response> {
|
|
19
|
+
let probe = false
|
|
20
|
+
if (this.openUntil !== undefined) {
|
|
21
|
+
if (this.now() < this.openUntil || this.probing)
|
|
22
|
+
throw new Error(
|
|
23
|
+
`unavailable after ${FAILURE_THRESHOLD} failures in a row; not asked again for ${COOLDOWN_MS / 1000} s`,
|
|
24
|
+
)
|
|
25
|
+
this.probing = probe = true
|
|
26
|
+
}
|
|
27
|
+
const generation = this.generation
|
|
28
|
+
let res: Response
|
|
29
|
+
try {
|
|
30
|
+
res = await request()
|
|
31
|
+
} catch (err) {
|
|
32
|
+
this.finish(generation, probe, true)
|
|
33
|
+
throw err
|
|
34
|
+
}
|
|
35
|
+
this.finish(generation, probe, res.status >= 500)
|
|
36
|
+
return res
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
private finish(generation: number, probe: boolean, outage: boolean): void {
|
|
40
|
+
if (generation !== this.generation) return
|
|
41
|
+
if (outage) {
|
|
42
|
+
this.failures++
|
|
43
|
+
if (probe || this.failures >= FAILURE_THRESHOLD) {
|
|
44
|
+
this.openUntil = this.now() + COOLDOWN_MS
|
|
45
|
+
this.probing = false
|
|
46
|
+
this.generation++
|
|
47
|
+
}
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
this.failures = 0
|
|
51
|
+
if (probe) {
|
|
52
|
+
this.openUntil = undefined
|
|
53
|
+
this.probing = false
|
|
54
|
+
this.generation++
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const FAILURE_THRESHOLD = 3
|
|
60
|
+
const COOLDOWN_MS = 30_000
|
package/src/shared-outputs.ts
CHANGED
|
@@ -413,6 +413,8 @@ export function excludeWorkspaceOutputs(
|
|
|
413
413
|
readonly dir: string
|
|
414
414
|
readonly tasks: GeneratedTask[]
|
|
415
415
|
}[],
|
|
416
|
+
/** The globs an opaque entry stands for (`vx-migrate`'s `...globalInputs` spread). */
|
|
417
|
+
expand: (entry: unknown) => readonly string[] = () => [],
|
|
416
418
|
): void {
|
|
417
419
|
const positive = (v: unknown): string[] =>
|
|
418
420
|
Array.isArray(v)
|
|
@@ -444,7 +446,7 @@ export function excludeWorkspaceOutputs(
|
|
|
444
446
|
?.inputs
|
|
445
447
|
if (inputs === undefined || !Array.isArray(inputs.workspaceFiles)) continue
|
|
446
448
|
const files = inputs.workspaceFiles as string[]
|
|
447
|
-
const reads = positive(files)
|
|
449
|
+
const reads = positive(files.flatMap((f) => (typeof f === 'string' ? [f] : expand(f))))
|
|
448
450
|
if (reads.length === 0) continue
|
|
449
451
|
for (const i of candidates(reads)) {
|
|
450
452
|
const w = writers[i]!
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The env names a package's tracked source spells: what stands in, in a
|
|
2
|
+
// written config, for a `*` env entry Turbo or Vite Task matched against
|
|
3
|
+
// the environment a run sees.
|
|
4
|
+
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { relPosix } from './paths.js'
|
|
7
|
+
|
|
8
|
+
/** Source and env-example files a framework build reads its variables from. */
|
|
9
|
+
const SPELLS_ENV =
|
|
10
|
+
/\.(c|m)?(j|t)sx?$|\.(vue|svelte|astro|html)$|(^|\/)\.env\.(example|sample|template)$/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The upper-case names the tracked source under `dirs` spells (`NEXT_PUBLIC_API`
|
|
14
|
+
* in `process.env.NEXT_PUBLIC_API` or an `.env.example`), sorted. Without
|
|
15
|
+
* git, none: the note still names the framework's prefix.
|
|
16
|
+
*/
|
|
17
|
+
export async function spelledNames(
|
|
18
|
+
root: string,
|
|
19
|
+
dirs: readonly string[],
|
|
20
|
+
tracked: readonly string[] | null,
|
|
21
|
+
): Promise<string[]> {
|
|
22
|
+
if (tracked === null) return []
|
|
23
|
+
const prefixes = dirs.map((dir) => {
|
|
24
|
+
const rel = relPosix(root, dir)
|
|
25
|
+
return rel === '' || rel === '.' ? '' : `${rel}/`
|
|
26
|
+
})
|
|
27
|
+
const names = new Set<string>()
|
|
28
|
+
for (const f of tracked) {
|
|
29
|
+
if (!SPELLS_ENV.test(f) || !prefixes.some((p) => f.startsWith(p))) continue
|
|
30
|
+
const text = await Bun.file(path.join(root, f))
|
|
31
|
+
.text()
|
|
32
|
+
.catch(() => '')
|
|
33
|
+
for (const m of text.matchAll(/\b[A-Z][A-Z0-9_]*_[A-Z0-9_]+\b/g)) names.add(m[0])
|
|
34
|
+
}
|
|
35
|
+
return [...names].sort()
|
|
36
|
+
}
|
package/src/tracked-outputs.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { lstat, readFile } from 'node:fs/promises'
|
|
11
11
|
import path from 'node:path'
|
|
12
|
-
import { outputsOverlap } from '@vzn/vx'
|
|
12
|
+
import { isLiteralPattern, outputsOverlap } from '@vzn/vx'
|
|
13
13
|
|
|
14
14
|
/** The files git tracks under `root`, root-relative; null outside a repo or without git. */
|
|
15
15
|
export async function trackedFiles(root: string): Promise<string[] | null> {
|
|
@@ -216,6 +216,27 @@ export function spareTrackedOutputs(
|
|
|
216
216
|
tracked: readonly string[],
|
|
217
217
|
): [string, string][] {
|
|
218
218
|
const todos: [string, string][] = []
|
|
219
|
+
const trackedSet = new Set(tracked)
|
|
220
|
+
/** Drop the `!` entries readers carry for a committed file no output names any more. */
|
|
221
|
+
const unhide = (owner: (typeof projects)[number] | null, rooted: string) => {
|
|
222
|
+
for (const q of projects) {
|
|
223
|
+
const qrel = path.relative(root, q.dir).split(path.sep).join('/')
|
|
224
|
+
for (const r of q.tasks) {
|
|
225
|
+
const inputs = (r.task?.['cache'] as { inputs?: Outputs } | undefined)?.inputs
|
|
226
|
+
if (inputs === undefined) continue
|
|
227
|
+
const own =
|
|
228
|
+
q === owner &&
|
|
229
|
+
Array.isArray(inputs.files) &&
|
|
230
|
+
(qrel === '' || rooted.startsWith(`${qrel}/`))
|
|
231
|
+
? rooted.slice(qrel === '' ? 0 : qrel.length + 1)
|
|
232
|
+
: undefined
|
|
233
|
+
if (own !== undefined)
|
|
234
|
+
inputs.files = (inputs.files as unknown[]).filter((g) => g !== `!${own}`)
|
|
235
|
+
if (Array.isArray(inputs.workspaceFiles))
|
|
236
|
+
inputs.workspaceFiles = inputs.workspaceFiles.filter((g) => g !== `!${rooted}`)
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
219
240
|
for (const p of projects) {
|
|
220
241
|
const rel = path.relative(root, p.dir).split(path.sep).join('/')
|
|
221
242
|
const own = rel === '' ? tracked : filesUnder(tracked, rel).map((f) => f.slice(rel.length + 1))
|
|
@@ -229,7 +250,20 @@ export function spareTrackedOutputs(
|
|
|
229
250
|
] as const) {
|
|
230
251
|
const globs = outputs[key]
|
|
231
252
|
if (!Array.isArray(globs) || globs.length === 0) continue
|
|
232
|
-
|
|
253
|
+
let strings = globs.filter((g): g is string => typeof g === 'string')
|
|
254
|
+
// An output naming one committed file (eslint's
|
|
255
|
+
// `eslint-suppressions.json`) is a source: its `!` would take the
|
|
256
|
+
// whole entry back, so the entry goes, and so does the `!` a
|
|
257
|
+
// reader's inputs got for it, or every reader ran uncached (hono).
|
|
258
|
+
const rootedOf = (g: string) => (key === 'files' && rel !== '' ? `${rel}/${g}` : g)
|
|
259
|
+
const sources = new Set(
|
|
260
|
+
strings.filter((g) => isLiteralPattern(g) && trackedSet.has(rootedOf(g))),
|
|
261
|
+
)
|
|
262
|
+
if (sources.size > 0) {
|
|
263
|
+
strings = strings.filter((g) => !sources.has(g.startsWith('!') ? g.slice(1) : g))
|
|
264
|
+
outputs[key] = strings
|
|
265
|
+
for (const g of sources) unhide(key === 'files' ? p : null, rootedOf(g))
|
|
266
|
+
}
|
|
233
267
|
const hit = coveredTracked(strings, base)
|
|
234
268
|
if (hit.length > 0) hits.push([key, strings, hit])
|
|
235
269
|
}
|
package/src/turbo/turbo-map.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { existsSync, readFileSync } from 'node:fs'
|
|
14
14
|
import path from 'node:path'
|
|
15
15
|
import { pruneOrphanPersistentNotes, type ProjectMeta, UserError } from '@vzn/vx'
|
|
16
|
-
import { minimatchToVx, withoutTakenBack } from '../glob-grammar.js'
|
|
16
|
+
import { literalGlob, minimatchToVx, withoutTakenBack } from '../glob-grammar.js'
|
|
17
17
|
import { shellQuote } from '../nx-command.js'
|
|
18
18
|
import { scriptCommand, yarnPnp } from '../script-command.js'
|
|
19
19
|
import {
|
|
@@ -65,6 +65,8 @@ interface TurboJson {
|
|
|
65
65
|
envMode?: unknown
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
+
const SHELL_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
69
|
+
|
|
68
70
|
/** A name core takes in `cache.inputs.env` and `exec.env.passThrough`. */
|
|
69
71
|
const keyable = (name: string): boolean =>
|
|
70
72
|
name.length > 0 && !name.startsWith('!') && !/[*?[\]{}=\0]/.test(name)
|
|
@@ -842,7 +844,7 @@ function rootDependencyGlobs(
|
|
|
842
844
|
const out: string[] = []
|
|
843
845
|
for (const name of seen) {
|
|
844
846
|
const rel = relPosix(root, byName.get(name)!.dir)
|
|
845
|
-
if (rel !== '' && rel !== '.') out.push(`${rel}/**`)
|
|
847
|
+
if (rel !== '' && rel !== '.') out.push(`${literalGlob(rel)}/**`)
|
|
846
848
|
}
|
|
847
849
|
return out.sort()
|
|
848
850
|
}
|
|
@@ -1110,22 +1112,15 @@ export async function mapTurboWorkspace(
|
|
|
1110
1112
|
// A name no package has a script for is an entry point of its own:
|
|
1111
1113
|
// `turbo run ci` over `ci: { dependsOn: ["lint", "build"] }`, or
|
|
1112
1114
|
// cal.com's `deploy` → `@calcom/web#build`, runs its edges, and vx
|
|
1113
|
-
// said no project declares it.
|
|
1114
|
-
//
|
|
1115
|
-
//
|
|
1116
|
-
//
|
|
1115
|
+
// said no project declares it. With no edge left (documenso's `lint`,
|
|
1116
|
+
// shadcn-ui's `check`) Turbo still runs a no-op node per package and
|
|
1117
|
+
// exits 0, so a CI step `vx run lint` must too. Its `pkg#task` edges
|
|
1118
|
+
// stay in that package alone: core's `--filter`/`--affected` reach
|
|
1119
|
+
// follows a task edge across packages, and cal.com's `deploy` in all
|
|
1120
|
+
// 116 would have made every package a dependent of web.
|
|
1117
1121
|
else if (!scripted.has(name)) {
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
if (envDependency(d) !== null || d.includes('$TURBO_ROOT$')) return false
|
|
1121
|
-
if (d.startsWith('^'))
|
|
1122
|
-
return d !== `^${name}` && [...runnable.values()].some((r) => r.has(d.slice(1)))
|
|
1123
|
-
return d !== name && (own.has(d) || defined.has(d))
|
|
1124
|
-
})
|
|
1125
|
-
if (keeps) {
|
|
1126
|
-
emitted.get(meta.name)!.add(name)
|
|
1127
|
-
entry.add(`${meta.name}#${name}`)
|
|
1128
|
-
}
|
|
1122
|
+
emitted.get(meta.name)!.add(name)
|
|
1123
|
+
entry.add(`${meta.name}#${name}`)
|
|
1129
1124
|
}
|
|
1130
1125
|
}
|
|
1131
1126
|
}
|
|
@@ -1495,7 +1490,14 @@ export async function mapTurboWorkspace(
|
|
|
1495
1490
|
)
|
|
1496
1491
|
literalEnvGapsOnce(projects, notes)
|
|
1497
1492
|
resolveSharedWorkspaceOutputs(root, projects)
|
|
1498
|
-
|
|
1493
|
+
// A written config spreads the preset (`...globalInputs`), which the
|
|
1494
|
+
// take-back pass must read as its globs: unread, opencode's configs
|
|
1495
|
+
// read `packages/plugin/**` with no `!packages/plugin/dist/**` and
|
|
1496
|
+
// core refused to load them.
|
|
1497
|
+
const opaque = new Map<string, readonly string[]>()
|
|
1498
|
+
for (const e of opts.splice('inputs', globals.inputs))
|
|
1499
|
+
if (typeof e !== 'string') opaque.set(JSON.stringify(e), globals.inputs)
|
|
1500
|
+
excludeWorkspaceOutputs(root, projects, (e) => opaque.get(JSON.stringify(e)) ?? [])
|
|
1499
1501
|
pruneUnreachedPersistentNotes(projects, metas, opts.persistentTodo)
|
|
1500
1502
|
pruneOrphanPersistentNotes(projects, opts.persistentTodo)
|
|
1501
1503
|
return { projects, notes, globals }
|
|
@@ -1730,7 +1732,7 @@ function buildTask(
|
|
|
1730
1732
|
const climbed = (glob: string): string | null => {
|
|
1731
1733
|
const body = glob.startsWith('!') ? glob.slice(1) : glob
|
|
1732
1734
|
if (!body.startsWith('../')) return null
|
|
1733
|
-
const anchored = path.posix.normalize(path.posix.join(pkgDir, body))
|
|
1735
|
+
const anchored = path.posix.normalize(path.posix.join(literalGlob(pkgDir), body))
|
|
1734
1736
|
if (anchored.startsWith('../')) return null
|
|
1735
1737
|
return (glob.startsWith('!') ? '!' : '') + anchored
|
|
1736
1738
|
}
|
|
@@ -1889,10 +1891,12 @@ function buildTask(
|
|
|
1889
1891
|
spelled,
|
|
1890
1892
|
)
|
|
1891
1893
|
|
|
1894
|
+
// A name that is no shell identifier never reached a task (`sh` drops it)
|
|
1895
|
+
// and core refuses it in passThrough.
|
|
1892
1896
|
const passThrough = uniq(
|
|
1893
1897
|
[...global('env'), ...global('pass'), ...envNames, ...passNames],
|
|
1894
1898
|
hidden('env', 'pass'),
|
|
1895
|
-
)
|
|
1899
|
+
).filter((n) => typeof n !== 'string' || SHELL_NAME.test(n))
|
|
1896
1900
|
|
|
1897
1901
|
const exec: Record<string, unknown> = { command }
|
|
1898
1902
|
if (passThrough.length > 0) exec.env = { passThrough }
|
package/src/turbo-cache/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
type PluginOptionKinds,
|
|
26
26
|
} from '@vzn/vx'
|
|
27
27
|
import { deadlineNamed } from '../remote-deadline.js'
|
|
28
|
+
import { OutageBreaker } from '../remote-breaker.js'
|
|
28
29
|
import { withRetry } from '../remote-retry.js'
|
|
29
30
|
import { headerValueFault } from '../remote-token.js'
|
|
30
31
|
|
|
@@ -55,6 +56,12 @@ export interface TurboCacheOptions {
|
|
|
55
56
|
uploadTimeoutMs?: number
|
|
56
57
|
/** Resends of a request answered 429 / 5xx or never connected (default 1, Turbo's); 0 turns them off. */
|
|
57
58
|
retries?: number
|
|
59
|
+
/**
|
|
60
|
+
* Send Turbo's preflight (`OPTIONS`) before each artifact request and
|
|
61
|
+
* follow its `Location`, or `TURBO_PREFLIGHT` / turbo.json's
|
|
62
|
+
* `remoteCache.preflight`; default off.
|
|
63
|
+
*/
|
|
64
|
+
preflight?: boolean
|
|
58
65
|
}
|
|
59
66
|
|
|
60
67
|
/** turbo.json's `remoteCache` block: the source below Turbo's environment, as in Turbo. */
|
|
@@ -66,6 +73,7 @@ export interface TurboJsonRemoteCache {
|
|
|
66
73
|
enabled?: unknown
|
|
67
74
|
timeout?: unknown
|
|
68
75
|
uploadTimeout?: unknown
|
|
76
|
+
preflight?: unknown
|
|
69
77
|
}
|
|
70
78
|
|
|
71
79
|
/** `.turbo/config.json`'s fields, under any of the spellings Turbo accepts. */
|
|
@@ -85,6 +93,7 @@ export interface TurboCacheConfig {
|
|
|
85
93
|
timeoutMs: number
|
|
86
94
|
uploadTimeoutMs: number
|
|
87
95
|
retries: number
|
|
96
|
+
preflight: boolean
|
|
88
97
|
}
|
|
89
98
|
|
|
90
99
|
/** Turbo's signature message prefix (`crates/turborepo-cache/src/signature_authentication.rs`). */
|
|
@@ -250,18 +259,14 @@ export function resolveTurboCacheConfig(
|
|
|
250
259
|
fromFile(file.teamId)
|
|
251
260
|
const teamSlug =
|
|
252
261
|
options.teamSlug ?? env['TURBO_TEAM'] ?? fromFile(local.teamSlug) ?? fromFile(file.teamSlug)
|
|
262
|
+
const preflight =
|
|
263
|
+
options.preflight ??
|
|
264
|
+
truthy(env['TURBO_PREFLIGHT'], 'TURBO_PREFLIGHT') ??
|
|
265
|
+
file.preflight === true
|
|
253
266
|
// Turbo signs only under `remoteCache.signature: true`; the env key alone
|
|
254
267
|
// signs nothing there, and a short one here refused the whole cache.
|
|
255
268
|
// `TURBO_SIGNATURE` (1/true, 0/false) sits above turbo.json, as in Turbo.
|
|
256
|
-
const
|
|
257
|
-
let signing = file.signature === true
|
|
258
|
-
if (signEnv !== undefined && signEnv !== '') {
|
|
259
|
-
if (!/^(1|0|true|false)$/.test(signEnv))
|
|
260
|
-
throw new Error(
|
|
261
|
-
`vx/turbo-cache: TURBO_SIGNATURE should be 1 or 0, got ${JSON.stringify(signEnv)}`,
|
|
262
|
-
)
|
|
263
|
-
signing = signEnv === '1' || signEnv === 'true'
|
|
264
|
-
}
|
|
269
|
+
const signing = truthy(env['TURBO_SIGNATURE'], 'TURBO_SIGNATURE') ?? file.signature === true
|
|
265
270
|
const signatureKey =
|
|
266
271
|
options.signatureKey ?? (signing ? env['TURBO_REMOTE_CACHE_SIGNATURE_KEY'] : undefined)
|
|
267
272
|
if (signatureKey !== undefined) {
|
|
@@ -303,9 +308,18 @@ export function resolveTurboCacheConfig(
|
|
|
303
308
|
seconds(file.uploadTimeout, 'remoteCache.uploadTimeout') ??
|
|
304
309
|
60_000,
|
|
305
310
|
retries,
|
|
311
|
+
preflight,
|
|
306
312
|
}
|
|
307
313
|
}
|
|
308
314
|
|
|
315
|
+
/** Turbo's `truth_env_var`: 1/true, 0/false; unset or empty is `undefined`. */
|
|
316
|
+
function truthy(v: string | undefined, name: string): boolean | undefined {
|
|
317
|
+
if (v === undefined || v === '') return undefined
|
|
318
|
+
if (!/^(1|0|true|false)$/.test(v))
|
|
319
|
+
throw new Error(`vx/turbo-cache: ${name} should be 1 or 0, got ${JSON.stringify(v)}`)
|
|
320
|
+
return v === '1' || v === 'true'
|
|
321
|
+
}
|
|
322
|
+
|
|
309
323
|
/**
|
|
310
324
|
* Turbo's timeout, whole seconds, as ms; `undefined` when unset. A value
|
|
311
325
|
* Turbo would refuse is refused here rather than read as some other wait.
|
|
@@ -353,7 +367,8 @@ const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
|
353
367
|
* The seam implementation: `has` is HEAD, `hasMany` is the batch query,
|
|
354
368
|
* `get`/`put` carry `x-artifact-duration` (and the tag when signing). An
|
|
355
369
|
* auth failure (401/403) throws ONCE — LayeredCache reports it — and then
|
|
356
|
-
* turns the layer off for the rest of the process
|
|
370
|
+
* turns the layer off for the rest of the process (an upload's 403 turns off
|
|
371
|
+
* uploads alone: the read-only token), so a bad token costs one
|
|
357
372
|
* line, not one per task — the requests already in flight when it lands
|
|
358
373
|
* degrade in silence rather than repeating it.
|
|
359
374
|
*
|
|
@@ -364,6 +379,14 @@ const MAX_SIGNED_BODY = 2 * 1024 ** 3 + ((2 * 1024 ** 3) >> 8) + 64 * 1024
|
|
|
364
379
|
*/
|
|
365
380
|
export class TurboRemoteCache implements RemoteCacheLayer {
|
|
366
381
|
private disabled = false
|
|
382
|
+
private readonly breaker = new OutageBreaker()
|
|
383
|
+
/**
|
|
384
|
+
* A `403` on an upload is a read-only token or server
|
|
385
|
+
* (turborepo-remote-cache's `READ_ONLY` and its JWTs without the write
|
|
386
|
+
* scope): uploads stop, reads go on. Read as a refused token, it turned
|
|
387
|
+
* the reads off too, and every task after the first upload missed.
|
|
388
|
+
*/
|
|
389
|
+
private writesRefused = false
|
|
367
390
|
private readonly key: Uint8Array | undefined
|
|
368
391
|
readonly endpoint: string
|
|
369
392
|
constructor(
|
|
@@ -388,14 +411,50 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
388
411
|
return u.toString()
|
|
389
412
|
}
|
|
390
413
|
|
|
391
|
-
private headers(extra: Record<string, string> = {}): Record<string, string> {
|
|
414
|
+
private headers(extra: Record<string, string> = {}, auth = true): Record<string, string> {
|
|
392
415
|
return {
|
|
393
|
-
Authorization: `Bearer ${this.config.token}
|
|
416
|
+
...(auth ? { Authorization: `Bearer ${this.config.token}` } : {}),
|
|
394
417
|
'x-artifact-client-interactive': process.stdout.isTTY ? '1' : '0',
|
|
395
418
|
...extra,
|
|
396
419
|
}
|
|
397
420
|
}
|
|
398
421
|
|
|
422
|
+
/**
|
|
423
|
+
* Turbo's preflight: an `OPTIONS` to the request's URL names the method
|
|
424
|
+
* and headers to come; the answer's `Location` (relative to `apiUrl`) is
|
|
425
|
+
* where the request goes, as given — a signed storage URL signs its own
|
|
426
|
+
* query, so nothing is appended — and the token goes along only when
|
|
427
|
+
* `Access-Control-Allow-Headers` admits `Authorization` (or `*`), so it
|
|
428
|
+
* never reaches a third-party store. The answer's status is not read, as
|
|
429
|
+
* in Turbo.
|
|
430
|
+
*/
|
|
431
|
+
private async preflight(
|
|
432
|
+
url: string,
|
|
433
|
+
method: string,
|
|
434
|
+
headers: Record<string, string>,
|
|
435
|
+
signal: () => AbortSignal | undefined,
|
|
436
|
+
): Promise<{ url: string; auth: boolean }> {
|
|
437
|
+
const s = signal()
|
|
438
|
+
const res = await this.fetchImpl(url, {
|
|
439
|
+
method: 'OPTIONS',
|
|
440
|
+
headers: {
|
|
441
|
+
Authorization: `Bearer ${this.config.token}`,
|
|
442
|
+
'Access-Control-Request-Method': method === 'HEAD' ? 'GET' : method,
|
|
443
|
+
'Access-Control-Request-Headers': ['Authorization', ...Object.keys(headers)].join(', '),
|
|
444
|
+
},
|
|
445
|
+
...(s === undefined ? {} : { signal: s }),
|
|
446
|
+
})
|
|
447
|
+
await res.body?.cancel()
|
|
448
|
+
const location = res.headers.get('location')
|
|
449
|
+
const allowed = res.headers.get('access-control-allow-headers') ?? ''
|
|
450
|
+
return {
|
|
451
|
+
url: location === null ? res.url || url : new URL(location, this.config.apiUrl).toString(),
|
|
452
|
+
auth:
|
|
453
|
+
allowed.trim() === '*' ||
|
|
454
|
+
allowed.split(',').some((h) => h.trim().toLowerCase() === 'authorization'),
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
399
458
|
/**
|
|
400
459
|
* `undefined` = the token was refused and the refusal is ALREADY
|
|
401
460
|
* reported, so the caller degrades to its miss value in silence. Only the
|
|
@@ -408,20 +467,41 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
408
467
|
init: { body?: Blob | string; headers?: Record<string, string>; timeoutMs?: number } = {},
|
|
409
468
|
): Promise<Response | undefined> {
|
|
410
469
|
const timeoutMs = init.timeoutMs ?? this.config.timeoutMs
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
470
|
+
// 0 is Turbo's "no deadline".
|
|
471
|
+
const signal = () => (timeoutMs === 0 ? undefined : AbortSignal.timeout(timeoutMs))
|
|
472
|
+
// The batch query is never preflighted, as in Turbo.
|
|
473
|
+
const preflight = this.config.preflight && method !== 'POST'
|
|
474
|
+
const res = await this.breaker
|
|
475
|
+
.send(() =>
|
|
476
|
+
withRetry(
|
|
477
|
+
async () => {
|
|
478
|
+
const extra = init.headers ?? {}
|
|
479
|
+
const to = preflight
|
|
480
|
+
? await this.preflight(this.url(pathname), method, extra, signal)
|
|
481
|
+
: { url: this.url(pathname), auth: true }
|
|
482
|
+
const s = signal()
|
|
483
|
+
return this.fetchImpl(to.url, {
|
|
484
|
+
method,
|
|
485
|
+
headers: this.headers(extra, to.auth),
|
|
486
|
+
...(init.body === undefined ? {} : { body: init.body }),
|
|
487
|
+
...(s === undefined ? {} : { signal: s }),
|
|
488
|
+
})
|
|
489
|
+
},
|
|
490
|
+
this.config.retries,
|
|
491
|
+
this.wait,
|
|
492
|
+
),
|
|
493
|
+
)
|
|
494
|
+
.catch((err: unknown) => {
|
|
495
|
+
throw deadlineNamed(err, timeoutMs)
|
|
496
|
+
})
|
|
497
|
+
if (method === 'PUT' && res.status === 403) {
|
|
498
|
+
const first = !this.writesRefused
|
|
499
|
+
this.writesRefused = true
|
|
500
|
+
if (!first) return undefined
|
|
501
|
+
throw new Error(
|
|
502
|
+
'HTTP 403: the upload was refused (a read-only token or server); uploads off for this run, reads go on',
|
|
503
|
+
)
|
|
504
|
+
}
|
|
425
505
|
if (res.status === 401 || res.status === 403) {
|
|
426
506
|
const first = !this.disabled
|
|
427
507
|
this.disabled = true
|
|
@@ -520,7 +600,7 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
520
600
|
}
|
|
521
601
|
|
|
522
602
|
async put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void> {
|
|
523
|
-
if (this.disabled) return
|
|
603
|
+
if (this.disabled || this.writesRefused) return
|
|
524
604
|
const headers: Record<string, string> = {
|
|
525
605
|
'Content-Type': 'application/octet-stream',
|
|
526
606
|
'Content-Length': String(body.size),
|
|
@@ -535,7 +615,9 @@ export class TurboRemoteCache implements RemoteCacheLayer {
|
|
|
535
615
|
timeoutMs: this.config.uploadTimeoutMs,
|
|
536
616
|
})
|
|
537
617
|
if (res === undefined) return
|
|
538
|
-
|
|
618
|
+
// Turbo's client takes any 2xx (`error_for_status`); a server's 201 or
|
|
619
|
+
// 204 stored the artifact and was counted a failed upload.
|
|
620
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
|
539
621
|
}
|
|
540
622
|
}
|
|
541
623
|
|
|
@@ -620,6 +702,7 @@ const TURBO_CACHE_KEYS: PluginOptionKinds<TurboCacheOptions> = {
|
|
|
620
702
|
timeoutMs: 'number',
|
|
621
703
|
uploadTimeoutMs: 'number',
|
|
622
704
|
retries: 'number',
|
|
705
|
+
preflight: 'boolean',
|
|
623
706
|
}
|
|
624
707
|
|
|
625
708
|
/**
|