@vzn/vx-migrate 0.0.578 → 0.0.580
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +2 -2
- package/src/nx/nx-map.ts +4 -1
- package/src/shared-outputs.ts +196 -92
- package/src/tracked-outputs.ts +36 -0
- package/src/turbo/turbo-map.ts +4 -1
package/README.md
CHANGED
|
@@ -63,7 +63,8 @@ Rules:
|
|
|
63
63
|
- `env` / `passThroughEnv`: explicit names go to `cache.inputs.env` (env only) and `exec.env.passThrough` (both, plus both globals); a wildcard expands over the run's environment under `turbo()`, which maps where the tasks run (a new variable name such a wildcard matches maps afresh); the CLI, which writes for another environment, lists the names the task's package and its workspace dependencies spell (a root task: the whole repo; clerk's `E2E_*`), noted once per entry, and one nothing spells is a todo; `*` is the only wildcard, as in Turbo, so `\*`, a leading `\!`, `?` and `[` are literal: unkey's `NEXT_PUBLIC_\*` names one variable, which vx cannot key, so it is dropped and reported once for the workspace with the count of tasks naming it (openstatus's root `build` env `\*` had written one todo into each of 45 configs; `turbo()`: only when the run's environment sets it, as Turbo hashes nothing otherwise); and a `!` entry (openstatus' `!NEXT_PUBLIC_VERCEL_URL`) removes the names it matches from its list, with no todo, since vx matches no name it is not given. A name both a global list and the task's own list carry is listed once.
|
|
64
64
|
- Framework inference (Turbo hashes and passes Next's `NEXT_PUBLIC_*`, Vite's `VITE_*`, … for a package that depends on the framework): `turbo()` expands the prefix over the run's environment; the CLI writes the names the tracked source and `.env.example` of the package and its workspace dependencies spell (the bare prefix, as a `startsWith` test spells it, is none) (Next bundles theirs: cal.com's web, 23 alone, 58 with them), and a note asks for any only an installed dependency reads.
|
|
65
65
|
|
|
66
|
-
- **Two tasks of one package on one output path** (strapi's `build`, `build:code` and `build:types`, all on `dist/**`): vx cleans a task's outputs before it runs and before a restore, so the loader refuses two cached tasks whose outputs provably overlap. The mapping resolves it before the file is written — the task with a `^` edge keeps its cache (the first declared when none has one);
|
|
66
|
+
- **Two tasks of one package on one output path** (strapi's `build`, `build:code` and `build:types`, all on `dist/**`): vx cleans a task's outputs before it runs and before a restore, so the loader refuses two cached tasks whose outputs provably overlap. Under vx's default `rules.exclusiveOutputs` an edge between the two changes nothing (twenty's `build:individual` writing into `build`'s `dist`). The mapping resolves it before the file is written — the task with a `^` edge keeps its cache (the first declared when none has one); the rest run uncached with a todo naming the keeper and the fix, their own output path. Same rule for Nx targets.
|
|
67
|
+
- **A task whose inputs read another task's outputs** (a `test` keyed on `**/*` beside a `build` writing `dist/**`): vx's default `rules.upfrontKeys` refuses it, since such a key cannot be known before the producer ran. Each overlapping output is taken back from the reader's inputs with `!` (`['**/*', '!dist/**']`), in package and workspace inputs alike; Turbo and Nx hash what git tracks, and a build's outputs are not, so the key reads what the adopted tool read. A literal input another task writes cannot be taken back: that task runs uncached with a todo. A committed file under such an output is still a source: a reader the `!` would hide it from runs uncached with a todo. Same for Nx targets.
|
|
67
68
|
- **Two packages' tasks on one workspace output** (cal.com's shared `post-install` writes `../../node_modules/@prisma/client/**` from every package with the script): core refuses two cached tasks on one path with no edge between them, so `turbo()` (and `nx()`) keeps the first (in package and task order) cached and runs the rest uncached, each with a todo naming the keeper. A package's own outputs count at their workspace path: typescript-eslint's root project caches `dist` and each package's typecheck `dist/packages/<name>`, one path twice. Edges across packages are not read, so an ordered pair loses its cache too.
|
|
68
69
|
- `cache: false` or `persistent: true` → no `cache` block; a persistent task gets `exec.persistent: {}` and, when some task depends on it, the consumer's `persistentTodo`.
|
|
69
70
|
- A task's `description` (Turbo 2.11.5's schema) is the vx task's `description`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx-migrate",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.580",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Adopt @vzn/vx from Turborepo or Nx: run the repo unchanged, migrate it to vx.config.ts, and keep your Turbo or Nx remote cache",
|
|
6
6
|
"keywords": [
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"access": "public"
|
|
40
40
|
},
|
|
41
41
|
"peerDependencies": {
|
|
42
|
-
"@vzn/vx": "^0.0.
|
|
42
|
+
"@vzn/vx": "^0.0.580"
|
|
43
43
|
},
|
|
44
44
|
"repository": {
|
|
45
45
|
"type": "git",
|
package/src/nx/nx-map.ts
CHANGED
|
@@ -28,6 +28,8 @@ import { mapRunCommands, shellQuote } from '../nx-command.js'
|
|
|
28
28
|
import { scriptCommand, yarnPnp } from '../script-command.js'
|
|
29
29
|
import {
|
|
30
30
|
ownFileTakeBacks,
|
|
31
|
+
excludeSiblingOutputs,
|
|
32
|
+
excludeWorkspaceOutputs,
|
|
31
33
|
resolveSharedOutputs,
|
|
32
34
|
resolveSharedWorkspaceOutputs,
|
|
33
35
|
wildcardOutput,
|
|
@@ -557,11 +559,12 @@ export async function mapNxWorkspace(
|
|
|
557
559
|
dir: meta.dir,
|
|
558
560
|
importLines: [],
|
|
559
561
|
...(tags.length > 0 ? { tags } : {}),
|
|
560
|
-
tasks: resolveSharedOutputs(tasks),
|
|
562
|
+
tasks: excludeSiblingOutputs(resolveSharedOutputs(tasks)),
|
|
561
563
|
})
|
|
562
564
|
}
|
|
563
565
|
|
|
564
566
|
resolveAtomizedWorkspaceOutputs(root, projects, split)
|
|
567
|
+
excludeWorkspaceOutputs(root, projects)
|
|
565
568
|
// A migration leaves Nx behind, so `nx sync` is no advice there: what a
|
|
566
569
|
// generator keeps (the TypeScript one's tsconfig `references`) is the
|
|
567
570
|
// user's to keep. `nx()` runs with Nx installed.
|
package/src/shared-outputs.ts
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
// Two targets of one project on one output path. vx cleans a task's
|
|
2
2
|
// declared outputs before it runs and before a cache-hit restore, so the
|
|
3
|
-
// loader refuses two cached tasks whose outputs provably overlap
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
3
|
+
// loader refuses two cached tasks whose outputs provably overlap, an edge
|
|
4
|
+
// between them or not under core's default `rules.exclusiveOutputs` (X-53).
|
|
5
|
+
// A workspace may turn the rule off (item 588's additive shape), but the
|
|
6
|
+
// mapping never sees the workspace's rules, so it maps for the default and
|
|
7
|
+
// loads under either.
|
|
8
|
+
// strapi (2026-09-11) declares `build`, `build:code` and `build:types` all
|
|
9
|
+
// on `dist/**`, and twenty's `build:individual` writes into `build`'s
|
|
10
|
+
// `dist`; the refusal came at load time, after the migration had reported
|
|
11
|
+
// clean. The mapping resolves it: the task with a `^` edge keeps its cache
|
|
12
|
+
// (it is the one a dependant waits for), or the first declared when none
|
|
13
|
+
// has one; every other task whose outputs overlap a kept task's runs
|
|
12
14
|
// uncached, with a todo that names the keeper and the fix (its own output
|
|
13
|
-
// path
|
|
15
|
+
// path).
|
|
14
16
|
//
|
|
15
17
|
// The overlap question is core's own `outputsOverlap`, asked through the
|
|
16
18
|
// façade. It used to be a COPY of it here, and the copy stopped being the
|
|
@@ -162,8 +164,6 @@ interface Cached {
|
|
|
162
164
|
name: string
|
|
163
165
|
files: string[]
|
|
164
166
|
hasUpstreamEdge: boolean
|
|
165
|
-
/** Same-project `dependsOn` names (no `^`), for the ordering test. */
|
|
166
|
-
localDeps: string[]
|
|
167
167
|
}
|
|
168
168
|
|
|
169
169
|
function cachedOutputs(t: GeneratedTask, index: number): Cached | null {
|
|
@@ -179,10 +179,7 @@ function cachedOutputs(t: GeneratedTask, index: number): Cached | null {
|
|
|
179
179
|
const deps = task['dependsOn']
|
|
180
180
|
const hasUpstreamEdge =
|
|
181
181
|
Array.isArray(deps) && deps.some((d) => typeof d === 'string' && d.startsWith('^'))
|
|
182
|
-
|
|
183
|
-
? deps.filter((d): d is string => typeof d === 'string' && !d.startsWith('^'))
|
|
184
|
-
: []
|
|
185
|
-
return { index, name: t.name, files: strings, hasUpstreamEdge, localDeps }
|
|
182
|
+
return { index, name: t.name, files: strings, hasUpstreamEdge }
|
|
186
183
|
}
|
|
187
184
|
|
|
188
185
|
/**
|
|
@@ -196,33 +193,6 @@ export function resolveSharedOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
|
|
|
196
193
|
const c = cachedOutputs(t, i)
|
|
197
194
|
if (c !== null) cached.push(c)
|
|
198
195
|
})
|
|
199
|
-
// Does `from` reach `to` through same-project edges? The generated
|
|
200
|
-
// tasks of one project are the whole graph here.
|
|
201
|
-
// Every task, cached or not: a hop through an uncached group task is an
|
|
202
|
-
// edge too.
|
|
203
|
-
const depsByName = new Map<string, string[]>()
|
|
204
|
-
for (const t of tasks) {
|
|
205
|
-
const deps = t.task?.['dependsOn']
|
|
206
|
-
depsByName.set(
|
|
207
|
-
t.name,
|
|
208
|
-
Array.isArray(deps)
|
|
209
|
-
? deps.filter((d): d is string => typeof d === 'string' && !d.startsWith('^'))
|
|
210
|
-
: [],
|
|
211
|
-
)
|
|
212
|
-
}
|
|
213
|
-
const reaches = (from: Cached, to: Cached): boolean => {
|
|
214
|
-
const seen = new Set<string>()
|
|
215
|
-
const stack = [...from.localDeps]
|
|
216
|
-
while (stack.length > 0) {
|
|
217
|
-
const n = stack.pop()!
|
|
218
|
-
if (n === to.name) return true
|
|
219
|
-
if (seen.has(n)) continue
|
|
220
|
-
seen.add(n)
|
|
221
|
-
for (const d of depsByName.get(n) ?? []) stack.push(d)
|
|
222
|
-
}
|
|
223
|
-
return false
|
|
224
|
-
}
|
|
225
|
-
const ordered = (x: Cached, y: Cached): boolean => reaches(x, y) || reaches(y, x)
|
|
226
196
|
const dropped = new Set<number>()
|
|
227
197
|
for (const a of cached) {
|
|
228
198
|
if (dropped.has(a.index)) continue
|
|
@@ -238,31 +208,128 @@ export function resolveSharedOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
|
|
|
238
208
|
const kept: Cached[] = [keeper]
|
|
239
209
|
for (const c of all) {
|
|
240
210
|
if (c === keeper) continue
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
const unordered = kept.find(
|
|
244
|
-
(k) => c.files.some((gb) => k.files.some((ga) => outputsOverlap(ga, gb))) && !ordered(c, k),
|
|
211
|
+
const clash = kept.find((k) =>
|
|
212
|
+
c.files.some((gb) => k.files.some((ga) => outputsOverlap(ga, gb))),
|
|
245
213
|
)
|
|
246
|
-
if (
|
|
214
|
+
if (clash === undefined) {
|
|
247
215
|
kept.push(c)
|
|
248
216
|
continue
|
|
249
217
|
}
|
|
250
218
|
dropped.add(c.index)
|
|
251
219
|
const t = tasks[c.index]!
|
|
252
|
-
const shared = c.files.find((gb) =>
|
|
220
|
+
const shared = c.files.find((gb) => clash.files.some((ga) => outputsOverlap(ga, gb)))!
|
|
253
221
|
delete t.task!['cache']
|
|
254
222
|
t.todos.push(
|
|
255
|
-
`declares the output ${JSON.stringify(shared)} that ${JSON.stringify(
|
|
223
|
+
`declares the output ${JSON.stringify(shared)} that ${JSON.stringify(clash.name)} also ` +
|
|
256
224
|
"declares — vx cleans a task's outputs before it runs and before a restore, so two " +
|
|
257
225
|
"cached tasks on one path would delete each other's work; this one runs uncached. " +
|
|
258
|
-
|
|
259
|
-
'so vx orders them and caches what this one adds.',
|
|
226
|
+
'Give it its own output path to cache it.',
|
|
260
227
|
)
|
|
261
228
|
}
|
|
262
229
|
}
|
|
263
230
|
return tasks
|
|
264
231
|
}
|
|
265
232
|
|
|
233
|
+
/**
|
|
234
|
+
* Two globs can match one path only when one's literal prefix is a
|
|
235
|
+
* directory of the other's (`packages/*\/dist/**` and `packages/a/dist/**`),
|
|
236
|
+
* so an indexed glob is a candidate only along its prefix's chain. Every
|
|
237
|
+
* pair was compared: 1,000 packages' builds and tests took 9 s to map.
|
|
238
|
+
* `candidates` is a superset of the indexed entries a glob can overlap.
|
|
239
|
+
*/
|
|
240
|
+
function prefixIndex(): {
|
|
241
|
+
index: (i: number, globs: readonly string[]) => void
|
|
242
|
+
candidates: (globs: readonly string[]) => number[]
|
|
243
|
+
} {
|
|
244
|
+
const at = new Map<string, number[]>()
|
|
245
|
+
const under = new Map<string, number[]>()
|
|
246
|
+
const chain = (g: string): string[] => {
|
|
247
|
+
const segs: string[] = []
|
|
248
|
+
for (const seg of g.split('/')) {
|
|
249
|
+
if (/[*?{}[\]()!]/.test(seg)) break
|
|
250
|
+
segs.push(seg)
|
|
251
|
+
}
|
|
252
|
+
return segs.map((_, i) => segs.slice(0, i + 1).join('/'))
|
|
253
|
+
}
|
|
254
|
+
return {
|
|
255
|
+
index(i, globs) {
|
|
256
|
+
for (const g of globs) {
|
|
257
|
+
const c = chain(g)
|
|
258
|
+
const push = (m: Map<string, number[]>, k: string) => {
|
|
259
|
+
const list = m.get(k)
|
|
260
|
+
if (list === undefined) m.set(k, [i])
|
|
261
|
+
else if (list.at(-1) !== i) list.push(i)
|
|
262
|
+
}
|
|
263
|
+
push(at, c.at(-1) ?? '')
|
|
264
|
+
for (const k of ['', ...c]) push(under, k)
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
candidates(globs) {
|
|
268
|
+
const found = new Set<number>()
|
|
269
|
+
for (const g of globs) {
|
|
270
|
+
const c = chain(g)
|
|
271
|
+
for (const k of ['', ...c]) for (const i of at.get(k) ?? []) found.add(i)
|
|
272
|
+
for (const i of under.get(c.at(-1) ?? '') ?? []) found.add(i)
|
|
273
|
+
}
|
|
274
|
+
return [...found].sort((a, b) => a - b)
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** A task's cached `inputs` block, when it has one with a `files` list. */
|
|
280
|
+
function inputsOf(t: GeneratedTask): { files: string[]; workspaceFiles?: unknown } | undefined {
|
|
281
|
+
const inputs = (t.task?.['cache'] as { inputs?: { files?: unknown } } | undefined)?.inputs
|
|
282
|
+
return Array.isArray(inputs?.files)
|
|
283
|
+
? (inputs as { files: string[]; workspaceFiles?: unknown })
|
|
284
|
+
: undefined
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Core refuses a task whose inputs can match another task's declared
|
|
289
|
+
* outputs (`rules.upfrontKeys`, on by default, core X-54): such a key
|
|
290
|
+
* reads what a producer writes this run and cannot be known before it ran.
|
|
291
|
+
* Turbo and Nx default a task's inputs to every file of its package
|
|
292
|
+
* (`**\/*`), which matches a sibling build's `dist`. Each overlapping
|
|
293
|
+
* output is taken back with a `!` entry, in place: Turbo and Nx hash the
|
|
294
|
+
* files git tracks, and a build's outputs are not, so the adopted tool
|
|
295
|
+
* never keyed them either. A literal input another task writes cannot be
|
|
296
|
+
* taken back (core refuses a `!` over a literal input), so that task runs
|
|
297
|
+
* uncached with a todo. Committed files under an output are the tracked
|
|
298
|
+
* pass's (`spareTrackedOutputs`).
|
|
299
|
+
*/
|
|
300
|
+
export function excludeSiblingOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
|
|
301
|
+
const writers = tasks.flatMap((t, index) => {
|
|
302
|
+
const c = cachedOutputs(t, index)
|
|
303
|
+
return c === null ? [] : [c]
|
|
304
|
+
})
|
|
305
|
+
if (writers.length === 0) return tasks
|
|
306
|
+
tasks.forEach((t, index) => {
|
|
307
|
+
const inputs = inputsOf(t)
|
|
308
|
+
if (inputs === undefined) return
|
|
309
|
+
const positive = inputs.files.filter((g) => typeof g === 'string' && !g.startsWith('!'))
|
|
310
|
+
for (const w of writers) {
|
|
311
|
+
if (w.index === index) continue
|
|
312
|
+
for (const out of w.files) {
|
|
313
|
+
if (inputs.files.includes(`!${out}`)) continue
|
|
314
|
+
const read = positive.filter((g) => outputsOverlap(g, out))
|
|
315
|
+
if (read.length === 0) continue
|
|
316
|
+
const literal = read.find((g) => isLiteralPattern(g))
|
|
317
|
+
if (literal !== undefined) {
|
|
318
|
+
delete t.task!['cache']
|
|
319
|
+
t.todos.push(
|
|
320
|
+
`reads ${JSON.stringify(literal)}, which ${JSON.stringify(w.name)} writes — vx keys a ` +
|
|
321
|
+
'task only on files no other task writes, so it runs uncached; read the source ' +
|
|
322
|
+
'instead in a vx.config to cache it',
|
|
323
|
+
)
|
|
324
|
+
return
|
|
325
|
+
}
|
|
326
|
+
inputs.files.push(`!${out}`)
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
})
|
|
330
|
+
return tasks
|
|
331
|
+
}
|
|
332
|
+
|
|
266
333
|
/**
|
|
267
334
|
* The same across projects, for workspace outputs: cal.com's shared
|
|
268
335
|
* `post-install` writes `../../node_modules/@prisma/client/**` from every
|
|
@@ -272,9 +339,8 @@ export function resolveSharedOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
|
|
|
272
339
|
* workspace path: typescript-eslint's root project caches `dist` and every
|
|
273
340
|
* package's typecheck `dist/packages/<name>`, and core refused the run
|
|
274
341
|
* over the nesting. Keepers are the first in project and task order;
|
|
275
|
-
* every later task on a kept path runs uncached, with a todo
|
|
276
|
-
*
|
|
277
|
-
* mapping does not hold, so an ordered pair loses its cache too.
|
|
342
|
+
* every later task on a kept path runs uncached, with a todo, edge or not,
|
|
343
|
+
* as core refuses the pair either way.
|
|
278
344
|
*/
|
|
279
345
|
export function resolveSharedWorkspaceOutputs(
|
|
280
346
|
root: string,
|
|
@@ -289,41 +355,7 @@ export function resolveSharedWorkspaceOutputs(
|
|
|
289
355
|
? v.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
|
|
290
356
|
: []
|
|
291
357
|
const kept: { id: string; project: string; ws: string[]; own: string[] }[] = []
|
|
292
|
-
|
|
293
|
-
// directory of the other's (`packages/*/dist/**` and `packages/a/dist/**`),
|
|
294
|
-
// so a kept glob is a candidate only along its prefix's chain. Every pair
|
|
295
|
-
// was compared: 1,000 packages' builds and tests took 9 s to map.
|
|
296
|
-
const at = new Map<string, number[]>()
|
|
297
|
-
const under = new Map<string, number[]>()
|
|
298
|
-
const chain = (g: string): string[] => {
|
|
299
|
-
const segs: string[] = []
|
|
300
|
-
for (const seg of g.split('/')) {
|
|
301
|
-
if (/[*?{}[\]()!]/.test(seg)) break
|
|
302
|
-
segs.push(seg)
|
|
303
|
-
}
|
|
304
|
-
return segs.map((_, i) => segs.slice(0, i + 1).join('/'))
|
|
305
|
-
}
|
|
306
|
-
const index = (i: number, globs: readonly string[]): void => {
|
|
307
|
-
for (const g of globs) {
|
|
308
|
-
const c = chain(g)
|
|
309
|
-
const push = (m: Map<string, number[]>, k: string) => {
|
|
310
|
-
const list = m.get(k)
|
|
311
|
-
if (list === undefined) m.set(k, [i])
|
|
312
|
-
else if (list.at(-1) !== i) list.push(i)
|
|
313
|
-
}
|
|
314
|
-
push(at, c.at(-1) ?? '')
|
|
315
|
-
for (const k of ['', ...c]) push(under, k)
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
const candidates = (globs: readonly string[]): number[] => {
|
|
319
|
-
const found = new Set<number>()
|
|
320
|
-
for (const g of globs) {
|
|
321
|
-
const c = chain(g)
|
|
322
|
-
for (const k of ['', ...c]) for (const i of at.get(k) ?? []) found.add(i)
|
|
323
|
-
for (const i of under.get(c.at(-1) ?? '') ?? []) found.add(i)
|
|
324
|
-
}
|
|
325
|
-
return [...found].sort((a, b) => a - b)
|
|
326
|
-
}
|
|
358
|
+
const { index, candidates } = prefixIndex()
|
|
327
359
|
for (const p of projects) {
|
|
328
360
|
const rel = relPosix(root, p.dir)
|
|
329
361
|
for (const t of p.tasks) {
|
|
@@ -335,8 +367,7 @@ export function resolveSharedWorkspaceOutputs(
|
|
|
335
367
|
rel === '' ? g.replace(/^(\.\/)+/, '') : `${rel}/${g.replace(/^(\.\/)+/, '')}`,
|
|
336
368
|
)
|
|
337
369
|
if (ws.length === 0 && own.length === 0) continue
|
|
338
|
-
// Two own outputs of one project are resolveSharedOutputs'
|
|
339
|
-
// reads the edges that order them.
|
|
370
|
+
// Two own outputs of one project are resolveSharedOutputs'.
|
|
340
371
|
const overlap = (a: string[], b: string[]): string | undefined =>
|
|
341
372
|
a.find((g) => b.some((h) => outputsOverlap(h, g)))
|
|
342
373
|
let clash: (typeof kept)[number] | undefined
|
|
@@ -367,6 +398,79 @@ export function resolveSharedWorkspaceOutputs(
|
|
|
367
398
|
}
|
|
368
399
|
}
|
|
369
400
|
|
|
401
|
+
/**
|
|
402
|
+
* `excludeSiblingOutputs` for root-anchored `cache.inputs.workspaceFiles`:
|
|
403
|
+
* every other task's `outputs.workspaceFiles`, and its `outputs.files` at
|
|
404
|
+
* their workspace path, is taken back from a reader's workspace inputs
|
|
405
|
+
* (Turbo's `globalDependencies` and `$TURBO_ROOT$` entries, Nx's
|
|
406
|
+
* `{workspaceRoot}` filesets). Run after `resolveSharedWorkspaceOutputs`,
|
|
407
|
+
* over what stays cached.
|
|
408
|
+
*/
|
|
409
|
+
export function excludeWorkspaceOutputs(
|
|
410
|
+
root: string,
|
|
411
|
+
projects: readonly {
|
|
412
|
+
readonly name: string
|
|
413
|
+
readonly dir: string
|
|
414
|
+
readonly tasks: GeneratedTask[]
|
|
415
|
+
}[],
|
|
416
|
+
): void {
|
|
417
|
+
const positive = (v: unknown): string[] =>
|
|
418
|
+
Array.isArray(v)
|
|
419
|
+
? v.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
|
|
420
|
+
: []
|
|
421
|
+
const writers: { task: GeneratedTask; id: string; globs: string[] }[] = []
|
|
422
|
+
const { index, candidates } = prefixIndex()
|
|
423
|
+
for (const p of projects) {
|
|
424
|
+
const rel = relPosix(root, p.dir)
|
|
425
|
+
for (const t of p.tasks) {
|
|
426
|
+
const outputs = (
|
|
427
|
+
t.task?.['cache'] as { outputs?: { files?: unknown; workspaceFiles?: unknown } } | undefined
|
|
428
|
+
)?.outputs
|
|
429
|
+
const globs = [
|
|
430
|
+
...positive(outputs?.workspaceFiles),
|
|
431
|
+
...positive(outputs?.files).map((g) =>
|
|
432
|
+
rel === '' ? g.replace(/^(\.\/)+/, '') : `${rel}/${g.replace(/^(\.\/)+/, '')}`,
|
|
433
|
+
),
|
|
434
|
+
]
|
|
435
|
+
if (globs.length === 0) continue
|
|
436
|
+
index(writers.length, globs)
|
|
437
|
+
writers.push({ task: t, id: `${p.name}#${t.name}`, globs })
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (writers.length === 0) return
|
|
441
|
+
for (const p of projects) {
|
|
442
|
+
for (const t of p.tasks) {
|
|
443
|
+
const inputs = (t.task?.['cache'] as { inputs?: { workspaceFiles?: unknown } } | undefined)
|
|
444
|
+
?.inputs
|
|
445
|
+
if (inputs === undefined || !Array.isArray(inputs.workspaceFiles)) continue
|
|
446
|
+
const files = inputs.workspaceFiles as string[]
|
|
447
|
+
const reads = positive(files)
|
|
448
|
+
if (reads.length === 0) continue
|
|
449
|
+
for (const i of candidates(reads)) {
|
|
450
|
+
const w = writers[i]!
|
|
451
|
+
if (w.task === t) continue
|
|
452
|
+
for (const out of w.globs) {
|
|
453
|
+
if (files.includes(`!${out}`)) continue
|
|
454
|
+
const read = reads.filter((g) => outputsOverlap(g, out))
|
|
455
|
+
if (read.length === 0) continue
|
|
456
|
+
const literal = read.find((g) => isLiteralPattern(g))
|
|
457
|
+
if (literal !== undefined) {
|
|
458
|
+
delete t.task!['cache']
|
|
459
|
+
t.todos.push(
|
|
460
|
+
`reads ${JSON.stringify(literal)}, which ${w.id} writes — vx keys a task only on ` +
|
|
461
|
+
'files no other task writes, so it runs uncached; read the source instead in a ' +
|
|
462
|
+
'vx.config to cache it',
|
|
463
|
+
)
|
|
464
|
+
break
|
|
465
|
+
}
|
|
466
|
+
files.push(`!${out}`)
|
|
467
|
+
}
|
|
468
|
+
if (t.task?.['cache'] === undefined) break
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
370
474
|
/** A `!` output with no positive beside it takes back nothing, and core refuses a list of them alone. */
|
|
371
475
|
export function takingBack(globs: readonly string[]): string[] {
|
|
372
476
|
return globs.some((g) => !g.startsWith('!')) ? [...globs] : []
|
package/src/tracked-outputs.ts
CHANGED
|
@@ -178,6 +178,18 @@ function coveredTracked(globs: readonly string[], files: readonly string[]): str
|
|
|
178
178
|
return [...hit].sort()
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
+
/** The first of `files` a `!` entry of `globs` takes back though a positive one reads it. */
|
|
182
|
+
function hidden(globs: unknown, files: readonly string[]): string | undefined {
|
|
183
|
+
if (!Array.isArray(globs) || files.length === 0) return undefined
|
|
184
|
+
const strings = globs.filter((g): g is string => typeof g === 'string')
|
|
185
|
+
const negative = strings.filter((g) => g.startsWith('!')).map((g) => g.slice(1))
|
|
186
|
+
if (negative.length === 0) return undefined
|
|
187
|
+
const positive = strings.filter((g) => !g.startsWith('!'))
|
|
188
|
+
return files.find(
|
|
189
|
+
(f) => negative.some((n) => outputsOverlap(n, f)) && positive.some((g) => outputsOverlap(g, f)),
|
|
190
|
+
)
|
|
191
|
+
}
|
|
192
|
+
|
|
181
193
|
/** Past this many, a task's take-backs cost its runs more than its cache saves. */
|
|
182
194
|
export const MAX_SPARED = 16
|
|
183
195
|
|
|
@@ -223,6 +235,30 @@ export function spareTrackedOutputs(
|
|
|
223
235
|
}
|
|
224
236
|
const spared = hits.flatMap(([, , hit]) => hit)
|
|
225
237
|
if (spared.length === 0) continue
|
|
238
|
+
// A committed file under these outputs is still a source, and the `!`
|
|
239
|
+
// entry `excludeSiblingOutputs` gave a reader over the whole output
|
|
240
|
+
// hides it from that reader's key: such a reader runs uncached (X-54).
|
|
241
|
+
const local = hits.flatMap(([key, , hit]) => (key === 'files' ? hit : []))
|
|
242
|
+
const rooted = hits.flatMap(([key, , hit]) =>
|
|
243
|
+
key === 'files' ? hit.map((f) => (rel === '' ? f : `${rel}/${f}`)) : hit,
|
|
244
|
+
)
|
|
245
|
+
for (const q of projects) {
|
|
246
|
+
for (const r of q.tasks) {
|
|
247
|
+
if (r === t) continue
|
|
248
|
+
const inputs = (r.task?.['cache'] as { inputs?: Outputs } | undefined)?.inputs
|
|
249
|
+
if (inputs === undefined) continue
|
|
250
|
+
const f =
|
|
251
|
+
(q === p ? hidden(inputs.files, local) : undefined) ??
|
|
252
|
+
hidden(inputs.workspaceFiles, rooted)
|
|
253
|
+
if (f === undefined) continue
|
|
254
|
+
delete r.task!['cache']
|
|
255
|
+
todos.push([
|
|
256
|
+
`${q.name}#${r.name}`,
|
|
257
|
+
`reads the committed ${f}, which its inputs take back with ${p.name}#${t.name}'s ` +
|
|
258
|
+
'outputs — task runs uncached; declare its inputs in a vx.config to cache it',
|
|
259
|
+
])
|
|
260
|
+
}
|
|
261
|
+
}
|
|
226
262
|
const shown = spared.slice(0, 3).join(', ') + (spared.length > 3 ? ', …' : '')
|
|
227
263
|
// Each `!` is matched against every output path on each save, clean
|
|
228
264
|
// and restore: 4,200 of them cost typescript-eslint's warm run 1.2 s.
|
package/src/turbo/turbo-map.ts
CHANGED
|
@@ -19,6 +19,8 @@ import { scriptCommand, yarnPnp } from '../script-command.js'
|
|
|
19
19
|
import {
|
|
20
20
|
ownFileOutput,
|
|
21
21
|
ownFileTodo,
|
|
22
|
+
excludeSiblingOutputs,
|
|
23
|
+
excludeWorkspaceOutputs,
|
|
22
24
|
resolveSharedOutputs,
|
|
23
25
|
resolveSharedWorkspaceOutputs,
|
|
24
26
|
takingBack,
|
|
@@ -1375,7 +1377,7 @@ export async function mapTurboWorkspace(
|
|
|
1375
1377
|
npmScriptEnv(mapped.task, name, command === script, meta, opts)
|
|
1376
1378
|
tasks.push(mapped)
|
|
1377
1379
|
}
|
|
1378
|
-
resolveSharedOutputs(tasks)
|
|
1380
|
+
excludeSiblingOutputs(resolveSharedOutputs(tasks))
|
|
1379
1381
|
projects.push({ name: meta.name, dir: meta.dir, tasks })
|
|
1380
1382
|
}
|
|
1381
1383
|
|
|
@@ -1398,6 +1400,7 @@ export async function mapTurboWorkspace(
|
|
|
1398
1400
|
)
|
|
1399
1401
|
literalEnvGapsOnce(projects, notes)
|
|
1400
1402
|
resolveSharedWorkspaceOutputs(root, projects)
|
|
1403
|
+
excludeWorkspaceOutputs(root, projects)
|
|
1401
1404
|
pruneUnreachedPersistentNotes(projects, metas, opts.persistentTodo)
|
|
1402
1405
|
pruneOrphanPersistentNotes(projects, opts.persistentTodo)
|
|
1403
1406
|
return { projects, notes, globals }
|