@vzn/vx-migrate 0.0.0 → 0.0.484
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +306 -0
- package/index.ts +4 -0
- package/package.json +44 -6
- package/src/adoption-plugin.ts +125 -0
- package/src/bin.ts +18 -0
- package/src/dotenv-probe.ts +86 -0
- package/src/glob-grammar.ts +63 -0
- package/src/index.ts +15 -0
- package/src/mapping-cache.ts +101 -0
- package/src/migrate-nx.ts +154 -0
- package/src/migrate-turbo.ts +400 -0
- package/src/migrate.ts +165 -0
- package/src/nx/export-graph.ts +29 -0
- package/src/nx/index.ts +638 -0
- package/src/nx/nx-deps.ts +272 -0
- package/src/nx/nx-dotenv.ts +106 -0
- package/src/nx/nx-inputs.ts +210 -0
- package/src/nx/nx-map.ts +1430 -0
- package/src/nx/nx-outputs.ts +231 -0
- package/src/nx/nx-upstream.ts +356 -0
- package/src/nx-cache/index.ts +236 -0
- package/src/nx-command.ts +458 -0
- package/src/nx-dotenv.cjs +41 -0
- package/src/nx-env.cjs +152 -0
- package/src/nx-exec.cjs +294 -0
- package/src/paths.ts +36 -0
- package/src/plugin-gaps.ts +48 -0
- package/src/remote-deadline.ts +11 -0
- package/src/remote-retry.ts +52 -0
- package/src/remote-token.ts +18 -0
- package/src/script-command.ts +69 -0
- package/src/shared-outputs.ts +373 -0
- package/src/tracked-outputs.ts +248 -0
- package/src/turbo/index.ts +291 -0
- package/src/turbo/turbo-map.ts +1976 -0
- package/src/turbo-cache/index.ts +661 -0
- package/src/workspace-notes.ts +54 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// The command a package.json script becomes. Inlining the body is the
|
|
2
|
+
// rule — one process less per task than `<pm> run <name>` — with two npm
|
|
3
|
+
// conventions mapped rather than lost, the way core's `vx init` maps
|
|
4
|
+
// them (workspace/migrate-scripts.ts):
|
|
5
|
+
//
|
|
6
|
+
// - `pre<name>` / `post<name>` hooks, which npm and pnpm run around
|
|
7
|
+
// `<name>` without being asked, are folded into the command in that
|
|
8
|
+
// order (novu's `prebuild` copies the CSS its `build` inlines,
|
|
9
|
+
// 2026-09-11; Nx and Turbo run `pnpm run build`, so they get the hooks
|
|
10
|
+
// for free). Lifecycle hooks of the package manager's own verbs
|
|
11
|
+
// (`preinstall`, `prepublishOnly`, …) never ride inside a task.
|
|
12
|
+
// - yarn ≥ 2 runs scripts in its own shell, where `run` is a builtin
|
|
13
|
+
// (`run -T rollup -c` is "the root's rollup", `run build:code` the
|
|
14
|
+
// sibling script); inlined into sh that body is `run: command not
|
|
15
|
+
// found` in every strapi package (2026-09-11), so such a script runs
|
|
16
|
+
// through `yarn run <name>`, exactly as Nx and Turbo run it — and yarn
|
|
17
|
+
// ≥ 2 runs no pre/post hooks, so none are folded there either.
|
|
18
|
+
// - A Yarn Plug'n'Play workspace has no node_modules: a dependency's
|
|
19
|
+
// `require` resolves only through `.pnp.cjs` and its bins only through
|
|
20
|
+
// the shims `yarn run` writes, so inlined, `node -e "require('left-pad')"`
|
|
21
|
+
// was MODULE_NOT_FOUND (2026-09-28). Every script runs as `yarn run <name>`.
|
|
22
|
+
|
|
23
|
+
import { readFileSync } from 'node:fs'
|
|
24
|
+
import path from 'node:path'
|
|
25
|
+
import { foldScriptHooks } from '@vzn/vx'
|
|
26
|
+
|
|
27
|
+
const YARN_RUN_BUILTIN = /(^|&&|\|\||;|\(|\|)\s*run\s/
|
|
28
|
+
const LIFECYCLE = /^(pre|post)(install|publish|pack|version)$|^(prepare|prepublishOnly|install)$/
|
|
29
|
+
|
|
30
|
+
function usable(value: unknown): value is string {
|
|
31
|
+
return typeof value === 'string' && value.length > 0
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function scriptCommand(
|
|
35
|
+
name: string,
|
|
36
|
+
body: string,
|
|
37
|
+
scripts: Readonly<Record<string, unknown>> = {},
|
|
38
|
+
pnp = false,
|
|
39
|
+
): string {
|
|
40
|
+
if (pnp) return `yarn run ${name}`
|
|
41
|
+
const hook = (h: string): string | undefined =>
|
|
42
|
+
!LIFECYCLE.test(h) && usable(scripts[h]) ? scripts[h] : undefined
|
|
43
|
+
const parts = [hook(`pre${name}`), body, hook(`post${name}`)]
|
|
44
|
+
if (parts.some((p) => p !== undefined && YARN_RUN_BUILTIN.test(p))) return `yarn run ${name}`
|
|
45
|
+
return foldScriptHooks(parts[0], body, parts[2])
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Yarn ≥ 2 links Plug'n'Play unless `.yarnrc.yml` names another `nodeLinker`.
|
|
50
|
+
* Read once per mapping; synchronous so every mapper, the synchronous ones
|
|
51
|
+
* included, can ask it.
|
|
52
|
+
*/
|
|
53
|
+
export function yarnPnp(root: string): boolean {
|
|
54
|
+
let rc: string
|
|
55
|
+
try {
|
|
56
|
+
rc = readFileSync(path.join(root, '.yarnrc.yml'), 'utf8')
|
|
57
|
+
} catch {
|
|
58
|
+
return false
|
|
59
|
+
}
|
|
60
|
+
const linker = /^nodeLinker:[ \t]*["']?([\w-]+)/m.exec(rc)?.[1]
|
|
61
|
+
return linker === undefined || linker === 'pnp'
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** `.yarnrc.yml` as a mapping cache reads it: `yarnPnp` decides every script command from it. */
|
|
65
|
+
export async function yarnrcText(root: string): Promise<string> {
|
|
66
|
+
return Bun.file(path.join(root, '.yarnrc.yml'))
|
|
67
|
+
.text()
|
|
68
|
+
.catch(() => '\0absent')
|
|
69
|
+
}
|
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
// Two targets of one project on one output path. vx cleans a task's
|
|
2
|
+
// declared outputs before it runs and before a cache-hit restore, so the
|
|
3
|
+
// loader refuses two cached tasks whose outputs provably overlap — UNLESS
|
|
4
|
+
// a same-project edge orders them (core item 588): then the dependant is
|
|
5
|
+
// ADDITIVE, keeps its cache, and owns only what its run added. strapi
|
|
6
|
+
// (2026-09-11) declares `build`, `build:code` and `build:types` all on
|
|
7
|
+
// `dist/**`, and the refusal came at load time, after the migration had
|
|
8
|
+
// reported clean. The mapping resolves it: the task with a `^` edge keeps
|
|
9
|
+
// its cache (it is the one a dependant waits for), or the first declared
|
|
10
|
+
// when none has one; every other task on that path stays cached when an
|
|
11
|
+
// edge orders it against every kept task it overlaps, and otherwise runs
|
|
12
|
+
// uncached, with a todo that names the keeper and the fix (its own output
|
|
13
|
+
// path, or the edge).
|
|
14
|
+
//
|
|
15
|
+
// The overlap question is core's own `outputsOverlap`, asked through the
|
|
16
|
+
// façade. It used to be a COPY of it here, and the copy stopped being the
|
|
17
|
+
// same test the moment core's grew: `./dist/**` against `dist/**` (item
|
|
18
|
+
// 441) and the literal `dist` against `dist/app.js` (item 442) are
|
|
19
|
+
// refused by the loader and were missed here — so the migration reported
|
|
20
|
+
// clean on a config core would not load, which is the exact failure this
|
|
21
|
+
// file exists to prevent (item 445).
|
|
22
|
+
|
|
23
|
+
import { isLiteralPattern, outputsOverlap, type GeneratedTask } from '@vzn/vx'
|
|
24
|
+
import { relPosix } from './paths.js'
|
|
25
|
+
import type { TrackedKinds } from './tracked-outputs.js'
|
|
26
|
+
|
|
27
|
+
// Core refuses an output glob that covers the project's own package.json
|
|
28
|
+
// or vx.config (config-schema's `ownFilesCovered`): vx cleans outputs before
|
|
29
|
+
// a run and restores them on a hit, so the manifest would go. Turbo caches
|
|
30
|
+
// it: trpc's client build lists `package.json` (it rewrites `exports`), and
|
|
31
|
+
// the one task made core refuse the whole run (2026-09-28). The façade does
|
|
32
|
+
// not export the check; tests/own-file-outputs.test.ts holds this copy to
|
|
33
|
+
// the loader in both directions.
|
|
34
|
+
const OWN_FILES = ['package.json', 'vx.config.ts', 'vx.config.mts', 'vx.config.js', 'vx.config.mjs']
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The first positive output glob that covers a project's own file, and that
|
|
38
|
+
* file. `config` is the config file the mapped task will live beside, as
|
|
39
|
+
* core checks it (null: none); absent, every spelling a user may add later.
|
|
40
|
+
* sanity's builds output `*.js` (top-level shims) beside no vx.config.js.
|
|
41
|
+
*/
|
|
42
|
+
export function ownFileOutput(
|
|
43
|
+
files: readonly string[],
|
|
44
|
+
config?: string | null,
|
|
45
|
+
): { glob: string; file: string } | undefined {
|
|
46
|
+
const own =
|
|
47
|
+
config === undefined ? OWN_FILES : config === null ? ['package.json'] : ['package.json', config]
|
|
48
|
+
for (const glob of files) {
|
|
49
|
+
if (glob.startsWith('!')) continue
|
|
50
|
+
const g = new Bun.Glob(glob.replace(/^(\.\/)+/, ''))
|
|
51
|
+
const file = own.find((f) => g.match(f))
|
|
52
|
+
if (file !== undefined) return { glob, file }
|
|
53
|
+
}
|
|
54
|
+
return undefined
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The `!` entries that take the project's own files back from `files`: each
|
|
59
|
+
* own file a positive glob covers and no `!` already takes back. Taken back,
|
|
60
|
+
* it is never cleaned, saved or restored (core's A-44), and core loads it.
|
|
61
|
+
*/
|
|
62
|
+
export function ownFileTakeBacks(files: readonly string[], config?: string | null): string[] {
|
|
63
|
+
const own =
|
|
64
|
+
config === undefined ? OWN_FILES : config === null ? ['package.json'] : ['package.json', config]
|
|
65
|
+
const glob = (g: string) => new Bun.Glob(g.replace(/^(\.\/)+/, ''))
|
|
66
|
+
const positive = files.filter((g) => !g.startsWith('!')).map(glob)
|
|
67
|
+
const taken = files.filter((g) => g.startsWith('!')).map((g) => glob(g.slice(1)))
|
|
68
|
+
return own
|
|
69
|
+
.filter((f) => positive.some((g) => g.match(f)) && !taken.some((g) => g.match(f)))
|
|
70
|
+
.map((f) => `!${f}`)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function ownFileTodo(own: { glob: string; file: string }): string {
|
|
74
|
+
return (
|
|
75
|
+
`output ${JSON.stringify(own.glob)} covers the project's own ${own.file}, which vx cleans ` +
|
|
76
|
+
'before every run — task runs uncached; declare the outputs without it in a vx.config to cache it'
|
|
77
|
+
)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Turbo and Nx never clean an output; vx cleans it before a run and a
|
|
81
|
+
// restore, so `**/*.d.ts` deleted a hand-written `src/env.d.ts` and an
|
|
82
|
+
// uncommitted edit to it was lost for good (item 1031). A wildcard first
|
|
83
|
+
// segment can reach the sources, and a `!` beside it (medusa: `*/**` minus
|
|
84
|
+
// `!src/**`) takes back only what it names. typescript-eslint's Nx tests
|
|
85
|
+
// output `{projectRoot}/**/*.shot`, 3,656 committed snapshots in one
|
|
86
|
+
// package, and nx() lacked the rule (2026-09-29).
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The first positive output glob whose first segment is a wildcard. Two
|
|
90
|
+
* shapes reach no source when `tracked` is known and says the project has
|
|
91
|
+
* none of their kind, and are let through: one segment ending in a literal
|
|
92
|
+
* extension no tracked file carries (n8n's `*.xml` junit reports, 143 test
|
|
93
|
+
* tasks), and `**` followed by `/<dir>/**` where no tracked file sits
|
|
94
|
+
* under a directory of that name (vercel/ai's builds, `dist` at any depth,
|
|
95
|
+
* 69 of them). A third: a first segment with no `**` that names no tracked
|
|
96
|
+
* top-level entry, and so reaches nothing tracked (tldraw's `dist-*`
|
|
97
|
+
* directories, 35 builds). `*.ts` beside tracked TypeScript, that form over
|
|
98
|
+
* `src`, or a bare `*` directory beside any tracked file is not.
|
|
99
|
+
*/
|
|
100
|
+
export function wildcardOutput(
|
|
101
|
+
files: readonly string[],
|
|
102
|
+
tracked?: TrackedKinds,
|
|
103
|
+
/** Let a literal-tail glob through once its tracked matches are taken back (`literalTailMatches`). */
|
|
104
|
+
literalTail = false,
|
|
105
|
+
): string | undefined {
|
|
106
|
+
return files.find(
|
|
107
|
+
(o) =>
|
|
108
|
+
!o.startsWith('!') &&
|
|
109
|
+
!isLiteralPattern(o.split('/')[0] ?? '') &&
|
|
110
|
+
!untrackedKind(o, tracked) &&
|
|
111
|
+
!(
|
|
112
|
+
literalTail &&
|
|
113
|
+
(literalTailMatches(o, tracked)?.every((m) => files.includes(`!${m}`)) ?? false)
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The tracked files a wildcard-first glob with a literal rest can reach
|
|
120
|
+
* (clerk's builds output `*\/package.json`, the subpath stubs it commits):
|
|
121
|
+
* it reaches no other file, so with each of these taken back by a `!` it
|
|
122
|
+
* reaches no source. Null for any other shape, or with `tracked` unknown.
|
|
123
|
+
*/
|
|
124
|
+
export function literalTailMatches(glob: string, tracked?: TrackedKinds): string[] | null {
|
|
125
|
+
if (tracked === undefined || glob.startsWith('!')) return null
|
|
126
|
+
const segs = glob.replace(/^(\.\/)+/, '').split('/')
|
|
127
|
+
const [first, ...rest] = segs
|
|
128
|
+
if (first === undefined || rest.length === 0 || first.includes('**')) return null
|
|
129
|
+
if (/[^A-Za-z0-9*?._@+-]/.test(first) || !rest.every((s) => isLiteralPattern(s) && s !== ''))
|
|
130
|
+
return null
|
|
131
|
+
const tail = `/${rest.join('/')}`
|
|
132
|
+
const head = new Bun.Glob(first)
|
|
133
|
+
return tracked.files.filter(
|
|
134
|
+
(f) => f.endsWith(tail) && f.split('/').length === segs.length && head.match(f.split('/')[0]!),
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function untrackedKind(glob: string, tracked: TrackedKinds | undefined): boolean {
|
|
139
|
+
if (tracked === undefined) return false
|
|
140
|
+
const g = glob.replace(/^(\.\/)+/, '')
|
|
141
|
+
const ext = /^[^/]*\.([A-Za-z0-9]+)$/.exec(g)?.[1]
|
|
142
|
+
if (ext !== undefined) return !tracked.exts.has(ext.toLowerCase())
|
|
143
|
+
const dir = /^\*\*\/([^*?[\]{}()!/]+)\/\*\*$/.exec(g)?.[1]
|
|
144
|
+
if (dir !== undefined) return !tracked.dirs.has(dir)
|
|
145
|
+
// Brackets, parens and `!` are left out: vx and Bun.Glob read them apart.
|
|
146
|
+
const first = /^([^/[\]()!\\]+)\//.exec(g)?.[1]
|
|
147
|
+
if (first === undefined || first.includes('**')) return false
|
|
148
|
+
const matcher = new Bun.Glob(first)
|
|
149
|
+
return ![...tracked.tops].some((top) => matcher.match(top))
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function wildcardTodo(glob: string): string {
|
|
153
|
+
return (
|
|
154
|
+
`output ${JSON.stringify(glob)}: a wildcard first segment reaches the sources, which ` +
|
|
155
|
+
'vx cleans before every run — task runs uncached; declare the exact outputs in a ' +
|
|
156
|
+
'vx.config to cache it'
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
interface Cached {
|
|
161
|
+
index: number
|
|
162
|
+
name: string
|
|
163
|
+
files: string[]
|
|
164
|
+
hasUpstreamEdge: boolean
|
|
165
|
+
/** Same-project `dependsOn` names (no `^`), for the ordering test. */
|
|
166
|
+
localDeps: string[]
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function cachedOutputs(t: GeneratedTask, index: number): Cached | null {
|
|
170
|
+
const task = t.task
|
|
171
|
+
if (task === null) return null
|
|
172
|
+
const cache = task['cache'] as { outputs?: { files?: unknown } } | undefined
|
|
173
|
+
const files = cache?.outputs?.files
|
|
174
|
+
if (!Array.isArray(files)) return null
|
|
175
|
+
// Where a task's outputs might land is its positive globs, as core reads
|
|
176
|
+
// them: a `!` compiled as a glob is true of every other path.
|
|
177
|
+
const strings = files.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
|
|
178
|
+
if (strings.length === 0) return null
|
|
179
|
+
const deps = task['dependsOn']
|
|
180
|
+
const hasUpstreamEdge =
|
|
181
|
+
Array.isArray(deps) && deps.some((d) => typeof d === 'string' && d.startsWith('^'))
|
|
182
|
+
const localDeps = Array.isArray(deps)
|
|
183
|
+
? deps.filter((d): d is string => typeof d === 'string' && !d.startsWith('^'))
|
|
184
|
+
: []
|
|
185
|
+
return { index, name: t.name, files: strings, hasUpstreamEdge, localDeps }
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Uncache every task whose declared outputs overlap a keeper's, in place,
|
|
190
|
+
* and return the tasks. Deterministic: keepers are chosen in declaration
|
|
191
|
+
* order, so the same graph maps the same way every run.
|
|
192
|
+
*/
|
|
193
|
+
export function resolveSharedOutputs(tasks: GeneratedTask[]): GeneratedTask[] {
|
|
194
|
+
const cached: Cached[] = []
|
|
195
|
+
tasks.forEach((t, i) => {
|
|
196
|
+
const c = cachedOutputs(t, i)
|
|
197
|
+
if (c !== null) cached.push(c)
|
|
198
|
+
})
|
|
199
|
+
// Does `from` reach `to` through same-project edges? The generated
|
|
200
|
+
// tasks of one project are the whole graph here.
|
|
201
|
+
// Every task, cached or not: a hop through an uncached group task is an
|
|
202
|
+
// edge too.
|
|
203
|
+
const depsByName = new Map<string, string[]>()
|
|
204
|
+
for (const t of tasks) {
|
|
205
|
+
const deps = t.task?.['dependsOn']
|
|
206
|
+
depsByName.set(
|
|
207
|
+
t.name,
|
|
208
|
+
Array.isArray(deps)
|
|
209
|
+
? deps.filter((d): d is string => typeof d === 'string' && !d.startsWith('^'))
|
|
210
|
+
: [],
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
const reaches = (from: Cached, to: Cached): boolean => {
|
|
214
|
+
const seen = new Set<string>()
|
|
215
|
+
const stack = [...from.localDeps]
|
|
216
|
+
while (stack.length > 0) {
|
|
217
|
+
const n = stack.pop()!
|
|
218
|
+
if (n === to.name) return true
|
|
219
|
+
if (seen.has(n)) continue
|
|
220
|
+
seen.add(n)
|
|
221
|
+
for (const d of depsByName.get(n) ?? []) stack.push(d)
|
|
222
|
+
}
|
|
223
|
+
return false
|
|
224
|
+
}
|
|
225
|
+
const ordered = (x: Cached, y: Cached): boolean => reaches(x, y) || reaches(y, x)
|
|
226
|
+
const dropped = new Set<number>()
|
|
227
|
+
for (const a of cached) {
|
|
228
|
+
if (dropped.has(a.index)) continue
|
|
229
|
+
const group = cached.filter(
|
|
230
|
+
(b) =>
|
|
231
|
+
b !== a &&
|
|
232
|
+
!dropped.has(b.index) &&
|
|
233
|
+
a.files.some((ga) => b.files.some((gb) => outputsOverlap(ga, gb))),
|
|
234
|
+
)
|
|
235
|
+
if (group.length === 0) continue
|
|
236
|
+
const all = [a, ...group]
|
|
237
|
+
const keeper = all.find((c) => c.hasUpstreamEdge) ?? all[0]!
|
|
238
|
+
const kept: Cached[] = [keeper]
|
|
239
|
+
for (const c of all) {
|
|
240
|
+
if (c === keeper) continue
|
|
241
|
+
// Additive under core's rule when an edge orders it against every
|
|
242
|
+
// kept task whose outputs it overlaps: it stays cached, no todo.
|
|
243
|
+
const unordered = kept.find(
|
|
244
|
+
(k) => c.files.some((gb) => k.files.some((ga) => outputsOverlap(ga, gb))) && !ordered(c, k),
|
|
245
|
+
)
|
|
246
|
+
if (unordered === undefined) {
|
|
247
|
+
kept.push(c)
|
|
248
|
+
continue
|
|
249
|
+
}
|
|
250
|
+
dropped.add(c.index)
|
|
251
|
+
const t = tasks[c.index]!
|
|
252
|
+
const shared = c.files.find((gb) => unordered.files.some((ga) => outputsOverlap(ga, gb)))!
|
|
253
|
+
delete t.task!['cache']
|
|
254
|
+
t.todos.push(
|
|
255
|
+
`declares the output ${JSON.stringify(shared)} that ${JSON.stringify(unordered.name)} also ` +
|
|
256
|
+
"declares — vx cleans a task's outputs before it runs and before a restore, so two " +
|
|
257
|
+
"cached tasks on one path would delete each other's work; this one runs uncached. " +
|
|
258
|
+
`Give it its own output path to cache it, or a dependsOn edge on ${JSON.stringify(unordered.name)} ` +
|
|
259
|
+
'so vx orders them and caches what this one adds.',
|
|
260
|
+
)
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return tasks
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* The same across projects, for workspace outputs: cal.com's shared
|
|
268
|
+
* `post-install` writes `../../node_modules/@prisma/client/**` from every
|
|
269
|
+
* package that has the script, one workspace path, and core refused the
|
|
270
|
+
* whole run over the first pair (two projects' outputs on one path with
|
|
271
|
+
* no edge between them). A project's own outputs take part at their
|
|
272
|
+
* workspace path: typescript-eslint's root project caches `dist` and every
|
|
273
|
+
* package's typecheck `dist/packages/<name>`, and core refused the run
|
|
274
|
+
* over the nesting. Keepers are the first in project and task order;
|
|
275
|
+
* every later task on a kept path runs uncached, with a todo. Edges are
|
|
276
|
+
* not read: across projects they are the package graph's, which the
|
|
277
|
+
* mapping does not hold, so an ordered pair loses its cache too.
|
|
278
|
+
*/
|
|
279
|
+
export function resolveSharedWorkspaceOutputs(
|
|
280
|
+
root: string,
|
|
281
|
+
projects: readonly {
|
|
282
|
+
readonly name: string
|
|
283
|
+
readonly dir: string
|
|
284
|
+
readonly tasks: GeneratedTask[]
|
|
285
|
+
}[],
|
|
286
|
+
): void {
|
|
287
|
+
const positive = (v: unknown): string[] =>
|
|
288
|
+
Array.isArray(v)
|
|
289
|
+
? v.filter((f): f is string => typeof f === 'string' && !f.startsWith('!'))
|
|
290
|
+
: []
|
|
291
|
+
const kept: { id: string; project: string; ws: string[]; own: string[] }[] = []
|
|
292
|
+
// Two globs can match one path only when one's literal prefix is a
|
|
293
|
+
// directory of the other's (`packages/*/dist/**` and `packages/a/dist/**`),
|
|
294
|
+
// so a kept glob is a candidate only along its prefix's chain. Every pair
|
|
295
|
+
// was compared: 1,000 packages' builds and tests took 9 s to map.
|
|
296
|
+
const at = new Map<string, number[]>()
|
|
297
|
+
const under = new Map<string, number[]>()
|
|
298
|
+
const chain = (g: string): string[] => {
|
|
299
|
+
const segs: string[] = []
|
|
300
|
+
for (const seg of g.split('/')) {
|
|
301
|
+
if (/[*?{}[\]()!]/.test(seg)) break
|
|
302
|
+
segs.push(seg)
|
|
303
|
+
}
|
|
304
|
+
return segs.map((_, i) => segs.slice(0, i + 1).join('/'))
|
|
305
|
+
}
|
|
306
|
+
const index = (i: number, globs: readonly string[]): void => {
|
|
307
|
+
for (const g of globs) {
|
|
308
|
+
const c = chain(g)
|
|
309
|
+
const push = (m: Map<string, number[]>, k: string) => {
|
|
310
|
+
const list = m.get(k)
|
|
311
|
+
if (list === undefined) m.set(k, [i])
|
|
312
|
+
else if (list.at(-1) !== i) list.push(i)
|
|
313
|
+
}
|
|
314
|
+
push(at, c.at(-1) ?? '')
|
|
315
|
+
for (const k of ['', ...c]) push(under, k)
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
const candidates = (globs: readonly string[]): number[] => {
|
|
319
|
+
const found = new Set<number>()
|
|
320
|
+
for (const g of globs) {
|
|
321
|
+
const c = chain(g)
|
|
322
|
+
for (const k of ['', ...c]) for (const i of at.get(k) ?? []) found.add(i)
|
|
323
|
+
for (const i of under.get(c.at(-1) ?? '') ?? []) found.add(i)
|
|
324
|
+
}
|
|
325
|
+
return [...found].sort((a, b) => a - b)
|
|
326
|
+
}
|
|
327
|
+
for (const p of projects) {
|
|
328
|
+
const rel = relPosix(root, p.dir)
|
|
329
|
+
for (const t of p.tasks) {
|
|
330
|
+
const cache = t.task?.['cache'] as
|
|
331
|
+
| { outputs?: { files?: unknown; workspaceFiles?: unknown } }
|
|
332
|
+
| undefined
|
|
333
|
+
const ws = positive(cache?.outputs?.workspaceFiles)
|
|
334
|
+
const own = positive(cache?.outputs?.files).map((g) =>
|
|
335
|
+
rel === '' ? g.replace(/^(\.\/)+/, '') : `${rel}/${g.replace(/^(\.\/)+/, '')}`,
|
|
336
|
+
)
|
|
337
|
+
if (ws.length === 0 && own.length === 0) continue
|
|
338
|
+
// Two own outputs of one project are resolveSharedOutputs', which
|
|
339
|
+
// reads the edges that order them.
|
|
340
|
+
const overlap = (a: string[], b: string[]): string | undefined =>
|
|
341
|
+
a.find((g) => b.some((h) => outputsOverlap(h, g)))
|
|
342
|
+
let clash: (typeof kept)[number] | undefined
|
|
343
|
+
let shared: string | undefined
|
|
344
|
+
for (const i of candidates([...ws, ...own])) {
|
|
345
|
+
const k = kept[i]!
|
|
346
|
+
shared =
|
|
347
|
+
overlap(ws, [...k.ws, ...k.own]) ??
|
|
348
|
+
overlap(own, k.project === p.name ? k.ws : [...k.ws, ...k.own])
|
|
349
|
+
if (shared !== undefined) {
|
|
350
|
+
clash = k
|
|
351
|
+
break
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (clash === undefined) {
|
|
355
|
+
index(kept.length, [...ws, ...own])
|
|
356
|
+
kept.push({ id: `${p.name}#${t.name}`, project: p.name, ws, own })
|
|
357
|
+
continue
|
|
358
|
+
}
|
|
359
|
+
delete t.task!['cache']
|
|
360
|
+
t.todos.push(
|
|
361
|
+
`declares the workspace output ${JSON.stringify(shared)} that ${clash.id} also declares — ` +
|
|
362
|
+
"vx cleans a task's outputs before it runs and before a restore, so two cached tasks " +
|
|
363
|
+
"on one path would delete each other's work; this one runs uncached. Give it its own " +
|
|
364
|
+
'output path to cache it.',
|
|
365
|
+
)
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** A `!` output with no positive beside it takes back nothing, and core refuses a list of them alone. */
|
|
371
|
+
export function takingBack(globs: readonly string[]): string[] {
|
|
372
|
+
return globs.some((g) => !g.startsWith('!')) ? [...globs] : []
|
|
373
|
+
}
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
// vx cleans a task's declared outputs before it runs (core's rule, kept on
|
|
2
|
+
// purpose: a stale file in an output is a stale artifact). Turbo and Nx
|
|
3
|
+
// never clean, so an output a repo declares over committed files is safe
|
|
4
|
+
// there and destructive here: typescript-eslint's website build caches
|
|
5
|
+
// `data`, which holds the committed `sponsors.json`, and the first vx run
|
|
6
|
+
// deleted it (2026-09-29). Each committed file under a mapped output is taken
|
|
7
|
+
// back with a `!` entry (core's A-44): not cleaned, not saved, not restored,
|
|
8
|
+
// still an input. The task keeps its cache.
|
|
9
|
+
|
|
10
|
+
import { lstat, readFile } from 'node:fs/promises'
|
|
11
|
+
import path from 'node:path'
|
|
12
|
+
import { outputsOverlap } from '@vzn/vx'
|
|
13
|
+
|
|
14
|
+
/** The files git tracks under `root`, root-relative; null outside a repo or without git. */
|
|
15
|
+
export async function trackedFiles(root: string): Promise<string[] | null> {
|
|
16
|
+
try {
|
|
17
|
+
const p = Bun.spawn(['git', 'ls-files', '-z'], { cwd: root, stdout: 'pipe', stderr: 'ignore' })
|
|
18
|
+
const out = await new Response(p.stdout).text()
|
|
19
|
+
if ((await p.exited) !== 0) return null
|
|
20
|
+
return out.split('\0').filter((f) => f !== '')
|
|
21
|
+
} catch {
|
|
22
|
+
return null
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const sortedMemo = new WeakMap<readonly string[], readonly string[]>()
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The files under `rel/` (all of them for the root). In a sorted list they
|
|
30
|
+
* are one run, found by a binary search: every project scanned the whole
|
|
31
|
+
* list, twice, and a cold mapping of 1,000 packages spent 177 ms there.
|
|
32
|
+
*/
|
|
33
|
+
function filesUnder(files: readonly string[], rel: string): readonly string[] {
|
|
34
|
+
if (rel === '' || rel === '.') return files
|
|
35
|
+
let sorted = sortedMemo.get(files)
|
|
36
|
+
if (sorted === undefined) sortedMemo.set(files, (sorted = [...files].sort()))
|
|
37
|
+
const prefix = `${rel}/`
|
|
38
|
+
let lo = 0
|
|
39
|
+
let hi = sorted.length
|
|
40
|
+
while (lo < hi) {
|
|
41
|
+
const mid = (lo + hi) >>> 1
|
|
42
|
+
if (sorted[mid]! < prefix) lo = mid + 1
|
|
43
|
+
else hi = mid
|
|
44
|
+
}
|
|
45
|
+
const out: string[] = []
|
|
46
|
+
for (let i = lo; i < sorted.length && sorted[i]!.startsWith(prefix); i++) out.push(sorted[i]!)
|
|
47
|
+
return out
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Which of `rels` (root-relative) git ignores: a path no glob over git's
|
|
52
|
+
* files can key. Untracked only (a tracked file is never ignored); empty
|
|
53
|
+
* outside a repo or without git.
|
|
54
|
+
*/
|
|
55
|
+
export async function gitIgnored(root: string, rels: readonly string[]): Promise<Set<string>> {
|
|
56
|
+
if (rels.length === 0) return new Set()
|
|
57
|
+
try {
|
|
58
|
+
const p = Bun.spawn(['git', 'check-ignore', '--stdin', '-z'], {
|
|
59
|
+
cwd: root,
|
|
60
|
+
stdin: new TextEncoder().encode(rels.map((r) => `${r}\0`).join('')),
|
|
61
|
+
stdout: 'pipe',
|
|
62
|
+
stderr: 'ignore',
|
|
63
|
+
})
|
|
64
|
+
const out = await new Response(p.stdout).text()
|
|
65
|
+
// 1: none ignored; anything else past 0 is no answer.
|
|
66
|
+
if ((await p.exited) !== 0) return new Set()
|
|
67
|
+
return new Set(out.split('\0').filter((f) => f !== ''))
|
|
68
|
+
} catch {
|
|
69
|
+
return new Set()
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* What a project's tracked files are: their extensions (lower case, no
|
|
75
|
+
* dot), directory names at any depth, and top-level entry names.
|
|
76
|
+
*/
|
|
77
|
+
export interface TrackedKinds {
|
|
78
|
+
readonly exts: ReadonlySet<string>
|
|
79
|
+
readonly dirs: ReadonlySet<string>
|
|
80
|
+
readonly tops: ReadonlySet<string>
|
|
81
|
+
/** The files themselves, project-relative. */
|
|
82
|
+
readonly files: readonly string[]
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Per project directory (root-relative, `.` or empty for the root), the
|
|
87
|
+
* kinds of the files git tracks under it at any depth: what a
|
|
88
|
+
* wildcard-first output glob may not claim (`wildcardOutput`).
|
|
89
|
+
*/
|
|
90
|
+
export function trackedKinds(tracked: readonly string[]): (rel: string) => TrackedKinds {
|
|
91
|
+
const memo = new Map<string, TrackedKinds>()
|
|
92
|
+
return (rel) => {
|
|
93
|
+
let kinds = memo.get(rel)
|
|
94
|
+
if (kinds === undefined) {
|
|
95
|
+
const exts = new Set<string>()
|
|
96
|
+
const dirs = new Set<string>()
|
|
97
|
+
const tops = new Set<string>()
|
|
98
|
+
const files: string[] = []
|
|
99
|
+
const all = rel === '' || rel === '.'
|
|
100
|
+
for (const f of filesUnder(tracked, rel)) {
|
|
101
|
+
const own = all ? f : f.slice(rel.length + 1)
|
|
102
|
+
files.push(own)
|
|
103
|
+
const ext = path.posix.extname(own)
|
|
104
|
+
if (ext !== '') exts.add(ext.slice(1).toLowerCase())
|
|
105
|
+
const segs = own.split('/')
|
|
106
|
+
tops.add(segs[0]!)
|
|
107
|
+
for (let i = 0; i < segs.length - 1; i++) dirs.add(segs[i]!)
|
|
108
|
+
}
|
|
109
|
+
memo.set(rel, (kinds = { exts, dirs, tops, files }))
|
|
110
|
+
}
|
|
111
|
+
return kinds
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* What moves when the tracked set can: the HEAD reflog's size (a commit, a
|
|
117
|
+
* checkout, a pull appends to it), HEAD itself, and the index's size and
|
|
118
|
+
* mtime (`git add`, `git rm`). Stats and a small read, so a kept mapping
|
|
119
|
+
* stays a hit between them; a run does not write the index. Without the
|
|
120
|
+
* index, a file `git add`ed under an output and not yet committed was not
|
|
121
|
+
* taken back by the kept mapping, and the run's clean deleted it. Under
|
|
122
|
+
* reftable storage HEAD reads `ref: refs/heads/.invalid` and no reflog file
|
|
123
|
+
* exists, so the ref stack's table list stands for both: every ref update
|
|
124
|
+
* names a new table in it.
|
|
125
|
+
*/
|
|
126
|
+
export async function headStamp(root: string): Promise<string> {
|
|
127
|
+
for (let dir = root; ;) {
|
|
128
|
+
const dotGit = path.join(dir, '.git')
|
|
129
|
+
const st = await lstat(dotGit).catch(() => null)
|
|
130
|
+
if (st !== null) {
|
|
131
|
+
let gitDir = dotGit
|
|
132
|
+
if (st.isFile()) {
|
|
133
|
+
const m = /^gitdir: (.*)$/m.exec(await readFile(dotGit, 'utf8').catch(() => ''))
|
|
134
|
+
if (m !== null) gitDir = path.resolve(dir, m[1]!.trim())
|
|
135
|
+
}
|
|
136
|
+
const head = await readFile(path.join(gitDir, 'HEAD'), 'utf8').catch(() => '')
|
|
137
|
+
const log = await lstat(path.join(gitDir, 'logs', 'HEAD')).catch(() => null)
|
|
138
|
+
const index = await lstat(path.join(gitDir, 'index')).catch(() => null)
|
|
139
|
+
// A worktree's own stack holds its HEAD; the common one, its branches.
|
|
140
|
+
const common = await readFile(path.join(gitDir, 'commondir'), 'utf8').catch(() => null)
|
|
141
|
+
const tables = await Promise.all(
|
|
142
|
+
[gitDir, ...(common === null ? [] : [path.resolve(gitDir, common.trim())])].map((d) =>
|
|
143
|
+
readFile(path.join(d, 'reftable', 'tables.list'), 'utf8').catch(() => ''),
|
|
144
|
+
),
|
|
145
|
+
)
|
|
146
|
+
return `${head.trim()}\0${log?.size ?? ''}\0${index?.size ?? ''}\0${index?.mtimeMs ?? ''}\0${tables.join('\0')}`
|
|
147
|
+
}
|
|
148
|
+
const up = path.dirname(dir)
|
|
149
|
+
if (up === dir) return 'no-git'
|
|
150
|
+
dir = up
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** A glob's leading literal segments: every path it can match starts there. */
|
|
155
|
+
function literalPrefix(glob: string): string {
|
|
156
|
+
const out: string[] = []
|
|
157
|
+
for (const seg of glob.replace(/^(\.\/)+/, '').split('/')) {
|
|
158
|
+
if (/[*?[\]{}()!]/.test(seg)) break
|
|
159
|
+
out.push(seg)
|
|
160
|
+
}
|
|
161
|
+
return out.join('/')
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** The tracked `files` (relative to the same base as `globs`) the positive globs cover and no `!` takes back. */
|
|
165
|
+
function coveredTracked(globs: readonly string[], files: readonly string[]): string[] {
|
|
166
|
+
const positive = globs.filter((g) => !g.startsWith('!'))
|
|
167
|
+
const taken = globs.filter((g) => g.startsWith('!')).map((g) => g.slice(1))
|
|
168
|
+
const hit = new Set<string>()
|
|
169
|
+
for (const g of positive) {
|
|
170
|
+
const prefix = literalPrefix(g)
|
|
171
|
+
for (const f of files) {
|
|
172
|
+
if (prefix !== '' && f !== prefix && !f.startsWith(`${prefix}/`)) continue
|
|
173
|
+
if (!outputsOverlap(g, f)) continue
|
|
174
|
+
if (taken.some((t) => outputsOverlap(t, f))) continue
|
|
175
|
+
hit.add(f)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return [...hit].sort()
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Past this many, a task's take-backs cost its runs more than its cache saves. */
|
|
182
|
+
export const MAX_SPARED = 16
|
|
183
|
+
|
|
184
|
+
interface Outputs {
|
|
185
|
+
files?: unknown
|
|
186
|
+
workspaceFiles?: unknown
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Take back every committed file a mapped task's outputs cover, in place,
|
|
191
|
+
* and return one `[pkg#task, todo]` per task that had any. `tracked` is
|
|
192
|
+
* root-relative.
|
|
193
|
+
*/
|
|
194
|
+
export function spareTrackedOutputs(
|
|
195
|
+
root: string,
|
|
196
|
+
projects: readonly {
|
|
197
|
+
readonly name: string
|
|
198
|
+
readonly dir: string
|
|
199
|
+
readonly tasks: readonly {
|
|
200
|
+
readonly name: string
|
|
201
|
+
readonly task: Record<string, unknown> | null
|
|
202
|
+
}[]
|
|
203
|
+
}[],
|
|
204
|
+
tracked: readonly string[],
|
|
205
|
+
): [string, string][] {
|
|
206
|
+
const todos: [string, string][] = []
|
|
207
|
+
for (const p of projects) {
|
|
208
|
+
const rel = path.relative(root, p.dir).split(path.sep).join('/')
|
|
209
|
+
const own = rel === '' ? tracked : filesUnder(tracked, rel).map((f) => f.slice(rel.length + 1))
|
|
210
|
+
for (const t of p.tasks) {
|
|
211
|
+
const outputs = (t.task?.['cache'] as { outputs?: Outputs } | undefined)?.outputs
|
|
212
|
+
if (outputs === undefined) continue
|
|
213
|
+
const hits: [key: 'files' | 'workspaceFiles', globs: string[], hit: string[]][] = []
|
|
214
|
+
for (const [key, base] of [
|
|
215
|
+
['files', own],
|
|
216
|
+
['workspaceFiles', tracked],
|
|
217
|
+
] as const) {
|
|
218
|
+
const globs = outputs[key]
|
|
219
|
+
if (!Array.isArray(globs) || globs.length === 0) continue
|
|
220
|
+
const strings = globs.filter((g): g is string => typeof g === 'string')
|
|
221
|
+
const hit = coveredTracked(strings, base)
|
|
222
|
+
if (hit.length > 0) hits.push([key, strings, hit])
|
|
223
|
+
}
|
|
224
|
+
const spared = hits.flatMap(([, , hit]) => hit)
|
|
225
|
+
if (spared.length === 0) continue
|
|
226
|
+
const shown = spared.slice(0, 3).join(', ') + (spared.length > 3 ? ', …' : '')
|
|
227
|
+
// Each `!` is matched against every output path on each save, clean
|
|
228
|
+
// and restore: 4,200 of them cost typescript-eslint's warm run 1.2 s.
|
|
229
|
+
if (spared.length > MAX_SPARED) {
|
|
230
|
+
delete t.task!['cache']
|
|
231
|
+
todos.push([
|
|
232
|
+
`${p.name}#${t.name}`,
|
|
233
|
+
`outputs cover ${spared.length} committed files (${shown}), which vx cleans before ` +
|
|
234
|
+
'every run — task runs uncached; declare the exact outputs in a vx.config to cache it',
|
|
235
|
+
])
|
|
236
|
+
continue
|
|
237
|
+
}
|
|
238
|
+
for (const [key, strings, hit] of hits)
|
|
239
|
+
outputs[key] = [...strings, ...hit.map((f) => `!${f}`)]
|
|
240
|
+
todos.push([
|
|
241
|
+
`${p.name}#${t.name}`,
|
|
242
|
+
`outputs cover ${spared.length} committed file(s) (${shown}) — vx cleans outputs ` +
|
|
243
|
+
'before a run, so they are taken back with `!` and kept',
|
|
244
|
+
])
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return todos
|
|
248
|
+
}
|