@vzn/vx-migrate 0.0.651 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -52
- package/package.json +2 -2
- package/src/migrate-scripts.ts +13 -7
- package/src/migrate-turbo.ts +8 -1
- package/src/migrate.ts +102 -18
- package/src/nx/index.ts +61 -11
- package/src/nx/nx-inputs.ts +48 -24
- package/src/nx/nx-map.ts +204 -20
- package/src/nx/nx-outputs.ts +6 -1
- package/src/nx/nx-upstream.ts +11 -11
- package/src/nx-cache/index.ts +42 -28
- package/src/packages.ts +51 -1
- package/src/remote-refusal.ts +49 -4
- package/src/shared-outputs.ts +86 -25
- package/src/tracked-outputs.ts +59 -20
- package/src/turbo/index.ts +2 -1
- package/src/turbo/turbo-map.ts +165 -29
- package/src/turbo-cache/index.ts +71 -40
- package/src/turbo-cache/oidc.ts +15 -7
- package/src/vite-task/vite-task-map.ts +3 -2
- package/src/workspace-notes.ts +19 -8
- package/src/workspace-plugins.ts +11 -3
package/src/nx/nx-map.ts
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
// default configuration is folded into the base task.
|
|
18
18
|
|
|
19
19
|
import { existsSync } from 'node:fs'
|
|
20
|
+
import { lstat } from 'node:fs/promises'
|
|
20
21
|
import path from 'node:path'
|
|
21
22
|
import { type Duration, type GeneratedProject, type GeneratedTask, UserError } from '@vzn/vx'
|
|
22
23
|
import type { PackageMeta } from '../packages.js'
|
|
@@ -34,8 +35,9 @@ import {
|
|
|
34
35
|
resolveSharedWorkspaceOutputs,
|
|
35
36
|
wildcardOutput,
|
|
36
37
|
wildcardTodo,
|
|
38
|
+
uncache,
|
|
37
39
|
} from '../shared-outputs.js'
|
|
38
|
-
import type
|
|
40
|
+
import { gitIgnored, type TrackedKinds } from '../tracked-outputs.js'
|
|
39
41
|
import { isRecord, packageScripts, relPosix } from '../paths.js'
|
|
40
42
|
import { withLernaOrder } from './lerna.js'
|
|
41
43
|
import { mapNxDeps, matchNxProjects, type TaskNameFor } from './nx-deps.js'
|
|
@@ -47,6 +49,7 @@ import {
|
|
|
47
49
|
ownerTargetOf,
|
|
48
50
|
} from './nx-dotenv.js'
|
|
49
51
|
import { emptyNxInputs, expandNxInputs } from './nx-inputs.js'
|
|
52
|
+
import { dotenvGlobsProbe } from '../dotenv-probe.js'
|
|
50
53
|
import { planNxUpstream, type NxUpstream } from './nx-upstream.js'
|
|
51
54
|
import { mapNxOutputs, nxDefaultOutputs, nxProjectOutputs } from './nx-outputs.js'
|
|
52
55
|
|
|
@@ -247,6 +250,13 @@ export interface MapNxOptions {
|
|
|
247
250
|
* `nx-exec`. The live plugin's only: a migration writes no function.
|
|
248
251
|
*/
|
|
249
252
|
readonly executors?: NxExecutors
|
|
253
|
+
/**
|
|
254
|
+
* The root lockfile when no plugin claims it (the live plugin's only):
|
|
255
|
+
* core keys no lockfile, so an `externalDependencies` input keys the
|
|
256
|
+
* file. Absent, a lockfile plugin keys the dependencies (a migration
|
|
257
|
+
* declares one).
|
|
258
|
+
*/
|
|
259
|
+
readonly lockfile?: string
|
|
250
260
|
}
|
|
251
261
|
|
|
252
262
|
/** One target of a translated executor, as the graph resolved it. */
|
|
@@ -294,6 +304,8 @@ type MapOpts = MapNxOptions & {
|
|
|
294
304
|
readonly hooks: (dir: string) => boolean
|
|
295
305
|
/** Tasks per `syncGenerators` list, said once in the notes. */
|
|
296
306
|
readonly syncTasks: Map<string, number>
|
|
307
|
+
/** Root-relative outputs git ignores (`ignoredOutputs`); unknown without git. */
|
|
308
|
+
readonly ignoredOutputs: ReadonlySet<string> | undefined
|
|
297
309
|
}
|
|
298
310
|
|
|
299
311
|
export interface NxMapping {
|
|
@@ -319,6 +331,46 @@ function nxTags(node: NxNode | undefined): string[] {
|
|
|
319
331
|
: []
|
|
320
332
|
}
|
|
321
333
|
|
|
334
|
+
/**
|
|
335
|
+
* The outputs of every project whose input Nx never hashes: git tracks
|
|
336
|
+
* nothing under it and ignores it, as Nx's file map skips gitignored files
|
|
337
|
+
* (a changed `dist` file is a hit under Nx 23.3, YL-1). One untracked but
|
|
338
|
+
* not ignored is in Nx's file map, so an input under it is hashed: taken
|
|
339
|
+
* for ignored, `build` keyed on no file and a generated source's edit
|
|
340
|
+
* replayed (stale hit). One `git check-ignore` per mapping.
|
|
341
|
+
*/
|
|
342
|
+
async function ignoredOutputs(
|
|
343
|
+
root: string,
|
|
344
|
+
nodes: Readonly<Record<string, NxNode>>,
|
|
345
|
+
tracked: (rel: string) => TrackedKinds,
|
|
346
|
+
): Promise<ReadonlySet<string>> {
|
|
347
|
+
const untracked = new Set<string>()
|
|
348
|
+
for (const [nodeName, node] of Object.entries(nodes)) {
|
|
349
|
+
const rel = normRel(node?.data?.root ?? '')
|
|
350
|
+
for (const o of nxProjectOutputs(node?.data?.targets, rel || '.', node?.name ?? nodeName)) {
|
|
351
|
+
const dir = path.posix.dirname(o)
|
|
352
|
+
if (
|
|
353
|
+
tracked(o).files.length === 0 &&
|
|
354
|
+
!tracked(dir).files.includes(dir === '.' ? o : o.slice(dir.length + 1))
|
|
355
|
+
)
|
|
356
|
+
untracked.add(o)
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// A missing `dist` matches a `dist/` rule only spelled as a directory;
|
|
360
|
+
// an existing file never does.
|
|
361
|
+
const hit = await gitIgnored(
|
|
362
|
+
root,
|
|
363
|
+
[...untracked].flatMap((o) => [o, `${o}/`]),
|
|
364
|
+
)
|
|
365
|
+
const out = new Set<string>()
|
|
366
|
+
for (const o of untracked) {
|
|
367
|
+
if (hit.has(o)) out.add(o)
|
|
368
|
+
else if (hit.has(`${o}/`) && !(await lstat(path.join(root, o)).catch(() => null))?.isFile())
|
|
369
|
+
out.add(o)
|
|
370
|
+
}
|
|
371
|
+
return out
|
|
372
|
+
}
|
|
373
|
+
|
|
322
374
|
function normRel(p: string): string {
|
|
323
375
|
return p === '' || p === '.' ? '.' : p.replace(/\/$/, '')
|
|
324
376
|
}
|
|
@@ -354,6 +406,8 @@ export async function mapNxWorkspace(
|
|
|
354
406
|
pnp: yarnPnp(root),
|
|
355
407
|
hooks: (dir) => runsScriptHooks(dir, hookMemo) !== null,
|
|
356
408
|
syncTasks: new Map(),
|
|
409
|
+
ignoredOutputs:
|
|
410
|
+
opts.tracked === undefined ? undefined : await ignoredOutputs(root, nodeMap, opts.tracked),
|
|
357
411
|
}
|
|
358
412
|
|
|
359
413
|
const metaByRel = new Map<string, PackageMeta>()
|
|
@@ -408,7 +462,13 @@ export async function mapNxWorkspace(
|
|
|
408
462
|
nodeNameOf.set(synthetic, nodeName)
|
|
409
463
|
}
|
|
410
464
|
|
|
411
|
-
const upstream = planNxUpstream(
|
|
465
|
+
const upstream = planNxUpstream(
|
|
466
|
+
nodeMap,
|
|
467
|
+
g.dependencies,
|
|
468
|
+
namedInputs,
|
|
469
|
+
metaByNode,
|
|
470
|
+
mapOpts.ignoredOutputs,
|
|
471
|
+
)
|
|
412
472
|
|
|
413
473
|
const taskNameFor: TaskNameFor = (project, target, configuration) => {
|
|
414
474
|
const t = nodeMap[project]?.data?.targets?.[target]
|
|
@@ -595,8 +655,15 @@ export async function mapNxWorkspace(
|
|
|
595
655
|
}
|
|
596
656
|
dropUnheldDeps(t, meta.name, emitted, emittedIds)
|
|
597
657
|
const reader = dependentReaders.get(t)
|
|
598
|
-
if (
|
|
599
|
-
|
|
658
|
+
if (
|
|
659
|
+
reader !== undefined &&
|
|
660
|
+
!reachesProducer(t, meta.name, emittedIds, reached, (target) =>
|
|
661
|
+
nodeName === undefined
|
|
662
|
+
? undefined
|
|
663
|
+
: nxEdges(nodeName, target).map((n) => metaByNode.get(n)?.name),
|
|
664
|
+
)
|
|
665
|
+
) {
|
|
666
|
+
uncache(t.task!)
|
|
600
667
|
t.todos.push(orphanReaderTodo(reader.globs, reader.dependsOn))
|
|
601
668
|
}
|
|
602
669
|
}
|
|
@@ -612,6 +679,7 @@ export async function mapNxWorkspace(
|
|
|
612
679
|
|
|
613
680
|
resolveAtomizedWorkspaceOutputs(root, projects, split)
|
|
614
681
|
excludeWorkspaceOutputs(root, projects)
|
|
682
|
+
keyUncachedProducerOutputs(projects, reached)
|
|
615
683
|
// A migration leaves Nx behind, so `nx sync` is no advice there: what a
|
|
616
684
|
// generator keeps (the TypeScript one's tsconfig `references`) is the
|
|
617
685
|
// user's to keep. `nx()` runs with Nx installed.
|
|
@@ -663,10 +731,7 @@ function resolveAtomizedWorkspaceOutputs(
|
|
|
663
731
|
const cached = (ps: readonly { name: string; dir: string; tasks: GeneratedTask[] }[]): number => {
|
|
664
732
|
const copy = structuredClone(ps.map((p) => ({ name: p.name, dir: p.dir, tasks: p.tasks })))
|
|
665
733
|
resolveSharedWorkspaceOutputs(root, copy)
|
|
666
|
-
return copy.reduce(
|
|
667
|
-
(n, p) => n + p.tasks.filter((t) => t.task?.['cache'] !== undefined).length,
|
|
668
|
-
0,
|
|
669
|
-
)
|
|
734
|
+
return copy.reduce((n, p) => n + p.tasks.filter((t) => t.task?.['cache'] === true).length, 0)
|
|
670
735
|
}
|
|
671
736
|
const better = split.size > 0 && cached(last) > cached(projects)
|
|
672
737
|
resolveSharedWorkspaceOutputs(root, better ? last : projects)
|
|
@@ -865,8 +930,9 @@ function buildTask(
|
|
|
865
930
|
rel: projectRel,
|
|
866
931
|
name: projectName,
|
|
867
932
|
outputs: nxProjectOutputs(nodeMap[nodeName]?.data?.targets, projectRel, projectName),
|
|
868
|
-
...(opts.
|
|
933
|
+
...(opts.ignoredOutputs === undefined ? {} : { ignoredOutputs: opts.ignoredOutputs }),
|
|
869
934
|
rootDeps: opts.rootDeps,
|
|
935
|
+
...(opts.lockfile === undefined ? {} : { lockfile: opts.lockfile }),
|
|
870
936
|
}
|
|
871
937
|
expandNxInputs(target.inputs ?? [], upstream.namedOf(nodeName), at, inputs, todos)
|
|
872
938
|
const cacheWanted =
|
|
@@ -1061,42 +1127,160 @@ function buildTask(
|
|
|
1061
1127
|
if (cacheEnabled) {
|
|
1062
1128
|
const outputs: Record<string, unknown> = { files: takingBack(outFiles) }
|
|
1063
1129
|
if (wsOutFiles.length > 0) outputs.workspaceFiles = takingBack(wsOutFiles)
|
|
1064
|
-
task.
|
|
1130
|
+
task.outputs = outputs
|
|
1131
|
+
task.cache = true
|
|
1065
1132
|
}
|
|
1066
1133
|
|
|
1067
1134
|
const generated: GeneratedTask = { name: variant.name, todos, task }
|
|
1068
1135
|
if (cacheEnabled && inputs.dependentOutputs.length > 0)
|
|
1069
|
-
dependentReaders.set(generated, {
|
|
1136
|
+
dependentReaders.set(generated, {
|
|
1137
|
+
globs: inputs.dependentOutputs,
|
|
1138
|
+
transitive: inputs.transitiveOutputs,
|
|
1139
|
+
dependsOn: target.dependsOn,
|
|
1140
|
+
})
|
|
1141
|
+
const produced = [
|
|
1142
|
+
...outFiles.filter((o) => !o.startsWith('!')).map((o) => path.posix.join(projectRel, o)),
|
|
1143
|
+
...wsOutFiles.filter((o) => !o.startsWith('!')),
|
|
1144
|
+
]
|
|
1145
|
+
if (produced.length > 0) producedPaths.set(generated, produced)
|
|
1070
1146
|
return generated
|
|
1071
1147
|
}
|
|
1072
1148
|
|
|
1149
|
+
/** Each task's Nx outputs, workspace-relative, cached or not: what a reader of them probes. */
|
|
1150
|
+
const producedPaths = new WeakMap<GeneratedTask, string[]>()
|
|
1151
|
+
|
|
1073
1152
|
/**
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1078
|
-
*
|
|
1153
|
+
* Nx hashes the files a `dependentTasksOutputFiles` reader names in its
|
|
1154
|
+
* producers' outputs; vx folds each producer's key, which covers what a
|
|
1155
|
+
* cached producer writes and not what an uncached one fetched from outside
|
|
1156
|
+
* (a `cache: false` `fetch` writing a gitignored schema): the reader hit
|
|
1157
|
+
* and replayed the old schema's code. Those files are keyed by a probe at
|
|
1158
|
+
* the root, as `turbo()` keys a `dependencyOutputs` input of an uncached
|
|
1159
|
+
* producer, and the producer is left out of the fold (`inputs.tasks`), so
|
|
1160
|
+
* core answers the probe after it ran.
|
|
1079
1161
|
*/
|
|
1080
|
-
|
|
1162
|
+
function keyUncachedProducerOutputs(
|
|
1163
|
+
projects: readonly GeneratedProject[],
|
|
1164
|
+
reached: (name: string) => ReadonlySet<string>,
|
|
1165
|
+
): void {
|
|
1166
|
+
const byId = new Map<string, { project: string; task: GeneratedTask }>()
|
|
1167
|
+
for (const p of projects)
|
|
1168
|
+
for (const t of p.tasks) byId.set(`${p.name}#${t.name}`, { project: p.name, task: t })
|
|
1169
|
+
const edgesOf = (project: string, deps: unknown): [id: string, filter: string][] =>
|
|
1170
|
+
(Array.isArray(deps) ? deps : []).flatMap((d): [string, string][] => {
|
|
1171
|
+
if (typeof d !== 'string' || d.includes('*') || d.includes('#inputs:')) return []
|
|
1172
|
+
return d.startsWith('^')
|
|
1173
|
+
? [...reached(project)].map((q) => [`${q}#${d.slice(1)}`, `${q}#${d.slice(1)}`])
|
|
1174
|
+
: [[d.includes('#') ? d : `${project}#${d}`, d]]
|
|
1175
|
+
})
|
|
1176
|
+
for (const p of projects) {
|
|
1177
|
+
for (const t of p.tasks) {
|
|
1178
|
+
const reader = dependentReaders.get(t)
|
|
1179
|
+
if (reader === undefined || t.task?.['cache'] !== true) continue
|
|
1180
|
+
const rels: string[] = []
|
|
1181
|
+
const unfold: string[] = []
|
|
1182
|
+
const keyOutputs = (producer: GeneratedTask, globs: readonly string[]): void => {
|
|
1183
|
+
for (const o of producedPaths.get(producer) ?? [])
|
|
1184
|
+
// A glob output (`dist/**/*.{js,d.ts}{,.map}`): every file under
|
|
1185
|
+
// its literal head, which the probe can walk.
|
|
1186
|
+
if (/[*?[{]/.test(o)) rels.push(`${globHead(o)}/**/*`)
|
|
1187
|
+
else
|
|
1188
|
+
for (const g of globs)
|
|
1189
|
+
// `**/<name>` is a find under the output; any other glob (a brace, a
|
|
1190
|
+
// class), every file there.
|
|
1191
|
+
rels.push(/^\*\*\/[^/*?[{]*\*?[^/*?[{]*$/.test(g) ? `${o}/${g}` : `${o}/**/*`)
|
|
1192
|
+
}
|
|
1193
|
+
const uncachedWriter = (producer: GeneratedTask | undefined): producer is GeneratedTask =>
|
|
1194
|
+
producer?.task?.['exec'] !== undefined &&
|
|
1195
|
+
producer.task['cache'] !== true &&
|
|
1196
|
+
(producedPaths.get(producer)?.length ?? 0) > 0
|
|
1197
|
+
for (const [id, filter] of edgesOf(p.name, t.task['dependsOn'])) {
|
|
1198
|
+
const producer = byId.get(id)?.task
|
|
1199
|
+
if (producer === undefined) continue
|
|
1200
|
+
// `transitive: true` reads the outputs of every task below, not
|
|
1201
|
+
// only the direct ones: an uncached `fetch` behind an `nx:noop`
|
|
1202
|
+
// `prep` hid its schema, and the reader replayed the old one (hunt,
|
|
1203
|
+
// 2026-11-08). A key folding the edge answers the probe before the
|
|
1204
|
+
// fetch ran, so the edge is left out and every output below it,
|
|
1205
|
+
// cached or not, is keyed by the probe.
|
|
1206
|
+
const below: GeneratedTask[] = []
|
|
1207
|
+
if (reader.transitive.length > 0) {
|
|
1208
|
+
const seen = new Set<string>([id])
|
|
1209
|
+
const stack = edgesOf(byId.get(id)!.project, producer.task?.['dependsOn'])
|
|
1210
|
+
while (stack.length > 0) {
|
|
1211
|
+
const [next] = stack.pop()!
|
|
1212
|
+
const node = byId.get(next)
|
|
1213
|
+
if (seen.has(next) || node === undefined) continue
|
|
1214
|
+
seen.add(next)
|
|
1215
|
+
below.push(node.task)
|
|
1216
|
+
stack.push(...edgesOf(node.project, node.task.task?.['dependsOn']))
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
if (!uncachedWriter(producer) && !below.some(uncachedWriter)) continue
|
|
1220
|
+
unfold.push(`!${filter}`)
|
|
1221
|
+
keyOutputs(producer, reader.globs)
|
|
1222
|
+
for (const b of below) keyOutputs(b, reader.transitive)
|
|
1223
|
+
}
|
|
1224
|
+
if (rels.length === 0) continue
|
|
1225
|
+
const probe = dotenvGlobsProbe([...new Set(rels)])
|
|
1226
|
+
if (probe === null) {
|
|
1227
|
+
uncache(t.task)
|
|
1228
|
+
t.todos.push(
|
|
1229
|
+
`reads ${reader.globs.map((g) => `\`${g}\``).join(', ')} from the outputs of an uncached task it depends on, ` +
|
|
1230
|
+
'which vx cannot key by a probe — uncached here; key it in a vx.config to cache it',
|
|
1231
|
+
)
|
|
1232
|
+
continue
|
|
1233
|
+
}
|
|
1234
|
+
const inputs = t.task['inputs'] as Record<string, unknown>
|
|
1235
|
+
inputs['workspaceRuntime'] = [...((inputs['workspaceRuntime'] as unknown[]) ?? []), probe]
|
|
1236
|
+
inputs['tasks'] = ['*', '^*', ...unfold]
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
function globHead(glob: string): string {
|
|
1242
|
+
const segs = glob.split('/')
|
|
1243
|
+
const i = segs.findIndex((s) => /[*?[{]/.test(s))
|
|
1244
|
+
return segs.slice(0, i).join('/')
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
const dependentReaders = new WeakMap<
|
|
1248
|
+
GeneratedTask,
|
|
1249
|
+
{ globs: string[]; transitive: string[]; dependsOn: unknown }
|
|
1250
|
+
>()
|
|
1081
1251
|
|
|
1082
1252
|
function orphanReaderTodo(globs: readonly string[], dependsOn: unknown): string {
|
|
1083
1253
|
return `reads ${globs.map((g) => `\`${g}\``).join(', ')} from the outputs of the tasks it depends on, and no task it depends on is migrated (dependsOn ${JSON.stringify(dependsOn ?? [])}) — uncached here, so it never replays output generated from an older input; migrate the producer and depend on it to cache this task`
|
|
1084
1254
|
}
|
|
1085
1255
|
|
|
1086
|
-
/**
|
|
1256
|
+
/**
|
|
1257
|
+
* Whether a cached reader of `dependentTasksOutputFiles` still reaches a
|
|
1258
|
+
* producer once the graph has its final edges: one of them reaches a task
|
|
1259
|
+
* vx runs, or is a `^` edge every project Nx links (`nxLinked`: their vx
|
|
1260
|
+
* names) runs it on. A `^build` kept because another project builds, while
|
|
1261
|
+
* Nx links a producer vx carries no task for, keyed nothing the task reads
|
|
1262
|
+
* (Growth, nx-codegen-dropped-producer.sh). With none linked, Nx hashes
|
|
1263
|
+
* nothing there either: a leaf library's inferred `build` (`^build`,
|
|
1264
|
+
* @nx/js/typescript) ran uncached every time where Nx caches it (hunt,
|
|
1265
|
+
* 2026-11-08).
|
|
1266
|
+
*/
|
|
1087
1267
|
function reachesProducer(
|
|
1088
1268
|
t: GeneratedTask,
|
|
1089
1269
|
project: string,
|
|
1090
1270
|
emittedIds: ReadonlySet<string>,
|
|
1091
1271
|
reached: (name: string) => ReadonlySet<string>,
|
|
1272
|
+
nxLinked: (target: string) => ReadonlyArray<string | undefined> | undefined,
|
|
1092
1273
|
): boolean {
|
|
1093
1274
|
const deps = t.task?.['dependsOn']
|
|
1094
1275
|
if (!Array.isArray(deps)) return false
|
|
1095
1276
|
return deps.some((d) => {
|
|
1096
1277
|
if (typeof d !== 'string' || d.includes('*')) return true
|
|
1097
1278
|
if (d.includes('#inputs:')) return false
|
|
1098
|
-
if (d.startsWith('^'))
|
|
1099
|
-
|
|
1279
|
+
if (d.startsWith('^')) {
|
|
1280
|
+
const target = d.slice(1)
|
|
1281
|
+
if ([...reached(project)].some((p) => emittedIds.has(`${p}#${target}`))) return true
|
|
1282
|
+
return nxLinked(target)?.every((p) => emittedIds.has(`${p}#${target}`)) === true
|
|
1283
|
+
}
|
|
1100
1284
|
return d.includes('#') || emittedIds.has(`${project}#${d}`)
|
|
1101
1285
|
})
|
|
1102
1286
|
}
|
package/src/nx/nx-outputs.ts
CHANGED
|
@@ -189,6 +189,8 @@ export function mapNxOutputs(
|
|
|
189
189
|
)
|
|
190
190
|
continue
|
|
191
191
|
}
|
|
192
|
+
const rootWide =
|
|
193
|
+
(projectRel === '' || projectRel === '.') && o.slice(neg.length).startsWith('{workspaceRoot}')
|
|
192
194
|
const globs = vxOutputGlobs(neg + s)
|
|
193
195
|
if (globs === null) {
|
|
194
196
|
todos.push(`output ${JSON.stringify(o)} has a glob form vx cannot spell — dropped`)
|
|
@@ -197,7 +199,10 @@ export function mapNxOutputs(
|
|
|
197
199
|
for (const g of globs) {
|
|
198
200
|
const gn = g.startsWith('!') ? '!' : ''
|
|
199
201
|
const gs = g.slice(gn.length)
|
|
200
|
-
|
|
202
|
+
// As the root project's inputs map (nx-inputs.ts): a `{workspaceRoot}`
|
|
203
|
+
// output names a workspace path, which may lie in a nested project a
|
|
204
|
+
// project glob never reaches (core refuses one there).
|
|
205
|
+
const own = rootWide ? null : underProject(gs, projectRel)
|
|
201
206
|
if (own !== null) {
|
|
202
207
|
outFiles.push(gn + (own === '' ? '**' : withProjectDir(own, projectRel, true)))
|
|
203
208
|
continue
|
package/src/nx/nx-upstream.ts
CHANGED
|
@@ -18,11 +18,10 @@
|
|
|
18
18
|
// and edges only out of the cycle.
|
|
19
19
|
import type { GeneratedTask } from '@vzn/vx'
|
|
20
20
|
import type { PackageMeta } from '../packages.js'
|
|
21
|
-
import type { TrackedKinds } from '../tracked-outputs.js'
|
|
22
21
|
import { emptyNxInputs, expandNxInputs, type NxInputs } from './nx-inputs.js'
|
|
23
22
|
import { nxProjectOutputs } from './nx-outputs.js'
|
|
24
23
|
import { matchNxProjects } from './nx-deps.js'
|
|
25
|
-
import { literalGlob } from '../glob-grammar.js'
|
|
24
|
+
import { apartTakeBacks, literalGlob } from '../glob-grammar.js'
|
|
26
25
|
|
|
27
26
|
interface GraphNode {
|
|
28
27
|
name?: string
|
|
@@ -73,7 +72,7 @@ export function planNxUpstream(
|
|
|
73
72
|
dependencies: unknown,
|
|
74
73
|
workspaceNamed: Readonly<Record<string, unknown[]>> | null,
|
|
75
74
|
metaByNode: ReadonlyMap<string, PackageMeta>,
|
|
76
|
-
|
|
75
|
+
ignoredOutputs?: ReadonlySet<string>,
|
|
77
76
|
): NxUpstream {
|
|
78
77
|
const isProject = (n: string): boolean => typeof nodes[n]?.data?.root === 'string'
|
|
79
78
|
const projectTags = Object.entries(nodes).map(([name, n]) => ({
|
|
@@ -194,13 +193,12 @@ export function planNxUpstream(
|
|
|
194
193
|
expandOwn(node, name, own, todos)
|
|
195
194
|
const rel = normRel(nodes[node]?.data?.root ?? '')
|
|
196
195
|
const inputs = emptyNxInputs()
|
|
197
|
-
|
|
196
|
+
const files = own.files.map((g) => {
|
|
198
197
|
const neg = g.startsWith('!') ? '!' : ''
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
inputs.wsFiles.push(...own.wsFiles)
|
|
198
|
+
return rel === '' ? g : `${neg}${literalGlob(rel)}/${neg === '' ? g : g.slice(1)}`
|
|
199
|
+
})
|
|
200
|
+
// Two of Nx's sets in one list, as in `expandNxInputs`.
|
|
201
|
+
inputs.wsFiles.push(...apartTakeBacks(files, own.wsFiles).flat())
|
|
204
202
|
inputs.envNames.push(...own.envNames)
|
|
205
203
|
inputs.runtimeCmds.push(...own.runtimeCmds)
|
|
206
204
|
inputs.upstream.push(...own.upstream)
|
|
@@ -223,13 +221,15 @@ export function planNxUpstream(
|
|
|
223
221
|
rel,
|
|
224
222
|
name: nm,
|
|
225
223
|
outputs: nxProjectOutputs(nodes[node]?.data?.targets, rel || '.', nm),
|
|
226
|
-
...(
|
|
224
|
+
...(ignoredOutputs === undefined ? {} : { ignoredOutputs }),
|
|
227
225
|
}
|
|
228
226
|
expandNxInputs([name], namedOf(node), at, into, todos)
|
|
229
227
|
}
|
|
230
228
|
|
|
231
229
|
const merge = (from: NxInputs, into: NxInputs): void => {
|
|
232
|
-
|
|
230
|
+
// A node's sets meet the reader's in one list: a `!` reaching another
|
|
231
|
+
// set's entry goes, as in `expandNxInputs`.
|
|
232
|
+
into.wsFiles.splice(0, Infinity, ...apartTakeBacks(into.wsFiles, from.wsFiles).flat())
|
|
233
233
|
for (const n of from.envNames) if (!into.envNames.includes(n)) into.envNames.push(n)
|
|
234
234
|
for (const c of from.runtimeCmds) if (!into.runtimeCmds.includes(c)) into.runtimeCmds.push(c)
|
|
235
235
|
}
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
} from '@vzn/vx'
|
|
23
23
|
import { deadline, MAX_TIMER_MS } from '../remote-deadline.js'
|
|
24
24
|
import { checkRemoteUrl } from '../remote-url.js'
|
|
25
|
-
import { refusal } from '../remote-refusal.js'
|
|
25
|
+
import { hideTokens, hidingErrors, refusal } from '../remote-refusal.js'
|
|
26
26
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
27
27
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
28
28
|
import {
|
|
@@ -112,6 +112,8 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
112
112
|
private token: string | undefined
|
|
113
113
|
/** Set by `open` when the token came from the keychain: a refusal deletes it. */
|
|
114
114
|
private forget: (() => Promise<void>) | undefined
|
|
115
|
+
/** The token as a refusal that quotes it says it: `***`. */
|
|
116
|
+
private readonly hidden = (text: string): string => hideTokens(text, [this.token])
|
|
115
117
|
private readonly keychainRemote: KeychainRemote
|
|
116
118
|
constructor(
|
|
117
119
|
private readonly config: NxCacheConfig,
|
|
@@ -156,21 +158,24 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
156
158
|
// every hit read as a corrupt artifact (nx#33092).
|
|
157
159
|
headers['Accept'] = 'application/octet-stream'
|
|
158
160
|
}
|
|
159
|
-
const res = await
|
|
160
|
-
(
|
|
161
|
-
|
|
162
|
-
(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
161
|
+
const res = await hidingErrors(
|
|
162
|
+
this.breaker.send(
|
|
163
|
+
() =>
|
|
164
|
+
withRetry(
|
|
165
|
+
() =>
|
|
166
|
+
this.fetchImpl(`${this.config.url}/v1/cache/${hash}`, {
|
|
167
|
+
method,
|
|
168
|
+
headers,
|
|
169
|
+
...(body === undefined ? {} : { body }),
|
|
170
|
+
signal: deadline(this.config.timeoutMs),
|
|
171
|
+
}),
|
|
172
|
+
this.config.retries,
|
|
173
|
+
this.wait,
|
|
174
|
+
this.cooldown,
|
|
175
|
+
),
|
|
176
|
+
method === 'GET',
|
|
177
|
+
),
|
|
178
|
+
this.hidden,
|
|
174
179
|
)
|
|
175
180
|
if (method === 'PUT' && res.status === 403) {
|
|
176
181
|
const first = !this.writesRefused
|
|
@@ -178,14 +183,14 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
178
183
|
if (!first) return undefined
|
|
179
184
|
// The status and the server's own words, as turboCache() says them.
|
|
180
185
|
throw new Error(
|
|
181
|
-
`${(await refusal(res)).message}: access forbidden (a read-only token cannot write); uploads off for this run, reads go on`,
|
|
186
|
+
`${(await refusal(res, this.hidden)).message}: access forbidden (a read-only token cannot write); uploads off for this run, reads go on`,
|
|
182
187
|
)
|
|
183
188
|
}
|
|
184
189
|
if (res.status === 401 || res.status === 403) {
|
|
185
190
|
const first = !this.disabled
|
|
186
191
|
this.disabled = true
|
|
187
192
|
if (!first) return undefined
|
|
188
|
-
const said = (await refusal(res)).message
|
|
193
|
+
const said = (await refusal(res, this.hidden)).message
|
|
189
194
|
const why =
|
|
190
195
|
res.status === 401
|
|
191
196
|
? 'missing or invalid token'
|
|
@@ -200,43 +205,52 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
200
205
|
return res
|
|
201
206
|
}
|
|
202
207
|
|
|
203
|
-
async get(
|
|
208
|
+
async get(
|
|
209
|
+
hash: string,
|
|
210
|
+
): Promise<{ body: Response; durationMs: number | undefined } | null | undefined> {
|
|
204
211
|
const res = await this.fetch(hash)
|
|
205
212
|
// The Nx wire carries no producing-task duration.
|
|
206
|
-
return res === null ?
|
|
213
|
+
return res === null || res === undefined ? res : { body: res, durationMs: undefined }
|
|
207
214
|
}
|
|
208
215
|
|
|
209
|
-
|
|
210
|
-
|
|
216
|
+
/** The record; null when the server holds none, undefined when off or refused (no answer). */
|
|
217
|
+
private async fetch(hash: string): Promise<Response | null | undefined> {
|
|
218
|
+
if (this.disabled) return undefined
|
|
211
219
|
const res = await this.request('GET', hash)
|
|
212
|
-
if (res === undefined) return
|
|
220
|
+
if (res === undefined) return undefined
|
|
213
221
|
if (res.status !== 200) {
|
|
214
222
|
if (res.status === 404) {
|
|
215
223
|
await res.body?.cancel()
|
|
216
224
|
return null
|
|
217
225
|
}
|
|
218
|
-
throw await refusal(res)
|
|
226
|
+
throw await refusal(res, this.hidden)
|
|
219
227
|
}
|
|
220
228
|
return res
|
|
221
229
|
}
|
|
222
230
|
|
|
223
231
|
async has(hash: string): Promise<{ size?: number } | null> {
|
|
224
232
|
const res = await this.fetch(hash)
|
|
225
|
-
if (res === null) return null
|
|
233
|
+
if (res === null || res === undefined) return null
|
|
226
234
|
await res.body?.cancel()
|
|
227
235
|
const size = Number(res.headers.get('content-length') ?? NaN)
|
|
228
236
|
return Number.isFinite(size) ? { size } : {}
|
|
229
237
|
}
|
|
230
238
|
|
|
231
239
|
// Nx's record carries no duration.
|
|
232
|
-
async put(hash: string, body: Blob): Promise<void | false> {
|
|
240
|
+
async put(hash: string, body: Blob, meta?: { replace?: true }): Promise<void | false> {
|
|
233
241
|
// Nothing sent, so no upload is counted.
|
|
234
242
|
if (this.disabled || this.writesRefused) return false
|
|
235
243
|
const res = await this.request('PUT', hash, body)
|
|
236
244
|
if (res === undefined) return false
|
|
237
245
|
// The spec answers 200; a store behind it may say 201 or 204.
|
|
238
|
-
if (res.ok
|
|
239
|
-
|
|
246
|
+
if (res.ok) return
|
|
247
|
+
// The spec's 409 keeps the held record: fine for a fresh put, but a
|
|
248
|
+
// `replace` came to write over a copy that did not read back.
|
|
249
|
+
if (res.status === 409) {
|
|
250
|
+
await res.body?.cancel()
|
|
251
|
+
return meta?.replace === true ? false : undefined
|
|
252
|
+
}
|
|
253
|
+
throw await refusal(res, this.hidden)
|
|
240
254
|
}
|
|
241
255
|
}
|
|
242
256
|
|
package/src/packages.ts
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
// `.vx/workspace.ts` at. Core reads neither (2026-10-14): a project is a
|
|
4
4
|
// directory holding a vx.config.
|
|
5
5
|
import { constants, existsSync, type Dirent } from 'node:fs'
|
|
6
|
-
import { access, readdir, realpath, stat } from 'node:fs/promises'
|
|
6
|
+
import { access, lstat, readdir, realpath, stat } from 'node:fs/promises'
|
|
7
|
+
import { isUtf8 } from 'node:buffer'
|
|
7
8
|
import path from 'node:path'
|
|
8
9
|
import { PROJECT_CONFIG_FILENAMES, ui, UserError } from '@vzn/vx'
|
|
9
10
|
import {
|
|
@@ -483,6 +484,55 @@ export async function reachesManifest(workspace: PackageWorkspace): Promise<bool
|
|
|
483
484
|
return false
|
|
484
485
|
}
|
|
485
486
|
|
|
487
|
+
/**
|
|
488
|
+
* The member directories the globs reach whose names are not valid UTF-8,
|
|
489
|
+
* root-relative, each such byte shown as `\xNN`: a name decodes lossily to
|
|
490
|
+
* one no directory has, so its package.json reads as absent. Asked only
|
|
491
|
+
* when the globs reach none, to say why.
|
|
492
|
+
*/
|
|
493
|
+
export async function undecodableMembers(workspace: PackageWorkspace): Promise<string[]> {
|
|
494
|
+
const { positive } = splitPackageGlobs(workspace.packageGlobs)
|
|
495
|
+
const parents = new Set<string>()
|
|
496
|
+
for (const pattern of positive) {
|
|
497
|
+
for (const dir of await memberDirs(workspace.root, pattern)) {
|
|
498
|
+
// A valid name may hold U+FFFD itself; a lossy one names nothing.
|
|
499
|
+
if (!relPosix(workspace.root, dir).includes('\uFFFD')) continue
|
|
500
|
+
let at = dir
|
|
501
|
+
while ((await lstat(at).catch((err: unknown) => absent(err, null))) === null)
|
|
502
|
+
at = path.dirname(at)
|
|
503
|
+
parents.add(at)
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
const out: string[] = []
|
|
507
|
+
for (const parent of parents) {
|
|
508
|
+
for (const name of await readdir(Buffer.from(parent), { encoding: 'buffer' })) {
|
|
509
|
+
if (isUtf8(name)) continue
|
|
510
|
+
const rel = relPosix(workspace.root, parent)
|
|
511
|
+
out.push(`${rel === '' ? '' : `${rel}/`}${escapeNotUtf8(name)}`)
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return out.sort()
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/** `name` with each byte that is not part of a valid UTF-8 sequence shown as `\xNN`. */
|
|
518
|
+
function escapeNotUtf8(name: Uint8Array): string {
|
|
519
|
+
let shown = ''
|
|
520
|
+
for (let i = 0; i < name.length;) {
|
|
521
|
+
const lead = name[i]!
|
|
522
|
+
const n =
|
|
523
|
+
lead < 0x80 ? 1 : lead < 0xc2 ? 0 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf5 ? 4 : 0
|
|
524
|
+
const seq = name.subarray(i, i + n)
|
|
525
|
+
if (n > 0 && seq.length === n && isUtf8(seq)) {
|
|
526
|
+
shown += Buffer.from(seq).toString()
|
|
527
|
+
i += n
|
|
528
|
+
} else {
|
|
529
|
+
shown += `\\x${lead.toString(16).padStart(2, '0')}`
|
|
530
|
+
i++
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return shown
|
|
534
|
+
}
|
|
535
|
+
|
|
486
536
|
/**
|
|
487
537
|
* The directories a workspace glob names. For the `<dir>/*` shape this is
|
|
488
538
|
* one readdir of `<dir>` — the same answer `Bun.Glob` gives, at a third of
|