@vzn/vx-migrate 0.0.652 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -33
- package/package.json +2 -2
- package/src/migrate-scripts.ts +11 -5
- package/src/migrate-turbo.ts +8 -1
- package/src/migrate.ts +102 -18
- package/src/nx/index.ts +61 -11
- package/src/nx/nx-inputs.ts +48 -24
- package/src/nx/nx-map.ts +199 -14
- package/src/nx/nx-outputs.ts +6 -1
- package/src/nx/nx-upstream.ts +11 -11
- package/src/nx-cache/index.ts +42 -28
- package/src/packages.ts +51 -1
- package/src/remote-refusal.ts +49 -4
- package/src/shared-outputs.ts +64 -9
- package/src/tracked-outputs.ts +58 -19
- package/src/turbo/index.ts +2 -1
- package/src/turbo/turbo-map.ts +157 -22
- package/src/turbo-cache/index.ts +71 -40
- package/src/turbo-cache/oidc.ts +15 -7
- package/src/workspace-notes.ts +19 -8
- package/src/workspace-plugins.ts +11 -3
package/src/nx/nx-map.ts
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
// default configuration is folded into the base task.
|
|
18
18
|
|
|
19
19
|
import { existsSync } from 'node:fs'
|
|
20
|
+
import { lstat } from 'node:fs/promises'
|
|
20
21
|
import path from 'node:path'
|
|
21
22
|
import { type Duration, type GeneratedProject, type GeneratedTask, UserError } from '@vzn/vx'
|
|
22
23
|
import type { PackageMeta } from '../packages.js'
|
|
@@ -36,7 +37,7 @@ import {
|
|
|
36
37
|
wildcardTodo,
|
|
37
38
|
uncache,
|
|
38
39
|
} from '../shared-outputs.js'
|
|
39
|
-
import type
|
|
40
|
+
import { gitIgnored, type TrackedKinds } from '../tracked-outputs.js'
|
|
40
41
|
import { isRecord, packageScripts, relPosix } from '../paths.js'
|
|
41
42
|
import { withLernaOrder } from './lerna.js'
|
|
42
43
|
import { mapNxDeps, matchNxProjects, type TaskNameFor } from './nx-deps.js'
|
|
@@ -48,6 +49,7 @@ import {
|
|
|
48
49
|
ownerTargetOf,
|
|
49
50
|
} from './nx-dotenv.js'
|
|
50
51
|
import { emptyNxInputs, expandNxInputs } from './nx-inputs.js'
|
|
52
|
+
import { dotenvGlobsProbe } from '../dotenv-probe.js'
|
|
51
53
|
import { planNxUpstream, type NxUpstream } from './nx-upstream.js'
|
|
52
54
|
import { mapNxOutputs, nxDefaultOutputs, nxProjectOutputs } from './nx-outputs.js'
|
|
53
55
|
|
|
@@ -248,6 +250,13 @@ export interface MapNxOptions {
|
|
|
248
250
|
* `nx-exec`. The live plugin's only: a migration writes no function.
|
|
249
251
|
*/
|
|
250
252
|
readonly executors?: NxExecutors
|
|
253
|
+
/**
|
|
254
|
+
* The root lockfile when no plugin claims it (the live plugin's only):
|
|
255
|
+
* core keys no lockfile, so an `externalDependencies` input keys the
|
|
256
|
+
* file. Absent, a lockfile plugin keys the dependencies (a migration
|
|
257
|
+
* declares one).
|
|
258
|
+
*/
|
|
259
|
+
readonly lockfile?: string
|
|
251
260
|
}
|
|
252
261
|
|
|
253
262
|
/** One target of a translated executor, as the graph resolved it. */
|
|
@@ -295,6 +304,8 @@ type MapOpts = MapNxOptions & {
|
|
|
295
304
|
readonly hooks: (dir: string) => boolean
|
|
296
305
|
/** Tasks per `syncGenerators` list, said once in the notes. */
|
|
297
306
|
readonly syncTasks: Map<string, number>
|
|
307
|
+
/** Root-relative outputs git ignores (`ignoredOutputs`); unknown without git. */
|
|
308
|
+
readonly ignoredOutputs: ReadonlySet<string> | undefined
|
|
298
309
|
}
|
|
299
310
|
|
|
300
311
|
export interface NxMapping {
|
|
@@ -320,6 +331,46 @@ function nxTags(node: NxNode | undefined): string[] {
|
|
|
320
331
|
: []
|
|
321
332
|
}
|
|
322
333
|
|
|
334
|
+
/**
|
|
335
|
+
* The outputs of every project whose input Nx never hashes: git tracks
|
|
336
|
+
* nothing under it and ignores it, as Nx's file map skips gitignored files
|
|
337
|
+
* (a changed `dist` file is a hit under Nx 23.3, YL-1). One untracked but
|
|
338
|
+
* not ignored is in Nx's file map, so an input under it is hashed: taken
|
|
339
|
+
* for ignored, `build` keyed on no file and a generated source's edit
|
|
340
|
+
* replayed (stale hit). One `git check-ignore` per mapping.
|
|
341
|
+
*/
|
|
342
|
+
async function ignoredOutputs(
|
|
343
|
+
root: string,
|
|
344
|
+
nodes: Readonly<Record<string, NxNode>>,
|
|
345
|
+
tracked: (rel: string) => TrackedKinds,
|
|
346
|
+
): Promise<ReadonlySet<string>> {
|
|
347
|
+
const untracked = new Set<string>()
|
|
348
|
+
for (const [nodeName, node] of Object.entries(nodes)) {
|
|
349
|
+
const rel = normRel(node?.data?.root ?? '')
|
|
350
|
+
for (const o of nxProjectOutputs(node?.data?.targets, rel || '.', node?.name ?? nodeName)) {
|
|
351
|
+
const dir = path.posix.dirname(o)
|
|
352
|
+
if (
|
|
353
|
+
tracked(o).files.length === 0 &&
|
|
354
|
+
!tracked(dir).files.includes(dir === '.' ? o : o.slice(dir.length + 1))
|
|
355
|
+
)
|
|
356
|
+
untracked.add(o)
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// A missing `dist` matches a `dist/` rule only spelled as a directory;
|
|
360
|
+
// an existing file never does.
|
|
361
|
+
const hit = await gitIgnored(
|
|
362
|
+
root,
|
|
363
|
+
[...untracked].flatMap((o) => [o, `${o}/`]),
|
|
364
|
+
)
|
|
365
|
+
const out = new Set<string>()
|
|
366
|
+
for (const o of untracked) {
|
|
367
|
+
if (hit.has(o)) out.add(o)
|
|
368
|
+
else if (hit.has(`${o}/`) && !(await lstat(path.join(root, o)).catch(() => null))?.isFile())
|
|
369
|
+
out.add(o)
|
|
370
|
+
}
|
|
371
|
+
return out
|
|
372
|
+
}
|
|
373
|
+
|
|
323
374
|
function normRel(p: string): string {
|
|
324
375
|
return p === '' || p === '.' ? '.' : p.replace(/\/$/, '')
|
|
325
376
|
}
|
|
@@ -355,6 +406,8 @@ export async function mapNxWorkspace(
|
|
|
355
406
|
pnp: yarnPnp(root),
|
|
356
407
|
hooks: (dir) => runsScriptHooks(dir, hookMemo) !== null,
|
|
357
408
|
syncTasks: new Map(),
|
|
409
|
+
ignoredOutputs:
|
|
410
|
+
opts.tracked === undefined ? undefined : await ignoredOutputs(root, nodeMap, opts.tracked),
|
|
358
411
|
}
|
|
359
412
|
|
|
360
413
|
const metaByRel = new Map<string, PackageMeta>()
|
|
@@ -409,7 +462,13 @@ export async function mapNxWorkspace(
|
|
|
409
462
|
nodeNameOf.set(synthetic, nodeName)
|
|
410
463
|
}
|
|
411
464
|
|
|
412
|
-
const upstream = planNxUpstream(
|
|
465
|
+
const upstream = planNxUpstream(
|
|
466
|
+
nodeMap,
|
|
467
|
+
g.dependencies,
|
|
468
|
+
namedInputs,
|
|
469
|
+
metaByNode,
|
|
470
|
+
mapOpts.ignoredOutputs,
|
|
471
|
+
)
|
|
413
472
|
|
|
414
473
|
const taskNameFor: TaskNameFor = (project, target, configuration) => {
|
|
415
474
|
const t = nodeMap[project]?.data?.targets?.[target]
|
|
@@ -596,7 +655,14 @@ export async function mapNxWorkspace(
|
|
|
596
655
|
}
|
|
597
656
|
dropUnheldDeps(t, meta.name, emitted, emittedIds)
|
|
598
657
|
const reader = dependentReaders.get(t)
|
|
599
|
-
if (
|
|
658
|
+
if (
|
|
659
|
+
reader !== undefined &&
|
|
660
|
+
!reachesProducer(t, meta.name, emittedIds, reached, (target) =>
|
|
661
|
+
nodeName === undefined
|
|
662
|
+
? undefined
|
|
663
|
+
: nxEdges(nodeName, target).map((n) => metaByNode.get(n)?.name),
|
|
664
|
+
)
|
|
665
|
+
) {
|
|
600
666
|
uncache(t.task!)
|
|
601
667
|
t.todos.push(orphanReaderTodo(reader.globs, reader.dependsOn))
|
|
602
668
|
}
|
|
@@ -613,6 +679,7 @@ export async function mapNxWorkspace(
|
|
|
613
679
|
|
|
614
680
|
resolveAtomizedWorkspaceOutputs(root, projects, split)
|
|
615
681
|
excludeWorkspaceOutputs(root, projects)
|
|
682
|
+
keyUncachedProducerOutputs(projects, reached)
|
|
616
683
|
// A migration leaves Nx behind, so `nx sync` is no advice there: what a
|
|
617
684
|
// generator keeps (the TypeScript one's tsconfig `references`) is the
|
|
618
685
|
// user's to keep. `nx()` runs with Nx installed.
|
|
@@ -863,8 +930,9 @@ function buildTask(
|
|
|
863
930
|
rel: projectRel,
|
|
864
931
|
name: projectName,
|
|
865
932
|
outputs: nxProjectOutputs(nodeMap[nodeName]?.data?.targets, projectRel, projectName),
|
|
866
|
-
...(opts.
|
|
933
|
+
...(opts.ignoredOutputs === undefined ? {} : { ignoredOutputs: opts.ignoredOutputs }),
|
|
867
934
|
rootDeps: opts.rootDeps,
|
|
935
|
+
...(opts.lockfile === undefined ? {} : { lockfile: opts.lockfile }),
|
|
868
936
|
}
|
|
869
937
|
expandNxInputs(target.inputs ?? [], upstream.namedOf(nodeName), at, inputs, todos)
|
|
870
938
|
const cacheWanted =
|
|
@@ -1065,37 +1133,154 @@ function buildTask(
|
|
|
1065
1133
|
|
|
1066
1134
|
const generated: GeneratedTask = { name: variant.name, todos, task }
|
|
1067
1135
|
if (cacheEnabled && inputs.dependentOutputs.length > 0)
|
|
1068
|
-
dependentReaders.set(generated, {
|
|
1136
|
+
dependentReaders.set(generated, {
|
|
1137
|
+
globs: inputs.dependentOutputs,
|
|
1138
|
+
transitive: inputs.transitiveOutputs,
|
|
1139
|
+
dependsOn: target.dependsOn,
|
|
1140
|
+
})
|
|
1141
|
+
const produced = [
|
|
1142
|
+
...outFiles.filter((o) => !o.startsWith('!')).map((o) => path.posix.join(projectRel, o)),
|
|
1143
|
+
...wsOutFiles.filter((o) => !o.startsWith('!')),
|
|
1144
|
+
]
|
|
1145
|
+
if (produced.length > 0) producedPaths.set(generated, produced)
|
|
1069
1146
|
return generated
|
|
1070
1147
|
}
|
|
1071
1148
|
|
|
1149
|
+
/** Each task's Nx outputs, workspace-relative, cached or not: what a reader of them probes. */
|
|
1150
|
+
const producedPaths = new WeakMap<GeneratedTask, string[]>()
|
|
1151
|
+
|
|
1072
1152
|
/**
|
|
1073
|
-
*
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1153
|
+
* Nx hashes the files a `dependentTasksOutputFiles` reader names in its
|
|
1154
|
+
* producers' outputs; vx folds each producer's key, which covers what a
|
|
1155
|
+
* cached producer writes and not what an uncached one fetched from outside
|
|
1156
|
+
* (a `cache: false` `fetch` writing a gitignored schema): the reader hit
|
|
1157
|
+
* and replayed the old schema's code. Those files are keyed by a probe at
|
|
1158
|
+
* the root, as `turbo()` keys a `dependencyOutputs` input of an uncached
|
|
1159
|
+
* producer, and the producer is left out of the fold (`inputs.tasks`), so
|
|
1160
|
+
* core answers the probe after it ran.
|
|
1078
1161
|
*/
|
|
1079
|
-
|
|
1162
|
+
function keyUncachedProducerOutputs(
|
|
1163
|
+
projects: readonly GeneratedProject[],
|
|
1164
|
+
reached: (name: string) => ReadonlySet<string>,
|
|
1165
|
+
): void {
|
|
1166
|
+
const byId = new Map<string, { project: string; task: GeneratedTask }>()
|
|
1167
|
+
for (const p of projects)
|
|
1168
|
+
for (const t of p.tasks) byId.set(`${p.name}#${t.name}`, { project: p.name, task: t })
|
|
1169
|
+
const edgesOf = (project: string, deps: unknown): [id: string, filter: string][] =>
|
|
1170
|
+
(Array.isArray(deps) ? deps : []).flatMap((d): [string, string][] => {
|
|
1171
|
+
if (typeof d !== 'string' || d.includes('*') || d.includes('#inputs:')) return []
|
|
1172
|
+
return d.startsWith('^')
|
|
1173
|
+
? [...reached(project)].map((q) => [`${q}#${d.slice(1)}`, `${q}#${d.slice(1)}`])
|
|
1174
|
+
: [[d.includes('#') ? d : `${project}#${d}`, d]]
|
|
1175
|
+
})
|
|
1176
|
+
for (const p of projects) {
|
|
1177
|
+
for (const t of p.tasks) {
|
|
1178
|
+
const reader = dependentReaders.get(t)
|
|
1179
|
+
if (reader === undefined || t.task?.['cache'] !== true) continue
|
|
1180
|
+
const rels: string[] = []
|
|
1181
|
+
const unfold: string[] = []
|
|
1182
|
+
const keyOutputs = (producer: GeneratedTask, globs: readonly string[]): void => {
|
|
1183
|
+
for (const o of producedPaths.get(producer) ?? [])
|
|
1184
|
+
// A glob output (`dist/**/*.{js,d.ts}{,.map}`): every file under
|
|
1185
|
+
// its literal head, which the probe can walk.
|
|
1186
|
+
if (/[*?[{]/.test(o)) rels.push(`${globHead(o)}/**/*`)
|
|
1187
|
+
else
|
|
1188
|
+
for (const g of globs)
|
|
1189
|
+
// `**/<name>` is a find under the output; any other glob (a brace, a
|
|
1190
|
+
// class), every file there.
|
|
1191
|
+
rels.push(/^\*\*\/[^/*?[{]*\*?[^/*?[{]*$/.test(g) ? `${o}/${g}` : `${o}/**/*`)
|
|
1192
|
+
}
|
|
1193
|
+
const uncachedWriter = (producer: GeneratedTask | undefined): producer is GeneratedTask =>
|
|
1194
|
+
producer?.task?.['exec'] !== undefined &&
|
|
1195
|
+
producer.task['cache'] !== true &&
|
|
1196
|
+
(producedPaths.get(producer)?.length ?? 0) > 0
|
|
1197
|
+
for (const [id, filter] of edgesOf(p.name, t.task['dependsOn'])) {
|
|
1198
|
+
const producer = byId.get(id)?.task
|
|
1199
|
+
if (producer === undefined) continue
|
|
1200
|
+
// `transitive: true` reads the outputs of every task below, not
|
|
1201
|
+
// only the direct ones: an uncached `fetch` behind an `nx:noop`
|
|
1202
|
+
// `prep` hid its schema, and the reader replayed the old one (hunt,
|
|
1203
|
+
// 2026-11-08). A key folding the edge answers the probe before the
|
|
1204
|
+
// fetch ran, so the edge is left out and every output below it,
|
|
1205
|
+
// cached or not, is keyed by the probe.
|
|
1206
|
+
const below: GeneratedTask[] = []
|
|
1207
|
+
if (reader.transitive.length > 0) {
|
|
1208
|
+
const seen = new Set<string>([id])
|
|
1209
|
+
const stack = edgesOf(byId.get(id)!.project, producer.task?.['dependsOn'])
|
|
1210
|
+
while (stack.length > 0) {
|
|
1211
|
+
const [next] = stack.pop()!
|
|
1212
|
+
const node = byId.get(next)
|
|
1213
|
+
if (seen.has(next) || node === undefined) continue
|
|
1214
|
+
seen.add(next)
|
|
1215
|
+
below.push(node.task)
|
|
1216
|
+
stack.push(...edgesOf(node.project, node.task.task?.['dependsOn']))
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
if (!uncachedWriter(producer) && !below.some(uncachedWriter)) continue
|
|
1220
|
+
unfold.push(`!${filter}`)
|
|
1221
|
+
keyOutputs(producer, reader.globs)
|
|
1222
|
+
for (const b of below) keyOutputs(b, reader.transitive)
|
|
1223
|
+
}
|
|
1224
|
+
if (rels.length === 0) continue
|
|
1225
|
+
const probe = dotenvGlobsProbe([...new Set(rels)])
|
|
1226
|
+
if (probe === null) {
|
|
1227
|
+
uncache(t.task)
|
|
1228
|
+
t.todos.push(
|
|
1229
|
+
`reads ${reader.globs.map((g) => `\`${g}\``).join(', ')} from the outputs of an uncached task it depends on, ` +
|
|
1230
|
+
'which vx cannot key by a probe — uncached here; key it in a vx.config to cache it',
|
|
1231
|
+
)
|
|
1232
|
+
continue
|
|
1233
|
+
}
|
|
1234
|
+
const inputs = t.task['inputs'] as Record<string, unknown>
|
|
1235
|
+
inputs['workspaceRuntime'] = [...((inputs['workspaceRuntime'] as unknown[]) ?? []), probe]
|
|
1236
|
+
inputs['tasks'] = ['*', '^*', ...unfold]
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
function globHead(glob: string): string {
|
|
1242
|
+
const segs = glob.split('/')
|
|
1243
|
+
const i = segs.findIndex((s) => /[*?[{]/.test(s))
|
|
1244
|
+
return segs.slice(0, i).join('/')
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
const dependentReaders = new WeakMap<
|
|
1248
|
+
GeneratedTask,
|
|
1249
|
+
{ globs: string[]; transitive: string[]; dependsOn: unknown }
|
|
1250
|
+
>()
|
|
1080
1251
|
|
|
1081
1252
|
function orphanReaderTodo(globs: readonly string[], dependsOn: unknown): string {
|
|
1082
1253
|
return `reads ${globs.map((g) => `\`${g}\``).join(', ')} from the outputs of the tasks it depends on, and no task it depends on is migrated (dependsOn ${JSON.stringify(dependsOn ?? [])}) — uncached here, so it never replays output generated from an older input; migrate the producer and depend on it to cache this task`
|
|
1083
1254
|
}
|
|
1084
1255
|
|
|
1085
|
-
/**
|
|
1256
|
+
/**
|
|
1257
|
+
* Whether a cached reader of `dependentTasksOutputFiles` still reaches a
|
|
1258
|
+
* producer once the graph has its final edges: one of them reaches a task
|
|
1259
|
+
* vx runs, or is a `^` edge every project Nx links (`nxLinked`: their vx
|
|
1260
|
+
* names) runs it on. A `^build` kept because another project builds, while
|
|
1261
|
+
* Nx links a producer vx carries no task for, keyed nothing the task reads
|
|
1262
|
+
* (Growth, nx-codegen-dropped-producer.sh). With none linked, Nx hashes
|
|
1263
|
+
* nothing there either: a leaf library's inferred `build` (`^build`,
|
|
1264
|
+
* @nx/js/typescript) ran uncached every time where Nx caches it (hunt,
|
|
1265
|
+
* 2026-11-08).
|
|
1266
|
+
*/
|
|
1086
1267
|
function reachesProducer(
|
|
1087
1268
|
t: GeneratedTask,
|
|
1088
1269
|
project: string,
|
|
1089
1270
|
emittedIds: ReadonlySet<string>,
|
|
1090
1271
|
reached: (name: string) => ReadonlySet<string>,
|
|
1272
|
+
nxLinked: (target: string) => ReadonlyArray<string | undefined> | undefined,
|
|
1091
1273
|
): boolean {
|
|
1092
1274
|
const deps = t.task?.['dependsOn']
|
|
1093
1275
|
if (!Array.isArray(deps)) return false
|
|
1094
1276
|
return deps.some((d) => {
|
|
1095
1277
|
if (typeof d !== 'string' || d.includes('*')) return true
|
|
1096
1278
|
if (d.includes('#inputs:')) return false
|
|
1097
|
-
if (d.startsWith('^'))
|
|
1098
|
-
|
|
1279
|
+
if (d.startsWith('^')) {
|
|
1280
|
+
const target = d.slice(1)
|
|
1281
|
+
if ([...reached(project)].some((p) => emittedIds.has(`${p}#${target}`))) return true
|
|
1282
|
+
return nxLinked(target)?.every((p) => emittedIds.has(`${p}#${target}`)) === true
|
|
1283
|
+
}
|
|
1099
1284
|
return d.includes('#') || emittedIds.has(`${project}#${d}`)
|
|
1100
1285
|
})
|
|
1101
1286
|
}
|
package/src/nx/nx-outputs.ts
CHANGED
|
@@ -189,6 +189,8 @@ export function mapNxOutputs(
|
|
|
189
189
|
)
|
|
190
190
|
continue
|
|
191
191
|
}
|
|
192
|
+
const rootWide =
|
|
193
|
+
(projectRel === '' || projectRel === '.') && o.slice(neg.length).startsWith('{workspaceRoot}')
|
|
192
194
|
const globs = vxOutputGlobs(neg + s)
|
|
193
195
|
if (globs === null) {
|
|
194
196
|
todos.push(`output ${JSON.stringify(o)} has a glob form vx cannot spell — dropped`)
|
|
@@ -197,7 +199,10 @@ export function mapNxOutputs(
|
|
|
197
199
|
for (const g of globs) {
|
|
198
200
|
const gn = g.startsWith('!') ? '!' : ''
|
|
199
201
|
const gs = g.slice(gn.length)
|
|
200
|
-
|
|
202
|
+
// As the root project's inputs map (nx-inputs.ts): a `{workspaceRoot}`
|
|
203
|
+
// output names a workspace path, which may lie in a nested project a
|
|
204
|
+
// project glob never reaches (core refuses one there).
|
|
205
|
+
const own = rootWide ? null : underProject(gs, projectRel)
|
|
201
206
|
if (own !== null) {
|
|
202
207
|
outFiles.push(gn + (own === '' ? '**' : withProjectDir(own, projectRel, true)))
|
|
203
208
|
continue
|
package/src/nx/nx-upstream.ts
CHANGED
|
@@ -18,11 +18,10 @@
|
|
|
18
18
|
// and edges only out of the cycle.
|
|
19
19
|
import type { GeneratedTask } from '@vzn/vx'
|
|
20
20
|
import type { PackageMeta } from '../packages.js'
|
|
21
|
-
import type { TrackedKinds } from '../tracked-outputs.js'
|
|
22
21
|
import { emptyNxInputs, expandNxInputs, type NxInputs } from './nx-inputs.js'
|
|
23
22
|
import { nxProjectOutputs } from './nx-outputs.js'
|
|
24
23
|
import { matchNxProjects } from './nx-deps.js'
|
|
25
|
-
import { literalGlob } from '../glob-grammar.js'
|
|
24
|
+
import { apartTakeBacks, literalGlob } from '../glob-grammar.js'
|
|
26
25
|
|
|
27
26
|
interface GraphNode {
|
|
28
27
|
name?: string
|
|
@@ -73,7 +72,7 @@ export function planNxUpstream(
|
|
|
73
72
|
dependencies: unknown,
|
|
74
73
|
workspaceNamed: Readonly<Record<string, unknown[]>> | null,
|
|
75
74
|
metaByNode: ReadonlyMap<string, PackageMeta>,
|
|
76
|
-
|
|
75
|
+
ignoredOutputs?: ReadonlySet<string>,
|
|
77
76
|
): NxUpstream {
|
|
78
77
|
const isProject = (n: string): boolean => typeof nodes[n]?.data?.root === 'string'
|
|
79
78
|
const projectTags = Object.entries(nodes).map(([name, n]) => ({
|
|
@@ -194,13 +193,12 @@ export function planNxUpstream(
|
|
|
194
193
|
expandOwn(node, name, own, todos)
|
|
195
194
|
const rel = normRel(nodes[node]?.data?.root ?? '')
|
|
196
195
|
const inputs = emptyNxInputs()
|
|
197
|
-
|
|
196
|
+
const files = own.files.map((g) => {
|
|
198
197
|
const neg = g.startsWith('!') ? '!' : ''
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
inputs.wsFiles.push(...own.wsFiles)
|
|
198
|
+
return rel === '' ? g : `${neg}${literalGlob(rel)}/${neg === '' ? g : g.slice(1)}`
|
|
199
|
+
})
|
|
200
|
+
// Two of Nx's sets in one list, as in `expandNxInputs`.
|
|
201
|
+
inputs.wsFiles.push(...apartTakeBacks(files, own.wsFiles).flat())
|
|
204
202
|
inputs.envNames.push(...own.envNames)
|
|
205
203
|
inputs.runtimeCmds.push(...own.runtimeCmds)
|
|
206
204
|
inputs.upstream.push(...own.upstream)
|
|
@@ -223,13 +221,15 @@ export function planNxUpstream(
|
|
|
223
221
|
rel,
|
|
224
222
|
name: nm,
|
|
225
223
|
outputs: nxProjectOutputs(nodes[node]?.data?.targets, rel || '.', nm),
|
|
226
|
-
...(
|
|
224
|
+
...(ignoredOutputs === undefined ? {} : { ignoredOutputs }),
|
|
227
225
|
}
|
|
228
226
|
expandNxInputs([name], namedOf(node), at, into, todos)
|
|
229
227
|
}
|
|
230
228
|
|
|
231
229
|
const merge = (from: NxInputs, into: NxInputs): void => {
|
|
232
|
-
|
|
230
|
+
// A node's sets meet the reader's in one list: a `!` reaching another
|
|
231
|
+
// set's entry goes, as in `expandNxInputs`.
|
|
232
|
+
into.wsFiles.splice(0, Infinity, ...apartTakeBacks(into.wsFiles, from.wsFiles).flat())
|
|
233
233
|
for (const n of from.envNames) if (!into.envNames.includes(n)) into.envNames.push(n)
|
|
234
234
|
for (const c of from.runtimeCmds) if (!into.runtimeCmds.includes(c)) into.runtimeCmds.push(c)
|
|
235
235
|
}
|
package/src/nx-cache/index.ts
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
} from '@vzn/vx'
|
|
23
23
|
import { deadline, MAX_TIMER_MS } from '../remote-deadline.js'
|
|
24
24
|
import { checkRemoteUrl } from '../remote-url.js'
|
|
25
|
-
import { refusal } from '../remote-refusal.js'
|
|
25
|
+
import { hideTokens, hidingErrors, refusal } from '../remote-refusal.js'
|
|
26
26
|
import { OutageBreaker } from '../remote-breaker.js'
|
|
27
27
|
import { Cooldown, withRetry } from '../remote-retry.js'
|
|
28
28
|
import {
|
|
@@ -112,6 +112,8 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
112
112
|
private token: string | undefined
|
|
113
113
|
/** Set by `open` when the token came from the keychain: a refusal deletes it. */
|
|
114
114
|
private forget: (() => Promise<void>) | undefined
|
|
115
|
+
/** The token as a refusal that quotes it says it: `***`. */
|
|
116
|
+
private readonly hidden = (text: string): string => hideTokens(text, [this.token])
|
|
115
117
|
private readonly keychainRemote: KeychainRemote
|
|
116
118
|
constructor(
|
|
117
119
|
private readonly config: NxCacheConfig,
|
|
@@ -156,21 +158,24 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
156
158
|
// every hit read as a corrupt artifact (nx#33092).
|
|
157
159
|
headers['Accept'] = 'application/octet-stream'
|
|
158
160
|
}
|
|
159
|
-
const res = await
|
|
160
|
-
(
|
|
161
|
-
|
|
162
|
-
(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
161
|
+
const res = await hidingErrors(
|
|
162
|
+
this.breaker.send(
|
|
163
|
+
() =>
|
|
164
|
+
withRetry(
|
|
165
|
+
() =>
|
|
166
|
+
this.fetchImpl(`${this.config.url}/v1/cache/${hash}`, {
|
|
167
|
+
method,
|
|
168
|
+
headers,
|
|
169
|
+
...(body === undefined ? {} : { body }),
|
|
170
|
+
signal: deadline(this.config.timeoutMs),
|
|
171
|
+
}),
|
|
172
|
+
this.config.retries,
|
|
173
|
+
this.wait,
|
|
174
|
+
this.cooldown,
|
|
175
|
+
),
|
|
176
|
+
method === 'GET',
|
|
177
|
+
),
|
|
178
|
+
this.hidden,
|
|
174
179
|
)
|
|
175
180
|
if (method === 'PUT' && res.status === 403) {
|
|
176
181
|
const first = !this.writesRefused
|
|
@@ -178,14 +183,14 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
178
183
|
if (!first) return undefined
|
|
179
184
|
// The status and the server's own words, as turboCache() says them.
|
|
180
185
|
throw new Error(
|
|
181
|
-
`${(await refusal(res)).message}: access forbidden (a read-only token cannot write); uploads off for this run, reads go on`,
|
|
186
|
+
`${(await refusal(res, this.hidden)).message}: access forbidden (a read-only token cannot write); uploads off for this run, reads go on`,
|
|
182
187
|
)
|
|
183
188
|
}
|
|
184
189
|
if (res.status === 401 || res.status === 403) {
|
|
185
190
|
const first = !this.disabled
|
|
186
191
|
this.disabled = true
|
|
187
192
|
if (!first) return undefined
|
|
188
|
-
const said = (await refusal(res)).message
|
|
193
|
+
const said = (await refusal(res, this.hidden)).message
|
|
189
194
|
const why =
|
|
190
195
|
res.status === 401
|
|
191
196
|
? 'missing or invalid token'
|
|
@@ -200,43 +205,52 @@ export class NxRemoteCache implements RemoteCacheLayer {
|
|
|
200
205
|
return res
|
|
201
206
|
}
|
|
202
207
|
|
|
203
|
-
async get(
|
|
208
|
+
async get(
|
|
209
|
+
hash: string,
|
|
210
|
+
): Promise<{ body: Response; durationMs: number | undefined } | null | undefined> {
|
|
204
211
|
const res = await this.fetch(hash)
|
|
205
212
|
// The Nx wire carries no producing-task duration.
|
|
206
|
-
return res === null ?
|
|
213
|
+
return res === null || res === undefined ? res : { body: res, durationMs: undefined }
|
|
207
214
|
}
|
|
208
215
|
|
|
209
|
-
|
|
210
|
-
|
|
216
|
+
/** The record; null when the server holds none, undefined when off or refused (no answer). */
|
|
217
|
+
private async fetch(hash: string): Promise<Response | null | undefined> {
|
|
218
|
+
if (this.disabled) return undefined
|
|
211
219
|
const res = await this.request('GET', hash)
|
|
212
|
-
if (res === undefined) return
|
|
220
|
+
if (res === undefined) return undefined
|
|
213
221
|
if (res.status !== 200) {
|
|
214
222
|
if (res.status === 404) {
|
|
215
223
|
await res.body?.cancel()
|
|
216
224
|
return null
|
|
217
225
|
}
|
|
218
|
-
throw await refusal(res)
|
|
226
|
+
throw await refusal(res, this.hidden)
|
|
219
227
|
}
|
|
220
228
|
return res
|
|
221
229
|
}
|
|
222
230
|
|
|
223
231
|
async has(hash: string): Promise<{ size?: number } | null> {
|
|
224
232
|
const res = await this.fetch(hash)
|
|
225
|
-
if (res === null) return null
|
|
233
|
+
if (res === null || res === undefined) return null
|
|
226
234
|
await res.body?.cancel()
|
|
227
235
|
const size = Number(res.headers.get('content-length') ?? NaN)
|
|
228
236
|
return Number.isFinite(size) ? { size } : {}
|
|
229
237
|
}
|
|
230
238
|
|
|
231
239
|
// Nx's record carries no duration.
|
|
232
|
-
async put(hash: string, body: Blob): Promise<void | false> {
|
|
240
|
+
async put(hash: string, body: Blob, meta?: { replace?: true }): Promise<void | false> {
|
|
233
241
|
// Nothing sent, so no upload is counted.
|
|
234
242
|
if (this.disabled || this.writesRefused) return false
|
|
235
243
|
const res = await this.request('PUT', hash, body)
|
|
236
244
|
if (res === undefined) return false
|
|
237
245
|
// The spec answers 200; a store behind it may say 201 or 204.
|
|
238
|
-
if (res.ok
|
|
239
|
-
|
|
246
|
+
if (res.ok) return
|
|
247
|
+
// The spec's 409 keeps the held record: fine for a fresh put, but a
|
|
248
|
+
// `replace` came to write over a copy that did not read back.
|
|
249
|
+
if (res.status === 409) {
|
|
250
|
+
await res.body?.cancel()
|
|
251
|
+
return meta?.replace === true ? false : undefined
|
|
252
|
+
}
|
|
253
|
+
throw await refusal(res, this.hidden)
|
|
240
254
|
}
|
|
241
255
|
}
|
|
242
256
|
|
package/src/packages.ts
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
// `.vx/workspace.ts` at. Core reads neither (2026-10-14): a project is a
|
|
4
4
|
// directory holding a vx.config.
|
|
5
5
|
import { constants, existsSync, type Dirent } from 'node:fs'
|
|
6
|
-
import { access, readdir, realpath, stat } from 'node:fs/promises'
|
|
6
|
+
import { access, lstat, readdir, realpath, stat } from 'node:fs/promises'
|
|
7
|
+
import { isUtf8 } from 'node:buffer'
|
|
7
8
|
import path from 'node:path'
|
|
8
9
|
import { PROJECT_CONFIG_FILENAMES, ui, UserError } from '@vzn/vx'
|
|
9
10
|
import {
|
|
@@ -483,6 +484,55 @@ export async function reachesManifest(workspace: PackageWorkspace): Promise<bool
|
|
|
483
484
|
return false
|
|
484
485
|
}
|
|
485
486
|
|
|
487
|
+
/**
|
|
488
|
+
* The member directories the globs reach whose names are not valid UTF-8,
|
|
489
|
+
* root-relative, each such byte shown as `\xNN`: a name decodes lossily to
|
|
490
|
+
* one no directory has, so its package.json reads as absent. Asked only
|
|
491
|
+
* when the globs reach none, to say why.
|
|
492
|
+
*/
|
|
493
|
+
export async function undecodableMembers(workspace: PackageWorkspace): Promise<string[]> {
|
|
494
|
+
const { positive } = splitPackageGlobs(workspace.packageGlobs)
|
|
495
|
+
const parents = new Set<string>()
|
|
496
|
+
for (const pattern of positive) {
|
|
497
|
+
for (const dir of await memberDirs(workspace.root, pattern)) {
|
|
498
|
+
// A valid name may hold U+FFFD itself; a lossy one names nothing.
|
|
499
|
+
if (!relPosix(workspace.root, dir).includes('\uFFFD')) continue
|
|
500
|
+
let at = dir
|
|
501
|
+
while ((await lstat(at).catch((err: unknown) => absent(err, null))) === null)
|
|
502
|
+
at = path.dirname(at)
|
|
503
|
+
parents.add(at)
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
const out: string[] = []
|
|
507
|
+
for (const parent of parents) {
|
|
508
|
+
for (const name of await readdir(Buffer.from(parent), { encoding: 'buffer' })) {
|
|
509
|
+
if (isUtf8(name)) continue
|
|
510
|
+
const rel = relPosix(workspace.root, parent)
|
|
511
|
+
out.push(`${rel === '' ? '' : `${rel}/`}${escapeNotUtf8(name)}`)
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return out.sort()
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/** `name` with each byte that is not part of a valid UTF-8 sequence shown as `\xNN`. */
|
|
518
|
+
function escapeNotUtf8(name: Uint8Array): string {
|
|
519
|
+
let shown = ''
|
|
520
|
+
for (let i = 0; i < name.length;) {
|
|
521
|
+
const lead = name[i]!
|
|
522
|
+
const n =
|
|
523
|
+
lead < 0x80 ? 1 : lead < 0xc2 ? 0 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf5 ? 4 : 0
|
|
524
|
+
const seq = name.subarray(i, i + n)
|
|
525
|
+
if (n > 0 && seq.length === n && isUtf8(seq)) {
|
|
526
|
+
shown += Buffer.from(seq).toString()
|
|
527
|
+
i += n
|
|
528
|
+
} else {
|
|
529
|
+
shown += `\\x${lead.toString(16).padStart(2, '0')}`
|
|
530
|
+
i++
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return shown
|
|
534
|
+
}
|
|
535
|
+
|
|
486
536
|
/**
|
|
487
537
|
* The directories a workspace glob names. For the `<dir>/*` shape this is
|
|
488
538
|
* one readdir of `<dir>` — the same answer `Bun.Glob` gives, at a third of
|
package/src/remote-refusal.ts
CHANGED
|
@@ -1,14 +1,57 @@
|
|
|
1
1
|
/** How long the body's first chunk is waited for: a refusal is not held on a body that never comes. */
|
|
2
2
|
const BODY_WAIT_MS = 1_000
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* The most of the first chunk read: a token that starts within the line's
|
|
6
|
+
* 200 characters must end within it to be hidden whole (a JWT is ~1 KB).
|
|
7
|
+
*/
|
|
8
|
+
const READ_BYTES = 4096
|
|
9
|
+
|
|
10
|
+
/** Shorter values are not hidden, as core masks none under 6 characters: each would blank common words. */
|
|
11
|
+
const MIN_TOKEN_CHARS = 6
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* `text` with each of `tokens` written `***`: a server that quoted a
|
|
15
|
+
* request's Authorization header printed the token to the CI log and
|
|
16
|
+
* cache.db, as core masks only secret-named variables.
|
|
17
|
+
*/
|
|
18
|
+
export function hideTokens(text: string, tokens: Iterable<string | undefined>): string {
|
|
19
|
+
let out = text
|
|
20
|
+
for (const token of tokens)
|
|
21
|
+
if (token !== undefined && token.length >= MIN_TOKEN_CHARS) out = out.replaceAll(token, '***')
|
|
22
|
+
return out
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** `work`, each error it throws with `hide` applied: a thrown fetch error can quote a request's header. */
|
|
26
|
+
export async function hidingErrors<T>(
|
|
27
|
+
work: Promise<T>,
|
|
28
|
+
hide: (text: string) => string,
|
|
29
|
+
): Promise<T> {
|
|
30
|
+
try {
|
|
31
|
+
return await work
|
|
32
|
+
} catch (err) {
|
|
33
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
34
|
+
const hidden = hide(message)
|
|
35
|
+
if (hidden === message) throw err
|
|
36
|
+
// The code is how core groups a failure (a reset connection), and a retry reads it.
|
|
37
|
+
const code = (err as { code?: unknown } | null)?.code
|
|
38
|
+
throw Object.assign(new Error(hidden), code === undefined ? {} : { code })
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
4
42
|
/**
|
|
5
43
|
* A remote's refusal, named by its status and its own words: "HTTP 500"
|
|
6
44
|
* alone said nothing, and a batch query Vercel refused with 400 "Too many
|
|
7
45
|
* hashes. Limit is 100." read as no answer and was never seen (main
|
|
8
|
-
* f71b377, 2026-10-23). One line of the body, bounded: it is remote text
|
|
9
|
-
*
|
|
46
|
+
* f71b377, 2026-10-23). One line of the body, bounded: it is remote text,
|
|
47
|
+
* each token `hide` knows written `***` before the cut, so none is left in
|
|
48
|
+
* part. Shared by `turboCache()` and `nxCache()`.
|
|
10
49
|
*/
|
|
11
|
-
export async function refusal(
|
|
50
|
+
export async function refusal(
|
|
51
|
+
res: Response,
|
|
52
|
+
hide: (text: string) => string,
|
|
53
|
+
what = '',
|
|
54
|
+
): Promise<Error> {
|
|
12
55
|
let said = ''
|
|
13
56
|
try {
|
|
14
57
|
const reader = res.body?.getReader()
|
|
@@ -22,7 +65,9 @@ export async function refusal(res: Response, what = ''): Promise<Error> {
|
|
|
22
65
|
]).finally(() => clearTimeout(timer))
|
|
23
66
|
// quiet: read no further; a cancel that fails has nothing left to free
|
|
24
67
|
void reader.cancel().catch(() => undefined)
|
|
25
|
-
said = new TextDecoder().decode(first?.value?.subarray(0,
|
|
68
|
+
said = hide(new TextDecoder().decode(first?.value?.subarray(0, READ_BYTES)))
|
|
69
|
+
.replace(/\s+/g, ' ')
|
|
70
|
+
.trim()
|
|
26
71
|
}
|
|
27
72
|
} catch {
|
|
28
73
|
// quiet: the status alone still names the refusal
|