@vzn/vx 0.0.616 → 0.0.618
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/cache/archive.ts +17 -0
- package/src/cache/cache.ts +2 -1
- package/src/cache/inputs.ts +11 -2
- package/src/cache/output-index.ts +24 -1
- package/src/graph/priorities.ts +11 -1
- package/src/graph/scheduler.ts +17 -4
- package/src/orchestrator/hit-restore.ts +12 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.618",
|
|
4
4
|
"description": "The fastest task runner and build cache for JavaScript monorepos.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.618",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.618",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.618",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.618"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/cache/archive.ts
CHANGED
|
@@ -689,6 +689,14 @@ class Extractor {
|
|
|
689
689
|
// resolve was a round trip spent on a comparison never made (item 627).
|
|
690
690
|
let probe = path.dirname(targetResolved)
|
|
691
691
|
while (probe.startsWith(baseResolved + path.sep)) {
|
|
692
|
+
// Nothing there resolves either: the lstat answers the common case
|
|
693
|
+
// (a clean pruned the directory) without the failed realpath's round
|
|
694
|
+
// trip and error (X-163). A dangling link is there and falls through,
|
|
695
|
+
// as does an lstat that fails otherwise: the realpath decides.
|
|
696
|
+
if (absent(probe)) {
|
|
697
|
+
probe = path.dirname(probe)
|
|
698
|
+
continue
|
|
699
|
+
}
|
|
692
700
|
const real = await realOf(probe).then(
|
|
693
701
|
(r) => r,
|
|
694
702
|
() => null,
|
|
@@ -918,6 +926,15 @@ async function linkOutError(base: string, probe: string, realBase: string): Prom
|
|
|
918
926
|
)
|
|
919
927
|
}
|
|
920
928
|
|
|
929
|
+
/** Whether nothing at all is at `p` (a dangling link is something). */
|
|
930
|
+
function absent(p: string): boolean {
|
|
931
|
+
try {
|
|
932
|
+
return lstatSync(p, { throwIfNoEntry: false }) === undefined
|
|
933
|
+
} catch {
|
|
934
|
+
return false
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
921
938
|
/**
|
|
922
939
|
* `realpath`, for a path holding a backslash too: Bun's answers ENOENT for
|
|
923
940
|
* any such path (1.4.2), so the containment walk took a link named
|
package/src/cache/cache.ts
CHANGED
|
@@ -1399,6 +1399,7 @@ export class Cache implements CacheLayer {
|
|
|
1399
1399
|
})
|
|
1400
1400
|
const liveHashes = live.map((r) => r.hash)
|
|
1401
1401
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1402
|
+
this.outputs.hold(fileRows)
|
|
1402
1403
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
1403
1404
|
for (const row of live) {
|
|
1404
1405
|
this.touched.add(row.hash)
|
|
@@ -1534,7 +1535,7 @@ export class Cache implements CacheLayer {
|
|
|
1534
1535
|
// same truncated expectation against the same truncated tree and agrees
|
|
1535
1536
|
// forever. Refuse instead of silently under-restoring — checked before
|
|
1536
1537
|
// anything is renamed into place.
|
|
1537
|
-
const rows = this.
|
|
1538
|
+
const rows = this.guard(() => this.outputs.rowsOf(hash))
|
|
1538
1539
|
const expected = rows
|
|
1539
1540
|
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
1540
1541
|
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
package/src/cache/inputs.ts
CHANGED
|
@@ -797,8 +797,15 @@ export async function cleanOutputs(args: {
|
|
|
797
797
|
projectDir: string
|
|
798
798
|
outputs: readonly string[]
|
|
799
799
|
nestedProjectDirs: string[]
|
|
800
|
+
/**
|
|
801
|
+
* What `resolveOutputs` returned for these same arguments, when the
|
|
802
|
+
* caller walked the tree with no await since: a hit that found its tree
|
|
803
|
+
* stale walked it to say so, and a second walk here was half the
|
|
804
|
+
* restore's tree work (X-161).
|
|
805
|
+
*/
|
|
806
|
+
resolved?: readonly string[] | undefined
|
|
800
807
|
}): Promise<string[]> {
|
|
801
|
-
const files = await resolveOutputs(args)
|
|
808
|
+
const files = args.resolved ?? (await resolveOutputs(args))
|
|
802
809
|
// `force: true` makes rm tolerate ENOENT (e.g. when two output
|
|
803
810
|
// globs overlap and a sibling already deleted a path mid-iteration).
|
|
804
811
|
// A symlink is unlinked, never followed.
|
|
@@ -1131,8 +1138,10 @@ export async function resolveWorkspaceOutputs(args: {
|
|
|
1131
1138
|
export async function cleanWorkspaceOutputs(args: {
|
|
1132
1139
|
workspaceRoot: string
|
|
1133
1140
|
outputs: readonly string[]
|
|
1141
|
+
/** `cleanOutputs`'s `resolved`, from `resolveWorkspaceOutputs`. */
|
|
1142
|
+
resolved?: readonly string[] | undefined
|
|
1134
1143
|
}): Promise<string[]> {
|
|
1135
|
-
const files = await resolveWorkspaceOutputs(args)
|
|
1144
|
+
const files = args.resolved ?? (await resolveWorkspaceOutputs(args))
|
|
1136
1145
|
await pruneEmptiedDirs(
|
|
1137
1146
|
args.workspaceRoot,
|
|
1138
1147
|
await removeAll(files, args.workspaceRoot),
|
|
@@ -49,6 +49,17 @@ export class OutputIndex {
|
|
|
49
49
|
private readonly pendingDirs = new Map<string, Array<[string, number]> | null>()
|
|
50
50
|
/** Stamps taken and not yet written, flushed with the directory snapshots. */
|
|
51
51
|
private readonly pendingStamps = new Map<string, Array<[string, number, number]>>()
|
|
52
|
+
/**
|
|
53
|
+
* The rows the last batched probe loaded, by hash: a hit's restore and
|
|
54
|
+
* the stamps after it read their rows here instead of the index, twice
|
|
55
|
+
* a restore (X-162). Read for path, size, mode and mtime only; a stamp
|
|
56
|
+
* taken since is the overlay `loadOutputFilesBatch` adds. Dropped when
|
|
57
|
+
* an entry's rows are replaced, and replaced by the next probe; rows
|
|
58
|
+
* held for an entry since pruned serve a restore that finds no artifact.
|
|
59
|
+
* Another process's save under the same key is the race the hit's own
|
|
60
|
+
* skip-restore check, which reads these rows, already runs.
|
|
61
|
+
*/
|
|
62
|
+
private readonly held = new Map<string, readonly OutputFileRow[]>()
|
|
52
63
|
|
|
53
64
|
constructor(private readonly db: Database) {
|
|
54
65
|
this.insertOutputFile = lazyStatement(
|
|
@@ -108,6 +119,7 @@ export class OutputIndex {
|
|
|
108
119
|
*/
|
|
109
120
|
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): void {
|
|
110
121
|
// A stamp taken for the rows this replaces describes their files, not these.
|
|
122
|
+
this.held.delete(hash)
|
|
111
123
|
this.pendingStamps.delete(hash)
|
|
112
124
|
this.deleteOutputFiles.run(hash)
|
|
113
125
|
this.deleteStamps.run(hash)
|
|
@@ -116,6 +128,17 @@ export class OutputIndex {
|
|
|
116
128
|
}
|
|
117
129
|
}
|
|
118
130
|
|
|
131
|
+
/** Hold a probe's rows for its restores (`held`), in place of the last probe's. */
|
|
132
|
+
hold(rows: ReadonlyMap<string, readonly OutputFileRow[]>): void {
|
|
133
|
+
this.held.clear()
|
|
134
|
+
for (const [hash, list] of rows) this.held.set(hash, list)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** `hash`'s rows: the probe's when held, else read. Path, size, mode and mtime only. */
|
|
138
|
+
rowsOf(hash: string): readonly OutputFileRow[] {
|
|
139
|
+
return this.held.get(hash) ?? this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
140
|
+
}
|
|
141
|
+
|
|
119
142
|
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
|
|
120
143
|
const out = new Map<string, OutputFileRow[]>()
|
|
121
144
|
if (hashes.length === 0) return out
|
|
@@ -298,7 +321,7 @@ export class OutputIndex {
|
|
|
298
321
|
* Written with the directory snapshots, in one transaction.
|
|
299
322
|
*/
|
|
300
323
|
recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
|
|
301
|
-
const rows = this.
|
|
324
|
+
const rows = this.rowsOf(hash)
|
|
302
325
|
const stamps: Array<[string, number, number]> = []
|
|
303
326
|
for (const e of rows) {
|
|
304
327
|
const ws = e.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
|
package/src/graph/priorities.ts
CHANGED
|
@@ -24,7 +24,17 @@ export function tieredReverseDepCount(
|
|
|
24
24
|
): Map<string, number> {
|
|
25
25
|
const exec = new Map<string, TaskNode>()
|
|
26
26
|
for (const [id, node] of nodes) if (!restoreTier.has(id)) exec.set(id, node)
|
|
27
|
-
|
|
27
|
+
// An exec tier with no edge inside it (a warm run's groups) counts 0
|
|
28
|
+
// everywhere: no closure to build.
|
|
29
|
+
let linked = false
|
|
30
|
+
for (const node of exec.values()) {
|
|
31
|
+
if (node.deps.some((d) => exec.has(d))) {
|
|
32
|
+
linked = true
|
|
33
|
+
break
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const counts = linked ? computeReverseDepCount(exec) : new Map<string, number>()
|
|
37
|
+
if (!linked) for (const id of exec.keys()) counts.set(id, 0)
|
|
28
38
|
let feeds = false
|
|
29
39
|
for (const node of exec.values()) {
|
|
30
40
|
const weight = 1 + counts.get(node.id)!
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -324,6 +324,9 @@ export interface ScheduleOptions {
|
|
|
324
324
|
class ReadyHeap {
|
|
325
325
|
private readonly ids: string[] = []
|
|
326
326
|
private readonly seq: number[] = []
|
|
327
|
+
// Each slot's priority, read once at push: a map lookup per comparison
|
|
328
|
+
// was two per heap level on every push and pop.
|
|
329
|
+
private readonly prio: number[] = []
|
|
327
330
|
private next = 0
|
|
328
331
|
constructor(private readonly priority: ReadonlyMap<string, number>) {}
|
|
329
332
|
get size(): number {
|
|
@@ -331,8 +334,8 @@ class ReadyHeap {
|
|
|
331
334
|
}
|
|
332
335
|
/** True if slot i outranks slot j (higher priority, or equal priority + earlier seq). */
|
|
333
336
|
private higher(i: number, j: number): boolean {
|
|
334
|
-
const pi = this.
|
|
335
|
-
const pj = this.
|
|
337
|
+
const pi = this.prio[i]!
|
|
338
|
+
const pj = this.prio[j]!
|
|
336
339
|
return pi !== pj ? pi > pj : this.seq[i]! < this.seq[j]!
|
|
337
340
|
}
|
|
338
341
|
private swap(i: number, j: number): void {
|
|
@@ -342,6 +345,9 @@ class ReadyHeap {
|
|
|
342
345
|
const si = this.seq[i]!
|
|
343
346
|
this.seq[i] = this.seq[j]!
|
|
344
347
|
this.seq[j] = si
|
|
348
|
+
const pi = this.prio[i]!
|
|
349
|
+
this.prio[i] = this.prio[j]!
|
|
350
|
+
this.prio[j] = pi
|
|
345
351
|
}
|
|
346
352
|
/**
|
|
347
353
|
* `seq` defaults to a fresh monotonic counter. A parked-then-repushed
|
|
@@ -361,6 +367,7 @@ class ReadyHeap {
|
|
|
361
367
|
push(id: string, seq: number = this.next++): void {
|
|
362
368
|
this.ids.push(id)
|
|
363
369
|
this.seq.push(seq)
|
|
370
|
+
this.prio.push(this.priority.get(id) ?? 0)
|
|
364
371
|
let i = this.ids.length - 1
|
|
365
372
|
while (i > 0) {
|
|
366
373
|
const parent = (i - 1) >> 1
|
|
@@ -379,9 +386,11 @@ class ReadyHeap {
|
|
|
379
386
|
const top = this.ids[0]!
|
|
380
387
|
const lastId = this.ids.pop()!
|
|
381
388
|
const lastSeq = this.seq.pop()!
|
|
389
|
+
const lastPrio = this.prio.pop()!
|
|
382
390
|
if (n > 1) {
|
|
383
391
|
this.ids[0] = lastId
|
|
384
392
|
this.seq[0] = lastSeq
|
|
393
|
+
this.prio[0] = lastPrio
|
|
385
394
|
let i = 0
|
|
386
395
|
for (;;) {
|
|
387
396
|
const l = 2 * i + 1
|
|
@@ -701,14 +710,18 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
701
710
|
// could fail anyway.
|
|
702
711
|
const aborted = (): boolean => options.signal?.aborted === true
|
|
703
712
|
const serverDied = options.serverDied ?? ((): boolean => false)
|
|
704
|
-
const deadServerVia = (id: string): string | undefined =>
|
|
705
|
-
deadServerBehind(nodes, serverDied, id)
|
|
706
713
|
const servers =
|
|
707
714
|
options.serverDied === undefined
|
|
708
715
|
? []
|
|
709
716
|
: [...nodes.values()]
|
|
710
717
|
.filter((n) => n.config.exec?.persistent !== undefined)
|
|
711
718
|
.map((n) => n.id)
|
|
719
|
+
// Only a server can die: with none in the graph, no walk (a stack and
|
|
720
|
+
// a set per dependency of every exec-tier task, X-171).
|
|
721
|
+
const deadServerVia =
|
|
722
|
+
servers.length === 0
|
|
723
|
+
? (): undefined => undefined
|
|
724
|
+
: (id: string): string | undefined => deadServerBehind(nodes, serverDied, id)
|
|
712
725
|
const willSkip = (id: string): boolean => {
|
|
713
726
|
if (failFastTripped || aborted()) return true
|
|
714
727
|
if (continueMode === 'never' && servers.some(serverDied)) {
|
|
@@ -102,6 +102,10 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
102
102
|
// — 1.7 ms each on a four-task real repo whose `link` and `element`
|
|
103
103
|
// tasks declare `outputs: []` (2026-09-10, VX_TIMING).
|
|
104
104
|
let skipRestore = !anyOutputs
|
|
105
|
+
// The walk that found the tree stale, handed to the clean when nothing
|
|
106
|
+
// was awaited between them: the clean's own walk would only repeat it
|
|
107
|
+
// (X-161).
|
|
108
|
+
let walked: { proj: string[]; ws: string[] } | undefined
|
|
105
109
|
if (anyOutputs) {
|
|
106
110
|
const endRows = span('output rows')
|
|
107
111
|
const expected = hit.outputRows ?? args.cache.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
@@ -180,6 +184,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
180
184
|
treeMatches = additive
|
|
181
185
|
? rowsPresent(actualRels, projExpected) && rowsPresent(actualWsRels, wsExpected)
|
|
182
186
|
: setsMatch(actualRels, projExpected) && setsMatch(actualWsRels, wsExpected)
|
|
187
|
+
if (!treeMatches) walked = { proj: actualAbs, ws: actualWsAbs }
|
|
183
188
|
}
|
|
184
189
|
if (treeMatches) {
|
|
185
190
|
const endStat = span('output stat')
|
|
@@ -216,6 +221,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
216
221
|
})
|
|
217
222
|
endGlob()
|
|
218
223
|
skipRestore = actual.length === 0 && actualWs.length === 0
|
|
224
|
+
walked = { proj: actual, ws: actualWs }
|
|
219
225
|
}
|
|
220
226
|
}
|
|
221
227
|
if (!skipRestore) {
|
|
@@ -233,8 +239,12 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
233
239
|
outputs: wsOutputs,
|
|
234
240
|
})
|
|
235
241
|
} else {
|
|
236
|
-
if (outputs.length > 0)
|
|
237
|
-
|
|
242
|
+
if (outputs.length > 0) {
|
|
243
|
+
cleanedRels = await cleanOutputs({ ...cleanArgs, resolved: walked?.proj })
|
|
244
|
+
}
|
|
245
|
+
if (wsOutputs.length > 0) {
|
|
246
|
+
cleanedWsRels = await cleanWorkspaceOutputs({ ...wsCleanArgs, resolved: walked?.ws })
|
|
247
|
+
}
|
|
238
248
|
}
|
|
239
249
|
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
|
|
240
250
|
// The tree now holds this entry's bytes: stamp them for the next hit
|