@vzn/vx 0.0.645 → 0.0.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -28
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/bin.ts +1 -1
- package/src/cache/cache.ts +66 -16
- package/src/cache/git-inputs.ts +55 -10
- package/src/cache/index.ts +4 -0
- package/src/cache/inputs.ts +61 -11
- package/src/cache/layer.ts +25 -2
- package/src/cache/layered-cache.ts +32 -2
- package/src/cache/output-index.ts +25 -11
- package/src/cache/remote-lane.ts +13 -4
- package/src/cache/run-history.ts +13 -6
- package/src/cache/schema.ts +19 -2
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/help.ts +4 -6
- package/src/cli/index.ts +0 -12
- package/src/cli/init.ts +43 -10
- package/src/cli/last.ts +4 -2
- package/src/cli/plugin-commands.ts +96 -3
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +2 -7
- package/src/cli/select.ts +5 -3
- package/src/cli/upgrade.ts +4 -5
- package/src/exec/executor.ts +112 -9
- package/src/exec/index.ts +2 -0
- package/src/exec/proc-sample.ts +67 -5
- package/src/exec/runner.ts +4 -0
- package/src/exec/sandbox-runtime.ts +11 -4
- package/src/graph/scheduler.ts +59 -8
- package/src/history/history.ts +119 -7
- package/src/history/metrics.ts +53 -16
- package/src/index.ts +15 -0
- package/src/orchestrator/execute-args.ts +3 -0
- package/src/orchestrator/execute-task.ts +28 -7
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +11 -4
- package/src/orchestrator/index.ts +17 -1
- package/src/orchestrator/local-floor.ts +104 -0
- package/src/orchestrator/local-shortcircuit.ts +26 -7
- package/src/orchestrator/miss-save.ts +10 -4
- package/src/orchestrator/persistent-task.ts +14 -0
- package/src/orchestrator/placement.ts +84 -9
- package/src/orchestrator/plugin-host.ts +21 -3
- package/src/orchestrator/plugin.ts +37 -0
- package/src/orchestrator/prepare.ts +11 -3
- package/src/orchestrator/run-keys.ts +164 -0
- package/src/orchestrator/run-records.ts +6 -0
- package/src/orchestrator/run.ts +93 -6
- package/src/orchestrator/sandbox-request.ts +9 -3
- package/src/orchestrator/task-artifacts.ts +142 -0
- package/src/orchestrator/task-hash.ts +11 -1
- package/src/orchestrator/task-process.ts +16 -0
- package/src/orchestrator/telemetry-host.ts +2 -1
- package/src/orchestrator/telemetry.ts +60 -17
- package/src/report/forecast.ts +6 -27
- package/src/report/framed-output.ts +41 -5
- package/src/report/logger.ts +60 -20
- package/src/report/status-line.ts +38 -18
- package/src/report/summary.ts +61 -7
- package/src/settings.ts +7 -0
- package/src/util/colors.ts +16 -1
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +30 -0
- package/src/util/ui.ts +4 -1
- package/src/workspace/config-bytecode.ts +276 -0
- package/src/workspace/index.ts +0 -1
- package/src/workspace/migration.ts +4 -32
- package/src/workspace/project-loader.ts +92 -14
- package/src/cli/foreign-flags.ts +0 -727
package/README.md
CHANGED
|
@@ -23,37 +23,19 @@ Task orchestration with a cache you can trust, on a core you can extend.
|
|
|
23
23
|
Each task is one shell command with declared inputs and outputs; vx runs
|
|
24
24
|
them in dependency order and replays results from a content-addressed cache.
|
|
25
25
|
|
|
26
|
-
## Install
|
|
27
|
-
|
|
28
|
-
```sh
|
|
29
|
-
npm install -D @vzn/vx # or: pnpm add -D -w · yarn add -D (-W on Yarn 1) · bun add -d
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and arm64; no Bun needed. The
|
|
33
|
-
[release](https://github.com/vznjs/vx/releases) binary needs no Node either.
|
|
34
|
-
Windows: run vx inside WSL2.
|
|
35
|
-
|
|
36
26
|
## Quick start
|
|
37
27
|
|
|
38
28
|
In a git repo:
|
|
39
29
|
|
|
40
30
|
```sh
|
|
41
|
-
|
|
42
|
-
vx run build --all # builds in dependency order and stores the results
|
|
43
|
-
vx run build --all # nothing changed: every task up-to-date
|
|
31
|
+
bunx @vzn/vx init # installs vx, writes an empty root vx.config.ts and the lockfile plugin
|
|
44
32
|
```
|
|
45
33
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
Without Turbo or Nx, tasks run uncached until you declare what they read (below).
|
|
50
|
-
|
|
51
|
-
`init` prints what it wrote and anything left to decide (a `TODO`). A miss you did not expect?
|
|
52
|
-
`npx @vzn/vx why <task>` names the input that moved.
|
|
53
|
-
|
|
54
|
-
### Without Turbo or Nx
|
|
34
|
+
A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and
|
|
35
|
+
arm64. Windows: run vx inside WSL2.
|
|
55
36
|
|
|
56
|
-
|
|
37
|
+
A task runs its tool directly, never through a package script. Cache one by
|
|
38
|
+
saying what it reads and writes:
|
|
57
39
|
|
|
58
40
|
```ts
|
|
59
41
|
// packages/app/vx.config.ts
|
|
@@ -74,10 +56,10 @@ export default defineProject({
|
|
|
74
56
|
```
|
|
75
57
|
|
|
76
58
|
```sh
|
|
77
|
-
|
|
78
|
-
|
|
59
|
+
vx run build --all # builds, in dependency order; stores the results
|
|
60
|
+
vx run build --all # nothing changed: up-to-date, in milliseconds
|
|
79
61
|
rm -rf packages/*/dist
|
|
80
|
-
|
|
62
|
+
vx run build --all # restored from the cache, not rebuilt
|
|
81
63
|
```
|
|
82
64
|
|
|
83
65
|
<img src="https://vznjs.github.io/vx/demo.svg" width="760" alt="vx run ci --all on examples/basic: cold, three tasks run; again, three up-to-date.">
|
|
@@ -86,7 +68,7 @@ npx @vzn/vx run build --all # restored from the cache, not rebuilt
|
|
|
86
68
|
|
|
87
69
|
`init` also writes `.vx/projects.lock.json`, every project's config as it evaluated.
|
|
88
70
|
Commit it. Every run reads its configs from the lock, so run
|
|
89
|
-
`
|
|
71
|
+
`vx lock` and commit the lock again after you:
|
|
90
72
|
|
|
91
73
|
- edit any `vx.config.ts` or `.vx/workspace.ts`,
|
|
92
74
|
- update dependencies or plugins,
|
|
@@ -134,7 +116,7 @@ Write your own: [plugin guide](https://vznjs.github.io/vx/guides/plugins/) ·
|
|
|
134
116
|
|
|
135
117
|
## Coming from Turbo or Nx
|
|
136
118
|
|
|
137
|
-
Beside `turbo.json` or `nx.json`, `
|
|
119
|
+
Beside `turbo.json` or `nx.json`, `bunx @vzn/vx init` runs `@vzn/vx-migrate`:
|
|
138
120
|
it writes a `vx.config.ts` per package from `turbo.json` or the Nx graph,
|
|
139
121
|
and you keep those and delete the old config. `vx init --keep` writes only
|
|
140
122
|
a `.vx/workspace.ts` that maps the old config instead, and its lock: a temporary start,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.646",
|
|
4
4
|
"description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.646",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.646",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.646",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.646"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/schemas/last.json
CHANGED
|
@@ -166,6 +166,9 @@
|
|
|
166
166
|
"hash",
|
|
167
167
|
"cpuMs",
|
|
168
168
|
"peakRssBytes",
|
|
169
|
+
"peakCores",
|
|
170
|
+
"avgRssBytes",
|
|
171
|
+
"placement",
|
|
169
172
|
"wallclockStartNs",
|
|
170
173
|
"wallclockEndNs",
|
|
171
174
|
"blockedBy",
|
|
@@ -199,6 +202,18 @@
|
|
|
199
202
|
"hash": { "type": "string" },
|
|
200
203
|
"cpuMs": { "type": ["number", "null"] },
|
|
201
204
|
"peakRssBytes": { "type": ["integer", "null"] },
|
|
205
|
+
"peakCores": {
|
|
206
|
+
"type": ["number", "null"],
|
|
207
|
+
"description": "most cores one sampling interval burned; null when nothing sampled"
|
|
208
|
+
},
|
|
209
|
+
"avgRssBytes": {
|
|
210
|
+
"type": ["integer", "null"],
|
|
211
|
+
"description": "mean resident memory over the samples; null when nothing sampled"
|
|
212
|
+
},
|
|
213
|
+
"placement": {
|
|
214
|
+
"type": ["string", "null"],
|
|
215
|
+
"description": "where an executor ran it (small#1); null when here"
|
|
216
|
+
},
|
|
202
217
|
"wallclockStartNs": {
|
|
203
218
|
"type": ["string", "null"],
|
|
204
219
|
"description": "decimal nanoseconds from the run's start"
|
package/skills/vx/SKILL.md
CHANGED
|
@@ -106,7 +106,7 @@ vx info --format json # vx and Bun versions, projects, config errors, plugins,
|
|
|
106
106
|
|
|
107
107
|
## Install
|
|
108
108
|
|
|
109
|
-
`
|
|
109
|
+
`bunx skills add vznjs/vx` installs this skill. To match the vx a workspace
|
|
110
110
|
has installed, copy it from there instead:
|
|
111
111
|
`cp node_modules/@vzn/vx/skills/vx/SKILL.md .claude/skills/vx/`.
|
|
112
112
|
`vx upgrade` inside a workspace moves the project's `@vzn/vx` and the
|
package/src/bin.ts
CHANGED
|
@@ -39,7 +39,7 @@ async function main(): Promise<void> {
|
|
|
39
39
|
// package, a workspace or project config — resolves to THIS core, not to
|
|
40
40
|
// a second copy from node_modules (core-alias.ts says why and what it
|
|
41
41
|
// costs). Registered before any verb runs; the façade loads on first use.
|
|
42
|
-
registerCoreAlias(() =>
|
|
42
|
+
registerCoreAlias(() => require('./index.js') as Record<string, unknown>)
|
|
43
43
|
const code = await run(argv)
|
|
44
44
|
// Not `process.exit` here: Bun drops what a pipe has not yet taken
|
|
45
45
|
// when it follows a large write (300 KB written, 64 KiB delivered;
|
package/src/cache/cache.ts
CHANGED
|
@@ -165,6 +165,8 @@ const VX_CACHE_NAMES = new Set([
|
|
|
165
165
|
'runs',
|
|
166
166
|
'failures',
|
|
167
167
|
'lockfile-claims',
|
|
168
|
+
// The workspace config's bytecode bundle (workspace/config-bytecode.ts).
|
|
169
|
+
'compiled',
|
|
168
170
|
])
|
|
169
171
|
|
|
170
172
|
/**
|
|
@@ -324,7 +326,16 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
|
|
|
324
326
|
// is unchanged.
|
|
325
327
|
// v42: fold_memo — each task's last key by its fold up to the files and
|
|
326
328
|
// a token of their git records. The cache KEY is unchanged.
|
|
327
|
-
|
|
329
|
+
// v43: runs.peak_cores / avg_rss_bytes — what a sampled run burned at
|
|
330
|
+
// its peak and held on average. The cache KEY is unchanged.
|
|
331
|
+
// v44: runs.placement — where an executor ran the task. The cache KEY
|
|
332
|
+
// is unchanged.
|
|
333
|
+
// v45: a 'remote' task_durations row also carries the task's measured
|
|
334
|
+
// cores, memory and saturation, for a run with no local history.
|
|
335
|
+
// The cache KEY is unchanged.
|
|
336
|
+
// v46: runs.keyless_by — the task with no inputs a key folded, so
|
|
337
|
+
// `vx why` names it. The cache KEY is unchanged.
|
|
338
|
+
export const SCHEMA_VERSION = 'v46'
|
|
328
339
|
|
|
329
340
|
/**
|
|
330
341
|
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
@@ -1699,6 +1710,10 @@ export class Cache implements CacheLayer {
|
|
|
1699
1710
|
return this.guard(() => this.readEntry(hash))
|
|
1700
1711
|
}
|
|
1701
1712
|
|
|
1713
|
+
getSaved(hash: string): Promise<CacheEntry | null> {
|
|
1714
|
+
return this.getIngested(hash)
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1702
1717
|
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
1703
1718
|
const row = this.selectEntry.get(hash) as EntryRow | null
|
|
1704
1719
|
if (row === null) return null
|
|
@@ -1956,7 +1971,14 @@ export class Cache implements CacheLayer {
|
|
|
1956
1971
|
)
|
|
1957
1972
|
}
|
|
1958
1973
|
|
|
1959
|
-
/**
|
|
1974
|
+
/**
|
|
1975
|
+
* The copy-out: false, having written nothing, where it is not the case it
|
|
1976
|
+
* serves. It writes only where nothing stands: each file's place is absent,
|
|
1977
|
+
* and every directory on the way either is a real directory or is absent
|
|
1978
|
+
* and made here. solid's build outputs `store/dist/**` beside the sources
|
|
1979
|
+
* in `store/`; asking that each output's TOP be absent sent every such
|
|
1980
|
+
* restore through the tar extractor, ~25 ms a task (2026-10-22).
|
|
1981
|
+
*/
|
|
1960
1982
|
private async copyTree(
|
|
1961
1983
|
tree: string,
|
|
1962
1984
|
rows: readonly OutputFileRow[],
|
|
@@ -1964,22 +1986,49 @@ export class Cache implements CacheLayer {
|
|
|
1964
1986
|
projectDir: string,
|
|
1965
1987
|
accept: ((name: string) => boolean) | undefined,
|
|
1966
1988
|
): Promise<boolean> {
|
|
1967
|
-
const tops = new Set<string>()
|
|
1968
1989
|
for (const r of rows) {
|
|
1969
1990
|
if (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) || r.mode === LINK_MODE) return false
|
|
1970
1991
|
if (accept !== undefined && !r.path.endsWith('/') && !accept(`outputs/${r.path}`))
|
|
1971
1992
|
return false
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
)
|
|
1980
|
-
|
|
1993
|
+
}
|
|
1994
|
+
// What stands at each path, asked once: a real directory, nothing, or
|
|
1995
|
+
// anything else (a file, a link), which the extractor's rules decide.
|
|
1996
|
+
const standing = new Map<string, Promise<'dir' | 'absent' | 'other'>>()
|
|
1997
|
+
const at = (rel: string): Promise<'dir' | 'absent' | 'other'> => {
|
|
1998
|
+
let known = standing.get(rel)
|
|
1999
|
+
if (known === undefined) {
|
|
2000
|
+
known = lstat(path.join(projectDir, rel)).then(
|
|
2001
|
+
(st) => (st.isDirectory() ? 'dir' : 'other'),
|
|
2002
|
+
(err: NodeJS.ErrnoException) => (err.code === 'ENOENT' ? 'absent' : 'other'),
|
|
2003
|
+
)
|
|
2004
|
+
standing.set(rel, known)
|
|
2005
|
+
}
|
|
2006
|
+
return known
|
|
2007
|
+
}
|
|
2008
|
+
// The topmost absent directories this copy makes, and the files it
|
|
2009
|
+
// writes into a directory that stood: what a failure removes, and only that.
|
|
2010
|
+
const made = new Set<string>()
|
|
2011
|
+
const placed: string[] = []
|
|
2012
|
+
const fits = await Promise.all(
|
|
2013
|
+
rows.map(async (r) => {
|
|
2014
|
+
const segs = r.path.replace(/\/$/, '').split('/')
|
|
2015
|
+
for (let i = 1; i <= segs.length; i++) {
|
|
2016
|
+
const rel = segs.slice(0, i).join('/')
|
|
2017
|
+
const isLeaf = i === segs.length
|
|
2018
|
+
const state = await at(rel)
|
|
2019
|
+
if (state === 'absent') {
|
|
2020
|
+
if (!isLeaf || r.path.endsWith('/')) made.add(rel)
|
|
2021
|
+
else placed.push(rel)
|
|
2022
|
+
return true
|
|
2023
|
+
}
|
|
2024
|
+
if (state === 'other' || (isLeaf && !r.path.endsWith('/'))) return false
|
|
2025
|
+
}
|
|
2026
|
+
return true
|
|
2027
|
+
}),
|
|
1981
2028
|
)
|
|
1982
|
-
if (
|
|
2029
|
+
if (fits.includes(false)) return false
|
|
2030
|
+
// A directory made under another one made here is the outer one's.
|
|
2031
|
+
const roots = [...made].filter((d) => ![...made].some((o) => d.startsWith(`${o}/`)))
|
|
1983
2032
|
readUmask()
|
|
1984
2033
|
const plain = restoredMode(0o666)
|
|
1985
2034
|
try {
|
|
@@ -1999,9 +2048,10 @@ export class Cache implements CacheLayer {
|
|
|
1999
2048
|
}),
|
|
2000
2049
|
)
|
|
2001
2050
|
} catch (err) {
|
|
2002
|
-
await Promise.all(
|
|
2003
|
-
|
|
2004
|
-
|
|
2051
|
+
await Promise.all([
|
|
2052
|
+
...roots.map((d) => rm(path.join(projectDir, d), { recursive: true, force: true })),
|
|
2053
|
+
...placed.map((f) => rm(path.join(projectDir, f), { force: true })),
|
|
2054
|
+
])
|
|
2005
2055
|
throw err
|
|
2006
2056
|
}
|
|
2007
2057
|
return true
|
package/src/cache/git-inputs.ts
CHANGED
|
@@ -204,7 +204,8 @@ function configObjectFormat(config: string): 'sha1' | 'sha256' | undefined {
|
|
|
204
204
|
interface PartitionDigest {
|
|
205
205
|
snapshot: readonly string[]
|
|
206
206
|
oids: OidSlot | undefined
|
|
207
|
-
|
|
207
|
+
/** Built on first read: a partition with untrusted files stats them then. */
|
|
208
|
+
digest: () => string | undefined
|
|
208
209
|
}
|
|
209
210
|
|
|
210
211
|
/**
|
|
@@ -515,11 +516,20 @@ export class GitFilesCache extends Map<string, readonly string[]> {
|
|
|
515
516
|
const d = this.digests.get(projectDir)
|
|
516
517
|
if (d === undefined || d.snapshot !== snapshot || this.get(projectDir) !== snapshot) return
|
|
517
518
|
if (this.oids.get(projectDir) !== d.oids || this.changed.has(projectDir)) return
|
|
518
|
-
return d.digest
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
noteFilesDigest(
|
|
522
|
-
|
|
519
|
+
return d.digest()
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
noteFilesDigest(
|
|
523
|
+
projectDir: string,
|
|
524
|
+
snapshot: readonly string[],
|
|
525
|
+
digest: () => string | undefined,
|
|
526
|
+
): void {
|
|
527
|
+
let memo: { value: string | undefined } | undefined
|
|
528
|
+
this.digests.set(projectDir, {
|
|
529
|
+
snapshot,
|
|
530
|
+
oids: this.oids.get(projectDir),
|
|
531
|
+
digest: () => (memo ??= { value: digest() }).value,
|
|
532
|
+
})
|
|
523
533
|
}
|
|
524
534
|
|
|
525
535
|
/** Trusted index OIDs for a project (abs path → oid), if any survive. */
|
|
@@ -1908,6 +1918,30 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
|
|
|
1908
1918
|
}
|
|
1909
1919
|
}
|
|
1910
1920
|
|
|
1921
|
+
/**
|
|
1922
|
+
* Each loose file's stat identity, `hashFile`'s memo key: mode, size,
|
|
1923
|
+
* mtime, ctime and inode. Undefined for a symlink (what it points to
|
|
1924
|
+
* folds too, and its stat does not move with that), a file gone, or one
|
|
1925
|
+
* changed within the racy window, where the memo trusts no stat either.
|
|
1926
|
+
*/
|
|
1927
|
+
function statRecords(
|
|
1928
|
+
rels: readonly string[],
|
|
1929
|
+
abs: (rel: string) => string,
|
|
1930
|
+
prefixLength: number,
|
|
1931
|
+
): string | undefined {
|
|
1932
|
+
if (rels.length === 0) return ''
|
|
1933
|
+
const now = Date.now()
|
|
1934
|
+
let out = ''
|
|
1935
|
+
for (const rel of rels) {
|
|
1936
|
+
const st = lstatSync(abs(rel), { throwIfNoEntry: false })
|
|
1937
|
+
if (st === undefined || !st.isFile()) return undefined
|
|
1938
|
+
if (now - Math.floor(st.ctimeMs) < racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) return
|
|
1939
|
+
const mode = (st.mode & 0o100) !== 0 ? '100755' : '100644'
|
|
1940
|
+
out += `${rel.slice(prefixLength)}\0stat:${mode}:${st.size}:${Math.floor(st.mtimeMs)}:${Math.floor(st.ctimeMs)}:${Number(st.ino)}\n`
|
|
1941
|
+
}
|
|
1942
|
+
return out
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1911
1945
|
/** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
|
|
1912
1946
|
export async function applyGitEnumeration(
|
|
1913
1947
|
enumeration: GitEnumeration,
|
|
@@ -2006,15 +2040,20 @@ export async function applyGitEnumeration(
|
|
|
2006
2040
|
}
|
|
2007
2041
|
// One hash of the records joined: one per record was ~25 ms of 125,000.
|
|
2008
2042
|
let records: string | undefined = ''
|
|
2043
|
+
// A file git vouches for no OID (untracked, or changed since the
|
|
2044
|
+
// index) is named by its stat instead, the identity `hashFile`'s memo
|
|
2045
|
+
// trusts: a just-added config or an edited file turned the fold memo
|
|
2046
|
+
// off for its whole project (+3-5 ms of solid's warm run, 2026-10-22).
|
|
2047
|
+
const loose: string[] = []
|
|
2009
2048
|
for (let i = start; i < end; i++) {
|
|
2010
2049
|
const rel = sorted[i]!
|
|
2011
2050
|
const own = rel.slice(prefix.length)
|
|
2012
2051
|
matches.push(own)
|
|
2013
2052
|
if (records === undefined) continue
|
|
2014
2053
|
const oid = trusted.get(rel)
|
|
2015
|
-
if (
|
|
2016
|
-
|
|
2017
|
-
|
|
2054
|
+
if (undecodable?.has(rel) === true || oid?.startsWith('120000:') === true) records = undefined
|
|
2055
|
+
else if (oid === undefined) loose.push(rel)
|
|
2056
|
+
else records += `${own}\0${oid}\n`
|
|
2018
2057
|
}
|
|
2019
2058
|
// An empty slice is a directory git did not see, not an empty project:
|
|
2020
2059
|
// a project has at least its package.json, tracked or untracked. A
|
|
@@ -2033,7 +2072,13 @@ export async function applyGitEnumeration(
|
|
|
2033
2072
|
// setLazyOids AFTER set — set() drops the project's OID slot.
|
|
2034
2073
|
cache.set(projectDir, matches)
|
|
2035
2074
|
cache.setLazyOids(projectDir, projOids)
|
|
2036
|
-
if (records !== undefined)
|
|
2075
|
+
if (records !== undefined) {
|
|
2076
|
+
const known = records
|
|
2077
|
+
cache.noteFilesDigest(projectDir, matches, () => {
|
|
2078
|
+
const statted = statRecords(loose, projAbs, prefix.length)
|
|
2079
|
+
return statted === undefined ? undefined : xxh3hex(`${known}\0${statted}`)
|
|
2080
|
+
})
|
|
2081
|
+
}
|
|
2037
2082
|
}
|
|
2038
2083
|
if (workspaceWide) {
|
|
2039
2084
|
cache.set(workspaceRoot, all)
|
package/src/cache/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ export {
|
|
|
29
29
|
WORKSPACE_OUTPUT_PREFIX,
|
|
30
30
|
type OutputDirRow,
|
|
31
31
|
OUTPUT_DIRS_CAP,
|
|
32
|
+
OUTPUT_DIRS_OVER_CAP,
|
|
32
33
|
FILE_HASH_RACY_MS,
|
|
33
34
|
OUTPUT_DIRS_RACY_MS,
|
|
34
35
|
racyWindowMs,
|
|
@@ -54,6 +55,7 @@ export {
|
|
|
54
55
|
type ResolveInputsArgs,
|
|
55
56
|
resolveOutputTree,
|
|
56
57
|
resolveOutputs,
|
|
58
|
+
outputWalkTakes,
|
|
57
59
|
resolveWorkspaceOutputs,
|
|
58
60
|
sweepRestoreTemps,
|
|
59
61
|
sweepStaging,
|
|
@@ -66,6 +68,7 @@ export {
|
|
|
66
68
|
stampOutputs,
|
|
67
69
|
stampWorkspaceOutputs,
|
|
68
70
|
stopRuntimeProbes,
|
|
71
|
+
answerRuntime,
|
|
69
72
|
type OutputStamp,
|
|
70
73
|
} from './inputs.js'
|
|
71
74
|
export {
|
|
@@ -84,6 +87,7 @@ export {
|
|
|
84
87
|
export {
|
|
85
88
|
type ArtifactInfo,
|
|
86
89
|
LayeredCache,
|
|
90
|
+
type LentAccess,
|
|
87
91
|
type RemoteCacheLayer,
|
|
88
92
|
type RemoteOpenContext,
|
|
89
93
|
} from './layered-cache.js'
|
package/src/cache/inputs.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
outputNever,
|
|
32
32
|
realpathOf,
|
|
33
33
|
relPosix,
|
|
34
|
+
restorableOutput,
|
|
34
35
|
shellArgv,
|
|
35
36
|
slashBraceExpansions,
|
|
36
37
|
braceAlternatives,
|
|
@@ -530,6 +531,28 @@ function projectBinDirs(projectDir: string, workspaceRoot: string): string[] {
|
|
|
530
531
|
return own === root ? [own] : [own, root]
|
|
531
532
|
}
|
|
532
533
|
|
|
534
|
+
/**
|
|
535
|
+
* One runtime probe answered on this machine as a key folds it: an
|
|
536
|
+
* `inputs.runtime` command run in `projectDir`, or with `workspace` a
|
|
537
|
+
* `workspaceRuntime` one run in `workspaceRoot`. For an executor that
|
|
538
|
+
* checks a task's probes where it runs it (`TaskExecutor.runtime`).
|
|
539
|
+
*/
|
|
540
|
+
export function answerRuntime(
|
|
541
|
+
command: string,
|
|
542
|
+
projectDir: string,
|
|
543
|
+
workspaceRoot: string,
|
|
544
|
+
workspace = false,
|
|
545
|
+
): Promise<string> {
|
|
546
|
+
return workspace
|
|
547
|
+
? runRuntimeCommand(
|
|
548
|
+
command,
|
|
549
|
+
workspaceRoot,
|
|
550
|
+
[path.join(workspaceRoot, 'node_modules', '.bin')],
|
|
551
|
+
undefined,
|
|
552
|
+
)
|
|
553
|
+
: runRuntimeCommand(command, projectDir, projectBinDirs(projectDir, workspaceRoot), undefined)
|
|
554
|
+
}
|
|
555
|
+
|
|
533
556
|
function resolveEnvValues(
|
|
534
557
|
names: readonly string[],
|
|
535
558
|
source: NodeJS.ProcessEnv,
|
|
@@ -748,6 +771,21 @@ export async function resolveOutputs(args: {
|
|
|
748
771
|
return (await scanOutputs(args)).files
|
|
749
772
|
}
|
|
750
773
|
|
|
774
|
+
/**
|
|
775
|
+
* Whether `resolveOutputs` takes the project-relative file `rel`, decided
|
|
776
|
+
* without the walk: a directory snapshot's file list is held to the
|
|
777
|
+
* entry's rows through it.
|
|
778
|
+
*/
|
|
779
|
+
export function outputWalkTakes(
|
|
780
|
+
projectDir: string,
|
|
781
|
+
outputs: readonly string[],
|
|
782
|
+
nestedProjectDirs: string[],
|
|
783
|
+
): (rel: string) => boolean {
|
|
784
|
+
const restorable = restorableOutput(outputs)
|
|
785
|
+
const nested = inNestedProject(projectDir, nestedProjectDirs)
|
|
786
|
+
return (rel) => restorable(rel) && !nested(rel)
|
|
787
|
+
}
|
|
788
|
+
|
|
751
789
|
/**
|
|
752
790
|
* `resolveOutputs`, plus the empty directories the globs match: a save
|
|
753
791
|
* keeps them, so a hit leaves the tree a miss left (Turbo and Nx restore
|
|
@@ -2103,21 +2141,31 @@ async function scanUnion(
|
|
|
2103
2141
|
emptyDirs?: Set<string>,
|
|
2104
2142
|
): Promise<Set<string>> {
|
|
2105
2143
|
const matches = new Set<string>()
|
|
2144
|
+
const take = (rel: string): void => {
|
|
2145
|
+
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
|
|
2146
|
+
const abs = path.resolve(cwd, rel)
|
|
2147
|
+
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2148
|
+
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
|
|
2149
|
+
else if (emptyDirs !== undefined && st?.isDirectory() === true) {
|
|
2150
|
+
if (readdirSync(abs).length === 0) emptyDirs.add(abs)
|
|
2151
|
+
} else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
|
|
2152
|
+
}
|
|
2153
|
+
const opts = { cwd, onlyFiles: false, followSymlinks: false, dot: true }
|
|
2106
2154
|
// `Bun.Glob`'s scan finds nothing for a brace whose alternatives hold a
|
|
2107
2155
|
// `/`: `{dist,lib/esm}/**` saved an empty artifact, and a hit restored
|
|
2108
2156
|
// nothing over a cleaned tree (A-10). Expanded here, as discovery does.
|
|
2109
|
-
|
|
2157
|
+
const patterns = positive.flatMap(slashBraceExpansions)
|
|
2158
|
+
// A leading `**` walks the whole project whatever else is listed, so the
|
|
2159
|
+
// list is one walk: Turbo's `dist/**`, `**/dist/**`, `**/dist` was three,
|
|
2160
|
+
// ~2 ms of a warm hit on solid.
|
|
2161
|
+
if (patterns.length > 1 && patterns.some((p) => p.startsWith('**'))) {
|
|
2162
|
+
const any = anyTaskGlob(patterns)
|
|
2163
|
+
for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
|
|
2164
|
+
return matches
|
|
2165
|
+
}
|
|
2166
|
+
for (const pattern of patterns) {
|
|
2110
2167
|
if (absentPrefix(cwd, pattern)) continue
|
|
2111
|
-
const
|
|
2112
|
-
for (const rel of glob.scanSync({ cwd, onlyFiles: false, followSymlinks: false, dot: true })) {
|
|
2113
|
-
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) continue
|
|
2114
|
-
const abs = path.resolve(cwd, rel)
|
|
2115
|
-
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2116
|
-
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
|
|
2117
|
-
else if (emptyDirs !== undefined && st?.isDirectory() === true) {
|
|
2118
|
-
if (readdirSync(abs).length === 0) emptyDirs.add(abs)
|
|
2119
|
-
} else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
|
|
2120
|
-
}
|
|
2168
|
+
for (const rel of globFor(pattern).scanSync(opts)) take(rel)
|
|
2121
2169
|
}
|
|
2122
2170
|
return matches
|
|
2123
2171
|
}
|
|
@@ -2139,6 +2187,8 @@ function absentPrefix(cwd: string, pattern: string): boolean {
|
|
|
2139
2187
|
}
|
|
2140
2188
|
}
|
|
2141
2189
|
|
|
2190
|
+
const WHOLE_TREE = new Bun.Glob('**')
|
|
2191
|
+
|
|
2142
2192
|
// Compiled once per pattern string for the life of the process: a `Bun.Glob`
|
|
2143
2193
|
// is immutable, and the same handful of declared globs are compiled again
|
|
2144
2194
|
// for every task otherwise (three sites, up to thousands of times per run).
|
package/src/cache/layer.ts
CHANGED
|
@@ -212,6 +212,16 @@ export interface RunRecord {
|
|
|
212
212
|
notReady?: 'timeout' | 'exited' | 'spawn'
|
|
213
213
|
/** Each remote read or write of its key that failed, `upload: <message>`, `; `-joined (v40). */
|
|
214
214
|
remoteError?: string
|
|
215
|
+
/** The sampled peak cores and mean resident memory of the run (v43). */
|
|
216
|
+
peakCores?: number
|
|
217
|
+
avgRssBytes?: number
|
|
218
|
+
/** Where an executor ran it (`small#1`); absent = here (v44). */
|
|
219
|
+
placement?: string
|
|
220
|
+
/** The task with no `inputs` its key folded, so it never read or saved (v46). */
|
|
221
|
+
keylessBy?: string
|
|
222
|
+
/** That machine's cores and memory (v44). */
|
|
223
|
+
placementCores?: number
|
|
224
|
+
placementMemoryBytes?: number
|
|
215
225
|
}
|
|
216
226
|
|
|
217
227
|
/**
|
|
@@ -384,6 +394,13 @@ export interface OutputDirRow {
|
|
|
384
394
|
*/
|
|
385
395
|
export const OUTPUT_DIRS_CAP = 8192
|
|
386
396
|
|
|
397
|
+
/**
|
|
398
|
+
* `mtime_ms` of the one row a snapshot over `OUTPUT_DIRS_CAP` leaves: never
|
|
399
|
+
* current, so the hit walks, and it tells the hit not to snapshot again.
|
|
400
|
+
* Cleared rows sent every hit through the walk AND a capped snapshot.
|
|
401
|
+
*/
|
|
402
|
+
export const OUTPUT_DIRS_OVER_CAP = -3
|
|
403
|
+
|
|
387
404
|
/**
|
|
388
405
|
* A directory whose mtime lies within this many ms of the snapshot is RACY
|
|
389
406
|
* and the whole snapshot is dropped. File timestamps are coarse on Linux
|
|
@@ -593,6 +610,12 @@ export interface CacheLayer {
|
|
|
593
610
|
stopUploads?(): void
|
|
594
611
|
key(input: CacheKeyInput): Promise<string>
|
|
595
612
|
get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
613
|
+
/**
|
|
614
|
+
* `get` past the run's read policy: an entry a machine running a task
|
|
615
|
+
* for this run saved is the run's own work, not a hit (`TaskArtifacts`).
|
|
616
|
+
* A layer without it is read through `get`.
|
|
617
|
+
*/
|
|
618
|
+
getSaved?(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
596
619
|
/**
|
|
597
620
|
* Optional batched `get` (same answers, fewer round trips). The
|
|
598
621
|
* short-circuit probe uses it when a layer offers one; a layer without it
|
|
@@ -651,8 +674,8 @@ export interface CacheLayer {
|
|
|
651
674
|
/**
|
|
652
675
|
* The directory-mtime short-circuit behind a warm hit (optional — a layer
|
|
653
676
|
* without it keeps the output walk). `recordOutputDirs` snapshots every
|
|
654
|
-
* directory under each of `prefixes` (project-relative,
|
|
655
|
-
*
|
|
677
|
+
* directory under each of `prefixes` (project-relative, `''` the project
|
|
678
|
+
* itself — see `outputDirPrefixes`) after a save or restore;
|
|
656
679
|
* `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
|
|
657
680
|
* every recorded directory still carries its recorded mtime, which proves
|
|
658
681
|
* no file was added or removed anywhere the glob could see. Machine-local
|
|
@@ -104,6 +104,13 @@ export interface ArtifactInfo {
|
|
|
104
104
|
readonly durationMs?: number
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
/** A run's access to a remote, lent to a machine that runs its tasks (`RemoteCacheLayer.lend`). */
|
|
108
|
+
export interface LentAccess {
|
|
109
|
+
readonly token: string
|
|
110
|
+
/** Epoch ms; asked again before it. */
|
|
111
|
+
readonly expiresAt?: number
|
|
112
|
+
}
|
|
113
|
+
|
|
107
114
|
/**
|
|
108
115
|
* What a remote cache layer must provide — THE plugin seam for remote
|
|
109
116
|
* caching. Core ships no wire client; a plugin's `cache` hook returns one
|
|
@@ -140,6 +147,14 @@ export interface RemoteCacheLayer {
|
|
|
140
147
|
* plugin's keychain and may ask on the terminal.
|
|
141
148
|
*/
|
|
142
149
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
150
|
+
/**
|
|
151
|
+
* Optional: this run's access, for a machine that runs its tasks
|
|
152
|
+
* elsewhere (`ExecutorContext.cacheFacts`); that machine's plugin reads
|
|
153
|
+
* it as `CacheContext.lent`. Only access made for this run (an OIDC
|
|
154
|
+
* exchange's), never a stored secret: the server's token decides what
|
|
155
|
+
* it may write. Asked after `open`.
|
|
156
|
+
*/
|
|
157
|
+
lend?(): Promise<LentAccess | undefined>
|
|
143
158
|
/**
|
|
144
159
|
* Existence probe: what the remote knows of the stored artifact, or null
|
|
145
160
|
* when it holds none. `--dry` and `--force` ask it per key when the
|
|
@@ -367,6 +382,16 @@ export class LayeredCache implements CacheLayer {
|
|
|
367
382
|
return this.asRemote(hash, materialized)
|
|
368
383
|
}
|
|
369
384
|
|
|
385
|
+
async getSaved(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
|
|
386
|
+
const here = await this.local.getIngested(hash)
|
|
387
|
+
if (here) return here
|
|
388
|
+
// Every remote, read or not, and past a probe's absence: the probe ran
|
|
389
|
+
// before the agent saved it.
|
|
390
|
+
for (const lane of this.lanes)
|
|
391
|
+
if (await this.pullFrom(lane, hash, ctx, true)) return this.local.getIngested(hash)
|
|
392
|
+
return null
|
|
393
|
+
}
|
|
394
|
+
|
|
370
395
|
/**
|
|
371
396
|
* A pull the task now waits on, given up when the rest of its body would
|
|
372
397
|
* take longer than running the task (its stored duration, doubled for
|
|
@@ -452,13 +477,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
452
477
|
return false
|
|
453
478
|
}
|
|
454
479
|
|
|
455
|
-
private async pullFrom(
|
|
480
|
+
private async pullFrom(
|
|
481
|
+
lane: RemoteLane,
|
|
482
|
+
hash: string,
|
|
483
|
+
ctx?: CacheGetContext,
|
|
484
|
+
fresh = false,
|
|
485
|
+
): Promise<boolean> {
|
|
456
486
|
const fetchStart = performance.now()
|
|
457
487
|
// A stopped run waits on no read, as on no upload: a lookup or download
|
|
458
488
|
// in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
|
|
459
489
|
// plugin's request is left to the exit, and its body is cancelled.
|
|
460
490
|
const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
|
|
461
|
-
const pending = lane.get(hash)
|
|
491
|
+
const pending = lane.get(hash, fresh)
|
|
462
492
|
const remoteBody = await Promise.race([pending, stopped])
|
|
463
493
|
if (remoteBody === STOPPED) {
|
|
464
494
|
// A late answer's body would hold its connection to the exit.
|