@vzn/vx 0.0.645 → 0.0.646

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +10 -28
  2. package/package.json +5 -5
  3. package/schemas/last.json +15 -0
  4. package/skills/vx/SKILL.md +1 -1
  5. package/src/bin.ts +1 -1
  6. package/src/cache/cache.ts +66 -16
  7. package/src/cache/git-inputs.ts +55 -10
  8. package/src/cache/index.ts +4 -0
  9. package/src/cache/inputs.ts +61 -11
  10. package/src/cache/layer.ts +25 -2
  11. package/src/cache/layered-cache.ts +32 -2
  12. package/src/cache/output-index.ts +25 -11
  13. package/src/cache/remote-lane.ts +13 -4
  14. package/src/cache/run-history.ts +13 -6
  15. package/src/cache/schema.ts +19 -2
  16. package/src/cli/core-alias.ts +5 -2
  17. package/src/cli/help.ts +4 -6
  18. package/src/cli/index.ts +0 -12
  19. package/src/cli/init.ts +43 -10
  20. package/src/cli/last.ts +4 -2
  21. package/src/cli/plugin-commands.ts +96 -3
  22. package/src/cli/plugin-templates.ts +1 -1
  23. package/src/cli/run.ts +2 -7
  24. package/src/cli/select.ts +5 -3
  25. package/src/cli/upgrade.ts +4 -5
  26. package/src/exec/executor.ts +112 -9
  27. package/src/exec/index.ts +2 -0
  28. package/src/exec/proc-sample.ts +67 -5
  29. package/src/exec/runner.ts +4 -0
  30. package/src/exec/sandbox-runtime.ts +11 -4
  31. package/src/graph/scheduler.ts +59 -8
  32. package/src/history/history.ts +119 -7
  33. package/src/history/metrics.ts +53 -16
  34. package/src/index.ts +15 -0
  35. package/src/orchestrator/execute-args.ts +3 -0
  36. package/src/orchestrator/execute-task.ts +28 -7
  37. package/src/orchestrator/held-servers.ts +86 -0
  38. package/src/orchestrator/hit-restore.ts +11 -4
  39. package/src/orchestrator/index.ts +17 -1
  40. package/src/orchestrator/local-floor.ts +104 -0
  41. package/src/orchestrator/local-shortcircuit.ts +26 -7
  42. package/src/orchestrator/miss-save.ts +10 -4
  43. package/src/orchestrator/persistent-task.ts +14 -0
  44. package/src/orchestrator/placement.ts +84 -9
  45. package/src/orchestrator/plugin-host.ts +21 -3
  46. package/src/orchestrator/plugin.ts +37 -0
  47. package/src/orchestrator/prepare.ts +11 -3
  48. package/src/orchestrator/run-keys.ts +164 -0
  49. package/src/orchestrator/run-records.ts +6 -0
  50. package/src/orchestrator/run.ts +93 -6
  51. package/src/orchestrator/sandbox-request.ts +9 -3
  52. package/src/orchestrator/task-artifacts.ts +142 -0
  53. package/src/orchestrator/task-hash.ts +11 -1
  54. package/src/orchestrator/task-process.ts +16 -0
  55. package/src/orchestrator/telemetry-host.ts +2 -1
  56. package/src/orchestrator/telemetry.ts +60 -17
  57. package/src/report/forecast.ts +6 -27
  58. package/src/report/framed-output.ts +41 -5
  59. package/src/report/logger.ts +60 -20
  60. package/src/report/status-line.ts +38 -18
  61. package/src/report/summary.ts +61 -7
  62. package/src/settings.ts +7 -0
  63. package/src/util/colors.ts +16 -1
  64. package/src/util/index.ts +1 -0
  65. package/src/util/paths.ts +30 -0
  66. package/src/util/ui.ts +4 -1
  67. package/src/workspace/config-bytecode.ts +276 -0
  68. package/src/workspace/index.ts +0 -1
  69. package/src/workspace/migration.ts +4 -32
  70. package/src/workspace/project-loader.ts +92 -14
  71. package/src/cli/foreign-flags.ts +0 -727
package/README.md CHANGED
@@ -23,37 +23,19 @@ Task orchestration with a cache you can trust, on a core you can extend.
23
23
  Each task is one shell command with declared inputs and outputs; vx runs
24
24
  them in dependency order and replays results from a content-addressed cache.
25
25
 
26
- ## Install
27
-
28
- ```sh
29
- npm install -D @vzn/vx # or: pnpm add -D -w · yarn add -D (-W on Yarn 1) · bun add -d
30
- ```
31
-
32
- A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and arm64; no Bun needed. The
33
- [release](https://github.com/vznjs/vx/releases) binary needs no Node either.
34
- Windows: run vx inside WSL2.
35
-
36
26
  ## Quick start
37
27
 
38
28
  In a git repo:
39
29
 
40
30
  ```sh
41
- npx @vzn/vx init # installs vx globally and in the repo; beside turbo.json or nx.json: migrates them; else a root vx.config.ts
42
- vx run build --all # builds in dependency order and stores the results
43
- vx run build --all # nothing changed: every task up-to-date
31
+ bunx @vzn/vx init # installs vx, writes an empty root vx.config.ts and the lockfile plugin
44
32
  ```
45
33
 
46
- Bun workspaces: `bun add -d @vzn/vx` and `bunx @vzn/vx` (npm refuses to run in
47
- a repo whose `devEngines` names Bun).
48
-
49
- Without Turbo or Nx, tasks run uncached until you declare what they read (below).
50
-
51
- `init` prints what it wrote and anything left to decide (a `TODO`). A miss you did not expect?
52
- `npx @vzn/vx why <task>` names the input that moved.
53
-
54
- ### Without Turbo or Nx
34
+ A prebuilt binary for Linux (glibc, not Alpine's musl) and macOS, x64 and
35
+ arm64. Windows: run vx inside WSL2.
55
36
 
56
- `init` writes one task per package script, uncached. Cache one by saying what it reads and writes:
37
+ A task runs its tool directly, never through a package script. Cache one by
38
+ saying what it reads and writes:
57
39
 
58
40
  ```ts
59
41
  // packages/app/vx.config.ts
@@ -74,10 +56,10 @@ export default defineProject({
74
56
  ```
75
57
 
76
58
  ```sh
77
- npx @vzn/vx run build --all # builds, in dependency order; stores the results
78
- npx @vzn/vx run build --all # nothing changed: up-to-date, in milliseconds
59
+ vx run build --all # builds, in dependency order; stores the results
60
+ vx run build --all # nothing changed: up-to-date, in milliseconds
79
61
  rm -rf packages/*/dist
80
- npx @vzn/vx run build --all # restored from the cache, not rebuilt
62
+ vx run build --all # restored from the cache, not rebuilt
81
63
  ```
82
64
 
83
65
  <img src="https://vznjs.github.io/vx/demo.svg" width="760" alt="vx run ci --all on examples/basic: cold, three tasks run; again, three up-to-date.">
@@ -86,7 +68,7 @@ npx @vzn/vx run build --all # restored from the cache, not rebuilt
86
68
 
87
69
  `init` also writes `.vx/projects.lock.json`, every project's config as it evaluated.
88
70
  Commit it. Every run reads its configs from the lock, so run
89
- `npx @vzn/vx lock` and commit the lock again after you:
71
+ `vx lock` and commit the lock again after you:
90
72
 
91
73
  - edit any `vx.config.ts` or `.vx/workspace.ts`,
92
74
  - update dependencies or plugins,
@@ -134,7 +116,7 @@ Write your own: [plugin guide](https://vznjs.github.io/vx/guides/plugins/) ·
134
116
 
135
117
  ## Coming from Turbo or Nx
136
118
 
137
- Beside `turbo.json` or `nx.json`, `npx @vzn/vx init` runs `@vzn/vx-migrate`:
119
+ Beside `turbo.json` or `nx.json`, `bunx @vzn/vx init` runs `@vzn/vx-migrate`:
138
120
  it writes a `vx.config.ts` per package from `turbo.json` or the Nx graph,
139
121
  and you keep those and delete the old config. `vx init --keep` writes only
140
122
  a `.vx/workspace.ts` that maps the old config instead, and its lock: a temporary start,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vzn/vx",
3
- "version": "0.0.645",
3
+ "version": "0.0.646",
4
4
  "description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -21,10 +21,10 @@
21
21
  "node": ">=18"
22
22
  },
23
23
  "optionalDependencies": {
24
- "@vzn/vx-linux-x64": "0.0.645",
25
- "@vzn/vx-linux-arm64": "0.0.645",
26
- "@vzn/vx-darwin-x64": "0.0.645",
27
- "@vzn/vx-darwin-arm64": "0.0.645"
24
+ "@vzn/vx-linux-x64": "0.0.646",
25
+ "@vzn/vx-linux-arm64": "0.0.646",
26
+ "@vzn/vx-darwin-x64": "0.0.646",
27
+ "@vzn/vx-darwin-arm64": "0.0.646"
28
28
  },
29
29
  "dependencies": {
30
30
  "@anthropic-ai/sandbox-runtime": "^0.0.76"
package/schemas/last.json CHANGED
@@ -166,6 +166,9 @@
166
166
  "hash",
167
167
  "cpuMs",
168
168
  "peakRssBytes",
169
+ "peakCores",
170
+ "avgRssBytes",
171
+ "placement",
169
172
  "wallclockStartNs",
170
173
  "wallclockEndNs",
171
174
  "blockedBy",
@@ -199,6 +202,18 @@
199
202
  "hash": { "type": "string" },
200
203
  "cpuMs": { "type": ["number", "null"] },
201
204
  "peakRssBytes": { "type": ["integer", "null"] },
205
+ "peakCores": {
206
+ "type": ["number", "null"],
207
+ "description": "most cores one sampling interval burned; null when nothing sampled"
208
+ },
209
+ "avgRssBytes": {
210
+ "type": ["integer", "null"],
211
+ "description": "mean resident memory over the samples; null when nothing sampled"
212
+ },
213
+ "placement": {
214
+ "type": ["string", "null"],
215
+ "description": "where an executor ran it (small#1); null when here"
216
+ },
202
217
  "wallclockStartNs": {
203
218
  "type": ["string", "null"],
204
219
  "description": "decimal nanoseconds from the run's start"
@@ -106,7 +106,7 @@ vx info --format json # vx and Bun versions, projects, config errors, plugins,
106
106
 
107
107
  ## Install
108
108
 
109
- `npx skills add vznjs/vx` installs this skill. To match the vx a workspace
109
+ `bunx skills add vznjs/vx` installs this skill. To match the vx a workspace
110
110
  has installed, copy it from there instead:
111
111
  `cp node_modules/@vzn/vx/skills/vx/SKILL.md .claude/skills/vx/`.
112
112
  `vx upgrade` inside a workspace moves the project's `@vzn/vx` and the
package/src/bin.ts CHANGED
@@ -39,7 +39,7 @@ async function main(): Promise<void> {
39
39
  // package, a workspace or project config — resolves to THIS core, not to
40
40
  // a second copy from node_modules (core-alias.ts says why and what it
41
41
  // costs). Registered before any verb runs; the façade loads on first use.
42
- registerCoreAlias(() => import('./index.js') as Promise<Record<string, unknown>>)
42
+ registerCoreAlias(() => require('./index.js') as Record<string, unknown>)
43
43
  const code = await run(argv)
44
44
  // Not `process.exit` here: Bun drops what a pipe has not yet taken
45
45
  // when it follows a large write (300 KB written, 64 KiB delivered;
@@ -165,6 +165,8 @@ const VX_CACHE_NAMES = new Set([
165
165
  'runs',
166
166
  'failures',
167
167
  'lockfile-claims',
168
+ // The workspace config's bytecode bundle (workspace/config-bytecode.ts).
169
+ 'compiled',
168
170
  ])
169
171
 
170
172
  /**
@@ -324,7 +326,16 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
324
326
  // is unchanged.
325
327
  // v42: fold_memo — each task's last key by its fold up to the files and
326
328
  // a token of their git records. The cache KEY is unchanged.
327
- export const SCHEMA_VERSION = 'v42'
329
+ // v43: runs.peak_cores / avg_rss_bytes — what a sampled run burned at
330
+ // its peak and held on average. The cache KEY is unchanged.
331
+ // v44: runs.placement — where an executor ran the task. The cache KEY
332
+ // is unchanged.
333
+ // v45: a 'remote' task_durations row also carries the task's measured
334
+ // cores, memory and saturation, for a run with no local history.
335
+ // The cache KEY is unchanged.
336
+ // v46: runs.keyless_by — the task with no inputs a key folded, so
337
+ // `vx why` names it. The cache KEY is unchanged.
338
+ export const SCHEMA_VERSION = 'v46'
328
339
 
329
340
  /**
330
341
  * An entry's key components as its one `entry_inputs` row (v34): JSON
@@ -1699,6 +1710,10 @@ export class Cache implements CacheLayer {
1699
1710
  return this.guard(() => this.readEntry(hash))
1700
1711
  }
1701
1712
 
1713
+ getSaved(hash: string): Promise<CacheEntry | null> {
1714
+ return this.getIngested(hash)
1715
+ }
1716
+
1702
1717
  private async readEntry(hash: string): Promise<CacheEntry | null> {
1703
1718
  const row = this.selectEntry.get(hash) as EntryRow | null
1704
1719
  if (row === null) return null
@@ -1956,7 +1971,14 @@ export class Cache implements CacheLayer {
1956
1971
  )
1957
1972
  }
1958
1973
 
1959
- /** The copy-out: false, having written nothing, where it is not the case it serves. */
1974
+ /**
1975
+ * The copy-out: false, having written nothing, where it is not the case it
1976
+ * serves. It writes only where nothing stands: each file's place is absent,
1977
+ * and every directory on the way either is a real directory or is absent
1978
+ * and made here. solid's build outputs `store/dist/**` beside the sources
1979
+ * in `store/`; asking that each output's TOP be absent sent every such
1980
+ * restore through the tar extractor, ~25 ms a task (2026-10-22).
1981
+ */
1960
1982
  private async copyTree(
1961
1983
  tree: string,
1962
1984
  rows: readonly OutputFileRow[],
@@ -1964,22 +1986,49 @@ export class Cache implements CacheLayer {
1964
1986
  projectDir: string,
1965
1987
  accept: ((name: string) => boolean) | undefined,
1966
1988
  ): Promise<boolean> {
1967
- const tops = new Set<string>()
1968
1989
  for (const r of rows) {
1969
1990
  if (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) || r.mode === LINK_MODE) return false
1970
1991
  if (accept !== undefined && !r.path.endsWith('/') && !accept(`outputs/${r.path}`))
1971
1992
  return false
1972
- tops.add(r.path.split('/')[0]!)
1973
- }
1974
- const held = await Promise.all(
1975
- [...tops].map((t) =>
1976
- lstat(path.join(projectDir, t)).then(
1977
- () => true,
1978
- () => false,
1979
- ),
1980
- ),
1993
+ }
1994
+ // What stands at each path, asked once: a real directory, nothing, or
1995
+ // anything else (a file, a link), which the extractor's rules decide.
1996
+ const standing = new Map<string, Promise<'dir' | 'absent' | 'other'>>()
1997
+ const at = (rel: string): Promise<'dir' | 'absent' | 'other'> => {
1998
+ let known = standing.get(rel)
1999
+ if (known === undefined) {
2000
+ known = lstat(path.join(projectDir, rel)).then(
2001
+ (st) => (st.isDirectory() ? 'dir' : 'other'),
2002
+ (err: NodeJS.ErrnoException) => (err.code === 'ENOENT' ? 'absent' : 'other'),
2003
+ )
2004
+ standing.set(rel, known)
2005
+ }
2006
+ return known
2007
+ }
2008
+ // The topmost absent directories this copy makes, and the files it
2009
+ // writes into a directory that stood: what a failure removes, and only that.
2010
+ const made = new Set<string>()
2011
+ const placed: string[] = []
2012
+ const fits = await Promise.all(
2013
+ rows.map(async (r) => {
2014
+ const segs = r.path.replace(/\/$/, '').split('/')
2015
+ for (let i = 1; i <= segs.length; i++) {
2016
+ const rel = segs.slice(0, i).join('/')
2017
+ const isLeaf = i === segs.length
2018
+ const state = await at(rel)
2019
+ if (state === 'absent') {
2020
+ if (!isLeaf || r.path.endsWith('/')) made.add(rel)
2021
+ else placed.push(rel)
2022
+ return true
2023
+ }
2024
+ if (state === 'other' || (isLeaf && !r.path.endsWith('/'))) return false
2025
+ }
2026
+ return true
2027
+ }),
1981
2028
  )
1982
- if (held.includes(true)) return false
2029
+ if (fits.includes(false)) return false
2030
+ // A directory made under another one made here is the outer one's.
2031
+ const roots = [...made].filter((d) => ![...made].some((o) => d.startsWith(`${o}/`)))
1983
2032
  readUmask()
1984
2033
  const plain = restoredMode(0o666)
1985
2034
  try {
@@ -1999,9 +2048,10 @@ export class Cache implements CacheLayer {
1999
2048
  }),
2000
2049
  )
2001
2050
  } catch (err) {
2002
- await Promise.all(
2003
- [...tops].map((t) => rm(path.join(projectDir, t), { recursive: true, force: true })),
2004
- )
2051
+ await Promise.all([
2052
+ ...roots.map((d) => rm(path.join(projectDir, d), { recursive: true, force: true })),
2053
+ ...placed.map((f) => rm(path.join(projectDir, f), { force: true })),
2054
+ ])
2005
2055
  throw err
2006
2056
  }
2007
2057
  return true
@@ -204,7 +204,8 @@ function configObjectFormat(config: string): 'sha1' | 'sha256' | undefined {
204
204
  interface PartitionDigest {
205
205
  snapshot: readonly string[]
206
206
  oids: OidSlot | undefined
207
- digest: string
207
+ /** Built on first read: a partition with untrusted files stats them then. */
208
+ digest: () => string | undefined
208
209
  }
209
210
 
210
211
  /**
@@ -515,11 +516,20 @@ export class GitFilesCache extends Map<string, readonly string[]> {
515
516
  const d = this.digests.get(projectDir)
516
517
  if (d === undefined || d.snapshot !== snapshot || this.get(projectDir) !== snapshot) return
517
518
  if (this.oids.get(projectDir) !== d.oids || this.changed.has(projectDir)) return
518
- return d.digest
519
- }
520
-
521
- noteFilesDigest(projectDir: string, snapshot: readonly string[], digest: string): void {
522
- this.digests.set(projectDir, { snapshot, oids: this.oids.get(projectDir), digest })
519
+ return d.digest()
520
+ }
521
+
522
+ noteFilesDigest(
523
+ projectDir: string,
524
+ snapshot: readonly string[],
525
+ digest: () => string | undefined,
526
+ ): void {
527
+ let memo: { value: string | undefined } | undefined
528
+ this.digests.set(projectDir, {
529
+ snapshot,
530
+ oids: this.oids.get(projectDir),
531
+ digest: () => (memo ??= { value: digest() }).value,
532
+ })
523
533
  }
524
534
 
525
535
  /** Trusted index OIDs for a project (abs path → oid), if any survive. */
@@ -1908,6 +1918,30 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
1908
1918
  }
1909
1919
  }
1910
1920
 
1921
+ /**
1922
+ * Each loose file's stat identity, `hashFile`'s memo key: mode, size,
1923
+ * mtime, ctime and inode. Undefined for a symlink (what it points to
1924
+ * folds too, and its stat does not move with that), a file gone, or one
1925
+ * changed within the racy window, where the memo trusts no stat either.
1926
+ */
1927
+ function statRecords(
1928
+ rels: readonly string[],
1929
+ abs: (rel: string) => string,
1930
+ prefixLength: number,
1931
+ ): string | undefined {
1932
+ if (rels.length === 0) return ''
1933
+ const now = Date.now()
1934
+ let out = ''
1935
+ for (const rel of rels) {
1936
+ const st = lstatSync(abs(rel), { throwIfNoEntry: false })
1937
+ if (st === undefined || !st.isFile()) return undefined
1938
+ if (now - Math.floor(st.ctimeMs) < racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) return
1939
+ const mode = (st.mode & 0o100) !== 0 ? '100755' : '100644'
1940
+ out += `${rel.slice(prefixLength)}\0stat:${mode}:${st.size}:${Math.floor(st.mtimeMs)}:${Math.floor(st.ctimeMs)}:${Number(st.ino)}\n`
1941
+ }
1942
+ return out
1943
+ }
1944
+
1911
1945
  /** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
1912
1946
  export async function applyGitEnumeration(
1913
1947
  enumeration: GitEnumeration,
@@ -2006,15 +2040,20 @@ export async function applyGitEnumeration(
2006
2040
  }
2007
2041
  // One hash of the records joined: one per record was ~25 ms of 125,000.
2008
2042
  let records: string | undefined = ''
2043
+ // A file git vouches for no OID (untracked, or changed since the
2044
+ // index) is named by its stat instead, the identity `hashFile`'s memo
2045
+ // trusts: a just-added config or an edited file turned the fold memo
2046
+ // off for its whole project (+3-5 ms of solid's warm run, 2026-10-22).
2047
+ const loose: string[] = []
2009
2048
  for (let i = start; i < end; i++) {
2010
2049
  const rel = sorted[i]!
2011
2050
  const own = rel.slice(prefix.length)
2012
2051
  matches.push(own)
2013
2052
  if (records === undefined) continue
2014
2053
  const oid = trusted.get(rel)
2015
- if (oid === undefined || oid.startsWith('120000:') || undecodable?.has(rel) === true) {
2016
- records = undefined
2017
- } else records += `${own}\0${oid}\n`
2054
+ if (undecodable?.has(rel) === true || oid?.startsWith('120000:') === true) records = undefined
2055
+ else if (oid === undefined) loose.push(rel)
2056
+ else records += `${own}\0${oid}\n`
2018
2057
  }
2019
2058
  // An empty slice is a directory git did not see, not an empty project:
2020
2059
  // a project has at least its package.json, tracked or untracked. A
@@ -2033,7 +2072,13 @@ export async function applyGitEnumeration(
2033
2072
  // setLazyOids AFTER set — set() drops the project's OID slot.
2034
2073
  cache.set(projectDir, matches)
2035
2074
  cache.setLazyOids(projectDir, projOids)
2036
- if (records !== undefined) cache.noteFilesDigest(projectDir, matches, xxh3hex(records))
2075
+ if (records !== undefined) {
2076
+ const known = records
2077
+ cache.noteFilesDigest(projectDir, matches, () => {
2078
+ const statted = statRecords(loose, projAbs, prefix.length)
2079
+ return statted === undefined ? undefined : xxh3hex(`${known}\0${statted}`)
2080
+ })
2081
+ }
2037
2082
  }
2038
2083
  if (workspaceWide) {
2039
2084
  cache.set(workspaceRoot, all)
@@ -29,6 +29,7 @@ export {
29
29
  WORKSPACE_OUTPUT_PREFIX,
30
30
  type OutputDirRow,
31
31
  OUTPUT_DIRS_CAP,
32
+ OUTPUT_DIRS_OVER_CAP,
32
33
  FILE_HASH_RACY_MS,
33
34
  OUTPUT_DIRS_RACY_MS,
34
35
  racyWindowMs,
@@ -54,6 +55,7 @@ export {
54
55
  type ResolveInputsArgs,
55
56
  resolveOutputTree,
56
57
  resolveOutputs,
58
+ outputWalkTakes,
57
59
  resolveWorkspaceOutputs,
58
60
  sweepRestoreTemps,
59
61
  sweepStaging,
@@ -66,6 +68,7 @@ export {
66
68
  stampOutputs,
67
69
  stampWorkspaceOutputs,
68
70
  stopRuntimeProbes,
71
+ answerRuntime,
69
72
  type OutputStamp,
70
73
  } from './inputs.js'
71
74
  export {
@@ -84,6 +87,7 @@ export {
84
87
  export {
85
88
  type ArtifactInfo,
86
89
  LayeredCache,
90
+ type LentAccess,
87
91
  type RemoteCacheLayer,
88
92
  type RemoteOpenContext,
89
93
  } from './layered-cache.js'
@@ -31,6 +31,7 @@ import {
31
31
  outputNever,
32
32
  realpathOf,
33
33
  relPosix,
34
+ restorableOutput,
34
35
  shellArgv,
35
36
  slashBraceExpansions,
36
37
  braceAlternatives,
@@ -530,6 +531,28 @@ function projectBinDirs(projectDir: string, workspaceRoot: string): string[] {
530
531
  return own === root ? [own] : [own, root]
531
532
  }
532
533
 
534
+ /**
535
+ * One runtime probe answered on this machine as a key folds it: an
536
+ * `inputs.runtime` command run in `projectDir`, or with `workspace` a
537
+ * `workspaceRuntime` one run in `workspaceRoot`. For an executor that
538
+ * checks a task's probes where it runs it (`TaskExecutor.runtime`).
539
+ */
540
+ export function answerRuntime(
541
+ command: string,
542
+ projectDir: string,
543
+ workspaceRoot: string,
544
+ workspace = false,
545
+ ): Promise<string> {
546
+ return workspace
547
+ ? runRuntimeCommand(
548
+ command,
549
+ workspaceRoot,
550
+ [path.join(workspaceRoot, 'node_modules', '.bin')],
551
+ undefined,
552
+ )
553
+ : runRuntimeCommand(command, projectDir, projectBinDirs(projectDir, workspaceRoot), undefined)
554
+ }
555
+
533
556
  function resolveEnvValues(
534
557
  names: readonly string[],
535
558
  source: NodeJS.ProcessEnv,
@@ -748,6 +771,21 @@ export async function resolveOutputs(args: {
748
771
  return (await scanOutputs(args)).files
749
772
  }
750
773
 
774
+ /**
775
+ * Whether `resolveOutputs` takes the project-relative file `rel`, decided
776
+ * without the walk: a directory snapshot's file list is held to the
777
+ * entry's rows through it.
778
+ */
779
+ export function outputWalkTakes(
780
+ projectDir: string,
781
+ outputs: readonly string[],
782
+ nestedProjectDirs: string[],
783
+ ): (rel: string) => boolean {
784
+ const restorable = restorableOutput(outputs)
785
+ const nested = inNestedProject(projectDir, nestedProjectDirs)
786
+ return (rel) => restorable(rel) && !nested(rel)
787
+ }
788
+
751
789
  /**
752
790
  * `resolveOutputs`, plus the empty directories the globs match: a save
753
791
  * keeps them, so a hit leaves the tree a miss left (Turbo and Nx restore
@@ -2103,21 +2141,31 @@ async function scanUnion(
2103
2141
  emptyDirs?: Set<string>,
2104
2142
  ): Promise<Set<string>> {
2105
2143
  const matches = new Set<string>()
2144
+ const take = (rel: string): void => {
2145
+ if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
2146
+ const abs = path.resolve(cwd, rel)
2147
+ const st = lstatSync(abs, { throwIfNoEntry: false })
2148
+ if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
2149
+ else if (emptyDirs !== undefined && st?.isDirectory() === true) {
2150
+ if (readdirSync(abs).length === 0) emptyDirs.add(abs)
2151
+ } else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
2152
+ }
2153
+ const opts = { cwd, onlyFiles: false, followSymlinks: false, dot: true }
2106
2154
  // `Bun.Glob`'s scan finds nothing for a brace whose alternatives hold a
2107
2155
  // `/`: `{dist,lib/esm}/**` saved an empty artifact, and a hit restored
2108
2156
  // nothing over a cleaned tree (A-10). Expanded here, as discovery does.
2109
- for (const pattern of positive.flatMap(slashBraceExpansions)) {
2157
+ const patterns = positive.flatMap(slashBraceExpansions)
2158
+ // A leading `**` walks the whole project whatever else is listed, so the
2159
+ // list is one walk: Turbo's `dist/**`, `**/dist/**`, `**/dist` was three,
2160
+ // ~2 ms of a warm hit on solid.
2161
+ if (patterns.length > 1 && patterns.some((p) => p.startsWith('**'))) {
2162
+ const any = anyTaskGlob(patterns)
2163
+ for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
2164
+ return matches
2165
+ }
2166
+ for (const pattern of patterns) {
2110
2167
  if (absentPrefix(cwd, pattern)) continue
2111
- const glob = globFor(pattern)
2112
- for (const rel of glob.scanSync({ cwd, onlyFiles: false, followSymlinks: false, dot: true })) {
2113
- if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) continue
2114
- const abs = path.resolve(cwd, rel)
2115
- const st = lstatSync(abs, { throwIfNoEntry: false })
2116
- if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
2117
- else if (emptyDirs !== undefined && st?.isDirectory() === true) {
2118
- if (readdirSync(abs).length === 0) emptyDirs.add(abs)
2119
- } else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
2120
- }
2168
+ for (const rel of globFor(pattern).scanSync(opts)) take(rel)
2121
2169
  }
2122
2170
  return matches
2123
2171
  }
@@ -2139,6 +2187,8 @@ function absentPrefix(cwd: string, pattern: string): boolean {
2139
2187
  }
2140
2188
  }
2141
2189
 
2190
+ const WHOLE_TREE = new Bun.Glob('**')
2191
+
2142
2192
  // Compiled once per pattern string for the life of the process: a `Bun.Glob`
2143
2193
  // is immutable, and the same handful of declared globs are compiled again
2144
2194
  // for every task otherwise (three sites, up to thousands of times per run).
@@ -212,6 +212,16 @@ export interface RunRecord {
212
212
  notReady?: 'timeout' | 'exited' | 'spawn'
213
213
  /** Each remote read or write of its key that failed, `upload: <message>`, `; `-joined (v40). */
214
214
  remoteError?: string
215
+ /** The sampled peak cores and mean resident memory of the run (v43). */
216
+ peakCores?: number
217
+ avgRssBytes?: number
218
+ /** Where an executor ran it (`small#1`); absent = here (v44). */
219
+ placement?: string
220
+ /** The task with no `inputs` its key folded, so it never read or saved (v46). */
221
+ keylessBy?: string
222
+ /** That machine's cores and memory (v44). */
223
+ placementCores?: number
224
+ placementMemoryBytes?: number
215
225
  }
216
226
 
217
227
  /**
@@ -384,6 +394,13 @@ export interface OutputDirRow {
384
394
  */
385
395
  export const OUTPUT_DIRS_CAP = 8192
386
396
 
397
+ /**
398
+ * `mtime_ms` of the one row a snapshot over `OUTPUT_DIRS_CAP` leaves: never
399
+ * current, so the hit walks, and it tells the hit not to snapshot again.
400
+ * Cleared rows sent every hit through the walk AND a capped snapshot.
401
+ */
402
+ export const OUTPUT_DIRS_OVER_CAP = -3
403
+
387
404
  /**
388
405
  * A directory whose mtime lies within this many ms of the snapshot is RACY
389
406
  * and the whole snapshot is dropped. File timestamps are coarse on Linux
@@ -593,6 +610,12 @@ export interface CacheLayer {
593
610
  stopUploads?(): void
594
611
  key(input: CacheKeyInput): Promise<string>
595
612
  get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
613
+ /**
614
+ * `get` past the run's read policy: an entry a machine running a task
615
+ * for this run saved is the run's own work, not a hit (`TaskArtifacts`).
616
+ * A layer without it is read through `get`.
617
+ */
618
+ getSaved?(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
596
619
  /**
597
620
  * Optional batched `get` (same answers, fewer round trips). The
598
621
  * short-circuit probe uses it when a layer offers one; a layer without it
@@ -651,8 +674,8 @@ export interface CacheLayer {
651
674
  /**
652
675
  * The directory-mtime short-circuit behind a warm hit (optional — a layer
653
676
  * without it keeps the output walk). `recordOutputDirs` snapshots every
654
- * directory under each of `prefixes` (project-relative, whole-subtree
655
- * globs only — see `wholeSubtreePrefixes`) after a save or restore;
677
+ * directory under each of `prefixes` (project-relative, `''` the project
678
+ * itself — see `outputDirPrefixes`) after a save or restore;
656
679
  * `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
657
680
  * every recorded directory still carries its recorded mtime, which proves
658
681
  * no file was added or removed anywhere the glob could see. Machine-local
@@ -104,6 +104,13 @@ export interface ArtifactInfo {
104
104
  readonly durationMs?: number
105
105
  }
106
106
 
107
+ /** A run's access to a remote, lent to a machine that runs its tasks (`RemoteCacheLayer.lend`). */
108
+ export interface LentAccess {
109
+ readonly token: string
110
+ /** Epoch ms; asked again before it. */
111
+ readonly expiresAt?: number
112
+ }
113
+
107
114
  /**
108
115
  * What a remote cache layer must provide — THE plugin seam for remote
109
116
  * caching. Core ships no wire client; a plugin's `cache` hook returns one
@@ -140,6 +147,14 @@ export interface RemoteCacheLayer {
140
147
  * plugin's keychain and may ask on the terminal.
141
148
  */
142
149
  open?(ctx: RemoteOpenContext): Promise<void>
150
+ /**
151
+ * Optional: this run's access, for a machine that runs its tasks
152
+ * elsewhere (`ExecutorContext.cacheFacts`); that machine's plugin reads
153
+ * it as `CacheContext.lent`. Only access made for this run (an OIDC
154
+ * exchange's), never a stored secret: the server's token decides what
155
+ * it may write. Asked after `open`.
156
+ */
157
+ lend?(): Promise<LentAccess | undefined>
143
158
  /**
144
159
  * Existence probe: what the remote knows of the stored artifact, or null
145
160
  * when it holds none. `--dry` and `--force` ask it per key when the
@@ -367,6 +382,16 @@ export class LayeredCache implements CacheLayer {
367
382
  return this.asRemote(hash, materialized)
368
383
  }
369
384
 
385
+ async getSaved(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
386
+ const here = await this.local.getIngested(hash)
387
+ if (here) return here
388
+ // Every remote, read or not, and past a probe's absence: the probe ran
389
+ // before the agent saved it.
390
+ for (const lane of this.lanes)
391
+ if (await this.pullFrom(lane, hash, ctx, true)) return this.local.getIngested(hash)
392
+ return null
393
+ }
394
+
370
395
  /**
371
396
  * A pull the task now waits on, given up when the rest of its body would
372
397
  * take longer than running the task (its stored duration, doubled for
@@ -452,13 +477,18 @@ export class LayeredCache implements CacheLayer {
452
477
  return false
453
478
  }
454
479
 
455
- private async pullFrom(lane: RemoteLane, hash: string, ctx?: CacheGetContext): Promise<boolean> {
480
+ private async pullFrom(
481
+ lane: RemoteLane,
482
+ hash: string,
483
+ ctx?: CacheGetContext,
484
+ fresh = false,
485
+ ): Promise<boolean> {
456
486
  const fetchStart = performance.now()
457
487
  // A stopped run waits on no read, as on no upload: a lookup or download
458
488
  // in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
459
489
  // plugin's request is left to the exit, and its body is cancelled.
460
490
  const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
461
- const pending = lane.get(hash)
491
+ const pending = lane.get(hash, fresh)
462
492
  const remoteBody = await Promise.race([pending, stopped])
463
493
  if (remoteBody === STOPPED) {
464
494
  // A late answer's body would hold its connection to the exit.