@vzn/vx 0.0.645 → 0.0.648

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +39 -50
  2. package/launcher.cjs +9 -2
  3. package/launcher.sh +2 -2
  4. package/package.json +5 -5
  5. package/schemas/cache.json +5 -5
  6. package/schemas/error.json +1 -1
  7. package/schemas/info.json +16 -2
  8. package/schemas/last.json +127 -5
  9. package/schemas/plan.json +28 -2
  10. package/schemas/show.json +10 -1
  11. package/skills/vx/SKILL.md +19 -3
  12. package/src/bin.ts +54 -13
  13. package/src/cache/archive.ts +324 -23
  14. package/src/cache/cache.ts +330 -65
  15. package/src/cache/git-inputs.ts +274 -97
  16. package/src/cache/index.ts +7 -2
  17. package/src/cache/inputs.ts +515 -83
  18. package/src/cache/key-fold.ts +15 -6
  19. package/src/cache/layer.ts +74 -11
  20. package/src/cache/layered-cache.ts +154 -59
  21. package/src/cache/linked-inputs.ts +1 -1
  22. package/src/cache/output-index.ts +44 -14
  23. package/src/cache/plugin-cache.ts +49 -23
  24. package/src/cache/remote-hits.ts +72 -0
  25. package/src/cache/remote-lane.ts +100 -15
  26. package/src/cache/run-history.ts +77 -23
  27. package/src/cache/run-store.ts +11 -10
  28. package/src/cache/schema.ts +42 -6
  29. package/src/cache/tar-stream.ts +5 -0
  30. package/src/cache/umask.ts +1 -2
  31. package/src/cli/cache.ts +58 -27
  32. package/src/cli/completions.ts +12 -5
  33. package/src/cli/core-alias.ts +5 -2
  34. package/src/cli/docs.ts +42 -16
  35. package/src/cli/help.ts +62 -25
  36. package/src/cli/index.ts +79 -45
  37. package/src/cli/info.ts +12 -6
  38. package/src/cli/init.ts +222 -34
  39. package/src/cli/last.ts +443 -47
  40. package/src/cli/lock.ts +197 -52
  41. package/src/cli/plan-format.ts +91 -49
  42. package/src/cli/plugin-commands.ts +160 -11
  43. package/src/cli/plugin-templates.ts +13 -13
  44. package/src/cli/registry.ts +209 -0
  45. package/src/cli/run.ts +111 -41
  46. package/src/cli/select.ts +148 -49
  47. package/src/cli/show.ts +59 -16
  48. package/src/cli/task-verb.ts +16 -6
  49. package/src/cli/upgrade.ts +155 -54
  50. package/src/cli/watch-cycle.ts +10 -5
  51. package/src/cli/watch-filter.ts +71 -3
  52. package/src/cli/watch-fs.ts +22 -6
  53. package/src/cli/watch-judge.ts +48 -6
  54. package/src/cli/watch.ts +163 -42
  55. package/src/cli/why.ts +211 -73
  56. package/src/cli/workspace-config.ts +54 -15
  57. package/src/config.ts +13 -1
  58. package/src/exec/env.ts +7 -2
  59. package/src/exec/executor.ts +262 -15
  60. package/src/exec/index.ts +8 -2
  61. package/src/exec/kill-tree.ts +15 -0
  62. package/src/exec/local-executor.ts +1 -1
  63. package/src/exec/proc-sample.ts +70 -6
  64. package/src/exec/runner.ts +256 -53
  65. package/src/exec/sandbox-binds.ts +7 -3
  66. package/src/exec/sandbox-paths.ts +1 -1
  67. package/src/exec/sandbox-runtime.ts +222 -28
  68. package/src/exec/sandbox-violations.ts +52 -20
  69. package/src/graph/critical-path.ts +79 -0
  70. package/src/graph/default-build.ts +19 -0
  71. package/src/graph/definition.ts +38 -0
  72. package/src/graph/folds.ts +38 -5
  73. package/src/graph/index.ts +4 -1
  74. package/src/graph/priorities.ts +74 -8
  75. package/src/graph/scheduler.ts +326 -65
  76. package/src/graph/task-graph.ts +228 -35
  77. package/src/handoff.ts +97 -9
  78. package/src/history/failure-mode.ts +6 -3
  79. package/src/history/history.ts +268 -50
  80. package/src/history/index.ts +2 -0
  81. package/src/history/metrics.ts +142 -45
  82. package/src/index.ts +42 -10
  83. package/src/orchestrator/admission.ts +10 -6
  84. package/src/orchestrator/affected-tasks.ts +11 -6
  85. package/src/orchestrator/deferred-outputs.ts +2 -0
  86. package/src/orchestrator/doctor.ts +134 -42
  87. package/src/orchestrator/execute-args.ts +6 -0
  88. package/src/orchestrator/execute-task.ts +155 -42
  89. package/src/orchestrator/fingerprint-watch.ts +1 -1
  90. package/src/orchestrator/graph-memo.ts +140 -0
  91. package/src/orchestrator/graph-stage.ts +18 -1
  92. package/src/orchestrator/held-servers.ts +86 -0
  93. package/src/orchestrator/hit-restore.ts +227 -122
  94. package/src/orchestrator/index.ts +20 -3
  95. package/src/orchestrator/local-floor.ts +198 -0
  96. package/src/orchestrator/local-shortcircuit.ts +51 -8
  97. package/src/orchestrator/lockfile-claim.ts +96 -16
  98. package/src/orchestrator/miss-save.ts +52 -12
  99. package/src/orchestrator/options.ts +8 -6
  100. package/src/orchestrator/output-host.ts +43 -14
  101. package/src/orchestrator/output-log.ts +90 -10
  102. package/src/orchestrator/persistent-task.ts +97 -28
  103. package/src/orchestrator/persistent.ts +16 -4
  104. package/src/orchestrator/placement.ts +257 -14
  105. package/src/orchestrator/plan.ts +64 -7
  106. package/src/orchestrator/plugin-host.ts +230 -40
  107. package/src/orchestrator/plugin.ts +150 -37
  108. package/src/orchestrator/prepare.ts +178 -32
  109. package/src/orchestrator/projects.ts +65 -40
  110. package/src/orchestrator/remote-prefetch.ts +119 -25
  111. package/src/orchestrator/run-artifacts.ts +95 -32
  112. package/src/orchestrator/run-context.ts +39 -41
  113. package/src/orchestrator/run-failures.ts +65 -23
  114. package/src/orchestrator/run-id.ts +6 -3
  115. package/src/orchestrator/run-keys.ts +300 -0
  116. package/src/orchestrator/run-live.ts +294 -0
  117. package/src/orchestrator/run-lock.ts +13 -7
  118. package/src/orchestrator/run-records.ts +35 -23
  119. package/src/orchestrator/run-report.ts +3 -3
  120. package/src/orchestrator/run-steps.ts +58 -22
  121. package/src/orchestrator/run-stop.ts +22 -5
  122. package/src/orchestrator/run.ts +409 -79
  123. package/src/orchestrator/sandbox-fix.ts +7 -5
  124. package/src/orchestrator/sandbox-request.ts +78 -22
  125. package/src/orchestrator/shell-verdict.ts +5 -5
  126. package/src/orchestrator/signals.ts +26 -1
  127. package/src/orchestrator/stable-keys.ts +92 -8
  128. package/src/orchestrator/steps.ts +2 -2
  129. package/src/orchestrator/task-artifacts.ts +206 -0
  130. package/src/orchestrator/task-hash.ts +89 -14
  131. package/src/orchestrator/task-log-buffer.ts +72 -36
  132. package/src/orchestrator/task-process.ts +22 -0
  133. package/src/orchestrator/telemetry-host.ts +24 -3
  134. package/src/orchestrator/telemetry.ts +113 -38
  135. package/src/orchestrator/upstream.ts +15 -0
  136. package/src/report/events.ts +32 -13
  137. package/src/report/forecast.ts +36 -35
  138. package/src/report/framed-output.ts +173 -46
  139. package/src/report/index.ts +1 -0
  140. package/src/report/logger.ts +217 -186
  141. package/src/report/path-links.ts +12 -4
  142. package/src/report/plain-output.ts +13 -5
  143. package/src/report/status-line.ts +94 -46
  144. package/src/report/summary.ts +174 -38
  145. package/src/report/terminal-signals.ts +13 -2
  146. package/src/settings.ts +14 -6
  147. package/src/util/bun-version.ts +4 -1
  148. package/src/util/colors.ts +20 -3
  149. package/src/util/config-frame.ts +41 -4
  150. package/src/{graph → util}/dependency-spec.ts +22 -13
  151. package/src/util/edit-distance.ts +13 -1
  152. package/src/util/env-flag.ts +9 -0
  153. package/src/util/errors.ts +59 -3
  154. package/src/util/essential-env.ts +57 -0
  155. package/src/util/git-ignored.ts +1 -1
  156. package/src/util/hash.ts +35 -0
  157. package/src/util/hide-env.ts +45 -5
  158. package/src/util/index.ts +22 -5
  159. package/src/util/num.ts +8 -0
  160. package/src/util/paths.ts +121 -21
  161. package/src/util/report.ts +6 -1
  162. package/src/util/secret-mask.ts +62 -1
  163. package/src/util/secrets.ts +7 -2
  164. package/src/util/size.ts +11 -3
  165. package/src/util/tail.ts +6 -1
  166. package/src/util/timing.ts +13 -1
  167. package/src/util/ui.ts +65 -4
  168. package/src/util/which.ts +1 -1
  169. package/src/util/write-file.ts +67 -0
  170. package/src/workspace/affected.ts +710 -107
  171. package/src/workspace/bracket-literal.ts +84 -0
  172. package/src/workspace/canonical-dir.ts +37 -0
  173. package/src/workspace/config-bytecode.ts +282 -0
  174. package/src/workspace/config-eval.ts +29 -8
  175. package/src/workspace/config-imports.ts +199 -58
  176. package/src/workspace/config-schema.ts +242 -39
  177. package/src/workspace/filter.ts +37 -8
  178. package/src/workspace/index.ts +11 -2
  179. package/src/workspace/json-data.ts +3 -4
  180. package/src/workspace/lockfile.ts +57 -8
  181. package/src/workspace/migration.ts +72 -49
  182. package/src/workspace/package-graph.ts +5 -4
  183. package/src/workspace/project-loader.ts +314 -47
  184. package/src/workspace/repo-id.ts +1 -1
  185. package/src/workspace/workspace.ts +176 -33
  186. package/src/cli/foreign-flags.ts +0 -727
@@ -1,9 +1,7 @@
1
1
  // The cache key, derived: `foldKey` is the whole of what `Cache.key` computes,
2
2
  // with the two things it read off the class (the file hasher and the
3
3
  // workspace-relative path) passed in. Nothing here touches a store, so the
4
- // fold runs wherever xxh3 does: the CLI, a remote layer, and the site's
5
- // playground, which bundles this file to derive the keys the CLI would
6
- // (design/playground-spike-2026-09.md, P1; item 691).
4
+ // fold runs wherever xxh3 does: the CLI and a remote layer (item 691).
7
5
 
8
6
  import { xxh3, xxh3hex } from '../util/index.js'
9
7
  import type { CacheKeyInput } from './layer.js'
@@ -113,14 +111,21 @@ import type { CacheKeyInput } from './layer.js'
113
111
  // v44: the sidecar lists the empty directories under the outputs; a v43
114
112
  // artifact holds none and a hit left them out (Growth's parity, 2026-10-16).
115
113
  // v45: the sidecar lists the symlinks under the outputs and a hit makes
116
- // them; a v44 artifact holds a link as a copy of its target (M, 2026-10-16).
114
+ // them; a v44 artifact holds a link as a copy of its target (2026-10-16).
117
115
  // v46: vx's names are reserved only at the project root; a v45 artifact
118
116
  // left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
119
117
  // `vx.lock.json` (Growth's parity, 2026-10-17).
120
118
  // v47: the local store keeps plain files (`cache.store`).
121
- // v48: a task glob is standard again (M, 2026-10-18): `app/[id]/page.js` is
119
+ // v48: a task glob is standard again (2026-10-18): `app/[id]/page.js` is
122
120
  // a class, and a v47 entry under its unchanged key holds the literal route.
123
- export const CACHE_VERSION = 'vx-cache-v48'
121
+ // v49: each brace alternative is read alone (2026-10-28): a v48 entry for
122
+ // `outputs: ['{dist,lib}']` under its unchanged key holds nothing.
123
+ // v49, on another branch at once: the sidecar keeps the mode of an output
124
+ // directory that withholds a permission 0755 grants, and a hit makes it so;
125
+ // a v48 artifact keeps none, and a hit gave `chmod 700 dist/keys` back as 0755.
126
+ // v50: both v49 changes in one build (2026-10-29): a v49 entry was written
127
+ // by a vx that had only one of them.
128
+ export const CACHE_VERSION = 'vx-cache-v51'
124
129
 
125
130
  /**
126
131
  * The digest a key folds for an input gone between its enumeration and its
@@ -321,5 +326,9 @@ export function foldHead(
321
326
  for (const [n, v] of pluginParts) cap.push({ kind: 'plugin', name: n, hash: digestValue(v) })
322
327
  }
323
328
  }
329
+ if (input.platform !== undefined) {
330
+ h = xxh3(`platform:${input.platform}`, h)
331
+ if (cap) cap.push({ kind: 'platform', name: 'platform', hash: input.platform })
332
+ }
324
333
  return h
325
334
  }
@@ -89,6 +89,12 @@ export interface CacheKeyInput {
89
89
  * derives exactly the key it always did.
90
90
  */
91
91
  pluginParts?: ReadonlyArray<readonly [name: string, value: string]>
92
+ /**
93
+ * The platform a task that declares `exec.platforms` runs on: its
94
+ * outputs are that platform's. Folded only when set, so a task that
95
+ * declares none derives the key it always did.
96
+ */
97
+ platform?: string
92
98
  /**
93
99
  * When set, `key()` pushes each component (kind, name, hash) it folds
94
100
  * — at the same fold sites, in fold order. Pure SIDE-CHANNEL: it does
@@ -212,6 +218,20 @@ export interface RunRecord {
212
218
  notReady?: 'timeout' | 'exited' | 'spawn'
213
219
  /** Each remote read or write of its key that failed, `upload: <message>`, `; `-joined (v40). */
214
220
  remoteError?: string
221
+ /** The sampled peak cores and mean resident memory of the run (v43). */
222
+ peakCores?: number
223
+ avgRssBytes?: number
224
+ /** Where an executor ran it (`small#1`); absent = here (v44). */
225
+ placement?: string
226
+ /** The task with no `inputs` its key folded, so it never read or saved (v46). */
227
+ keylessBy?: string
228
+ /** The task's definition (`taskDefinition`): what its history is kept under (v48). */
229
+ def?: string
230
+ /** That machine's cores and memory (v44). */
231
+ placementCores?: number
232
+ placementMemoryBytes?: number
233
+ /** The share of a core each runnable thread got there (`ExecuteResult.machine.cpuShare`, v49). */
234
+ cpuShare?: number
215
235
  }
216
236
 
217
237
  /**
@@ -307,7 +327,7 @@ export interface PruneOptions {
307
327
  * Pick the victims and count the orphans, delete nothing: the result
308
328
  * says what a real prune with the same policy would reap right now.
309
329
  */
310
- dryRun?: boolean
330
+ dry?: boolean
311
331
  }
312
332
 
313
333
  export interface PruneResult {
@@ -384,6 +404,13 @@ export interface OutputDirRow {
384
404
  */
385
405
  export const OUTPUT_DIRS_CAP = 8192
386
406
 
407
+ /**
408
+ * `mtime_ms` of the one row a snapshot over `OUTPUT_DIRS_CAP` leaves: never
409
+ * current, so the hit walks, and it tells the hit not to snapshot again.
410
+ * Cleared rows sent every hit through the walk AND a capped snapshot.
411
+ */
412
+ export const OUTPUT_DIRS_OVER_CAP = -3
413
+
387
414
  /**
388
415
  * A directory whose mtime lies within this many ms of the snapshot is RACY
389
416
  * and the whole snapshot is dropped. File timestamps are coarse on Linux
@@ -557,12 +584,28 @@ export interface CacheLayer {
557
584
  * batch omits it and the prefetch pass falls back to per-hash GETs.
558
585
  */
559
586
  remoteHasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
587
+ /**
588
+ * `remoteHasMany`, with a remote that answers no batch query asked each
589
+ * key, `concurrency` at a time: a complete answer or `null`. For a run
590
+ * that plans on the answer (an executor's demand); others pull instead.
591
+ */
592
+ remoteHasEach?(
593
+ hashes: readonly string[],
594
+ concurrency: number,
595
+ ): Promise<Map<string, ArtifactInfo> | null>
560
596
  /**
561
597
  * The subset of `hashes` the local store already holds. The prefetch
562
598
  * pass probes the remote only for the rest, so a run the local cache
563
599
  * answers makes no remote call. Absent reads as "none held".
564
600
  */
565
601
  localHasMany?(hashes: readonly string[]): Promise<Set<string>>
602
+ /**
603
+ * The subset of `hashes` a remote said it holds, as the local index
604
+ * keeps it across runs and processes: trusted now, with no round-trip.
605
+ * A pull of one starts at once, waiting on no batch probe. Absent, a
606
+ * run learns its remote hits from `remoteHasMany` alone.
607
+ */
608
+ remoteKnown?(hashes: readonly string[]): Map<string, ArtifactInfo>
566
609
  /**
567
610
  * Record that the remote has NO artifact for each of `hashes` (from a
568
611
  * batch probe), so a later `get`/`prefetch` short-circuits to a miss with
@@ -576,12 +619,6 @@ export interface CacheLayer {
576
619
  * `hasRemote`.
577
620
  */
578
621
  probeBeforeUpload?(hashes: Promise<readonly string[]>): void
579
- /**
580
- * The run's batch probe starts when `started` settles: until then a pull
581
- * waits, so a lookup that lands first does not GET a key the probe is
582
- * about to call absent. Only meaningful with `hasRemote`.
583
- */
584
- probeFirst?(started: Promise<void>): void
585
622
  /**
586
623
  * Resolve once every background write-through upload settles. The
587
624
  * orchestrator awaits it before `close()` — an upload reading layer state
@@ -593,6 +630,28 @@ export interface CacheLayer {
593
630
  stopUploads?(): void
594
631
  key(input: CacheKeyInput): Promise<string>
595
632
  get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
633
+ /**
634
+ * `get` past the run's read policy: an entry a machine running a task
635
+ * for this run saved is the run's own work, not a hit (`TaskArtifacts`).
636
+ * A layer without it is read through `get`.
637
+ */
638
+ getSaved?(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
639
+ /**
640
+ * True once a remote the run reads holds entry `hash`, the local entry
641
+ * uploaded to those it writes when none does (`TaskArtifacts.share`). A
642
+ * layer without it shares nothing.
643
+ */
644
+ share?(hash: string): Promise<boolean>
645
+ /**
646
+ * The local entry uploaded over each written remote's copy, which another
647
+ * machine found corrupt (`TaskArtifacts.resend`). True when one took it.
648
+ */
649
+ resend?(hash: string): Promise<boolean>
650
+ /**
651
+ * Whether a remote this run writes took entry `hash` when `save` uploaded
652
+ * it, or already held it; undefined when this run sent it to none.
653
+ */
654
+ uploaded?(hash: string): Promise<boolean | undefined>
596
655
  /**
597
656
  * Optional batched `get` (same answers, fewer round trips). The
598
657
  * short-circuit probe uses it when a layer offers one; a layer without it
@@ -647,12 +706,12 @@ export interface CacheLayer {
647
706
  * out-of-band file edits or deletions and falls through to a real
648
707
  * restore.
649
708
  */
650
- isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean>
709
+ isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): boolean
651
710
  /**
652
711
  * The directory-mtime short-circuit behind a warm hit (optional — a layer
653
712
  * without it keeps the output walk). `recordOutputDirs` snapshots every
654
- * directory under each of `prefixes` (project-relative, whole-subtree
655
- * globs only — see `wholeSubtreePrefixes`) after a save or restore;
713
+ * directory under each of `prefixes` (project-relative, `''` the project
714
+ * itself — see `outputDirPrefixes`) after a save or restore;
656
715
  * `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
657
716
  * every recorded directory still carries its recorded mtime, which proves
658
717
  * no file was added or removed anywhere the glob could see. Machine-local
@@ -676,7 +735,7 @@ export interface CacheLayer {
676
735
  */
677
736
  recordOutputStamps?(hash: string, projectDir: string, workspaceRoot: string): void
678
737
  loadOutputDirsBatch?(hashes: readonly string[]): Map<string, OutputDirRow[]>
679
- outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean>
738
+ outputDirsCurrent?(projectDir: string, rows: readonly OutputDirRow[]): boolean
680
739
  /**
681
740
  * Extract the artifact's `outputs/` entries into `projectDir` and —
682
741
  * when `workspaceRoot` is given — its `workspace-outputs/` entries
@@ -713,6 +772,8 @@ export interface CacheLayer {
713
772
  * format.
714
773
  */
715
774
  workspaceOutputFiles?: string[]
775
+ /** The empty directories under `workspaceOutputFiles`' globs (absolute). */
776
+ workspaceOutputDirs?: string[]
716
777
  workspaceRoot?: string
717
778
  /**
718
779
  * Cache-key components for this entry (the Tier-3 input
@@ -722,6 +783,8 @@ export interface CacheLayer {
722
783
  * Only computed on the miss/save path — never on a hit.
723
784
  */
724
785
  inputComponents?: readonly TaskInputRow[]
786
+ /** No remote held `hash` when the run asked: an upload asks none first. */
787
+ absent?: boolean
725
788
  }): Promise<void>
726
789
  /**
727
790
  * Adopt an artifact produced elsewhere — the remote-hit path. Streams
@@ -35,6 +35,7 @@ import type {
35
35
  SaveArgs,
36
36
  } from './cache.js'
37
37
  import type { OutputDirRow } from './cache.js'
38
+ import { CorruptArtifactError } from './layer.js'
38
39
  import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
39
40
  import type { PluginSecrets } from '../util/index.js'
40
41
 
@@ -57,6 +58,7 @@ function endsOnStop(
57
58
  // which the flag turns into an error rather than a short body.
58
59
  void stopped.then(() => {
59
60
  stop = true
61
+ // quiet: stopping: a reader already done has nothing to cancel
60
62
  void reader.cancel().catch(() => undefined)
61
63
  })
62
64
  const stream = new ReadableStream<Uint8Array>({
@@ -80,6 +82,15 @@ const judged = (i: ArtifactInfo): i is { size: number; durationMs: number } =>
80
82
  /** How often a task waiting on its download weighs running instead. */
81
83
  const JUDGE_EVERY_MS = 25
82
84
 
85
+ /**
86
+ * The least a download is given against running its task. A stored
87
+ * duration is the command's alone, in whole ms (Turbo's wire sends 0 for
88
+ * a sub-ms task), and a budget under one weighing is none: a duration of
89
+ * 0 gave up every body not in by the first, 99% in or not, and ran the
90
+ * task beside the bytes already here.
91
+ */
92
+ const LEAST_BUDGET_MS = JUDGE_EVERY_MS
93
+
83
94
  interface Download {
84
95
  readonly start: number
85
96
  received: number
@@ -104,6 +115,13 @@ export interface ArtifactInfo {
104
115
  readonly durationMs?: number
105
116
  }
106
117
 
118
+ /** A run's access to a remote, lent to a machine that runs its tasks (`RemoteCacheLayer.lend`). */
119
+ export interface LentAccess {
120
+ readonly token: string
121
+ /** Epoch ms; asked again before it. */
122
+ readonly expiresAt?: number
123
+ }
124
+
107
125
  /**
108
126
  * What a remote cache layer must provide — THE plugin seam for remote
109
127
  * caching. Core ships no wire client; a plugin's `cache` hook returns one
@@ -140,6 +158,14 @@ export interface RemoteCacheLayer {
140
158
  * plugin's keychain and may ask on the terminal.
141
159
  */
142
160
  open?(ctx: RemoteOpenContext): Promise<void>
161
+ /**
162
+ * Optional: this run's access, for a machine that runs its tasks
163
+ * elsewhere (`ExecutorContext.cacheFacts`); that machine's plugin reads
164
+ * it as `CacheContext.lent`. Only access made for this run (an OIDC
165
+ * exchange's), never a stored secret: the server's token decides what
166
+ * it may write. Asked after `open`.
167
+ */
168
+ lend?(): Promise<LentAccess | undefined>
143
169
  /**
144
170
  * Existence probe: what the remote knows of the stored artifact, or null
145
171
  * when it holds none. `--dry` and `--force` ask it per key when the
@@ -171,9 +197,16 @@ export interface RemoteCacheLayer {
171
197
  * artifact, so a plugin that streams it never holds it whole, and every
172
198
  * read of it sees the same bytes. An entry is immutable: a `put` of a
173
199
  * key the remote holds keeps the stored bytes. Core never asks `has`
174
- * first; a wire that would overwrite checks inside.
200
+ * first; a wire that would overwrite checks inside, unless told `missing`.
201
+ * `false`: nothing was sent and nothing failed (a layer that writes
202
+ * nothing this run, such as one with no token), so no upload is counted.
175
203
  */
176
- put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
204
+ put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void | false>
205
+ /**
206
+ * The run found `hash` in no remote, and a `put` of it follows: a wire
207
+ * that checks before it overwrites can skip that request.
208
+ */
209
+ missing?(hash: string): void
177
210
  }
178
211
 
179
212
  export class LayeredCache implements CacheLayer {
@@ -181,6 +214,8 @@ export class LayeredCache implements CacheLayer {
181
214
  readonly hasRemote = true
182
215
 
183
216
  private readonly lanes: readonly RemoteLane[]
217
+ /** Each key `save` uploaded: whether a remote took it (`uploaded`). */
218
+ private readonly sent = new Map<string, Promise<boolean>>()
184
219
 
185
220
  /**
186
221
  * In-flight remote pulls keyed by hash. `prefetch` and `get` both go
@@ -194,17 +229,6 @@ export class LayeredCache implements CacheLayer {
194
229
  */
195
230
  private readonly inflight = new Map<string, Promise<boolean>>()
196
231
 
197
- /**
198
- * Hashes a batch probe is asking about, each resolving whether a pull may
199
- * still find it (false: no remote holds it). A `get` that lands while the
200
- * probe is out waits for its answer instead of spending a GET on a key
201
- * the probe is about to call absent.
202
- */
203
- private readonly probing = new Map<string, Promise<boolean>>()
204
-
205
- /** Settles once the run's batch probe has started (`probeFirst`); a pull waits on it. */
206
- private probeStarted: Promise<void> | null = null
207
-
208
232
  /**
209
233
  * Hashes whose local artifact was materialized FROM a remote this run
210
234
  * (by `prefetch` or `get`'s read-through). A later `get` finds them as
@@ -252,6 +276,13 @@ export class LayeredCache implements CacheLayer {
252
276
  return this.local.localHasMany(hashes)
253
277
  }
254
278
 
279
+ remoteKnown(hashes: readonly string[]): Map<string, ArtifactInfo> {
280
+ if (this.readable.length === 0) return new Map()
281
+ const known = this.local.remoteHits.get(hashes)
282
+ for (const [h, info] of known) if (!this.infos.has(h)) this.infos.set(h, info)
283
+ return known
284
+ }
285
+
255
286
  /**
256
287
  * Batch existence probe over the readable remotes — the subset of
257
288
  * `hashes` some remote holds, one round-trip each, or `null` when any
@@ -265,27 +296,34 @@ export class LayeredCache implements CacheLayer {
265
296
  */
266
297
  async remoteHasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
267
298
  if (this.readable.length === 0) return null
268
- const answer = this.probeMany(hashes).then((found) => {
269
- if (found !== null) for (const [h, info] of found) this.infos.set(h, info)
270
- return found
271
- })
272
- const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
273
- for (const h of waiting) {
274
- this.probing.set(
275
- h,
276
- answer.then(
277
- (found) => {
278
- this.probing.delete(h)
279
- return found === null || found.has(h)
280
- },
281
- () => {
282
- this.probing.delete(h)
283
- return true
284
- },
285
- ),
286
- )
299
+ const found = await this.probeMany(hashes)
300
+ if (found !== null) {
301
+ for (const [h, info] of found) this.infos.set(h, info)
302
+ // The whole answer, kept for the next run and the next process.
303
+ this.local.remoteHits.record(hashes, found)
304
+ }
305
+ return found
306
+ }
307
+
308
+ /**
309
+ * Each readable remote asked key by key (`RemoteLane.hasEach`), a key
310
+ * its batch answer already covered not asked again: a complete answer,
311
+ * for a run that plans on it. Null when a probe failed or none may be read.
312
+ */
313
+ async remoteHasEach(
314
+ hashes: readonly string[],
315
+ concurrency: number,
316
+ ): Promise<Map<string, ArtifactInfo> | null> {
317
+ if (this.readable.length === 0) return null
318
+ const out = new Map<string, ArtifactInfo>()
319
+ for (const lane of this.readable) {
320
+ const found = await lane.hasEach(hashes, concurrency)
321
+ if (found === null) return null
322
+ for (const [h, info] of found) if (!out.has(h)) out.set(h, info)
287
323
  }
288
- return answer
324
+ for (const [h, info] of out) this.infos.set(h, info)
325
+ this.local.remoteHits.record(hashes, out)
326
+ return out
289
327
  }
290
328
 
291
329
  private async probeMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
@@ -312,17 +350,6 @@ export class LayeredCache implements CacheLayer {
312
350
  }
313
351
  }
314
352
 
315
- probeFirst(started: Promise<void>): void {
316
- const gate = started.then(
317
- () => {},
318
- () => {},
319
- )
320
- this.probeStarted = gate
321
- void gate.then(() => {
322
- if (this.probeStarted === gate) this.probeStarted = null
323
- })
324
- }
325
-
326
353
  probeBeforeUpload(hashes: Promise<readonly string[]>): void {
327
354
  for (const lane of this.lanes) lane.probeBeforeUpload(hashes)
328
355
  }
@@ -332,7 +359,9 @@ export class LayeredCache implements CacheLayer {
332
359
  if (localHit) {
333
360
  // A prefetch this run may have materialized this entry FROM
334
361
  // remote; the local row exists now, but the work was saved by
335
- // the remote cache, so the provenance stays 'remote'.
362
+ // the remote cache, so the provenance stays 'remote'. Its row lands
363
+ // a tick before the pull settles and says so: wait for that.
364
+ await this.inflight.get(hash)
336
365
  return this.asRemote(hash, localHit)
337
366
  }
338
367
 
@@ -345,6 +374,9 @@ export class LayeredCache implements CacheLayer {
345
374
  // remote walk per key: a settled `false` means no remote had it
346
375
  // (no second probe), and a settled/in-flight `true` means the
347
376
  // artifact was/will-be ingested locally — re-read below.
377
+ // A lookup never waits on the run's batch probe: a hit then paid two
378
+ // round-trips before its first byte (the probe, then the GET), ~300 ms
379
+ // an up-to-date task on CI. A GET answers a miss as fast as the probe.
348
380
  const ingested = await this.awaitOrRun(hash, this.pullFromRemote(hash, ctx))
349
381
  if (!ingested) return null
350
382
 
@@ -367,6 +399,16 @@ export class LayeredCache implements CacheLayer {
367
399
  return this.asRemote(hash, materialized)
368
400
  }
369
401
 
402
+ async getSaved(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
403
+ const here = await this.local.getIngested(hash)
404
+ if (here) return here
405
+ // Every remote, read or not, and past a probe's absence: the probe ran
406
+ // before the agent saved it.
407
+ for (const lane of this.lanes)
408
+ if (await this.pullFrom(lane, hash, ctx, true)) return this.local.getIngested(hash)
409
+ return null
410
+ }
411
+
370
412
  /**
371
413
  * A pull the task now waits on, given up when the rest of its body would
372
414
  * take longer than running the task (its stored duration, doubled for
@@ -382,7 +424,7 @@ export class LayeredCache implements CacheLayer {
382
424
  const info = this.infos.get(hash)
383
425
  if (info === undefined) continue
384
426
  if (!judged(info)) return pull
385
- const budget = info.durationMs * 2
427
+ const budget = Math.max(info.durationMs * 2, LEAST_BUDGET_MS)
386
428
  const dl = this.downloads.get(hash)
387
429
  if (dl === undefined) continue
388
430
  const elapsed = performance.now() - dl.start
@@ -426,11 +468,6 @@ export class LayeredCache implements CacheLayer {
426
468
  private pullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
427
469
  const existing = this.inflight.get(hash)
428
470
  if (existing) return existing
429
- const probed = this.probing.get(hash)
430
- if (probed !== undefined)
431
- return probed.then((found) => (found ? this.pullFromRemote(hash, ctx) : false))
432
- if (this.probeStarted !== null)
433
- return this.probeStarted.then(() => this.pullFromRemote(hash, ctx))
434
471
  const p = this.doPullFromRemote(hash, ctx)
435
472
  this.inflight.set(hash, p)
436
473
  return p
@@ -446,26 +483,38 @@ export class LayeredCache implements CacheLayer {
446
483
  // caller re-reads it, keeping whatever provenance it already had (local,
447
484
  // or remote if a concurrent prefetch set it).
448
485
  if ((await this.local.has(hash)) === 'local') return true
486
+ let absent = true
449
487
  for (const lane of this.readable) {
450
- if (await this.pullFrom(lane, hash, ctx)) return true
488
+ const got = await this.pullFrom(lane, hash, ctx)
489
+ if (got === true) return true
490
+ if (got === false) absent = false
451
491
  }
492
+ // Every remote answered it holds none: a hit the index kept is gone.
493
+ if (absent) this.local.remoteHits.forget(hash)
452
494
  return false
453
495
  }
454
496
 
455
- private async pullFrom(lane: RemoteLane, hash: string, ctx?: CacheGetContext): Promise<boolean> {
497
+ private async pullFrom(
498
+ lane: RemoteLane,
499
+ hash: string,
500
+ ctx?: CacheGetContext,
501
+ fresh = false,
502
+ ): Promise<boolean | null> {
456
503
  const fetchStart = performance.now()
457
504
  // A stopped run waits on no read, as on no upload: a lookup or download
458
- // in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
505
+ // in flight held a Ctrl-C until its last byte (2026-10-17). The
459
506
  // plugin's request is left to the exit, and its body is cancelled.
460
507
  const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
461
- const pending = lane.get(hash)
508
+ const pending = lane.get(hash, fresh)
462
509
  const remoteBody = await Promise.race([pending, stopped])
463
510
  if (remoteBody === STOPPED) {
464
511
  // A late answer's body would hold its connection to the exit.
465
512
  void pending.then((r) => (r?.body instanceof Response ? r.body.body?.cancel() : undefined))
466
513
  return false
467
514
  }
468
- if (remoteBody === null) return false
515
+ // The remote holds none: the one answer that says the key is gone.
516
+ if (remoteBody === null) return null
517
+ if (remoteBody === undefined) return false
469
518
 
470
519
  // Ingest the remote body into local using the caller-supplied
471
520
  // taskId/command plus the remote-reported durationMs, which `get`
@@ -495,6 +544,7 @@ export class LayeredCache implements CacheLayer {
495
544
  endsOnStop(remoteBody.body, ends, (n) => (dl.received += n)),
496
545
  meta,
497
546
  )
547
+ // quiet: raced and awaited below, where its error is reported
498
548
  ingest.catch(() => undefined)
499
549
  if ((await Promise.race([ingest, ends])) === STOPPED) return false
500
550
  } catch (err) {
@@ -503,6 +553,9 @@ export class LayeredCache implements CacheLayer {
503
553
  // crash the run. A corrupt LOCAL artifact is a miss too, one level
504
554
  // down: `Cache.restoreOutputs` drops it (A-52).
505
555
  lane.report('download', hash, err)
556
+ // Bad bytes there: the re-run's save sends good ones over them, or
557
+ // every machine rebuilt the task and none repaired it (2026-10-24).
558
+ if (err instanceof CorruptArtifactError) lane.distrust(hash)
506
559
  return false
507
560
  } finally {
508
561
  this.downloads.delete(hash)
@@ -568,6 +621,7 @@ export class LayeredCache implements CacheLayer {
568
621
  packed = await this.local.packArtifactBytes(args)
569
622
  } catch (err) {
570
623
  for (const lane of writable) lane.report('upload', hash, err)
624
+ this.sent.set(hash, Promise.resolve(false))
571
625
  return
572
626
  }
573
627
  }
@@ -575,7 +629,48 @@ export class LayeredCache implements CacheLayer {
575
629
  packed !== undefined
576
630
  ? Promise.resolve({ body: new Blob([packed]) })
577
631
  : this.local.pinArtifact(hash)
578
- for (const lane of writable) lane.upload(hash, durationMs, body, this.local.uploads)
632
+ const sent = writable.map((lane) =>
633
+ lane.upload(hash, durationMs, body, this.local.uploads, args.absent === true),
634
+ )
635
+ this.sent.set(
636
+ hash,
637
+ Promise.all(sent).then((held) => held.includes(true)),
638
+ )
639
+ }
640
+
641
+ uploaded(hash: string): Promise<boolean | undefined> {
642
+ return this.sent.get(hash) ?? Promise.resolve(undefined)
643
+ }
644
+
645
+ /**
646
+ * True once a remote this run reads holds entry `hash`, uploading the
647
+ * local entry to each one it also writes when none does. Another machine
648
+ * of the run (a DTE worker) restores from the remotes alone: a hit only
649
+ * this cache held, the remote having evicted it, failed its dependant
650
+ * there.
651
+ */
652
+ async share(hash: string): Promise<boolean> {
653
+ for (const lane of this.readable) if ((await lane.has(hash)) !== null) return true
654
+ return this.upload(hash, false)
655
+ }
656
+
657
+ /** `share`, over a remote copy another machine of the run found corrupt. */
658
+ resend(hash: string): Promise<boolean> {
659
+ return this.upload(hash, true)
660
+ }
661
+
662
+ private async upload(hash: string, over: boolean): Promise<boolean> {
663
+ const lanes = this.readable.filter((l) => l.write)
664
+ if (lanes.length === 0) return false
665
+ const entry = await this.local.getIngested(hash)
666
+ if (entry === null) return false
667
+ if (over) for (const lane of lanes) lane.distrust(hash)
668
+ const held = await Promise.all(
669
+ lanes.map((l) =>
670
+ l.upload(hash, entry.durationMs, () => this.local.pinArtifact(hash), this.local.uploads),
671
+ ),
672
+ )
673
+ return held.includes(true)
579
674
  }
580
675
 
581
676
  /**
@@ -602,7 +697,7 @@ export class LayeredCache implements CacheLayer {
602
697
  return this.local.loadOutputFilesBatch(hashes)
603
698
  }
604
699
 
605
- async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
700
+ isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): boolean {
606
701
  return this.local.isOutputsCurrent(projectDir, expected)
607
702
  }
608
703
 
@@ -622,7 +717,7 @@ export class LayeredCache implements CacheLayer {
622
717
  return this.local.loadOutputDirsBatch(hashes)
623
718
  }
624
719
 
625
- outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
720
+ outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): boolean {
626
721
  return this.local.outputDirsCurrent(projectDir, rows)
627
722
  }
628
723
 
@@ -1,5 +1,5 @@
1
1
  // A symlinked input keys the content behind it, as Nx reads a linked file
2
- // where it points (owner, 2026-10-16: "Do like nx"). Its own fold stays the
2
+ // where it points (2026-10-16: "Do like nx"). Its own fold stays the
3
3
  // blob of its target STRING (file-hashes.ts), so a retarget to equal bytes
4
4
  // moves the key; this is what follows it. The target string alone let an
5
5
  // edit behind the link replay the old output (Growth, 2026-10-16), wherever