@vzn/vx 0.0.644 → 0.0.646

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +22 -48
  3. package/package.json +5 -5
  4. package/schemas/init.json +5 -0
  5. package/schemas/last.json +15 -0
  6. package/skills/vx/SKILL.md +1 -1
  7. package/src/bin.ts +1 -1
  8. package/src/cache/archive.ts +12 -76
  9. package/src/cache/cache.ts +72 -42
  10. package/src/cache/git-inputs.ts +55 -10
  11. package/src/cache/index.ts +5 -2
  12. package/src/cache/inputs.ts +61 -11
  13. package/src/cache/layer.ts +30 -8
  14. package/src/cache/layered-cache.ts +138 -44
  15. package/src/cache/output-index.ts +25 -11
  16. package/src/cache/plugin-cache.ts +9 -14
  17. package/src/cache/remote-lane.ts +60 -22
  18. package/src/cache/run-history.ts +13 -6
  19. package/src/cache/schema.ts +19 -2
  20. package/src/cache/scoped-cache.ts +3 -80
  21. package/src/cli/core-alias.ts +5 -2
  22. package/src/cli/help.ts +8 -10
  23. package/src/cli/index.ts +0 -12
  24. package/src/cli/init.ts +159 -25
  25. package/src/cli/last.ts +4 -2
  26. package/src/cli/plan-format.ts +6 -2
  27. package/src/cli/plugin-commands.ts +98 -6
  28. package/src/cli/plugin-templates.ts +2 -2
  29. package/src/cli/run.ts +2 -8
  30. package/src/cli/select.ts +5 -3
  31. package/src/cli/upgrade.ts +5 -6
  32. package/src/config.ts +2 -4
  33. package/src/exec/executor.ts +112 -9
  34. package/src/exec/index.ts +2 -0
  35. package/src/exec/proc-sample.ts +67 -5
  36. package/src/exec/runner.ts +4 -0
  37. package/src/exec/sandbox-runtime.ts +11 -4
  38. package/src/graph/scheduler.ts +59 -8
  39. package/src/history/history.ts +119 -7
  40. package/src/history/metrics.ts +53 -16
  41. package/src/index.ts +16 -1
  42. package/src/orchestrator/affected-tasks.ts +0 -18
  43. package/src/orchestrator/execute-args.ts +3 -0
  44. package/src/orchestrator/execute-task.ts +28 -7
  45. package/src/orchestrator/held-servers.ts +86 -0
  46. package/src/orchestrator/hit-restore.ts +11 -4
  47. package/src/orchestrator/index.ts +18 -1
  48. package/src/orchestrator/local-floor.ts +104 -0
  49. package/src/orchestrator/local-shortcircuit.ts +26 -7
  50. package/src/orchestrator/miss-save.ts +10 -4
  51. package/src/orchestrator/options.ts +0 -8
  52. package/src/orchestrator/persistent-task.ts +14 -0
  53. package/src/orchestrator/placement.ts +84 -9
  54. package/src/orchestrator/plan.ts +7 -1
  55. package/src/orchestrator/plugin-host.ts +22 -7
  56. package/src/orchestrator/plugin.ts +48 -7
  57. package/src/orchestrator/prepare.ts +14 -49
  58. package/src/orchestrator/run-context.ts +39 -1
  59. package/src/orchestrator/run-keys.ts +164 -0
  60. package/src/orchestrator/run-records.ts +6 -0
  61. package/src/orchestrator/run.ts +95 -6
  62. package/src/orchestrator/sandbox-request.ts +9 -3
  63. package/src/orchestrator/task-artifacts.ts +142 -0
  64. package/src/orchestrator/task-hash.ts +12 -4
  65. package/src/orchestrator/task-process.ts +16 -0
  66. package/src/orchestrator/telemetry-host.ts +2 -1
  67. package/src/orchestrator/telemetry.ts +60 -17
  68. package/src/report/forecast.ts +6 -27
  69. package/src/report/framed-output.ts +41 -5
  70. package/src/report/logger.ts +60 -20
  71. package/src/report/status-line.ts +38 -18
  72. package/src/report/summary.ts +75 -9
  73. package/src/settings.ts +8 -7
  74. package/src/util/colors.ts +16 -1
  75. package/src/util/index.ts +1 -0
  76. package/src/util/paths.ts +30 -0
  77. package/src/util/ui.ts +4 -1
  78. package/src/workspace/affected.ts +0 -39
  79. package/src/workspace/config-bytecode.ts +276 -0
  80. package/src/workspace/index.ts +0 -2
  81. package/src/workspace/migration.ts +4 -32
  82. package/src/workspace/project-loader.ts +92 -14
  83. package/src/cli/foreign-flags.ts +0 -727
@@ -204,7 +204,8 @@ function configObjectFormat(config: string): 'sha1' | 'sha256' | undefined {
204
204
  interface PartitionDigest {
205
205
  snapshot: readonly string[]
206
206
  oids: OidSlot | undefined
207
- digest: string
207
+ /** Built on first read: a partition with untrusted files stats them then. */
208
+ digest: () => string | undefined
208
209
  }
209
210
 
210
211
  /**
@@ -515,11 +516,20 @@ export class GitFilesCache extends Map<string, readonly string[]> {
515
516
  const d = this.digests.get(projectDir)
516
517
  if (d === undefined || d.snapshot !== snapshot || this.get(projectDir) !== snapshot) return
517
518
  if (this.oids.get(projectDir) !== d.oids || this.changed.has(projectDir)) return
518
- return d.digest
519
- }
520
-
521
- noteFilesDigest(projectDir: string, snapshot: readonly string[], digest: string): void {
522
- this.digests.set(projectDir, { snapshot, oids: this.oids.get(projectDir), digest })
519
+ return d.digest()
520
+ }
521
+
522
+ noteFilesDigest(
523
+ projectDir: string,
524
+ snapshot: readonly string[],
525
+ digest: () => string | undefined,
526
+ ): void {
527
+ let memo: { value: string | undefined } | undefined
528
+ this.digests.set(projectDir, {
529
+ snapshot,
530
+ oids: this.oids.get(projectDir),
531
+ digest: () => (memo ??= { value: digest() }).value,
532
+ })
523
533
  }
524
534
 
525
535
  /** Trusted index OIDs for a project (abs path → oid), if any survive. */
@@ -1908,6 +1918,30 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
1908
1918
  }
1909
1919
  }
1910
1920
 
1921
+ /**
1922
+ * Each loose file's stat identity, `hashFile`'s memo key: mode, size,
1923
+ * mtime, ctime and inode. Undefined for a symlink (what it points to
1924
+ * folds too, and its stat does not move with that), a file gone, or one
1925
+ * changed within the racy window, where the memo trusts no stat either.
1926
+ */
1927
+ function statRecords(
1928
+ rels: readonly string[],
1929
+ abs: (rel: string) => string,
1930
+ prefixLength: number,
1931
+ ): string | undefined {
1932
+ if (rels.length === 0) return ''
1933
+ const now = Date.now()
1934
+ let out = ''
1935
+ for (const rel of rels) {
1936
+ const st = lstatSync(abs(rel), { throwIfNoEntry: false })
1937
+ if (st === undefined || !st.isFile()) return undefined
1938
+ if (now - Math.floor(st.ctimeMs) < racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) return
1939
+ const mode = (st.mode & 0o100) !== 0 ? '100755' : '100644'
1940
+ out += `${rel.slice(prefixLength)}\0stat:${mode}:${st.size}:${Math.floor(st.mtimeMs)}:${Math.floor(st.ctimeMs)}:${Number(st.ino)}\n`
1941
+ }
1942
+ return out
1943
+ }
1944
+
1911
1945
  /** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
1912
1946
  export async function applyGitEnumeration(
1913
1947
  enumeration: GitEnumeration,
@@ -2006,15 +2040,20 @@ export async function applyGitEnumeration(
2006
2040
  }
2007
2041
  // One hash of the records joined: one per record was ~25 ms of 125,000.
2008
2042
  let records: string | undefined = ''
2043
+ // A file git vouches for no OID (untracked, or changed since the
2044
+ // index) is named by its stat instead, the identity `hashFile`'s memo
2045
+ // trusts: a just-added config or an edited file turned the fold memo
2046
+ // off for its whole project (+3-5 ms of solid's warm run, 2026-10-22).
2047
+ const loose: string[] = []
2009
2048
  for (let i = start; i < end; i++) {
2010
2049
  const rel = sorted[i]!
2011
2050
  const own = rel.slice(prefix.length)
2012
2051
  matches.push(own)
2013
2052
  if (records === undefined) continue
2014
2053
  const oid = trusted.get(rel)
2015
- if (oid === undefined || oid.startsWith('120000:') || undecodable?.has(rel) === true) {
2016
- records = undefined
2017
- } else records += `${own}\0${oid}\n`
2054
+ if (undecodable?.has(rel) === true || oid?.startsWith('120000:') === true) records = undefined
2055
+ else if (oid === undefined) loose.push(rel)
2056
+ else records += `${own}\0${oid}\n`
2018
2057
  }
2019
2058
  // An empty slice is a directory git did not see, not an empty project:
2020
2059
  // a project has at least its package.json, tracked or untracked. A
@@ -2033,7 +2072,13 @@ export async function applyGitEnumeration(
2033
2072
  // setLazyOids AFTER set — set() drops the project's OID slot.
2034
2073
  cache.set(projectDir, matches)
2035
2074
  cache.setLazyOids(projectDir, projOids)
2036
- if (records !== undefined) cache.noteFilesDigest(projectDir, matches, xxh3hex(records))
2075
+ if (records !== undefined) {
2076
+ const known = records
2077
+ cache.noteFilesDigest(projectDir, matches, () => {
2078
+ const statted = statRecords(loose, projAbs, prefix.length)
2079
+ return statted === undefined ? undefined : xxh3hex(`${known}\0${statted}`)
2080
+ })
2081
+ }
2037
2082
  }
2038
2083
  if (workspaceWide) {
2039
2084
  cache.set(workspaceRoot, all)
@@ -29,6 +29,7 @@ export {
29
29
  WORKSPACE_OUTPUT_PREFIX,
30
30
  type OutputDirRow,
31
31
  OUTPUT_DIRS_CAP,
32
+ OUTPUT_DIRS_OVER_CAP,
32
33
  FILE_HASH_RACY_MS,
33
34
  OUTPUT_DIRS_RACY_MS,
34
35
  racyWindowMs,
@@ -54,6 +55,7 @@ export {
54
55
  type ResolveInputsArgs,
55
56
  resolveOutputTree,
56
57
  resolveOutputs,
58
+ outputWalkTakes,
57
59
  resolveWorkspaceOutputs,
58
60
  sweepRestoreTemps,
59
61
  sweepStaging,
@@ -66,6 +68,7 @@ export {
66
68
  stampOutputs,
67
69
  stampWorkspaceOutputs,
68
70
  stopRuntimeProbes,
71
+ answerRuntime,
69
72
  type OutputStamp,
70
73
  } from './inputs.js'
71
74
  export {
@@ -82,14 +85,14 @@ export {
82
85
  type BlobSizeMemo,
83
86
  } from './git-inputs.js'
84
87
  export {
88
+ type ArtifactInfo,
85
89
  LayeredCache,
90
+ type LentAccess,
86
91
  type RemoteCacheLayer,
87
- type RemoteKey,
88
92
  type RemoteOpenContext,
89
93
  } from './layered-cache.js'
90
94
  export { networkConcurrency } from './remote-lane.js'
91
95
  export type { ResolvedRemote } from './remote-lane.js'
92
- export { BranchScope } from './scoped-cache.js'
93
96
  export {
94
97
  absentPluginCache,
95
98
  pluginCache,
@@ -31,6 +31,7 @@ import {
31
31
  outputNever,
32
32
  realpathOf,
33
33
  relPosix,
34
+ restorableOutput,
34
35
  shellArgv,
35
36
  slashBraceExpansions,
36
37
  braceAlternatives,
@@ -530,6 +531,28 @@ function projectBinDirs(projectDir: string, workspaceRoot: string): string[] {
530
531
  return own === root ? [own] : [own, root]
531
532
  }
532
533
 
534
+ /**
535
+ * One runtime probe answered on this machine as a key folds it: an
536
+ * `inputs.runtime` command run in `projectDir`, or with `workspace` a
537
+ * `workspaceRuntime` one run in `workspaceRoot`. For an executor that
538
+ * checks a task's probes where it runs it (`TaskExecutor.runtime`).
539
+ */
540
+ export function answerRuntime(
541
+ command: string,
542
+ projectDir: string,
543
+ workspaceRoot: string,
544
+ workspace = false,
545
+ ): Promise<string> {
546
+ return workspace
547
+ ? runRuntimeCommand(
548
+ command,
549
+ workspaceRoot,
550
+ [path.join(workspaceRoot, 'node_modules', '.bin')],
551
+ undefined,
552
+ )
553
+ : runRuntimeCommand(command, projectDir, projectBinDirs(projectDir, workspaceRoot), undefined)
554
+ }
555
+
533
556
  function resolveEnvValues(
534
557
  names: readonly string[],
535
558
  source: NodeJS.ProcessEnv,
@@ -748,6 +771,21 @@ export async function resolveOutputs(args: {
748
771
  return (await scanOutputs(args)).files
749
772
  }
750
773
 
774
+ /**
775
+ * Whether `resolveOutputs` takes the project-relative file `rel`, decided
776
+ * without the walk: a directory snapshot's file list is held to the
777
+ * entry's rows through it.
778
+ */
779
+ export function outputWalkTakes(
780
+ projectDir: string,
781
+ outputs: readonly string[],
782
+ nestedProjectDirs: string[],
783
+ ): (rel: string) => boolean {
784
+ const restorable = restorableOutput(outputs)
785
+ const nested = inNestedProject(projectDir, nestedProjectDirs)
786
+ return (rel) => restorable(rel) && !nested(rel)
787
+ }
788
+
751
789
  /**
752
790
  * `resolveOutputs`, plus the empty directories the globs match: a save
753
791
  * keeps them, so a hit leaves the tree a miss left (Turbo and Nx restore
@@ -2103,21 +2141,31 @@ async function scanUnion(
2103
2141
  emptyDirs?: Set<string>,
2104
2142
  ): Promise<Set<string>> {
2105
2143
  const matches = new Set<string>()
2144
+ const take = (rel: string): void => {
2145
+ if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
2146
+ const abs = path.resolve(cwd, rel)
2147
+ const st = lstatSync(abs, { throwIfNoEntry: false })
2148
+ if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
2149
+ else if (emptyDirs !== undefined && st?.isDirectory() === true) {
2150
+ if (readdirSync(abs).length === 0) emptyDirs.add(abs)
2151
+ } else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
2152
+ }
2153
+ const opts = { cwd, onlyFiles: false, followSymlinks: false, dot: true }
2106
2154
  // `Bun.Glob`'s scan finds nothing for a brace whose alternatives hold a
2107
2155
  // `/`: `{dist,lib/esm}/**` saved an empty artifact, and a hit restored
2108
2156
  // nothing over a cleaned tree (A-10). Expanded here, as discovery does.
2109
- for (const pattern of positive.flatMap(slashBraceExpansions)) {
2157
+ const patterns = positive.flatMap(slashBraceExpansions)
2158
+ // A leading `**` walks the whole project whatever else is listed, so the
2159
+ // list is one walk: Turbo's `dist/**`, `**/dist/**`, `**/dist` was three,
2160
+ // ~2 ms of a warm hit on solid.
2161
+ if (patterns.length > 1 && patterns.some((p) => p.startsWith('**'))) {
2162
+ const any = anyTaskGlob(patterns)
2163
+ for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
2164
+ return matches
2165
+ }
2166
+ for (const pattern of patterns) {
2110
2167
  if (absentPrefix(cwd, pattern)) continue
2111
- const glob = globFor(pattern)
2112
- for (const rel of glob.scanSync({ cwd, onlyFiles: false, followSymlinks: false, dot: true })) {
2113
- if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) continue
2114
- const abs = path.resolve(cwd, rel)
2115
- const st = lstatSync(abs, { throwIfNoEntry: false })
2116
- if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
2117
- else if (emptyDirs !== undefined && st?.isDirectory() === true) {
2118
- if (readdirSync(abs).length === 0) emptyDirs.add(abs)
2119
- } else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
2120
- }
2168
+ for (const rel of globFor(pattern).scanSync(opts)) take(rel)
2121
2169
  }
2122
2170
  return matches
2123
2171
  }
@@ -2139,6 +2187,8 @@ function absentPrefix(cwd: string, pattern: string): boolean {
2139
2187
  }
2140
2188
  }
2141
2189
 
2190
+ const WHOLE_TREE = new Bun.Glob('**')
2191
+
2142
2192
  // Compiled once per pattern string for the life of the process: a `Bun.Glob`
2143
2193
  // is immutable, and the same handful of declared globs are compiled again
2144
2194
  // for every task otherwise (three sites, up to thousands of times per run).
@@ -5,6 +5,7 @@
5
5
  // reference to `Cache` below is a type (the local handle a layer may wrap).
6
6
 
7
7
  import type { Cache } from './cache.js'
8
+ import type { ArtifactInfo } from './layered-cache.js'
8
9
  import { fsClockLag, stampGrainMs } from './fs-clock.js'
9
10
 
10
11
  export interface CacheKeyInput {
@@ -211,6 +212,16 @@ export interface RunRecord {
211
212
  notReady?: 'timeout' | 'exited' | 'spawn'
212
213
  /** Each remote read or write of its key that failed, `upload: <message>`, `; `-joined (v40). */
213
214
  remoteError?: string
215
+ /** The sampled peak cores and mean resident memory of the run (v43). */
216
+ peakCores?: number
217
+ avgRssBytes?: number
218
+ /** Where an executor ran it (`small#1`); absent = here (v44). */
219
+ placement?: string
220
+ /** The task with no `inputs` its key folded, so it never read or saved (v46). */
221
+ keylessBy?: string
222
+ /** That machine's cores and memory (v44). */
223
+ placementCores?: number
224
+ placementMemoryBytes?: number
214
225
  }
215
226
 
216
227
  /**
@@ -383,6 +394,13 @@ export interface OutputDirRow {
383
394
  */
384
395
  export const OUTPUT_DIRS_CAP = 8192
385
396
 
397
+ /**
398
+ * `mtime_ms` of the one row a snapshot over `OUTPUT_DIRS_CAP` leaves: never
399
+ * current, so the hit walks, and it tells the hit not to snapshot again.
400
+ * Cleared rows sent every hit through the walk AND a capped snapshot.
401
+ */
402
+ export const OUTPUT_DIRS_OVER_CAP = -3
403
+
386
404
  /**
387
405
  * A directory whose mtime lies within this many ms of the snapshot is RACY
388
406
  * and the whole snapshot is dropped. File timestamps are coarse on Linux
@@ -550,12 +568,12 @@ export interface CacheLayer {
550
568
  */
551
569
  readonly hasRemote?: boolean
552
570
  /**
553
- * Batch existence probe over the remote — the subset of `hashes` stored
554
- * remotely, in ONE round-trip, or `null` for "no batch info; use
555
- * per-hash". Only meaningful with `hasRemote`; a remote layer that can't
571
+ * Batch existence probe over the remote — the `hashes` stored remotely
572
+ * and what the first remote holding each knows of it (`ArtifactInfo`), in
573
+ * ONE round-trip, or `null` for "no batch info; use per-hash". Only meaningful with `hasRemote`; a remote layer that can't
556
574
  * batch omits it and the prefetch pass falls back to per-hash GETs.
557
575
  */
558
- remoteHasMany?(hashes: readonly string[]): Promise<Set<string> | null>
576
+ remoteHasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
559
577
  /**
560
578
  * The subset of `hashes` the local store already holds. The prefetch
561
579
  * pass probes the remote only for the rest, so a run the local cache
@@ -581,8 +599,6 @@ export interface CacheLayer {
581
599
  * about to call absent. Only meaningful with `hasRemote`.
582
600
  */
583
601
  probeFirst?(started: Promise<void>): void
584
- /** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
585
- noteKey?(taskId: string, hash: string): void
586
602
  /**
587
603
  * Resolve once every background write-through upload settles. The
588
604
  * orchestrator awaits it before `close()` — an upload reading layer state
@@ -594,6 +610,12 @@ export interface CacheLayer {
594
610
  stopUploads?(): void
595
611
  key(input: CacheKeyInput): Promise<string>
596
612
  get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
613
+ /**
614
+ * `get` past the run's read policy: an entry a machine running a task
615
+ * for this run saved is the run's own work, not a hit (`TaskArtifacts`).
616
+ * A layer without it is read through `get`.
617
+ */
618
+ getSaved?(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
597
619
  /**
598
620
  * Optional batched `get` (same answers, fewer round trips). The
599
621
  * short-circuit probe uses it when a layer offers one; a layer without it
@@ -652,8 +674,8 @@ export interface CacheLayer {
652
674
  /**
653
675
  * The directory-mtime short-circuit behind a warm hit (optional — a layer
654
676
  * without it keeps the output walk). `recordOutputDirs` snapshots every
655
- * directory under each of `prefixes` (project-relative, whole-subtree
656
- * globs only — see `wholeSubtreePrefixes`) after a save or restore;
677
+ * directory under each of `prefixes` (project-relative, `''` the project
678
+ * itself — see `outputDirPrefixes`) after a save or restore;
657
679
  * `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
658
680
  * every recorded directory still carries its recorded mtime, which proves
659
681
  * no file was added or removed anywhere the glob could see. Machine-local
@@ -36,7 +36,6 @@ import type {
36
36
  } from './cache.js'
37
37
  import type { OutputDirRow } from './cache.js'
38
38
  import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
39
- import type { BranchScope } from './scoped-cache.js'
40
39
  import type { PluginSecrets } from '../util/index.js'
41
40
 
42
41
  const STOPPED: unique symbol = Symbol('stopped')
@@ -46,7 +45,11 @@ const STOPPED: unique symbol = Symbol('stopped')
46
45
  * fails at once, removes its temp, and releases the connection; the cache
47
46
  * then closes under no write.
48
47
  */
49
- function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Blob | Response {
48
+ function endsOnStop(
49
+ body: Blob | Response,
50
+ stopped: Promise<typeof STOPPED>,
51
+ onChunk: (bytes: number) => void = () => {},
52
+ ): Blob | Response {
50
53
  if (body instanceof Blob || body.body === null) return body
51
54
  const reader = body.body.getReader()
52
55
  let stop = false
@@ -61,13 +64,28 @@ function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Bl
61
64
  const next = await reader.read()
62
65
  if (stop) controller.error(new Error('the run was stopped'))
63
66
  else if (next.done) controller.close()
64
- else controller.enqueue(next.value)
67
+ else {
68
+ onChunk(next.value.byteLength)
69
+ controller.enqueue(next.value)
70
+ }
65
71
  },
66
72
  cancel: (reason) => reader.cancel(reason),
67
73
  })
68
74
  return new Response(stream, { headers: body.headers })
69
75
  }
70
76
 
77
+ const judged = (i: ArtifactInfo): i is { size: number; durationMs: number } =>
78
+ i.size !== undefined && i.durationMs !== undefined
79
+
80
+ /** How often a task waiting on its download weighs running instead. */
81
+ const JUDGE_EVERY_MS = 25
82
+
83
+ interface Download {
84
+ readonly start: number
85
+ received: number
86
+ abandon(): void
87
+ }
88
+
71
89
  /** What a remote's `open` is handed: the run's CI verdict and its plugin's keychain. */
72
90
  export interface RemoteOpenContext extends PluginSecrets {
73
91
  /** The run is on CI: nobody may be asked. */
@@ -77,13 +95,20 @@ export interface RemoteOpenContext extends PluginSecrets {
77
95
  }
78
96
 
79
97
  /**
80
- * Where a remote call's key lives: `scope` names the branch scope a scoped
81
- * key was derived in (`scopedKey`), absent for a task's own key. The key
82
- * alone is enough; a backend that keeps one store per scope may use it to
83
- * go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
98
+ * What a remote knows of an artifact it holds, before it is downloaded:
99
+ * the body's size in bytes and the producing task's duration, each when
100
+ * the wire carries it (Turbo's batch query and `HEAD` do).
84
101
  */
85
- export interface RemoteKey {
86
- readonly scope?: string
102
+ export interface ArtifactInfo {
103
+ readonly size?: number
104
+ readonly durationMs?: number
105
+ }
106
+
107
+ /** A run's access to a remote, lent to a machine that runs its tasks (`RemoteCacheLayer.lend`). */
108
+ export interface LentAccess {
109
+ readonly token: string
110
+ /** Epoch ms; asked again before it. */
111
+ readonly expiresAt?: number
87
112
  }
88
113
 
89
114
  /**
@@ -122,17 +147,29 @@ export interface RemoteCacheLayer {
122
147
  * plugin's keychain and may ask on the terminal.
123
148
  */
124
149
  open?(ctx: RemoteOpenContext): Promise<void>
125
- /** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
126
- has(hash: string, at?: RemoteKey): Promise<boolean>
127
150
  /**
128
- * Optional batch existence probe: given N hashes, return the subset stored
129
- * remotely in ONE round-trip. Its presence says the wire can batch: a run
130
- * asks it once for its keys, `--dry` before its per-key answers, and
131
- * `--force` in the background to skip uploads of held keys. Without it a
132
- * run GETs each key directly (no `has`). Never throws for control flow:
133
- * `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
151
+ * Optional: this run's access, for a machine that runs its tasks
152
+ * elsewhere (`ExecutorContext.cacheFacts`); that machine's plugin reads
153
+ * it as `CacheContext.lent`. Only access made for this run (an OIDC
154
+ * exchange's), never a stored secret: the server's token decides what
155
+ * it may write. Asked after `open`.
134
156
  */
135
- hasMany?(hashes: readonly string[], at?: readonly RemoteKey[]): Promise<Set<string> | null>
157
+ lend?(): Promise<LentAccess | undefined>
158
+ /**
159
+ * Existence probe: what the remote knows of the stored artifact, or null
160
+ * when it holds none. `--dry` and `--force` ask it per key when the
161
+ * remote cannot batch.
162
+ */
163
+ has(hash: string): Promise<ArtifactInfo | null>
164
+ /**
165
+ * Optional batch existence probe: given N hashes, the stored ones and what
166
+ * the remote knows of each, in ONE round-trip. Its presence says the wire
167
+ * can batch: a run asks it once for its keys, `--dry` before its per-key
168
+ * answers, and `--force` in the background to skip uploads of held keys.
169
+ * Without it a run GETs each key directly (no `has`). Never throws for
170
+ * control flow: `null` means "no batch info; use per-hash".
171
+ */
172
+ hasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
136
173
  /**
137
174
  * Fetch an artifact; `null` = miss. `body` is read once, by core,
138
175
  * straight into the local cache — an HTTP wire returns its `fetch`
@@ -142,10 +179,7 @@ export interface RemoteCacheLayer {
142
179
  * nothing and the read is a miss, so a layer can check bytes as they pass
143
180
  * (a digest, a signature) and refuse at the end.
144
181
  */
145
- get(
146
- hash: string,
147
- at?: RemoteKey,
148
- ): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
182
+ get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
149
183
  /**
150
184
  * Store an artifact (fire-and-forget from LayeredCache's perspective).
151
185
  * `body` is file-backed (`Bun.file`) when the local store holds the
@@ -154,7 +188,7 @@ export interface RemoteCacheLayer {
154
188
  * key the remote holds keeps the stored bytes. Core never asks `has`
155
189
  * first; a wire that would overwrite checks inside.
156
190
  */
157
- put(hash: string, body: Blob, meta: { durationMs: number } & RemoteKey): Promise<void>
191
+ put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
158
192
  }
159
193
 
160
194
  export class LayeredCache implements CacheLayer {
@@ -196,23 +230,18 @@ export class LayeredCache implements CacheLayer {
196
230
  * its download and ingest took.
197
231
  */
198
232
  private readonly remoteSourced = new Map<string, number>()
233
+ /** What the batch probe said of each held key: its size and stored duration. */
234
+ private readonly infos = new Map<string, ArtifactInfo>()
235
+ /** Bodies being read now, by key: how far along, and how to give one up. */
236
+ private readonly downloads = new Map<string, Download>()
199
237
 
200
238
  /** Remotes in declared order: the first readable one holding a hash answers it. */
201
239
  constructor(
202
240
  readonly local: Cache,
203
241
  remotes: readonly ResolvedRemote[],
204
- /** The run's branch scope: it picks each remote key; the local store never sees it. */
205
- private readonly branch?: BranchScope,
206
242
  uploadConcurrency?: number,
207
243
  ) {
208
- this.lanes = remotes.map(
209
- (r) =>
210
- new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
211
- )
212
- }
213
-
214
- noteKey(taskId: string, hash: string): void {
215
- this.branch?.noteKey(taskId, hash)
244
+ this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic, uploadConcurrency))
216
245
  }
217
246
 
218
247
  // Asked, not kept: a remote whose `open` failed leaves the run after this is built.
@@ -249,9 +278,12 @@ export class LayeredCache implements CacheLayer {
249
278
  * absences either way, so a hash only a sibling holds costs it no GET.
250
279
  * Never throws.
251
280
  */
252
- async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
281
+ async remoteHasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
253
282
  if (this.readable.length === 0) return null
254
- const answer = this.probeMany(hashes)
283
+ const answer = this.probeMany(hashes).then((found) => {
284
+ if (found !== null) for (const [h, info] of found) this.infos.set(h, info)
285
+ return found
286
+ })
255
287
  const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
256
288
  for (const h of waiting) {
257
289
  this.probing.set(
@@ -271,12 +303,12 @@ export class LayeredCache implements CacheLayer {
271
303
  return answer
272
304
  }
273
305
 
274
- private async probeMany(hashes: readonly string[]): Promise<Set<string> | null> {
275
- let out: Set<string> | null = new Set()
306
+ private async probeMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
307
+ let out: Map<string, ArtifactInfo> | null = new Map()
276
308
  for (const lane of this.readable) {
277
309
  const found = await lane.hasMany(hashes)
278
310
  if (found === null) out = null
279
- else if (out !== null) for (const h of found) out.add(h)
311
+ else if (out !== null) for (const [h, info] of found) if (!out.has(h)) out.set(h, info)
280
312
  }
281
313
  return out
282
314
  }
@@ -328,7 +360,7 @@ export class LayeredCache implements CacheLayer {
328
360
  // remote walk per key: a settled `false` means no remote had it
329
361
  // (no second probe), and a settled/in-flight `true` means the
330
362
  // artifact was/will-be ingested locally — re-read below.
331
- const ingested = await this.pullFromRemote(hash, ctx)
363
+ const ingested = await this.awaitOrRun(hash, this.pullFromRemote(hash, ctx))
332
364
  if (!ingested) return null
333
365
 
334
366
  // Read past the local READ gate: `ingest` is deliberately ungated, so
@@ -350,6 +382,49 @@ export class LayeredCache implements CacheLayer {
350
382
  return this.asRemote(hash, materialized)
351
383
  }
352
384
 
385
+ async getSaved(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
386
+ const here = await this.local.getIngested(hash)
387
+ if (here) return here
388
+ // Every remote, read or not, and past a probe's absence: the probe ran
389
+ // before the agent saved it.
390
+ for (const lane of this.lanes)
391
+ if (await this.pullFrom(lane, hash, ctx, true)) return this.local.getIngested(hash)
392
+ return null
393
+ }
394
+
395
+ /**
396
+ * A pull the task now waits on, given up when the rest of its body would
397
+ * take longer than running the task (its stored duration, doubled for
398
+ * the save a run adds): the task then runs here, a miss. Only a key the
399
+ * batch probe sized and timed is judged, from its own body's pace so far.
400
+ * The facts are read each tick: the probe can answer after the task asks.
401
+ */
402
+ private async awaitOrRun(hash: string, pull: Promise<boolean>): Promise<boolean> {
403
+ if (this.infos.has(hash) && !judged(this.infos.get(hash)!)) return pull
404
+ for (;;) {
405
+ const done = await Promise.race([pull, Bun.sleep(JUDGE_EVERY_MS).then(() => undefined)])
406
+ if (done !== undefined) return done
407
+ const info = this.infos.get(hash)
408
+ if (info === undefined) continue
409
+ if (!judged(info)) return pull
410
+ const budget = info.durationMs * 2
411
+ const dl = this.downloads.get(hash)
412
+ if (dl === undefined) continue
413
+ const elapsed = performance.now() - dl.start
414
+ const left =
415
+ dl.received === 0
416
+ ? elapsed > budget
417
+ ? Infinity
418
+ : 0
419
+ : ((info.size - dl.received) * elapsed) / dl.received
420
+ if (left > budget) {
421
+ dl.abandon()
422
+ await pull
423
+ return false
424
+ }
425
+ }
426
+ }
427
+
353
428
  private asRemote(hash: string, entry: CacheEntry): CacheEntry {
354
429
  const fetchMs = this.remoteSourced.get(hash)
355
430
  return fetchMs === undefined ? entry : { ...entry, source: 'remote', fetchMs }
@@ -360,7 +435,7 @@ export class LayeredCache implements CacheLayer {
360
435
  // a remote cache. Remote errors degrade to a miss (never throw).
361
436
  async has(hash: string): Promise<'local' | 'remote' | null> {
362
437
  if ((await this.local.has(hash)) === 'local') return 'local'
363
- for (const lane of this.readable) if (await lane.has(hash)) return 'remote'
438
+ for (const lane of this.readable) if ((await lane.has(hash)) !== null) return 'remote'
364
439
  return null
365
440
  }
366
441
 
@@ -402,13 +477,18 @@ export class LayeredCache implements CacheLayer {
402
477
  return false
403
478
  }
404
479
 
405
- private async pullFrom(lane: RemoteLane, hash: string, ctx?: CacheGetContext): Promise<boolean> {
480
+ private async pullFrom(
481
+ lane: RemoteLane,
482
+ hash: string,
483
+ ctx?: CacheGetContext,
484
+ fresh = false,
485
+ ): Promise<boolean> {
406
486
  const fetchStart = performance.now()
407
487
  // A stopped run waits on no read, as on no upload: a lookup or download
408
488
  // in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
409
489
  // plugin's request is left to the exit, and its body is cancelled.
410
490
  const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
411
- const pending = lane.get(hash)
491
+ const pending = lane.get(hash, fresh)
412
492
  const remoteBody = await Promise.race([pending, stopped])
413
493
  if (remoteBody === STOPPED) {
414
494
  // A late answer's body would hold its connection to the exit.
@@ -434,10 +514,19 @@ export class LayeredCache implements CacheLayer {
434
514
  durationMs: durationOf(remoteBody.durationMs),
435
515
  ...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
436
516
  }
517
+ let abandon = (): void => {}
518
+ const abandoned = new Promise<typeof STOPPED>((resolve) => (abandon = () => resolve(STOPPED)))
519
+ const dl: Download = { start: performance.now(), received: 0, abandon }
520
+ this.downloads.set(hash, dl)
521
+ const ends = Promise.race([stopped, abandoned])
437
522
  try {
438
- const ingest = this.local.ingest(hash, endsOnStop(remoteBody.body, stopped), meta)
523
+ const ingest = this.local.ingest(
524
+ hash,
525
+ endsOnStop(remoteBody.body, ends, (n) => (dl.received += n)),
526
+ meta,
527
+ )
439
528
  ingest.catch(() => undefined)
440
- if ((await Promise.race([ingest, stopped])) === STOPPED) return false
529
+ if ((await Promise.race([ingest, ends])) === STOPPED) return false
441
530
  } catch (err) {
442
531
  // The bytes came off the network — a corrupt/truncated remote
443
532
  // artifact must degrade to a cache miss (task re-executes), not
@@ -445,8 +534,13 @@ export class LayeredCache implements CacheLayer {
445
534
  // down: `Cache.restoreOutputs` drops it (A-52).
446
535
  lane.report('download', hash, err)
447
536
  return false
537
+ } finally {
538
+ this.downloads.delete(hash)
448
539
  }
449
540
  this.remoteSourced.set(hash, Math.round(performance.now() - fetchStart))
541
+ this.local.downloads.count++
542
+ this.local.downloads.bytes +=
543
+ remoteBody.body instanceof Blob ? remoteBody.body.size : dl.received
450
544
  return true
451
545
  }
452
546