@vzn/vx 0.0.644 → 0.0.646
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +22 -48
- package/package.json +5 -5
- package/schemas/init.json +5 -0
- package/schemas/last.json +15 -0
- package/skills/vx/SKILL.md +1 -1
- package/src/bin.ts +1 -1
- package/src/cache/archive.ts +12 -76
- package/src/cache/cache.ts +72 -42
- package/src/cache/git-inputs.ts +55 -10
- package/src/cache/index.ts +5 -2
- package/src/cache/inputs.ts +61 -11
- package/src/cache/layer.ts +30 -8
- package/src/cache/layered-cache.ts +138 -44
- package/src/cache/output-index.ts +25 -11
- package/src/cache/plugin-cache.ts +9 -14
- package/src/cache/remote-lane.ts +60 -22
- package/src/cache/run-history.ts +13 -6
- package/src/cache/schema.ts +19 -2
- package/src/cache/scoped-cache.ts +3 -80
- package/src/cli/core-alias.ts +5 -2
- package/src/cli/help.ts +8 -10
- package/src/cli/index.ts +0 -12
- package/src/cli/init.ts +159 -25
- package/src/cli/last.ts +4 -2
- package/src/cli/plan-format.ts +6 -2
- package/src/cli/plugin-commands.ts +98 -6
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +2 -8
- package/src/cli/select.ts +5 -3
- package/src/cli/upgrade.ts +5 -6
- package/src/config.ts +2 -4
- package/src/exec/executor.ts +112 -9
- package/src/exec/index.ts +2 -0
- package/src/exec/proc-sample.ts +67 -5
- package/src/exec/runner.ts +4 -0
- package/src/exec/sandbox-runtime.ts +11 -4
- package/src/graph/scheduler.ts +59 -8
- package/src/history/history.ts +119 -7
- package/src/history/metrics.ts +53 -16
- package/src/index.ts +16 -1
- package/src/orchestrator/affected-tasks.ts +0 -18
- package/src/orchestrator/execute-args.ts +3 -0
- package/src/orchestrator/execute-task.ts +28 -7
- package/src/orchestrator/held-servers.ts +86 -0
- package/src/orchestrator/hit-restore.ts +11 -4
- package/src/orchestrator/index.ts +18 -1
- package/src/orchestrator/local-floor.ts +104 -0
- package/src/orchestrator/local-shortcircuit.ts +26 -7
- package/src/orchestrator/miss-save.ts +10 -4
- package/src/orchestrator/options.ts +0 -8
- package/src/orchestrator/persistent-task.ts +14 -0
- package/src/orchestrator/placement.ts +84 -9
- package/src/orchestrator/plan.ts +7 -1
- package/src/orchestrator/plugin-host.ts +22 -7
- package/src/orchestrator/plugin.ts +48 -7
- package/src/orchestrator/prepare.ts +14 -49
- package/src/orchestrator/run-context.ts +39 -1
- package/src/orchestrator/run-keys.ts +164 -0
- package/src/orchestrator/run-records.ts +6 -0
- package/src/orchestrator/run.ts +95 -6
- package/src/orchestrator/sandbox-request.ts +9 -3
- package/src/orchestrator/task-artifacts.ts +142 -0
- package/src/orchestrator/task-hash.ts +12 -4
- package/src/orchestrator/task-process.ts +16 -0
- package/src/orchestrator/telemetry-host.ts +2 -1
- package/src/orchestrator/telemetry.ts +60 -17
- package/src/report/forecast.ts +6 -27
- package/src/report/framed-output.ts +41 -5
- package/src/report/logger.ts +60 -20
- package/src/report/status-line.ts +38 -18
- package/src/report/summary.ts +75 -9
- package/src/settings.ts +8 -7
- package/src/util/colors.ts +16 -1
- package/src/util/index.ts +1 -0
- package/src/util/paths.ts +30 -0
- package/src/util/ui.ts +4 -1
- package/src/workspace/affected.ts +0 -39
- package/src/workspace/config-bytecode.ts +276 -0
- package/src/workspace/index.ts +0 -2
- package/src/workspace/migration.ts +4 -32
- package/src/workspace/project-loader.ts +92 -14
- package/src/cli/foreign-flags.ts +0 -727
package/src/cache/git-inputs.ts
CHANGED
|
@@ -204,7 +204,8 @@ function configObjectFormat(config: string): 'sha1' | 'sha256' | undefined {
|
|
|
204
204
|
interface PartitionDigest {
|
|
205
205
|
snapshot: readonly string[]
|
|
206
206
|
oids: OidSlot | undefined
|
|
207
|
-
|
|
207
|
+
/** Built on first read: a partition with untrusted files stats them then. */
|
|
208
|
+
digest: () => string | undefined
|
|
208
209
|
}
|
|
209
210
|
|
|
210
211
|
/**
|
|
@@ -515,11 +516,20 @@ export class GitFilesCache extends Map<string, readonly string[]> {
|
|
|
515
516
|
const d = this.digests.get(projectDir)
|
|
516
517
|
if (d === undefined || d.snapshot !== snapshot || this.get(projectDir) !== snapshot) return
|
|
517
518
|
if (this.oids.get(projectDir) !== d.oids || this.changed.has(projectDir)) return
|
|
518
|
-
return d.digest
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
noteFilesDigest(
|
|
522
|
-
|
|
519
|
+
return d.digest()
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
noteFilesDigest(
|
|
523
|
+
projectDir: string,
|
|
524
|
+
snapshot: readonly string[],
|
|
525
|
+
digest: () => string | undefined,
|
|
526
|
+
): void {
|
|
527
|
+
let memo: { value: string | undefined } | undefined
|
|
528
|
+
this.digests.set(projectDir, {
|
|
529
|
+
snapshot,
|
|
530
|
+
oids: this.oids.get(projectDir),
|
|
531
|
+
digest: () => (memo ??= { value: digest() }).value,
|
|
532
|
+
})
|
|
523
533
|
}
|
|
524
534
|
|
|
525
535
|
/** Trusted index OIDs for a project (abs path → oid), if any survive. */
|
|
@@ -1908,6 +1918,30 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
|
|
|
1908
1918
|
}
|
|
1909
1919
|
}
|
|
1910
1920
|
|
|
1921
|
+
/**
|
|
1922
|
+
* Each loose file's stat identity, `hashFile`'s memo key: mode, size,
|
|
1923
|
+
* mtime, ctime and inode. Undefined for a symlink (what it points to
|
|
1924
|
+
* folds too, and its stat does not move with that), a file gone, or one
|
|
1925
|
+
* changed within the racy window, where the memo trusts no stat either.
|
|
1926
|
+
*/
|
|
1927
|
+
function statRecords(
|
|
1928
|
+
rels: readonly string[],
|
|
1929
|
+
abs: (rel: string) => string,
|
|
1930
|
+
prefixLength: number,
|
|
1931
|
+
): string | undefined {
|
|
1932
|
+
if (rels.length === 0) return ''
|
|
1933
|
+
const now = Date.now()
|
|
1934
|
+
let out = ''
|
|
1935
|
+
for (const rel of rels) {
|
|
1936
|
+
const st = lstatSync(abs(rel), { throwIfNoEntry: false })
|
|
1937
|
+
if (st === undefined || !st.isFile()) return undefined
|
|
1938
|
+
if (now - Math.floor(st.ctimeMs) < racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) return
|
|
1939
|
+
const mode = (st.mode & 0o100) !== 0 ? '100755' : '100644'
|
|
1940
|
+
out += `${rel.slice(prefixLength)}\0stat:${mode}:${st.size}:${Math.floor(st.mtimeMs)}:${Math.floor(st.ctimeMs)}:${Number(st.ino)}\n`
|
|
1941
|
+
}
|
|
1942
|
+
return out
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1911
1945
|
/** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
|
|
1912
1946
|
export async function applyGitEnumeration(
|
|
1913
1947
|
enumeration: GitEnumeration,
|
|
@@ -2006,15 +2040,20 @@ export async function applyGitEnumeration(
|
|
|
2006
2040
|
}
|
|
2007
2041
|
// One hash of the records joined: one per record was ~25 ms of 125,000.
|
|
2008
2042
|
let records: string | undefined = ''
|
|
2043
|
+
// A file git vouches for no OID (untracked, or changed since the
|
|
2044
|
+
// index) is named by its stat instead, the identity `hashFile`'s memo
|
|
2045
|
+
// trusts: a just-added config or an edited file turned the fold memo
|
|
2046
|
+
// off for its whole project (+3-5 ms of solid's warm run, 2026-10-22).
|
|
2047
|
+
const loose: string[] = []
|
|
2009
2048
|
for (let i = start; i < end; i++) {
|
|
2010
2049
|
const rel = sorted[i]!
|
|
2011
2050
|
const own = rel.slice(prefix.length)
|
|
2012
2051
|
matches.push(own)
|
|
2013
2052
|
if (records === undefined) continue
|
|
2014
2053
|
const oid = trusted.get(rel)
|
|
2015
|
-
if (
|
|
2016
|
-
|
|
2017
|
-
|
|
2054
|
+
if (undecodable?.has(rel) === true || oid?.startsWith('120000:') === true) records = undefined
|
|
2055
|
+
else if (oid === undefined) loose.push(rel)
|
|
2056
|
+
else records += `${own}\0${oid}\n`
|
|
2018
2057
|
}
|
|
2019
2058
|
// An empty slice is a directory git did not see, not an empty project:
|
|
2020
2059
|
// a project has at least its package.json, tracked or untracked. A
|
|
@@ -2033,7 +2072,13 @@ export async function applyGitEnumeration(
|
|
|
2033
2072
|
// setLazyOids AFTER set — set() drops the project's OID slot.
|
|
2034
2073
|
cache.set(projectDir, matches)
|
|
2035
2074
|
cache.setLazyOids(projectDir, projOids)
|
|
2036
|
-
if (records !== undefined)
|
|
2075
|
+
if (records !== undefined) {
|
|
2076
|
+
const known = records
|
|
2077
|
+
cache.noteFilesDigest(projectDir, matches, () => {
|
|
2078
|
+
const statted = statRecords(loose, projAbs, prefix.length)
|
|
2079
|
+
return statted === undefined ? undefined : xxh3hex(`${known}\0${statted}`)
|
|
2080
|
+
})
|
|
2081
|
+
}
|
|
2037
2082
|
}
|
|
2038
2083
|
if (workspaceWide) {
|
|
2039
2084
|
cache.set(workspaceRoot, all)
|
package/src/cache/index.ts
CHANGED
|
@@ -29,6 +29,7 @@ export {
|
|
|
29
29
|
WORKSPACE_OUTPUT_PREFIX,
|
|
30
30
|
type OutputDirRow,
|
|
31
31
|
OUTPUT_DIRS_CAP,
|
|
32
|
+
OUTPUT_DIRS_OVER_CAP,
|
|
32
33
|
FILE_HASH_RACY_MS,
|
|
33
34
|
OUTPUT_DIRS_RACY_MS,
|
|
34
35
|
racyWindowMs,
|
|
@@ -54,6 +55,7 @@ export {
|
|
|
54
55
|
type ResolveInputsArgs,
|
|
55
56
|
resolveOutputTree,
|
|
56
57
|
resolveOutputs,
|
|
58
|
+
outputWalkTakes,
|
|
57
59
|
resolveWorkspaceOutputs,
|
|
58
60
|
sweepRestoreTemps,
|
|
59
61
|
sweepStaging,
|
|
@@ -66,6 +68,7 @@ export {
|
|
|
66
68
|
stampOutputs,
|
|
67
69
|
stampWorkspaceOutputs,
|
|
68
70
|
stopRuntimeProbes,
|
|
71
|
+
answerRuntime,
|
|
69
72
|
type OutputStamp,
|
|
70
73
|
} from './inputs.js'
|
|
71
74
|
export {
|
|
@@ -82,14 +85,14 @@ export {
|
|
|
82
85
|
type BlobSizeMemo,
|
|
83
86
|
} from './git-inputs.js'
|
|
84
87
|
export {
|
|
88
|
+
type ArtifactInfo,
|
|
85
89
|
LayeredCache,
|
|
90
|
+
type LentAccess,
|
|
86
91
|
type RemoteCacheLayer,
|
|
87
|
-
type RemoteKey,
|
|
88
92
|
type RemoteOpenContext,
|
|
89
93
|
} from './layered-cache.js'
|
|
90
94
|
export { networkConcurrency } from './remote-lane.js'
|
|
91
95
|
export type { ResolvedRemote } from './remote-lane.js'
|
|
92
|
-
export { BranchScope } from './scoped-cache.js'
|
|
93
96
|
export {
|
|
94
97
|
absentPluginCache,
|
|
95
98
|
pluginCache,
|
package/src/cache/inputs.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
outputNever,
|
|
32
32
|
realpathOf,
|
|
33
33
|
relPosix,
|
|
34
|
+
restorableOutput,
|
|
34
35
|
shellArgv,
|
|
35
36
|
slashBraceExpansions,
|
|
36
37
|
braceAlternatives,
|
|
@@ -530,6 +531,28 @@ function projectBinDirs(projectDir: string, workspaceRoot: string): string[] {
|
|
|
530
531
|
return own === root ? [own] : [own, root]
|
|
531
532
|
}
|
|
532
533
|
|
|
534
|
+
/**
|
|
535
|
+
* One runtime probe answered on this machine as a key folds it: an
|
|
536
|
+
* `inputs.runtime` command run in `projectDir`, or with `workspace` a
|
|
537
|
+
* `workspaceRuntime` one run in `workspaceRoot`. For an executor that
|
|
538
|
+
* checks a task's probes where it runs it (`TaskExecutor.runtime`).
|
|
539
|
+
*/
|
|
540
|
+
export function answerRuntime(
|
|
541
|
+
command: string,
|
|
542
|
+
projectDir: string,
|
|
543
|
+
workspaceRoot: string,
|
|
544
|
+
workspace = false,
|
|
545
|
+
): Promise<string> {
|
|
546
|
+
return workspace
|
|
547
|
+
? runRuntimeCommand(
|
|
548
|
+
command,
|
|
549
|
+
workspaceRoot,
|
|
550
|
+
[path.join(workspaceRoot, 'node_modules', '.bin')],
|
|
551
|
+
undefined,
|
|
552
|
+
)
|
|
553
|
+
: runRuntimeCommand(command, projectDir, projectBinDirs(projectDir, workspaceRoot), undefined)
|
|
554
|
+
}
|
|
555
|
+
|
|
533
556
|
function resolveEnvValues(
|
|
534
557
|
names: readonly string[],
|
|
535
558
|
source: NodeJS.ProcessEnv,
|
|
@@ -748,6 +771,21 @@ export async function resolveOutputs(args: {
|
|
|
748
771
|
return (await scanOutputs(args)).files
|
|
749
772
|
}
|
|
750
773
|
|
|
774
|
+
/**
|
|
775
|
+
* Whether `resolveOutputs` takes the project-relative file `rel`, decided
|
|
776
|
+
* without the walk: a directory snapshot's file list is held to the
|
|
777
|
+
* entry's rows through it.
|
|
778
|
+
*/
|
|
779
|
+
export function outputWalkTakes(
|
|
780
|
+
projectDir: string,
|
|
781
|
+
outputs: readonly string[],
|
|
782
|
+
nestedProjectDirs: string[],
|
|
783
|
+
): (rel: string) => boolean {
|
|
784
|
+
const restorable = restorableOutput(outputs)
|
|
785
|
+
const nested = inNestedProject(projectDir, nestedProjectDirs)
|
|
786
|
+
return (rel) => restorable(rel) && !nested(rel)
|
|
787
|
+
}
|
|
788
|
+
|
|
751
789
|
/**
|
|
752
790
|
* `resolveOutputs`, plus the empty directories the globs match: a save
|
|
753
791
|
* keeps them, so a hit leaves the tree a miss left (Turbo and Nx restore
|
|
@@ -2103,21 +2141,31 @@ async function scanUnion(
|
|
|
2103
2141
|
emptyDirs?: Set<string>,
|
|
2104
2142
|
): Promise<Set<string>> {
|
|
2105
2143
|
const matches = new Set<string>()
|
|
2144
|
+
const take = (rel: string): void => {
|
|
2145
|
+
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
|
|
2146
|
+
const abs = path.resolve(cwd, rel)
|
|
2147
|
+
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2148
|
+
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
|
|
2149
|
+
else if (emptyDirs !== undefined && st?.isDirectory() === true) {
|
|
2150
|
+
if (readdirSync(abs).length === 0) emptyDirs.add(abs)
|
|
2151
|
+
} else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
|
|
2152
|
+
}
|
|
2153
|
+
const opts = { cwd, onlyFiles: false, followSymlinks: false, dot: true }
|
|
2106
2154
|
// `Bun.Glob`'s scan finds nothing for a brace whose alternatives hold a
|
|
2107
2155
|
// `/`: `{dist,lib/esm}/**` saved an empty artifact, and a hit restored
|
|
2108
2156
|
// nothing over a cleaned tree (A-10). Expanded here, as discovery does.
|
|
2109
|
-
|
|
2157
|
+
const patterns = positive.flatMap(slashBraceExpansions)
|
|
2158
|
+
// A leading `**` walks the whole project whatever else is listed, so the
|
|
2159
|
+
// list is one walk: Turbo's `dist/**`, `**/dist/**`, `**/dist` was three,
|
|
2160
|
+
// ~2 ms of a warm hit on solid.
|
|
2161
|
+
if (patterns.length > 1 && patterns.some((p) => p.startsWith('**'))) {
|
|
2162
|
+
const any = anyTaskGlob(patterns)
|
|
2163
|
+
for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
|
|
2164
|
+
return matches
|
|
2165
|
+
}
|
|
2166
|
+
for (const pattern of patterns) {
|
|
2110
2167
|
if (absentPrefix(cwd, pattern)) continue
|
|
2111
|
-
const
|
|
2112
|
-
for (const rel of glob.scanSync({ cwd, onlyFiles: false, followSymlinks: false, dot: true })) {
|
|
2113
|
-
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) continue
|
|
2114
|
-
const abs = path.resolve(cwd, rel)
|
|
2115
|
-
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2116
|
-
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) matches.add(abs)
|
|
2117
|
-
else if (emptyDirs !== undefined && st?.isDirectory() === true) {
|
|
2118
|
-
if (readdirSync(abs).length === 0) emptyDirs.add(abs)
|
|
2119
|
-
} else if (st === undefined && rel.includes('\ufffd')) refuseUndecodableOutput(abs)
|
|
2120
|
-
}
|
|
2168
|
+
for (const rel of globFor(pattern).scanSync(opts)) take(rel)
|
|
2121
2169
|
}
|
|
2122
2170
|
return matches
|
|
2123
2171
|
}
|
|
@@ -2139,6 +2187,8 @@ function absentPrefix(cwd: string, pattern: string): boolean {
|
|
|
2139
2187
|
}
|
|
2140
2188
|
}
|
|
2141
2189
|
|
|
2190
|
+
const WHOLE_TREE = new Bun.Glob('**')
|
|
2191
|
+
|
|
2142
2192
|
// Compiled once per pattern string for the life of the process: a `Bun.Glob`
|
|
2143
2193
|
// is immutable, and the same handful of declared globs are compiled again
|
|
2144
2194
|
// for every task otherwise (three sites, up to thousands of times per run).
|
package/src/cache/layer.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// reference to `Cache` below is a type (the local handle a layer may wrap).
|
|
6
6
|
|
|
7
7
|
import type { Cache } from './cache.js'
|
|
8
|
+
import type { ArtifactInfo } from './layered-cache.js'
|
|
8
9
|
import { fsClockLag, stampGrainMs } from './fs-clock.js'
|
|
9
10
|
|
|
10
11
|
export interface CacheKeyInput {
|
|
@@ -211,6 +212,16 @@ export interface RunRecord {
|
|
|
211
212
|
notReady?: 'timeout' | 'exited' | 'spawn'
|
|
212
213
|
/** Each remote read or write of its key that failed, `upload: <message>`, `; `-joined (v40). */
|
|
213
214
|
remoteError?: string
|
|
215
|
+
/** The sampled peak cores and mean resident memory of the run (v43). */
|
|
216
|
+
peakCores?: number
|
|
217
|
+
avgRssBytes?: number
|
|
218
|
+
/** Where an executor ran it (`small#1`); absent = here (v44). */
|
|
219
|
+
placement?: string
|
|
220
|
+
/** The task with no `inputs` its key folded, so it never read or saved (v46). */
|
|
221
|
+
keylessBy?: string
|
|
222
|
+
/** That machine's cores and memory (v44). */
|
|
223
|
+
placementCores?: number
|
|
224
|
+
placementMemoryBytes?: number
|
|
214
225
|
}
|
|
215
226
|
|
|
216
227
|
/**
|
|
@@ -383,6 +394,13 @@ export interface OutputDirRow {
|
|
|
383
394
|
*/
|
|
384
395
|
export const OUTPUT_DIRS_CAP = 8192
|
|
385
396
|
|
|
397
|
+
/**
|
|
398
|
+
* `mtime_ms` of the one row a snapshot over `OUTPUT_DIRS_CAP` leaves: never
|
|
399
|
+
* current, so the hit walks, and it tells the hit not to snapshot again.
|
|
400
|
+
* Cleared rows sent every hit through the walk AND a capped snapshot.
|
|
401
|
+
*/
|
|
402
|
+
export const OUTPUT_DIRS_OVER_CAP = -3
|
|
403
|
+
|
|
386
404
|
/**
|
|
387
405
|
* A directory whose mtime lies within this many ms of the snapshot is RACY
|
|
388
406
|
* and the whole snapshot is dropped. File timestamps are coarse on Linux
|
|
@@ -550,12 +568,12 @@ export interface CacheLayer {
|
|
|
550
568
|
*/
|
|
551
569
|
readonly hasRemote?: boolean
|
|
552
570
|
/**
|
|
553
|
-
* Batch existence probe over the remote — the
|
|
554
|
-
*
|
|
555
|
-
* per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
571
|
+
* Batch existence probe over the remote — the `hashes` stored remotely
|
|
572
|
+
* and what the first remote holding each knows of it (`ArtifactInfo`), in
|
|
573
|
+
* ONE round-trip, or `null` for "no batch info; use per-hash". Only meaningful with `hasRemote`; a remote layer that can't
|
|
556
574
|
* batch omits it and the prefetch pass falls back to per-hash GETs.
|
|
557
575
|
*/
|
|
558
|
-
remoteHasMany?(hashes: readonly string[]): Promise<
|
|
576
|
+
remoteHasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
|
|
559
577
|
/**
|
|
560
578
|
* The subset of `hashes` the local store already holds. The prefetch
|
|
561
579
|
* pass probes the remote only for the rest, so a run the local cache
|
|
@@ -581,8 +599,6 @@ export interface CacheLayer {
|
|
|
581
599
|
* about to call absent. Only meaningful with `hasRemote`.
|
|
582
600
|
*/
|
|
583
601
|
probeFirst?(started: Promise<void>): void
|
|
584
|
-
/** `taskId`'s key is `hash`: a scoped run picks each remote key by its task (`BranchScope`). */
|
|
585
|
-
noteKey?(taskId: string, hash: string): void
|
|
586
602
|
/**
|
|
587
603
|
* Resolve once every background write-through upload settles. The
|
|
588
604
|
* orchestrator awaits it before `close()` — an upload reading layer state
|
|
@@ -594,6 +610,12 @@ export interface CacheLayer {
|
|
|
594
610
|
stopUploads?(): void
|
|
595
611
|
key(input: CacheKeyInput): Promise<string>
|
|
596
612
|
get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
613
|
+
/**
|
|
614
|
+
* `get` past the run's read policy: an entry a machine running a task
|
|
615
|
+
* for this run saved is the run's own work, not a hit (`TaskArtifacts`).
|
|
616
|
+
* A layer without it is read through `get`.
|
|
617
|
+
*/
|
|
618
|
+
getSaved?(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null>
|
|
597
619
|
/**
|
|
598
620
|
* Optional batched `get` (same answers, fewer round trips). The
|
|
599
621
|
* short-circuit probe uses it when a layer offers one; a layer without it
|
|
@@ -652,8 +674,8 @@ export interface CacheLayer {
|
|
|
652
674
|
/**
|
|
653
675
|
* The directory-mtime short-circuit behind a warm hit (optional — a layer
|
|
654
676
|
* without it keeps the output walk). `recordOutputDirs` snapshots every
|
|
655
|
-
* directory under each of `prefixes` (project-relative,
|
|
656
|
-
*
|
|
677
|
+
* directory under each of `prefixes` (project-relative, `''` the project
|
|
678
|
+
* itself — see `outputDirPrefixes`) after a save or restore;
|
|
657
679
|
* `loadOutputDirsBatch` reads them back; `outputDirsCurrent` is true iff
|
|
658
680
|
* every recorded directory still carries its recorded mtime, which proves
|
|
659
681
|
* no file was added or removed anywhere the glob could see. Machine-local
|
|
@@ -36,7 +36,6 @@ import type {
|
|
|
36
36
|
} from './cache.js'
|
|
37
37
|
import type { OutputDirRow } from './cache.js'
|
|
38
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
-
import type { BranchScope } from './scoped-cache.js'
|
|
40
39
|
import type { PluginSecrets } from '../util/index.js'
|
|
41
40
|
|
|
42
41
|
const STOPPED: unique symbol = Symbol('stopped')
|
|
@@ -46,7 +45,11 @@ const STOPPED: unique symbol = Symbol('stopped')
|
|
|
46
45
|
* fails at once, removes its temp, and releases the connection; the cache
|
|
47
46
|
* then closes under no write.
|
|
48
47
|
*/
|
|
49
|
-
function endsOnStop(
|
|
48
|
+
function endsOnStop(
|
|
49
|
+
body: Blob | Response,
|
|
50
|
+
stopped: Promise<typeof STOPPED>,
|
|
51
|
+
onChunk: (bytes: number) => void = () => {},
|
|
52
|
+
): Blob | Response {
|
|
50
53
|
if (body instanceof Blob || body.body === null) return body
|
|
51
54
|
const reader = body.body.getReader()
|
|
52
55
|
let stop = false
|
|
@@ -61,13 +64,28 @@ function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Bl
|
|
|
61
64
|
const next = await reader.read()
|
|
62
65
|
if (stop) controller.error(new Error('the run was stopped'))
|
|
63
66
|
else if (next.done) controller.close()
|
|
64
|
-
else
|
|
67
|
+
else {
|
|
68
|
+
onChunk(next.value.byteLength)
|
|
69
|
+
controller.enqueue(next.value)
|
|
70
|
+
}
|
|
65
71
|
},
|
|
66
72
|
cancel: (reason) => reader.cancel(reason),
|
|
67
73
|
})
|
|
68
74
|
return new Response(stream, { headers: body.headers })
|
|
69
75
|
}
|
|
70
76
|
|
|
77
|
+
const judged = (i: ArtifactInfo): i is { size: number; durationMs: number } =>
|
|
78
|
+
i.size !== undefined && i.durationMs !== undefined
|
|
79
|
+
|
|
80
|
+
/** How often a task waiting on its download weighs running instead. */
|
|
81
|
+
const JUDGE_EVERY_MS = 25
|
|
82
|
+
|
|
83
|
+
interface Download {
|
|
84
|
+
readonly start: number
|
|
85
|
+
received: number
|
|
86
|
+
abandon(): void
|
|
87
|
+
}
|
|
88
|
+
|
|
71
89
|
/** What a remote's `open` is handed: the run's CI verdict and its plugin's keychain. */
|
|
72
90
|
export interface RemoteOpenContext extends PluginSecrets {
|
|
73
91
|
/** The run is on CI: nobody may be asked. */
|
|
@@ -77,13 +95,20 @@ export interface RemoteOpenContext extends PluginSecrets {
|
|
|
77
95
|
}
|
|
78
96
|
|
|
79
97
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* go straight to that store. turboCache(), nxCache() and vx-reapi ignore it.
|
|
98
|
+
* What a remote knows of an artifact it holds, before it is downloaded:
|
|
99
|
+
* the body's size in bytes and the producing task's duration, each when
|
|
100
|
+
* the wire carries it (Turbo's batch query and `HEAD` do).
|
|
84
101
|
*/
|
|
85
|
-
export interface
|
|
86
|
-
readonly
|
|
102
|
+
export interface ArtifactInfo {
|
|
103
|
+
readonly size?: number
|
|
104
|
+
readonly durationMs?: number
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** A run's access to a remote, lent to a machine that runs its tasks (`RemoteCacheLayer.lend`). */
|
|
108
|
+
export interface LentAccess {
|
|
109
|
+
readonly token: string
|
|
110
|
+
/** Epoch ms; asked again before it. */
|
|
111
|
+
readonly expiresAt?: number
|
|
87
112
|
}
|
|
88
113
|
|
|
89
114
|
/**
|
|
@@ -122,17 +147,29 @@ export interface RemoteCacheLayer {
|
|
|
122
147
|
* plugin's keychain and may ask on the terminal.
|
|
123
148
|
*/
|
|
124
149
|
open?(ctx: RemoteOpenContext): Promise<void>
|
|
125
|
-
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
126
|
-
has(hash: string, at?: RemoteKey): Promise<boolean>
|
|
127
150
|
/**
|
|
128
|
-
* Optional
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
* `null` means "no batch info; use per-hash". `at[i]` is `hashes[i]`'s.
|
|
151
|
+
* Optional: this run's access, for a machine that runs its tasks
|
|
152
|
+
* elsewhere (`ExecutorContext.cacheFacts`); that machine's plugin reads
|
|
153
|
+
* it as `CacheContext.lent`. Only access made for this run (an OIDC
|
|
154
|
+
* exchange's), never a stored secret: the server's token decides what
|
|
155
|
+
* it may write. Asked after `open`.
|
|
134
156
|
*/
|
|
135
|
-
|
|
157
|
+
lend?(): Promise<LentAccess | undefined>
|
|
158
|
+
/**
|
|
159
|
+
* Existence probe: what the remote knows of the stored artifact, or null
|
|
160
|
+
* when it holds none. `--dry` and `--force` ask it per key when the
|
|
161
|
+
* remote cannot batch.
|
|
162
|
+
*/
|
|
163
|
+
has(hash: string): Promise<ArtifactInfo | null>
|
|
164
|
+
/**
|
|
165
|
+
* Optional batch existence probe: given N hashes, the stored ones and what
|
|
166
|
+
* the remote knows of each, in ONE round-trip. Its presence says the wire
|
|
167
|
+
* can batch: a run asks it once for its keys, `--dry` before its per-key
|
|
168
|
+
* answers, and `--force` in the background to skip uploads of held keys.
|
|
169
|
+
* Without it a run GETs each key directly (no `has`). Never throws for
|
|
170
|
+
* control flow: `null` means "no batch info; use per-hash".
|
|
171
|
+
*/
|
|
172
|
+
hasMany?(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null>
|
|
136
173
|
/**
|
|
137
174
|
* Fetch an artifact; `null` = miss. `body` is read once, by core,
|
|
138
175
|
* straight into the local cache — an HTTP wire returns its `fetch`
|
|
@@ -142,10 +179,7 @@ export interface RemoteCacheLayer {
|
|
|
142
179
|
* nothing and the read is a miss, so a layer can check bytes as they pass
|
|
143
180
|
* (a digest, a signature) and refuse at the end.
|
|
144
181
|
*/
|
|
145
|
-
get(
|
|
146
|
-
hash: string,
|
|
147
|
-
at?: RemoteKey,
|
|
148
|
-
): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
182
|
+
get(hash: string): Promise<{ body: Blob | Response; durationMs: number | undefined } | null>
|
|
149
183
|
/**
|
|
150
184
|
* Store an artifact (fire-and-forget from LayeredCache's perspective).
|
|
151
185
|
* `body` is file-backed (`Bun.file`) when the local store holds the
|
|
@@ -154,7 +188,7 @@ export interface RemoteCacheLayer {
|
|
|
154
188
|
* key the remote holds keeps the stored bytes. Core never asks `has`
|
|
155
189
|
* first; a wire that would overwrite checks inside.
|
|
156
190
|
*/
|
|
157
|
-
put(hash: string, body: Blob, meta: { durationMs: number }
|
|
191
|
+
put(hash: string, body: Blob, meta: { durationMs: number }): Promise<void>
|
|
158
192
|
}
|
|
159
193
|
|
|
160
194
|
export class LayeredCache implements CacheLayer {
|
|
@@ -196,23 +230,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
196
230
|
* its download and ingest took.
|
|
197
231
|
*/
|
|
198
232
|
private readonly remoteSourced = new Map<string, number>()
|
|
233
|
+
/** What the batch probe said of each held key: its size and stored duration. */
|
|
234
|
+
private readonly infos = new Map<string, ArtifactInfo>()
|
|
235
|
+
/** Bodies being read now, by key: how far along, and how to give one up. */
|
|
236
|
+
private readonly downloads = new Map<string, Download>()
|
|
199
237
|
|
|
200
238
|
/** Remotes in declared order: the first readable one holding a hash answers it. */
|
|
201
239
|
constructor(
|
|
202
240
|
readonly local: Cache,
|
|
203
241
|
remotes: readonly ResolvedRemote[],
|
|
204
|
-
/** The run's branch scope: it picks each remote key; the local store never sees it. */
|
|
205
|
-
private readonly branch?: BranchScope,
|
|
206
242
|
uploadConcurrency?: number,
|
|
207
243
|
) {
|
|
208
|
-
this.lanes = remotes.map(
|
|
209
|
-
(r) =>
|
|
210
|
-
new RemoteLane(r, local.traffic, branch?.remote(r.remote) ?? r.remote, uploadConcurrency),
|
|
211
|
-
)
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
noteKey(taskId: string, hash: string): void {
|
|
215
|
-
this.branch?.noteKey(taskId, hash)
|
|
244
|
+
this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic, uploadConcurrency))
|
|
216
245
|
}
|
|
217
246
|
|
|
218
247
|
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
@@ -249,9 +278,12 @@ export class LayeredCache implements CacheLayer {
|
|
|
249
278
|
* absences either way, so a hash only a sibling holds costs it no GET.
|
|
250
279
|
* Never throws.
|
|
251
280
|
*/
|
|
252
|
-
async remoteHasMany(hashes: readonly string[]): Promise<
|
|
281
|
+
async remoteHasMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
253
282
|
if (this.readable.length === 0) return null
|
|
254
|
-
const answer = this.probeMany(hashes)
|
|
283
|
+
const answer = this.probeMany(hashes).then((found) => {
|
|
284
|
+
if (found !== null) for (const [h, info] of found) this.infos.set(h, info)
|
|
285
|
+
return found
|
|
286
|
+
})
|
|
255
287
|
const waiting = hashes.filter((h) => !this.inflight.has(h) && !this.probing.has(h))
|
|
256
288
|
for (const h of waiting) {
|
|
257
289
|
this.probing.set(
|
|
@@ -271,12 +303,12 @@ export class LayeredCache implements CacheLayer {
|
|
|
271
303
|
return answer
|
|
272
304
|
}
|
|
273
305
|
|
|
274
|
-
private async probeMany(hashes: readonly string[]): Promise<
|
|
275
|
-
let out:
|
|
306
|
+
private async probeMany(hashes: readonly string[]): Promise<Map<string, ArtifactInfo> | null> {
|
|
307
|
+
let out: Map<string, ArtifactInfo> | null = new Map()
|
|
276
308
|
for (const lane of this.readable) {
|
|
277
309
|
const found = await lane.hasMany(hashes)
|
|
278
310
|
if (found === null) out = null
|
|
279
|
-
else if (out !== null) for (const h of found) out.
|
|
311
|
+
else if (out !== null) for (const [h, info] of found) if (!out.has(h)) out.set(h, info)
|
|
280
312
|
}
|
|
281
313
|
return out
|
|
282
314
|
}
|
|
@@ -328,7 +360,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
328
360
|
// remote walk per key: a settled `false` means no remote had it
|
|
329
361
|
// (no second probe), and a settled/in-flight `true` means the
|
|
330
362
|
// artifact was/will-be ingested locally — re-read below.
|
|
331
|
-
const ingested = await this.pullFromRemote(hash, ctx)
|
|
363
|
+
const ingested = await this.awaitOrRun(hash, this.pullFromRemote(hash, ctx))
|
|
332
364
|
if (!ingested) return null
|
|
333
365
|
|
|
334
366
|
// Read past the local READ gate: `ingest` is deliberately ungated, so
|
|
@@ -350,6 +382,49 @@ export class LayeredCache implements CacheLayer {
|
|
|
350
382
|
return this.asRemote(hash, materialized)
|
|
351
383
|
}
|
|
352
384
|
|
|
385
|
+
async getSaved(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
|
|
386
|
+
const here = await this.local.getIngested(hash)
|
|
387
|
+
if (here) return here
|
|
388
|
+
// Every remote, read or not, and past a probe's absence: the probe ran
|
|
389
|
+
// before the agent saved it.
|
|
390
|
+
for (const lane of this.lanes)
|
|
391
|
+
if (await this.pullFrom(lane, hash, ctx, true)) return this.local.getIngested(hash)
|
|
392
|
+
return null
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* A pull the task now waits on, given up when the rest of its body would
|
|
397
|
+
* take longer than running the task (its stored duration, doubled for
|
|
398
|
+
* the save a run adds): the task then runs here, a miss. Only a key the
|
|
399
|
+
* batch probe sized and timed is judged, from its own body's pace so far.
|
|
400
|
+
* The facts are read each tick: the probe can answer after the task asks.
|
|
401
|
+
*/
|
|
402
|
+
private async awaitOrRun(hash: string, pull: Promise<boolean>): Promise<boolean> {
|
|
403
|
+
if (this.infos.has(hash) && !judged(this.infos.get(hash)!)) return pull
|
|
404
|
+
for (;;) {
|
|
405
|
+
const done = await Promise.race([pull, Bun.sleep(JUDGE_EVERY_MS).then(() => undefined)])
|
|
406
|
+
if (done !== undefined) return done
|
|
407
|
+
const info = this.infos.get(hash)
|
|
408
|
+
if (info === undefined) continue
|
|
409
|
+
if (!judged(info)) return pull
|
|
410
|
+
const budget = info.durationMs * 2
|
|
411
|
+
const dl = this.downloads.get(hash)
|
|
412
|
+
if (dl === undefined) continue
|
|
413
|
+
const elapsed = performance.now() - dl.start
|
|
414
|
+
const left =
|
|
415
|
+
dl.received === 0
|
|
416
|
+
? elapsed > budget
|
|
417
|
+
? Infinity
|
|
418
|
+
: 0
|
|
419
|
+
: ((info.size - dl.received) * elapsed) / dl.received
|
|
420
|
+
if (left > budget) {
|
|
421
|
+
dl.abandon()
|
|
422
|
+
await pull
|
|
423
|
+
return false
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
353
428
|
private asRemote(hash: string, entry: CacheEntry): CacheEntry {
|
|
354
429
|
const fetchMs = this.remoteSourced.get(hash)
|
|
355
430
|
return fetchMs === undefined ? entry : { ...entry, source: 'remote', fetchMs }
|
|
@@ -360,7 +435,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
360
435
|
// a remote cache. Remote errors degrade to a miss (never throw).
|
|
361
436
|
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
362
437
|
if ((await this.local.has(hash)) === 'local') return 'local'
|
|
363
|
-
for (const lane of this.readable) if (await lane.has(hash)) return 'remote'
|
|
438
|
+
for (const lane of this.readable) if ((await lane.has(hash)) !== null) return 'remote'
|
|
364
439
|
return null
|
|
365
440
|
}
|
|
366
441
|
|
|
@@ -402,13 +477,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
402
477
|
return false
|
|
403
478
|
}
|
|
404
479
|
|
|
405
|
-
private async pullFrom(
|
|
480
|
+
private async pullFrom(
|
|
481
|
+
lane: RemoteLane,
|
|
482
|
+
hash: string,
|
|
483
|
+
ctx?: CacheGetContext,
|
|
484
|
+
fresh = false,
|
|
485
|
+
): Promise<boolean> {
|
|
406
486
|
const fetchStart = performance.now()
|
|
407
487
|
// A stopped run waits on no read, as on no upload: a lookup or download
|
|
408
488
|
// in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
|
|
409
489
|
// plugin's request is left to the exit, and its body is cancelled.
|
|
410
490
|
const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
|
|
411
|
-
const pending = lane.get(hash)
|
|
491
|
+
const pending = lane.get(hash, fresh)
|
|
412
492
|
const remoteBody = await Promise.race([pending, stopped])
|
|
413
493
|
if (remoteBody === STOPPED) {
|
|
414
494
|
// A late answer's body would hold its connection to the exit.
|
|
@@ -434,10 +514,19 @@ export class LayeredCache implements CacheLayer {
|
|
|
434
514
|
durationMs: durationOf(remoteBody.durationMs),
|
|
435
515
|
...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
436
516
|
}
|
|
517
|
+
let abandon = (): void => {}
|
|
518
|
+
const abandoned = new Promise<typeof STOPPED>((resolve) => (abandon = () => resolve(STOPPED)))
|
|
519
|
+
const dl: Download = { start: performance.now(), received: 0, abandon }
|
|
520
|
+
this.downloads.set(hash, dl)
|
|
521
|
+
const ends = Promise.race([stopped, abandoned])
|
|
437
522
|
try {
|
|
438
|
-
const ingest = this.local.ingest(
|
|
523
|
+
const ingest = this.local.ingest(
|
|
524
|
+
hash,
|
|
525
|
+
endsOnStop(remoteBody.body, ends, (n) => (dl.received += n)),
|
|
526
|
+
meta,
|
|
527
|
+
)
|
|
439
528
|
ingest.catch(() => undefined)
|
|
440
|
-
if ((await Promise.race([ingest,
|
|
529
|
+
if ((await Promise.race([ingest, ends])) === STOPPED) return false
|
|
441
530
|
} catch (err) {
|
|
442
531
|
// The bytes came off the network — a corrupt/truncated remote
|
|
443
532
|
// artifact must degrade to a cache miss (task re-executes), not
|
|
@@ -445,8 +534,13 @@ export class LayeredCache implements CacheLayer {
|
|
|
445
534
|
// down: `Cache.restoreOutputs` drops it (A-52).
|
|
446
535
|
lane.report('download', hash, err)
|
|
447
536
|
return false
|
|
537
|
+
} finally {
|
|
538
|
+
this.downloads.delete(hash)
|
|
448
539
|
}
|
|
449
540
|
this.remoteSourced.set(hash, Math.round(performance.now() - fetchStart))
|
|
541
|
+
this.local.downloads.count++
|
|
542
|
+
this.local.downloads.bytes +=
|
|
543
|
+
remoteBody.body instanceof Blob ? remoteBody.body.size : dl.received
|
|
450
544
|
return true
|
|
451
545
|
}
|
|
452
546
|
|