@vzn/vx 0.0.650 → 0.0.652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -23
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/schemas/show.json +14 -20
- package/src/bin.ts +21 -31
- package/src/cache/archive.ts +1 -1
- package/src/cache/cache.ts +221 -30
- package/src/cache/file-hashes.ts +3 -0
- package/src/cache/git-inputs.ts +188 -39
- package/src/cache/history-stream.ts +134 -0
- package/src/cache/history-writer.ts +76 -0
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +77 -14
- package/src/cache/layer.ts +17 -6
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/linked-inputs.ts +3 -0
- package/src/cache/lockfile-memo.ts +5 -3
- package/src/cache/output-index.ts +5 -2
- package/src/cache/remote-lane.ts +2 -1
- package/src/cache/run-history.ts +56 -17
- package/src/cache/schema.ts +15 -2
- package/src/cli/cache.ts +28 -18
- package/src/cli/completions.ts +4 -2
- package/src/cli/docs.ts +10 -11
- package/src/cli/help.ts +14 -25
- package/src/cli/index.ts +28 -9
- package/src/cli/info.ts +16 -8
- package/src/cli/init.ts +23 -15
- package/src/cli/last.ts +246 -204
- package/src/cli/lock.ts +55 -5
- package/src/cli/plan-format.ts +100 -118
- package/src/cli/plugin-commands.ts +8 -2
- package/src/cli/plugin-templates.ts +36 -45
- package/src/cli/run.ts +46 -27
- package/src/cli/select.ts +43 -5
- package/src/cli/show.ts +45 -52
- package/src/cli/task-verb.ts +22 -0
- package/src/cli/upgrade.ts +3 -1
- package/src/cli/watch-fs.ts +6 -1
- package/src/cli/watch-judge.ts +3 -2
- package/src/cli/watch-set.ts +4 -4
- package/src/cli/watch.ts +94 -52
- package/src/cli/why.ts +123 -89
- package/src/cli/workspace-config.ts +2 -1
- package/src/config.ts +47 -30
- package/src/exec/executor.ts +42 -2
- package/src/exec/index.ts +1 -0
- package/src/exec/kill-tree.ts +58 -5
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/pipes.ts +152 -0
- package/src/exec/runner.ts +199 -182
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-paths.ts +3 -1
- package/src/exec/sandbox-runtime.ts +317 -23
- package/src/exec/spawn-off.ts +154 -0
- package/src/exec/task-proxy.ts +93 -0
- package/src/exec/warm-shell.ts +146 -0
- package/src/graph/critical-path.ts +40 -28
- package/src/graph/definition.ts +16 -3
- package/src/graph/priorities.ts +98 -44
- package/src/graph/scheduler.ts +17 -11
- package/src/graph/task-graph.ts +51 -24
- package/src/handoff.ts +10 -7
- package/src/history/failure-mode.ts +24 -1
- package/src/history/history.ts +5 -0
- package/src/history/index.ts +1 -0
- package/src/history/metrics.ts +14 -6
- package/src/index.ts +7 -3
- package/src/orchestrator/admission.ts +1 -1
- package/src/orchestrator/command-windows.ts +1 -1
- package/src/orchestrator/deferred-outputs.ts +1 -1
- package/src/orchestrator/doctor.ts +1 -1
- package/src/orchestrator/download-policy.ts +5 -7
- package/src/orchestrator/excluded-keys.ts +1 -1
- package/src/orchestrator/execute-task.ts +67 -20
- package/src/orchestrator/graph-memo.ts +50 -1
- package/src/orchestrator/graph-stage.ts +6 -0
- package/src/orchestrator/hit-restore.ts +5 -6
- package/src/orchestrator/index.ts +3 -0
- package/src/orchestrator/keyed-projects.ts +28 -0
- package/src/orchestrator/live-usage.ts +16 -11
- package/src/orchestrator/local-shortcircuit.ts +2 -2
- package/src/orchestrator/locked-digests.ts +63 -0
- package/src/orchestrator/lockfile-claim.ts +108 -29
- package/src/orchestrator/miss-save.ts +58 -11
- package/src/orchestrator/persistent-task.ts +5 -1
- package/src/orchestrator/placement.ts +1 -1
- package/src/orchestrator/plan.ts +7 -3
- package/src/orchestrator/plugin-host.ts +67 -2
- package/src/orchestrator/plugin.ts +53 -2
- package/src/orchestrator/prepare.ts +16 -1
- package/src/orchestrator/projects.ts +4 -1
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-context.ts +29 -10
- package/src/orchestrator/run-failures.ts +62 -13
- package/src/orchestrator/run-keys.ts +1 -1
- package/src/orchestrator/run-records.ts +88 -59
- package/src/orchestrator/run-report.ts +1 -1
- package/src/orchestrator/run.ts +145 -41
- package/src/orchestrator/sandbox-request.ts +52 -9
- package/src/orchestrator/shell-verdict.ts +9 -0
- package/src/orchestrator/signals.ts +8 -5
- package/src/orchestrator/stable-keys.ts +17 -18
- package/src/orchestrator/task-artifacts.ts +1 -1
- package/src/orchestrator/task-hash.ts +21 -7
- package/src/orchestrator/telemetry.ts +5 -1
- package/src/report/events.ts +6 -14
- package/src/report/forecast.ts +10 -32
- package/src/report/framed-output.ts +283 -175
- package/src/report/index.ts +8 -3
- package/src/report/logger.ts +74 -39
- package/src/report/plain-output.ts +3 -1
- package/src/report/status-line.ts +49 -357
- package/src/report/summary.ts +105 -136
- package/src/settings.ts +8 -8
- package/src/ui/ansi.ts +48 -0
- package/src/ui/components.ts +506 -0
- package/src/ui/index.ts +79 -0
- package/src/ui/kit.ts +37 -0
- package/src/ui/live-region.ts +363 -0
- package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
- package/src/ui/terminal.ts +51 -0
- package/src/ui/theme.ts +224 -0
- package/src/ui/version-line.ts +28 -0
- package/src/util/config-frame.ts +191 -14
- package/src/util/edit-distance.ts +6 -1
- package/src/util/index.ts +2 -15
- package/src/util/paths.ts +21 -3
- package/src/util/secret-mask.ts +39 -3
- package/src/util/settle.ts +33 -1
- package/src/util/tail.ts +4 -1
- package/src/util/timing.ts +4 -16
- package/src/workspace/affected.ts +89 -30
- package/src/workspace/bracket-literal.ts +2 -2
- package/src/workspace/config-schema.ts +70 -63
- package/src/workspace/dependencies-lock.ts +191 -0
- package/src/workspace/filter.ts +36 -1
- package/src/workspace/index.ts +9 -0
- package/src/workspace/lockfile.ts +159 -6
- package/src/workspace/migration.ts +16 -9
- package/src/workspace/workspace.ts +3 -0
- package/src/util/colors.ts +0 -133
- package/src/util/report.ts +0 -44
- package/src/util/ui.ts +0 -258
- /package/src/{report/path-links.ts → ui/links.ts} +0 -0
- /package/src/{report → ui}/terminal-signals.ts +0 -0
package/src/cache/inputs.ts
CHANGED
|
@@ -17,7 +17,15 @@
|
|
|
17
17
|
// be a git work tree; non-git environments are not supported.
|
|
18
18
|
|
|
19
19
|
import path from 'node:path'
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
type Dirent,
|
|
22
|
+
lstatSync,
|
|
23
|
+
readdirSync,
|
|
24
|
+
realpathSync,
|
|
25
|
+
rmdirSync,
|
|
26
|
+
rmSync,
|
|
27
|
+
statSync,
|
|
28
|
+
} from 'node:fs'
|
|
21
29
|
import { rm, rmdir } from 'node:fs/promises'
|
|
22
30
|
import type { InputsConfig, TaskConfig } from '../config.js'
|
|
23
31
|
import {
|
|
@@ -101,6 +109,9 @@ const ALWAYS_IGNORE = [
|
|
|
101
109
|
// must not select every `**` reader under --affected.
|
|
102
110
|
'.vx/.gitattributes',
|
|
103
111
|
'.vx/.gitignore',
|
|
112
|
+
// `vx lock`'s record of the dependency edges and digests: the key folds
|
|
113
|
+
// the digests through `fingerprint` already.
|
|
114
|
+
'.vx/dependencies.lock.json',
|
|
104
115
|
// Where restores stage (archive.ts): vx's own, never an input.
|
|
105
116
|
'.vx/tmp/**',
|
|
106
117
|
]
|
|
@@ -834,15 +845,23 @@ export async function resolveOutputTree(args: {
|
|
|
834
845
|
projectDir: string
|
|
835
846
|
outputs: readonly string[]
|
|
836
847
|
nestedProjectDirs: string[]
|
|
837
|
-
|
|
848
|
+
/** Also list the FIFOs, sockets and devices the globs name (a save refuses them). */
|
|
849
|
+
special?: boolean
|
|
850
|
+
}): Promise<{ files: string[]; emptyDirs: string[]; special: string[] }> {
|
|
838
851
|
const emptyDirs = new Set<string>()
|
|
839
|
-
const
|
|
840
|
-
|
|
852
|
+
const special = args.special === true ? new Set<string>() : undefined
|
|
853
|
+
const { files } = await scanOutputs(args, emptyDirs, special)
|
|
854
|
+
return {
|
|
855
|
+
files,
|
|
856
|
+
emptyDirs: containedIn(args.projectDir, [...emptyDirs]).sort(),
|
|
857
|
+
special: special === undefined ? [] : containedIn(args.projectDir, [...special]).sort(),
|
|
858
|
+
}
|
|
841
859
|
}
|
|
842
860
|
|
|
843
861
|
async function scanOutputs(
|
|
844
862
|
args: { projectDir: string; outputs: readonly string[]; nestedProjectDirs: string[] },
|
|
845
863
|
emptyDirs?: Set<string>,
|
|
864
|
+
special?: Set<string>,
|
|
846
865
|
): Promise<{ files: string[] }> {
|
|
847
866
|
const { positive, negative } = splitNegations(args.outputs)
|
|
848
867
|
if (positive.length === 0) return { files: [] }
|
|
@@ -856,6 +875,7 @@ async function scanOutputs(
|
|
|
856
875
|
args.projectDir,
|
|
857
876
|
inNestedProject(args.projectDir, args.nestedProjectDirs),
|
|
858
877
|
emptyDirs,
|
|
878
|
+
special,
|
|
859
879
|
)),
|
|
860
880
|
]
|
|
861
881
|
// Containment, enforced HERE and not only at the loader. `cleanOutputs`
|
|
@@ -1255,7 +1275,7 @@ export async function stampOutputs(args: {
|
|
|
1255
1275
|
return stampFiles(await resolveOutputs(args))
|
|
1256
1276
|
}
|
|
1257
1277
|
|
|
1258
|
-
/** `stampOutputs` for root-anchored `
|
|
1278
|
+
/** `stampOutputs` for root-anchored `outputs.workspaceFiles` (A-43). */
|
|
1259
1279
|
export async function stampWorkspaceOutputs(args: {
|
|
1260
1280
|
workspaceRoot: string
|
|
1261
1281
|
outputs: readonly string[]
|
|
@@ -1295,7 +1315,7 @@ export async function ownOutputsSince(
|
|
|
1295
1315
|
return changedSince(await resolveOutputs(args), before)
|
|
1296
1316
|
}
|
|
1297
1317
|
|
|
1298
|
-
/** `ownOutputsSince` for root-anchored `
|
|
1318
|
+
/** `ownOutputsSince` for root-anchored `outputs.workspaceFiles` (A-43). */
|
|
1299
1319
|
export async function ownWorkspaceOutputsSince(
|
|
1300
1320
|
args: { workspaceRoot: string; outputs: readonly string[] },
|
|
1301
1321
|
before: ReadonlyMap<string, OutputStamp>,
|
|
@@ -1524,6 +1544,7 @@ export async function resolveWorkspaceOutputs(
|
|
|
1524
1544
|
outputs: readonly string[]
|
|
1525
1545
|
},
|
|
1526
1546
|
emptyDirs?: Set<string>,
|
|
1547
|
+
special?: Set<string>,
|
|
1527
1548
|
): Promise<string[]> {
|
|
1528
1549
|
const { positive, negative } = splitNegations(args.outputs)
|
|
1529
1550
|
if (positive.length === 0) return []
|
|
@@ -1531,7 +1552,7 @@ export async function resolveWorkspaceOutputs(
|
|
|
1531
1552
|
const trees = asTrees(positive)
|
|
1532
1553
|
refuseDirLinkedOut(args.workspaceRoot, trees, 'workspace')
|
|
1533
1554
|
const scanned = [
|
|
1534
|
-
...(await scanUnion(trees, excludeGlobs, args.workspaceRoot, undefined, emptyDirs)),
|
|
1555
|
+
...(await scanUnion(trees, excludeGlobs, args.workspaceRoot, undefined, emptyDirs, special)),
|
|
1535
1556
|
]
|
|
1536
1557
|
// Same containment as the project twin, anchored one level out. These globs
|
|
1537
1558
|
// deliberately ignore PROJECT boundaries — that is the escape hatch — but
|
|
@@ -1544,10 +1565,17 @@ export async function resolveWorkspaceOutputs(
|
|
|
1544
1565
|
export async function resolveWorkspaceOutputTree(args: {
|
|
1545
1566
|
workspaceRoot: string
|
|
1546
1567
|
outputs: readonly string[]
|
|
1547
|
-
|
|
1568
|
+
/** As `resolveOutputTree`'s. */
|
|
1569
|
+
special?: boolean
|
|
1570
|
+
}): Promise<{ files: string[]; emptyDirs: string[]; special: string[] }> {
|
|
1548
1571
|
const emptyDirs = new Set<string>()
|
|
1549
|
-
const
|
|
1550
|
-
|
|
1572
|
+
const special = args.special === true ? new Set<string>() : undefined
|
|
1573
|
+
const files = await resolveWorkspaceOutputs(args, emptyDirs, special)
|
|
1574
|
+
return {
|
|
1575
|
+
files,
|
|
1576
|
+
emptyDirs: containedIn(args.workspaceRoot, [...emptyDirs]).sort(),
|
|
1577
|
+
special: special === undefined ? [] : containedIn(args.workspaceRoot, [...special]).sort(),
|
|
1578
|
+
}
|
|
1551
1579
|
}
|
|
1552
1580
|
|
|
1553
1581
|
/**
|
|
@@ -1962,7 +1990,7 @@ export function declaresInput(
|
|
|
1962
1990
|
// A changed path under the task's own outputs is a committed file
|
|
1963
1991
|
// (`git diff` lists only what git tracks) or a stray `git status`
|
|
1964
1992
|
// lists: either way it may be an input, so it counts.
|
|
1965
|
-
const plan = filesPlan(inputs.files, task.
|
|
1993
|
+
const plan = filesPlan(inputs.files, task.outputs?.files ?? [])
|
|
1966
1994
|
if (plan !== null && (inPlan(plan, projectRel) || ownOnly(plan, projectRel))) return true
|
|
1967
1995
|
if (plan !== null && dirLink && plan.mayHold(projectRel) && !plan.isExcluded(projectRel)) {
|
|
1968
1996
|
return true
|
|
@@ -1992,8 +2020,7 @@ export function workspaceFilesReachInto(task: TaskConfig, dir: string): boolean
|
|
|
1992
2020
|
*/
|
|
1993
2021
|
export function namedOutright(task: TaskConfig): { files: string[]; workspaceFiles: string[] } {
|
|
1994
2022
|
const inputs = task.inputs
|
|
1995
|
-
const plan =
|
|
1996
|
-
inputs === undefined ? null : filesPlan(inputs.files, task.cache?.outputs.files ?? [])
|
|
2023
|
+
const plan = inputs === undefined ? null : filesPlan(inputs.files, task.outputs?.files ?? [])
|
|
1997
2024
|
const ws = inputs?.workspaceFiles ?? []
|
|
1998
2025
|
return {
|
|
1999
2026
|
files: plan === null ? [] : outright(plan),
|
|
@@ -2239,7 +2266,7 @@ function listingFor(
|
|
|
2239
2266
|
function refuseUndecodableOutput(abs: string): void {
|
|
2240
2267
|
if (!undecodableOnDisk(abs)) return
|
|
2241
2268
|
throw new UserError(
|
|
2242
|
-
`
|
|
2269
|
+
`outputs matched ${JSON.stringify(abs)}: the name is not valid UTF-8 (shown with ` +
|
|
2243
2270
|
`\ufffd), and vx cannot save a file by it. Rename it, or keep it out of the outputs.`,
|
|
2244
2271
|
)
|
|
2245
2272
|
}
|
|
@@ -2453,13 +2480,19 @@ async function scanUnion(
|
|
|
2453
2480
|
cwd: string,
|
|
2454
2481
|
nested: (rel: string) => boolean = () => false,
|
|
2455
2482
|
emptyDirs?: Set<string>,
|
|
2483
|
+
special?: Set<string>,
|
|
2456
2484
|
): Promise<Set<string>> {
|
|
2457
2485
|
const matches = new Set<string>()
|
|
2458
2486
|
const links: string[] = []
|
|
2487
|
+
// Directories the walk entered: Bun's scan skips a FIFO, socket or
|
|
2488
|
+
// device outright, so `special` is found by listing them.
|
|
2489
|
+
const walked = special === undefined ? undefined : new Set<string>()
|
|
2459
2490
|
const take = (rel: string): void => {
|
|
2460
2491
|
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
|
|
2461
2492
|
const abs = path.resolve(cwd, rel)
|
|
2493
|
+
walked?.add(path.dirname(abs))
|
|
2462
2494
|
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2495
|
+
if (walked !== undefined && st?.isDirectory() === true) walked.add(abs)
|
|
2463
2496
|
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) {
|
|
2464
2497
|
matches.add(abs)
|
|
2465
2498
|
if (st.isSymbolicLink()) links.push(abs)
|
|
@@ -2484,17 +2517,22 @@ async function scanUnion(
|
|
|
2484
2517
|
) {
|
|
2485
2518
|
const any = anyTaskGlob(patterns)
|
|
2486
2519
|
for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
|
|
2520
|
+
if (special !== undefined) findSpecial(walked!, cwd, any, excludeGlobs, nested, special)
|
|
2487
2521
|
return matches
|
|
2488
2522
|
}
|
|
2489
2523
|
for (const pattern of patterns) {
|
|
2490
2524
|
if (absentPrefix(cwd, pattern)) continue
|
|
2491
2525
|
const from = scanRoot(cwd, pattern)
|
|
2492
2526
|
if (from === null) continue
|
|
2527
|
+
walked?.add(from === undefined ? cwd : from.dir)
|
|
2493
2528
|
if (from === undefined) for (const rel of globFor(pattern).scanSync(opts)) take(rel)
|
|
2494
2529
|
else
|
|
2495
2530
|
for (const rel of globFor(from.rest).scanSync({ ...opts, cwd: from.dir }))
|
|
2496
2531
|
take(`${from.prefix}/${rel}`)
|
|
2497
2532
|
}
|
|
2533
|
+
if (special !== undefined) {
|
|
2534
|
+
findSpecial(walked!, cwd, anyTaskGlob(patterns), excludeGlobs, nested, special)
|
|
2535
|
+
}
|
|
2498
2536
|
// A literal names the path and its tree (`dist`, `dist/**`), and a tree
|
|
2499
2537
|
// whose head is a directory link is walked through it: the link and the
|
|
2500
2538
|
// files under it were one artifact holding `dist` as a file and a
|
|
@@ -2510,6 +2548,31 @@ async function scanUnion(
|
|
|
2510
2548
|
return matches
|
|
2511
2549
|
}
|
|
2512
2550
|
|
|
2551
|
+
/** The FIFOs, sockets and devices in `dirs` that the output globs name. */
|
|
2552
|
+
function findSpecial(
|
|
2553
|
+
dirs: ReadonlySet<string>,
|
|
2554
|
+
cwd: string,
|
|
2555
|
+
any: (rel: string) => boolean,
|
|
2556
|
+
excludeGlobs: readonly Bun.Glob[],
|
|
2557
|
+
nested: (rel: string) => boolean,
|
|
2558
|
+
into: Set<string>,
|
|
2559
|
+
): void {
|
|
2560
|
+
for (const dir of dirs) {
|
|
2561
|
+
let entries: Dirent[]
|
|
2562
|
+
try {
|
|
2563
|
+
entries = readdirSync(dir, { withFileTypes: true })
|
|
2564
|
+
} catch {
|
|
2565
|
+
continue
|
|
2566
|
+
}
|
|
2567
|
+
for (const e of entries) {
|
|
2568
|
+
if (e.isFile() || e.isDirectory() || e.isSymbolicLink()) continue
|
|
2569
|
+
const abs = path.join(dir, e.name)
|
|
2570
|
+
const rel = path.relative(cwd, abs).split(path.sep).join('/')
|
|
2571
|
+
if (any(rel) && !nested(rel) && !excludeGlobs.some((g) => g.match(rel))) into.add(abs)
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2513
2576
|
/** `pattern`'s static directory passes through a symlink under `cwd`. */
|
|
2514
2577
|
function headLinked(cwd: string, pattern: string): boolean {
|
|
2515
2578
|
const prefix = staticPrefix(pattern)
|
package/src/cache/layer.ts
CHANGED
|
@@ -199,7 +199,7 @@ export interface RunRecord {
|
|
|
199
199
|
restored?: boolean
|
|
200
200
|
attempts?: number // >1 when the task retried; the direct within-run flaky signal
|
|
201
201
|
/**
|
|
202
|
-
* Whether the task declared
|
|
202
|
+
* Whether the task declared `cache: true`. An uncached task executes
|
|
203
203
|
* on every run and its key exists only for dependents to fold, so a
|
|
204
204
|
* reader (`vx why`, `vx last`) must not present its re-run as a cache
|
|
205
205
|
* decision. Absent on rows written before the column existed.
|
|
@@ -232,6 +232,15 @@ export interface RunRecord {
|
|
|
232
232
|
placementMemoryBytes?: number
|
|
233
233
|
/** The share of a core each runnable thread got there (`ExecuteResult.machine.cpuShare`, v49). */
|
|
234
234
|
cpuShare?: number
|
|
235
|
+
/** How long it waited ready for a slot (`TaskOutcome.queuedMs`, v51). */
|
|
236
|
+
queuedMs?: number
|
|
237
|
+
/** Its output files' bytes (`TaskOutcome.outputBytes`, v51). */
|
|
238
|
+
outputBytes?: number
|
|
239
|
+
/** The tasks and groups downstream of it in the run, the scheduler's count (v51). */
|
|
240
|
+
blocks?: number
|
|
241
|
+
/** The flaky record its row's note quoted (`TaskOutcome.flaky`, v51). */
|
|
242
|
+
flakyPasses?: number
|
|
243
|
+
flakyFailures?: number
|
|
235
244
|
}
|
|
236
245
|
|
|
237
246
|
/**
|
|
@@ -349,7 +358,7 @@ export interface PruneResult {
|
|
|
349
358
|
|
|
350
359
|
/**
|
|
351
360
|
* Artifact + `output_files` namespace prefix for workspace-root-
|
|
352
|
-
* anchored outputs (`
|
|
361
|
+
* anchored outputs (`outputs.workspaceFiles`). Project outputs
|
|
353
362
|
* keep their bare project-relative `path` rows; workspace rows store
|
|
354
363
|
* the full `workspace-outputs/<rel-to-root>` tar entry name as the
|
|
355
364
|
* discriminator — least-invasive row format, no schema change. A
|
|
@@ -458,7 +467,7 @@ export interface CacheGetContext {
|
|
|
458
467
|
taskId: string
|
|
459
468
|
command: string
|
|
460
469
|
/**
|
|
461
|
-
* The task's declared `
|
|
470
|
+
* The task's declared `outputs`. A remote artifact is ingested
|
|
462
471
|
* only when every file it carries is one of them: bytes off the network
|
|
463
472
|
* named `outputs/src/in.txt` or `workspace-outputs/.git/hooks/…` wrote
|
|
464
473
|
* an input and a git hook on a green `cache-hit-remote` (item 942).
|
|
@@ -795,9 +804,9 @@ export interface CacheLayer {
|
|
|
795
804
|
*/
|
|
796
805
|
ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void>
|
|
797
806
|
/**
|
|
798
|
-
* The
|
|
799
|
-
*
|
|
800
|
-
* transaction (one fsync). Input-fingerprint rows are NOT written here — they
|
|
807
|
+
* The run-history write at a run's end, atomically: the per-task `runs`
|
|
808
|
+
* rows (less the hits a `HistoryStream` already wrote) and the one
|
|
809
|
+
* `invocations` header row — in a SINGLE transaction (one fsync). Input-fingerprint rows are NOT written here — they
|
|
801
810
|
* ride the entry-save transaction (`save`/`ingest`) so a warm
|
|
802
811
|
* all-cache-hit run writes nothing.
|
|
803
812
|
*/
|
|
@@ -823,6 +832,8 @@ export interface CacheLayer {
|
|
|
823
832
|
outputsPath(hash: string): string
|
|
824
833
|
/** The stored artifact's compressed size, wherever it lives; undefined when none is stored. */
|
|
825
834
|
artifactSize?(hash: string): number | undefined
|
|
835
|
+
/** The output files' bytes, uncompressed, of the entry this process just saved; asked once. */
|
|
836
|
+
outputBytes?(hash: string): number | undefined
|
|
826
837
|
prune(options: PruneOptions): Promise<PruneResult>
|
|
827
838
|
close(): void
|
|
828
839
|
}
|
|
@@ -575,6 +575,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
575
575
|
return this.local.artifactSize(hash)
|
|
576
576
|
}
|
|
577
577
|
|
|
578
|
+
outputBytes(hash: string): number | undefined {
|
|
579
|
+
return this.local.outputBytes(hash)
|
|
580
|
+
}
|
|
581
|
+
|
|
578
582
|
hashFile(filePath: string): Promise<string> {
|
|
579
583
|
return this.local.hashFile(filePath)
|
|
580
584
|
}
|
|
@@ -71,6 +71,9 @@ export async function behindLink(
|
|
|
71
71
|
return NOTHING_BEHIND
|
|
72
72
|
}
|
|
73
73
|
if (seen.has(real)) return NOTHING_BEHIND
|
|
74
|
+
// A repository's own store, as a `.git/**` input names nothing: walked,
|
|
75
|
+
// every commit and index write re-keyed the task. Its target string keys.
|
|
76
|
+
if (real.split(path.sep).includes('.git')) return NOTHING_BEHIND
|
|
74
77
|
if (st.isFile()) return `file\0${await hashFile(real).catch(absentOr)}`
|
|
75
78
|
if (!st.isDirectory()) return 'special'
|
|
76
79
|
const inner = new Set(seen).add(real)
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
import { Database } from 'bun:sqlite'
|
|
12
12
|
import { existsSync } from 'node:fs'
|
|
13
13
|
import path from 'node:path'
|
|
14
|
-
import { closeDb, SCHEMA_VERSION
|
|
14
|
+
import { closeDb, SCHEMA_VERSION } from './cache.js'
|
|
15
|
+
import { SQLITE_FCNTL_PERSIST_WAL } from './schema.js'
|
|
15
16
|
|
|
16
17
|
/** One claimant's parse of one lockfile's content. */
|
|
17
18
|
export interface LockfileMemo {
|
|
@@ -21,7 +22,8 @@ export interface LockfileMemo {
|
|
|
21
22
|
lock: string
|
|
22
23
|
/** Each extra file the digests read → its content hash then. */
|
|
23
24
|
extras: Record<string, string>
|
|
24
|
-
|
|
25
|
+
/** Each importer's digest; null when the parse that wrote it left them out (a run's). */
|
|
26
|
+
importers: Record<string, string> | null
|
|
25
27
|
edges: Record<string, unknown>
|
|
26
28
|
/** Each importer's recorded dependency names; absent when the claimant reads none. */
|
|
27
29
|
recorded?: Record<string, readonly string[]>
|
|
@@ -77,7 +79,7 @@ export function readLockfileMemo(
|
|
|
77
79
|
version: row.version,
|
|
78
80
|
lock: row.lock,
|
|
79
81
|
extras: JSON.parse(row.extras) as Record<string, string>,
|
|
80
|
-
importers: JSON.parse(row.importers) as Record<string, string
|
|
82
|
+
importers: JSON.parse(row.importers) as Record<string, string> | null,
|
|
81
83
|
edges: JSON.parse(row.edges) as Record<string, unknown>,
|
|
82
84
|
...(row.recorded !== null
|
|
83
85
|
? { recorded: JSON.parse(row.recorded) as Record<string, readonly string[]> }
|
|
@@ -143,17 +143,20 @@ export class OutputIndex {
|
|
|
143
143
|
/**
|
|
144
144
|
* Replace an entry's file rows. Called INSIDE the store's save transaction,
|
|
145
145
|
* so an entry and its fingerprints land together (an UPDATE on the same
|
|
146
|
-
* hash refreshes rather than appends).
|
|
146
|
+
* hash refreshes rather than appends). Answers the files' bytes.
|
|
147
147
|
*/
|
|
148
|
-
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>):
|
|
148
|
+
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): number {
|
|
149
149
|
// A stamp taken for the rows this replaces describes their files, not these.
|
|
150
150
|
this.held.delete(hash)
|
|
151
151
|
this.pendingStamps.delete(hash)
|
|
152
152
|
this.deleteOutputFiles.run(hash)
|
|
153
153
|
this.deleteStamps.run(hash)
|
|
154
|
+
let bytes = 0
|
|
154
155
|
for (const [rel, size, mode, mtime] of rows) {
|
|
155
156
|
this.insertOutputFile.run(hash, rel, size, mode, mtime)
|
|
157
|
+
bytes += size
|
|
156
158
|
}
|
|
159
|
+
return bytes
|
|
157
160
|
}
|
|
158
161
|
|
|
159
162
|
/** Hold a probe's rows for its restores (`held`), in place of the last probe's. */
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// class with the endpoint named. The order across remotes, the local store
|
|
5
5
|
// and provenance are `LayeredCache`'s; nothing here knows another remote.
|
|
6
6
|
|
|
7
|
-
import { machineParallelism, thrownMessage
|
|
7
|
+
import { machineParallelism, thrownMessage } from '../util/index.js'
|
|
8
|
+
import * as ui from '../ui/index.js'
|
|
8
9
|
import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
|
|
9
10
|
import type { ArtifactInfo, RemoteCacheLayer } from './layered-cache.js'
|
|
10
11
|
|
package/src/cache/run-history.ts
CHANGED
|
@@ -41,23 +41,29 @@ const RUN_COLUMNS = [
|
|
|
41
41
|
'keyless_by',
|
|
42
42
|
'def',
|
|
43
43
|
'cpu_share',
|
|
44
|
+
'queued_ms',
|
|
45
|
+
'output_bytes',
|
|
46
|
+
'blocks',
|
|
47
|
+
'flaky_passes',
|
|
48
|
+
'flaky_failures',
|
|
44
49
|
]
|
|
45
50
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
/**
|
|
52
|
+
* An INSERT of `n` `runs` rows over the columns `kept` names (indexes into
|
|
53
|
+
* `RUN_COLUMNS`); a column it leaves out is NULL, as no `runs` column has
|
|
54
|
+
* a default.
|
|
55
|
+
*/
|
|
56
|
+
function insertRunsSql(kept: readonly number[], n: number): string {
|
|
57
|
+
const tuple = `(${kept.map(() => '?').join(', ')})`
|
|
58
|
+
return `INSERT INTO runs(${kept.map((i) => RUN_COLUMNS[i]).join(', ')}) VALUES ${Array(n).fill(tuple).join(', ')}`
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* rows, 40 at a time 6.4 (2026-10-03).
|
|
62
|
+
* Bound variables per INSERT: within 999, the oldest default ceiling. A
|
|
63
|
+
* statement per row was 8.7 ms of 1,000 rows, 40 at a time 6.4
|
|
64
|
+
* (2026-10-03).
|
|
59
65
|
*/
|
|
60
|
-
const
|
|
66
|
+
const VARIABLES_PER_INSERT = 999
|
|
61
67
|
|
|
62
68
|
/** Rows per `task_durations` INSERT: 7 variables each, under 999. */
|
|
63
69
|
const DURATIONS_PER_INSERT = 140
|
|
@@ -147,6 +153,21 @@ export class RunHistory {
|
|
|
147
153
|
.immediate()
|
|
148
154
|
}
|
|
149
155
|
|
|
156
|
+
/**
|
|
157
|
+
* Hits written while the run goes on (history-stream.ts): their rows and
|
|
158
|
+
* the flaky record's first hit of a key. A hit times nothing, so it
|
|
159
|
+
* leaves no `task_durations` row; IMMEDIATE, as `recordRunBundle` says.
|
|
160
|
+
*/
|
|
161
|
+
recordHits(runs: readonly RunRecord[]): void {
|
|
162
|
+
if (runs.length === 0) return
|
|
163
|
+
this.db
|
|
164
|
+
.transaction(() => {
|
|
165
|
+
this.insertRuns(runs)
|
|
166
|
+
this.insertOutcomes(runs)
|
|
167
|
+
})
|
|
168
|
+
.immediate()
|
|
169
|
+
}
|
|
170
|
+
|
|
150
171
|
/** One `task_durations` row per executed success, as if run alone: what a forecast reads. */
|
|
151
172
|
private insertDurations(runs: readonly RunRecord[], source: string, sha: string | null): void {
|
|
152
173
|
const done = runs.filter(
|
|
@@ -204,7 +225,13 @@ export class RunHistory {
|
|
|
204
225
|
}
|
|
205
226
|
}
|
|
206
227
|
|
|
207
|
-
/**
|
|
228
|
+
/**
|
|
229
|
+
* `runs` as few INSERTs; each distinct forward-args list digested once.
|
|
230
|
+
* Only the columns some row fills are bound: a hit leaves half of them
|
|
231
|
+
* NULL, and binding 18 of 36 took the 8,002 rows of a warm run from
|
|
232
|
+
* 31.7 to 24.0 ms alone, the whole record stage 86.8 to 83.3 ms (median
|
|
233
|
+
* of 15 interleaved runs, 2026-11-06).
|
|
234
|
+
*/
|
|
208
235
|
private insertRuns(runs: readonly RunRecord[]): void {
|
|
209
236
|
const digests = new Map<string, string>()
|
|
210
237
|
const digest = (value: string): string => {
|
|
@@ -212,12 +239,19 @@ export class RunHistory {
|
|
|
212
239
|
if (d === undefined) digests.set(value, (d = this.digestValue(value)))
|
|
213
240
|
return d
|
|
214
241
|
}
|
|
215
|
-
|
|
216
|
-
|
|
242
|
+
const bound = runs.map((r) => bindRun(r, digest))
|
|
243
|
+
const filled = RUN_COLUMNS.map(() => false)
|
|
244
|
+
for (const row of bound)
|
|
245
|
+
for (let c = 0; c < row.length; c++) if (row[c] !== null) filled[c] = true
|
|
246
|
+
const kept: number[] = []
|
|
247
|
+
for (let c = 0; c < filled.length; c++) if (filled[c]) kept.push(c)
|
|
248
|
+
const perInsert = Math.floor(VARIABLES_PER_INSERT / kept.length)
|
|
249
|
+
for (let i = 0; i < bound.length; i += perInsert) {
|
|
250
|
+
const block = bound.slice(i, i + perInsert)
|
|
217
251
|
const params: SQLQueryBindings[] = []
|
|
218
|
-
for (const
|
|
252
|
+
for (const row of block) for (const c of kept) params.push(row[c] as SQLQueryBindings)
|
|
219
253
|
// `query` caches by text: one statement for every full block.
|
|
220
|
-
this.db.query(insertRunsSql(block.length)).run(...params)
|
|
254
|
+
this.db.query(insertRunsSql(kept, block.length)).run(...params)
|
|
221
255
|
}
|
|
222
256
|
}
|
|
223
257
|
|
|
@@ -282,7 +316,7 @@ export class RunHistory {
|
|
|
282
316
|
}
|
|
283
317
|
}
|
|
284
318
|
|
|
285
|
-
/**
|
|
319
|
+
/** A RunRecord as one value per `RUN_COLUMNS` entry, in its order. */
|
|
286
320
|
function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBindings[] {
|
|
287
321
|
return [
|
|
288
322
|
// The ONE place the no-key sentinel is applied, so the column's
|
|
@@ -321,6 +355,11 @@ function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBi
|
|
|
321
355
|
run.keylessBy ?? null,
|
|
322
356
|
run.def ?? null,
|
|
323
357
|
run.cpuShare ?? null,
|
|
358
|
+
run.queuedMs ?? null,
|
|
359
|
+
run.outputBytes ?? null,
|
|
360
|
+
run.blocks ?? null,
|
|
361
|
+
run.flakyPasses ?? null,
|
|
362
|
+
run.flakyFailures ?? null,
|
|
324
363
|
]
|
|
325
364
|
}
|
|
326
365
|
|
package/src/cache/schema.ts
CHANGED
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
|
|
7
7
|
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
8
8
|
|
|
9
|
+
/** SQLite's file-control opcode (sqlite3.h). */
|
|
10
|
+
export const SQLITE_FCNTL_PERSIST_WAL = 10
|
|
11
|
+
|
|
9
12
|
/**
|
|
10
13
|
* A statement prepared on its first `run` or `get`. A warm run executes few
|
|
11
14
|
* of the index's statements, and preparing every one at the open cost each
|
|
@@ -84,7 +87,7 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
84
87
|
-- v23: attempts a retried task took (>1); NULL for a once-run task.
|
|
85
88
|
-- The direct within-run flaky signal.
|
|
86
89
|
attempts INTEGER,
|
|
87
|
-
-- v25: 1 when the task declared
|
|
90
|
+
-- v25: 1 when the task declared cache: true, 0 when it runs every
|
|
88
91
|
-- time by design; NULL on rows older than the column.
|
|
89
92
|
cached INTEGER,
|
|
90
93
|
-- v27: why a task failed or was skipped, as the run's own footer
|
|
@@ -119,7 +122,17 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
119
122
|
def TEXT,
|
|
120
123
|
-- v49: the share of a core each runnable thread got where it ran
|
|
121
124
|
-- (ExecuteResult.machine.cpuShare); NULL = all it wanted.
|
|
122
|
-
cpu_share REAL
|
|
125
|
+
cpu_share REAL,
|
|
126
|
+
-- v51: the rest of the results row. How long the task waited ready
|
|
127
|
+
-- for a slot (NULL: it started at once); its output files' bytes,
|
|
128
|
+
-- uncompressed (a hit's restored, a miss's saved; NULL: no entry);
|
|
129
|
+
-- the tasks downstream of it in the run; the flaky record its note
|
|
130
|
+
-- quoted (NULL: not proven flaky).
|
|
131
|
+
queued_ms INTEGER,
|
|
132
|
+
output_bytes INTEGER,
|
|
133
|
+
blocks INTEGER,
|
|
134
|
+
flaky_passes INTEGER,
|
|
135
|
+
flaky_failures INTEGER
|
|
123
136
|
);
|
|
124
137
|
-- Two whole-table indexes only, and both APPEND: every row of a run carries the
|
|
125
138
|
-- same run_id and a started_at newer than everything before it, so
|
package/src/cli/cache.ts
CHANGED
|
@@ -11,8 +11,8 @@ import {
|
|
|
11
11
|
parseSize,
|
|
12
12
|
relPosix,
|
|
13
13
|
treeBytes,
|
|
14
|
-
ui,
|
|
15
14
|
} from '../util/index.js'
|
|
15
|
+
import * as ui from '../ui/index.js'
|
|
16
16
|
import { acquireRunLock } from '../orchestrator/index.js'
|
|
17
17
|
import { findWorkspaceRoot } from '../workspace/index.js'
|
|
18
18
|
import {
|
|
@@ -105,7 +105,7 @@ interface PruneArgs {
|
|
|
105
105
|
format?: 'pretty' | 'json'
|
|
106
106
|
/** `--cache-dir`: prune the cache a run with the same flag uses. */
|
|
107
107
|
cacheDir?: string
|
|
108
|
-
/** `--global`: clear every
|
|
108
|
+
/** `--global`: clear every workspace's store under `~/.vx`. */
|
|
109
109
|
global?: boolean
|
|
110
110
|
error?: string
|
|
111
111
|
}
|
|
@@ -181,7 +181,7 @@ export function parsePruneArgs(args: readonly string[]): PruneArgs {
|
|
|
181
181
|
}
|
|
182
182
|
if (out.global === true && (out.olderThanMs !== undefined || out.maxBytes !== undefined)) {
|
|
183
183
|
return {
|
|
184
|
-
error: `--global clears every
|
|
184
|
+
error: `--global clears every workspace's store and takes no --older-than or --max-size${seeHelp('cache')}`,
|
|
185
185
|
}
|
|
186
186
|
}
|
|
187
187
|
if (out.global === true && out.cacheDir !== undefined) {
|
|
@@ -270,9 +270,7 @@ function printPruned(
|
|
|
270
270
|
): void {
|
|
271
271
|
if (json) {
|
|
272
272
|
const { evicted, bytesFreed, orphans, orphanBytes } = result
|
|
273
|
-
|
|
274
|
-
`${JSON.stringify({ dry: dry, evicted, bytesFreed, orphans, orphanBytes })}\n`,
|
|
275
|
-
)
|
|
273
|
+
ui.data('out', `${JSON.stringify({ dry: dry, evicted, bytesFreed, orphans, orphanBytes })}\n`)
|
|
276
274
|
return
|
|
277
275
|
}
|
|
278
276
|
const plural = (n: number, one: string, many: string): string => `${n} ${n === 1 ? one : many}`
|
|
@@ -321,6 +319,25 @@ async function clear(
|
|
|
321
319
|
storeRoot: string | null,
|
|
322
320
|
parsed: PruneArgs,
|
|
323
321
|
): Promise<number> {
|
|
322
|
+
// A clear beside a run deletes what the run is restoring or saving; it
|
|
323
|
+
// waits for the run's lock, as a trim does, and only then reads what to
|
|
324
|
+
// delete: read first, what the run saved meanwhile went uncounted.
|
|
325
|
+
const release =
|
|
326
|
+
parsed.dry === true ? async () => {} : await acquireRunLock(root, { log: warnToStderr })
|
|
327
|
+
try {
|
|
328
|
+
return clearHeld(args, root, dir, storeRoot, parsed)
|
|
329
|
+
} finally {
|
|
330
|
+
await release()
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function clearHeld(
|
|
335
|
+
args: readonly string[],
|
|
336
|
+
root: string,
|
|
337
|
+
dir: string,
|
|
338
|
+
storeRoot: string | null,
|
|
339
|
+
parsed: PruneArgs,
|
|
340
|
+
): number {
|
|
324
341
|
// The store the index recorded and today's (a changed HOME, an origin
|
|
325
342
|
// added) both go; a directory with no index is not a vx cache to delete.
|
|
326
343
|
const targets: string[] = []
|
|
@@ -376,23 +393,16 @@ async function clear(
|
|
|
376
393
|
? readdirSync(t).map((e) => path.join(t, e))
|
|
377
394
|
: [t]
|
|
378
395
|
const bytes = doomed.reduce((n, t) => n + entries(t).reduce((m, e) => m + treeBytes(e), 0), 0)
|
|
379
|
-
if (!dry
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
try {
|
|
384
|
-
for (const t of doomed) {
|
|
385
|
-
for (const e of entries(t)) rmSync(e, { recursive: true, force: true })
|
|
386
|
-
if (t === named && readdirSync(t).length === 0) rmSync(t, { recursive: true })
|
|
387
|
-
}
|
|
388
|
-
} finally {
|
|
389
|
-
await release()
|
|
396
|
+
if (!dry) {
|
|
397
|
+
for (const t of doomed) {
|
|
398
|
+
for (const e of entries(t)) rmSync(e, { recursive: true, force: true })
|
|
399
|
+
if (t === named && readdirSync(t).length === 0) rmSync(t, { recursive: true })
|
|
390
400
|
}
|
|
391
401
|
}
|
|
392
402
|
const shown = doomed.map((t) => (path.relative(root, t).startsWith('..') ? t : relPosix(root, t)))
|
|
393
403
|
if (parsed.format === 'json') {
|
|
394
404
|
const result: ClearResult = { dry: dry, cleared: shown, bytesFreed: bytes }
|
|
395
|
-
|
|
405
|
+
ui.data('out', `${JSON.stringify(result)}\n`)
|
|
396
406
|
} else
|
|
397
407
|
ui.print(
|
|
398
408
|
'out',
|
package/src/cli/completions.ts
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
// Tab is the wrong price).
|
|
7
7
|
|
|
8
8
|
import { acceptedFlags, CORE_VERBS, flagHint, refuse, seeHelp } from './help.js'
|
|
9
|
-
import { nearest
|
|
9
|
+
import { nearest } from '../util/index.js'
|
|
10
|
+
import * as ui from '../ui/index.js'
|
|
11
|
+
import { sayNotice } from '../ui/index.js'
|
|
10
12
|
import { setting } from '../settings.js'
|
|
11
13
|
|
|
12
14
|
function enumValues(path: string): readonly string[] {
|
|
@@ -167,6 +169,6 @@ export async function completionsCmd(
|
|
|
167
169
|
const verbs = [...CORE_VERBS.filter((v) => v !== 'help' && v !== 'version'), 'help', 'version']
|
|
168
170
|
// No overlap to skip: the workspace load refuses a plugin verb named like a core one.
|
|
169
171
|
verbs.push(...pluginVerbs)
|
|
170
|
-
|
|
172
|
+
ui.data('out', completionScript(shell as CompletionShell, verbs))
|
|
171
173
|
return 0
|
|
172
174
|
}
|