@vzn/vx 0.0.649 → 0.0.651
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -20
- package/package.json +5 -5
- package/schemas/last.json +15 -0
- package/src/bin.ts +9 -31
- package/src/cache/cache.ts +225 -51
- package/src/cache/file-hashes.ts +3 -0
- package/src/cache/fold-memo.ts +8 -1
- package/src/cache/git-inputs.ts +188 -39
- package/src/cache/index.ts +12 -0
- package/src/cache/inputs.ts +3 -0
- package/src/cache/layer.ts +11 -0
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/linked-inputs.ts +3 -0
- package/src/cache/lockfile-memo.ts +120 -0
- package/src/cache/output-index.ts +5 -2
- package/src/cache/remote-hits.ts +9 -1
- package/src/cache/remote-lane.ts +2 -1
- package/src/cache/run-history.ts +11 -1
- package/src/cache/run-notes.ts +123 -0
- package/src/cache/schema.ts +54 -1
- package/src/cli/cache.ts +26 -16
- package/src/cli/completions.ts +4 -2
- package/src/cli/docs.ts +10 -11
- package/src/cli/help.ts +12 -23
- package/src/cli/index.ts +4 -9
- package/src/cli/info.ts +15 -7
- package/src/cli/init.ts +23 -15
- package/src/cli/last.ts +224 -223
- package/src/cli/lock.ts +55 -5
- package/src/cli/plan-format.ts +100 -118
- package/src/cli/plugin-commands.ts +10 -3
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +46 -27
- package/src/cli/select.ts +43 -5
- package/src/cli/show.ts +38 -42
- package/src/cli/task-verb.ts +22 -0
- package/src/cli/upgrade.ts +3 -1
- package/src/cli/watch-fs.ts +6 -1
- package/src/cli/watch-judge.ts +2 -1
- package/src/cli/watch.ts +83 -44
- package/src/cli/why.ts +126 -92
- package/src/cli/workspace-config.ts +2 -1
- package/src/exec/executor.ts +2 -1
- package/src/exec/kill-tree.ts +58 -5
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/pipes.ts +152 -0
- package/src/exec/runner.ts +199 -182
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-runtime.ts +11 -2
- package/src/exec/spawn-off.ts +154 -0
- package/src/exec/warm-shell.ts +146 -0
- package/src/graph/critical-path.ts +40 -28
- package/src/graph/priorities.ts +98 -44
- package/src/graph/scheduler.ts +12 -2
- package/src/graph/task-graph.ts +27 -1
- package/src/handoff.ts +10 -7
- package/src/history/history.ts +39 -2
- package/src/history/index.ts +3 -0
- package/src/history/metrics.ts +9 -1
- package/src/index.ts +12 -3
- package/src/orchestrator/doctor.ts +3 -3
- package/src/orchestrator/execute-task.ts +16 -2
- package/src/orchestrator/graph-memo.ts +50 -1
- package/src/orchestrator/graph-stage.ts +6 -0
- package/src/orchestrator/hit-restore.ts +3 -0
- package/src/orchestrator/index.ts +3 -1
- package/src/orchestrator/live-usage.ts +98 -0
- package/src/orchestrator/locked-digests.ts +63 -0
- package/src/orchestrator/lockfile-claim.ts +176 -102
- package/src/orchestrator/miss-save.ts +31 -3
- package/src/orchestrator/persistent-task.ts +3 -1
- package/src/orchestrator/plan.ts +5 -1
- package/src/orchestrator/plugin-host.ts +61 -0
- package/src/orchestrator/plugin.ts +49 -4
- package/src/orchestrator/prepare.ts +18 -2
- package/src/orchestrator/projects.ts +4 -1
- package/src/orchestrator/run-context.ts +29 -10
- package/src/orchestrator/run-failures.ts +35 -114
- package/src/orchestrator/run-records.ts +10 -0
- package/src/orchestrator/run.ts +89 -26
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/signals.ts +8 -5
- package/src/orchestrator/steps.ts +2 -2
- package/src/orchestrator/task-hash.ts +17 -3
- package/src/report/forecast.ts +12 -33
- package/src/report/framed-output.ts +388 -161
- package/src/report/index.ts +8 -2
- package/src/report/logger.ts +100 -36
- package/src/report/status-line.ts +81 -365
- package/src/report/summary.ts +104 -131
- package/src/ui/ansi.ts +48 -0
- package/src/ui/components.ts +506 -0
- package/src/ui/index.ts +79 -0
- package/src/ui/kit.ts +37 -0
- package/src/ui/live-region.ts +363 -0
- package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
- package/src/ui/terminal.ts +51 -0
- package/src/ui/theme.ts +224 -0
- package/src/ui/version-line.ts +28 -0
- package/src/util/config-frame.ts +65 -10
- package/src/util/edit-distance.ts +6 -1
- package/src/util/hidden-prompt.ts +8 -2
- package/src/util/index.ts +2 -15
- package/src/util/paths.ts +20 -2
- package/src/util/secret-mask.ts +39 -3
- package/src/util/secrets.ts +3 -0
- package/src/util/settle.ts +31 -0
- package/src/util/timing.ts +4 -16
- package/src/workspace/affected.ts +89 -30
- package/src/workspace/config-schema.ts +4 -2
- package/src/workspace/dependencies-lock.ts +191 -0
- package/src/workspace/filter.ts +28 -0
- package/src/workspace/index.ts +9 -0
- package/src/workspace/lockfile.ts +159 -6
- package/src/workspace/migration.ts +4 -3
- package/src/workspace/workspace.ts +3 -0
- package/src/orchestrator/run-steps.ts +0 -93
- package/src/util/colors.ts +0 -130
- package/src/util/report.ts +0 -44
- package/src/util/ui.ts +0 -258
- /package/src/{report/path-links.ts → ui/links.ts} +0 -0
- /package/src/{report → ui}/terminal-signals.ts +0 -0
package/src/cache/fold-memo.ts
CHANGED
|
@@ -13,7 +13,14 @@ export interface FoldMemoRow {
|
|
|
13
13
|
key: string
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
/** What a key fold asks of the memo; the rows stay core's (`IndexFoldMemo`). */
|
|
17
|
+
export interface FoldMemo {
|
|
18
|
+
/** The key remembered for `taskId` under `probe`, if that is the probe it was folded under. */
|
|
19
|
+
get(taskId: string, probe: string): string | undefined
|
|
20
|
+
set(taskId: string, probe: string, key: string): void
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class IndexFoldMemo implements FoldMemo {
|
|
17
24
|
private rows: Map<string, FoldMemoRow> | undefined
|
|
18
25
|
private readonly dirty = new Map<string, FoldMemoRow>()
|
|
19
26
|
|
package/src/cache/git-inputs.ts
CHANGED
|
@@ -8,11 +8,15 @@
|
|
|
8
8
|
import os from 'node:os'
|
|
9
9
|
import path from 'node:path'
|
|
10
10
|
import {
|
|
11
|
+
closeSync,
|
|
11
12
|
existsSync,
|
|
13
|
+
fstatSync,
|
|
12
14
|
lstatSync,
|
|
13
15
|
mkdtempSync,
|
|
16
|
+
openSync,
|
|
14
17
|
readdirSync,
|
|
15
18
|
readFileSync,
|
|
19
|
+
readSync,
|
|
16
20
|
realpathSync,
|
|
17
21
|
renameSync,
|
|
18
22
|
rmSync,
|
|
@@ -892,7 +896,12 @@ export function fileIdentity(mode: string, oid: string): string {
|
|
|
892
896
|
function parseLsFilesOutput(
|
|
893
897
|
records: readonly string[],
|
|
894
898
|
undecodableRecords: ReadonlySet<string>,
|
|
895
|
-
debug?: {
|
|
899
|
+
debug?: {
|
|
900
|
+
sizes: readonly number[]
|
|
901
|
+
ctimes: readonly number[]
|
|
902
|
+
mtimes: readonly number[]
|
|
903
|
+
indexed: Map<string, { oid: string; size: number; ctime: number; mtime: number }>
|
|
904
|
+
},
|
|
896
905
|
): GitLsResult {
|
|
897
906
|
const files: string[] = []
|
|
898
907
|
const oids = new Map<string, string>()
|
|
@@ -914,7 +923,12 @@ function parseLsFilesOutput(
|
|
|
914
923
|
if ((mode === '100644' || mode === '100755' || mode === '120000') && stage === '0') {
|
|
915
924
|
oids.set(filePath, fileIdentity(mode, m[3]!))
|
|
916
925
|
if (debug !== undefined && mode !== '120000') {
|
|
917
|
-
debug.indexed.set(filePath, {
|
|
926
|
+
debug.indexed.set(filePath, {
|
|
927
|
+
oid: m[3]!,
|
|
928
|
+
size: debug.sizes[i]!,
|
|
929
|
+
ctime: debug.ctimes[i]!,
|
|
930
|
+
mtime: debug.mtimes[i]!,
|
|
931
|
+
})
|
|
918
932
|
}
|
|
919
933
|
} else if (mode === '160000') {
|
|
920
934
|
gitlinks.add(filePath)
|
|
@@ -940,11 +954,15 @@ function parseLsFilesOutput(
|
|
|
940
954
|
function splitLsDebug(run: GitRun): {
|
|
941
955
|
records: string[]
|
|
942
956
|
sizes: number[]
|
|
957
|
+
ctimes: number[]
|
|
958
|
+
mtimes: number[]
|
|
943
959
|
undecodable: Set<string>
|
|
944
960
|
} {
|
|
945
961
|
const segs = run.stdout.split('\0')
|
|
946
962
|
const records: string[] = []
|
|
947
963
|
const sizes: number[] = []
|
|
964
|
+
const ctimes: number[] = []
|
|
965
|
+
const mtimes: number[] = []
|
|
948
966
|
const undecodable = new Set<string>()
|
|
949
967
|
let record = segs[0]!
|
|
950
968
|
let recordRaw = record
|
|
@@ -952,12 +970,22 @@ function splitLsDebug(run: GitRun): {
|
|
|
952
970
|
const seg = segs[i]
|
|
953
971
|
let rest = ''
|
|
954
972
|
let size = -1
|
|
973
|
+
let ctime = -1
|
|
974
|
+
let mtime = -1
|
|
955
975
|
if (seg !== undefined) {
|
|
956
976
|
let at = 0
|
|
957
977
|
for (let k = 0; k < 5 && at >= 0; k++) {
|
|
958
978
|
const nl = seg.indexOf('\n', at)
|
|
959
979
|
if (nl < 0) at = -1
|
|
960
980
|
else {
|
|
981
|
+
// ` ctime: <sec>:<nsec>`, then ` mtime: <sec>:<nsec>`.
|
|
982
|
+
if (k < 2) {
|
|
983
|
+
const n = parseInt(seg.slice(seg.indexOf(':', at) + 1, nl), 10)
|
|
984
|
+
if (!Number.isNaN(n)) {
|
|
985
|
+
if (k === 0) ctime = n
|
|
986
|
+
else mtime = n
|
|
987
|
+
}
|
|
988
|
+
}
|
|
961
989
|
if (k === 4) {
|
|
962
990
|
const field = seg.indexOf('size: ', at)
|
|
963
991
|
if (field >= 0 && field < nl) {
|
|
@@ -973,12 +1001,14 @@ function splitLsDebug(run: GitRun): {
|
|
|
973
1001
|
if (record.length > 0) {
|
|
974
1002
|
records.push(record)
|
|
975
1003
|
sizes.push(size)
|
|
1004
|
+
ctimes.push(ctime)
|
|
1005
|
+
mtimes.push(mtime)
|
|
976
1006
|
if (run.undecodable.has(recordRaw)) undecodable.add(record)
|
|
977
1007
|
}
|
|
978
1008
|
record = rest
|
|
979
1009
|
recordRaw = seg ?? ''
|
|
980
1010
|
}
|
|
981
|
-
return { records, sizes, undecodable }
|
|
1011
|
+
return { records, sizes, ctimes, mtimes, undecodable }
|
|
982
1012
|
}
|
|
983
1013
|
|
|
984
1014
|
/**
|
|
@@ -998,6 +1028,8 @@ export interface IndexBlobs {
|
|
|
998
1028
|
paths: string[]
|
|
999
1029
|
oids: string[]
|
|
1000
1030
|
sizes: number[]
|
|
1031
|
+
/** Entries git cannot see a same-second rewrite of (`sameSecond`). */
|
|
1032
|
+
sameSecond: string[]
|
|
1001
1033
|
}
|
|
1002
1034
|
|
|
1003
1035
|
/** What the blob-size check reads, lazily: the verdict's key, and the listing a miss needs. */
|
|
@@ -1010,6 +1042,31 @@ export interface IndexBlobCheck {
|
|
|
1010
1042
|
rekey(): Promise<string | undefined>
|
|
1011
1043
|
}
|
|
1012
1044
|
|
|
1045
|
+
/** At most `n` contiguous parts of `items`, none under `CAT_FILE_MIN`: a spawn costs more than a few lookups. */
|
|
1046
|
+
function chunks<T>(items: readonly T[], n: number): T[][] {
|
|
1047
|
+
const parts = Math.min(n, Math.ceil(items.length / CAT_FILE_MIN))
|
|
1048
|
+
const size = Math.ceil(items.length / Math.max(1, parts))
|
|
1049
|
+
const out: T[][] = []
|
|
1050
|
+
for (let at = 0; at < items.length; at += size) out.push(items.slice(at, at + size))
|
|
1051
|
+
return out
|
|
1052
|
+
}
|
|
1053
|
+
const CAT_FILE_MIN = 256
|
|
1054
|
+
|
|
1055
|
+
const sizeChecks = new WeakMap<GitEnumeration, Promise<void>>()
|
|
1056
|
+
/**
|
|
1057
|
+
* `dropResizedOids` once per enumeration: a run starts it as soon as its
|
|
1058
|
+
* cache opens, so the check runs beside the config load and graph build
|
|
1059
|
+
* instead of after them, and `applyGitEnumeration` waits on the same one.
|
|
1060
|
+
*/
|
|
1061
|
+
export function checkBlobSizes(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
|
|
1062
|
+
let check = sizeChecks.get(enumeration)
|
|
1063
|
+
if (check === undefined) {
|
|
1064
|
+
check = dropResizedOids(enumeration, memo)
|
|
1065
|
+
sizeChecks.set(enumeration, check)
|
|
1066
|
+
}
|
|
1067
|
+
return check
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1013
1070
|
/**
|
|
1014
1071
|
* Drop each trusted OID whose blob is not the size the index recorded for
|
|
1015
1072
|
* the worktree file (A-60). A clean filter's blob differs from the file it
|
|
@@ -1026,6 +1083,8 @@ export interface IndexBlobCheck {
|
|
|
1026
1083
|
* (racy) entry records 0 and is hashed from disk. An answer that cannot be
|
|
1027
1084
|
* read trusts nothing. A filter that keeps the size (`tr a-z A-Z`) is not
|
|
1028
1085
|
* caught: only a read of every trusted file would be (caching.md, step 12).
|
|
1086
|
+
* The verdict also holds each entry git cannot see a same-second rewrite
|
|
1087
|
+
* of (`sameSecond`), read from the same listing.
|
|
1029
1088
|
*/
|
|
1030
1089
|
async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
|
|
1031
1090
|
const { trusted, blobCheck } = enumeration
|
|
@@ -1033,31 +1092,47 @@ async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo)
|
|
|
1033
1092
|
const key = blobCheck.key
|
|
1034
1093
|
let resized = key === undefined ? undefined : memo?.blobVerdict(key)
|
|
1035
1094
|
if (resized === undefined) {
|
|
1036
|
-
|
|
1095
|
+
// The sizes are asked beside the listing: both read the index the
|
|
1096
|
+
// enumeration listed (the listing refuses one written since), so the
|
|
1097
|
+
// OIDs are the listing's. One after the other, a cold 9,603-task run
|
|
1098
|
+
// waited on the listing's 70-95 ms before the first spawn (2026-11-01).
|
|
1099
|
+
const listing = blobCheck.blobs()
|
|
1100
|
+
// Only what is still trusted: a path dropped already needs no size.
|
|
1101
|
+
// Built here, not with the listing: a run whose verdict is memoized
|
|
1102
|
+
// never needs it, and on 9,603 tasks it cost 17-35 ms (2026-11-01).
|
|
1103
|
+
// A non-regular mode rides its value (`100755:<oid>`); cat-file takes the OID.
|
|
1104
|
+
const oids = [...new Set(Array.from(trusted.values(), (v) => v.slice(v.lastIndexOf(':') + 1)))]
|
|
1105
|
+
const sizes = memo?.knownBlobSizes(oids) ?? new Map<string, number>()
|
|
1106
|
+
const unknown = oids.filter((oid) => !sizes.has(oid))
|
|
1107
|
+
// One cat-file per core: a fresh clone's objects are loose, and one
|
|
1108
|
+
// process opened and inflated 2,505 of them in 30 ms before the first
|
|
1109
|
+
// task of a cold 100-package run; four took 14 (2026-11-03).
|
|
1110
|
+
const asked = Promise.all(
|
|
1111
|
+
chunks(unknown, Math.max(1, navigator.hardwareConcurrency)).map((part) =>
|
|
1112
|
+
enumeration.catFile(part.join('\n') + '\n'),
|
|
1113
|
+
),
|
|
1114
|
+
)
|
|
1115
|
+
const [blobs, runs] = await Promise.all([listing, asked])
|
|
1037
1116
|
if (blobs === null) {
|
|
1038
1117
|
trusted.clear()
|
|
1039
1118
|
return
|
|
1040
1119
|
}
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
return
|
|
1049
|
-
}
|
|
1050
|
-
const learned = new Map<string, number>()
|
|
1051
|
-
for (const line of run.stdout.split('\n')) {
|
|
1120
|
+
if (runs.some((run) => run === null || run.exitCode !== 0)) {
|
|
1121
|
+
trusted.clear()
|
|
1122
|
+
return
|
|
1123
|
+
}
|
|
1124
|
+
const learned = new Map<string, number>()
|
|
1125
|
+
for (const run of runs) {
|
|
1126
|
+
for (const line of run!.stdout.split('\n')) {
|
|
1052
1127
|
const sp = line.indexOf(' ')
|
|
1053
1128
|
const n = Number(line.slice(sp + 1))
|
|
1054
1129
|
if (sp > 0 && line.slice(sp + 1) !== '' && Number.isInteger(n))
|
|
1055
1130
|
learned.set(line.slice(0, sp), n)
|
|
1056
1131
|
}
|
|
1057
|
-
for (const [oid, n] of learned) sizes.set(oid, n)
|
|
1058
|
-
memo?.rememberBlobSizes(learned)
|
|
1059
1132
|
}
|
|
1060
|
-
|
|
1133
|
+
for (const [oid, n] of learned) sizes.set(oid, n)
|
|
1134
|
+
if (learned.size > 0) memo?.rememberBlobSizes(learned)
|
|
1135
|
+
resized = [...blobs.sameSecond]
|
|
1061
1136
|
for (let i = 0; i < blobs.oids.length; i++) {
|
|
1062
1137
|
const size = sizes.get(blobs.oids[i]!)
|
|
1063
1138
|
// The index keeps the low 32 bits of a size.
|
|
@@ -1626,6 +1701,19 @@ function hasRacyEntry(index: Uint8Array, mtimeSec: number, oidBytes: number): bo
|
|
|
1626
1701
|
return false
|
|
1627
1702
|
}
|
|
1628
1703
|
|
|
1704
|
+
/**
|
|
1705
|
+
* git compares a file's ctime to the second only, and its racy rule reads
|
|
1706
|
+
* mtime alone: a same-size rewrite that keeps (or restores) an older mtime,
|
|
1707
|
+
* made in the second the index recorded the file, keeps every stat git
|
|
1708
|
+
* compares, and `status` calls it clean. An entry whose recorded ctime, not
|
|
1709
|
+
* its mtime, reaches the second its index was written can hide one; git
|
|
1710
|
+
* re-hashes an entry racy by mtime. Its OID is not trusted. The stamps are
|
|
1711
|
+
* git's own (`ls-files --debug`), judged once per index (the verdict).
|
|
1712
|
+
*/
|
|
1713
|
+
function sameSecond(ctimeSec: number, mtimeSec: number, writtenSec: number): boolean {
|
|
1714
|
+
return ctimeSec < 0 || (ctimeSec >= writtenSec && mtimeSec < writtenSec)
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1629
1717
|
/** A private copy a run older than this may still have handed to `status`. */
|
|
1630
1718
|
const STALE_INDEX_MS = 60_000
|
|
1631
1719
|
|
|
@@ -1711,7 +1799,7 @@ export async function startGitEnumeration(
|
|
|
1711
1799
|
stdin?: string,
|
|
1712
1800
|
env?: Record<string, string>,
|
|
1713
1801
|
): Promise<GitRun | null> => {
|
|
1714
|
-
const spawn = <Out extends 'pipe' |
|
|
1802
|
+
const spawn = <Out extends 'pipe' | number>(stdout: Out) =>
|
|
1715
1803
|
Bun.spawn({
|
|
1716
1804
|
// `status` refreshes the index when it can take `index.lock`, so
|
|
1717
1805
|
// the user's own `git add` or `commit` failed on the lock while a
|
|
@@ -1728,23 +1816,30 @@ export async function startGitEnumeration(
|
|
|
1728
1816
|
// loading configs while git lists, and a pipe held a 10 MB listing
|
|
1729
1817
|
// until the thread came back to drain it 64 KiB at a time (~90 ms of
|
|
1730
1818
|
// a warm 5,000-project run). In a directory of its own, so no link
|
|
1731
|
-
// another user planted in a shared temp directory takes the bytes
|
|
1732
|
-
//
|
|
1733
|
-
|
|
1819
|
+
// another user planted in a shared temp directory takes the bytes, and
|
|
1820
|
+
// the directory goes the moment the file is open: a run stopped
|
|
1821
|
+
// mid-listing (a Ctrl-C before the run's signal handlers, or a kill)
|
|
1822
|
+
// runs no `finally`, and each left a `vx-git-*` behind. A temp
|
|
1823
|
+
// directory that takes none falls back to the pipe.
|
|
1824
|
+
let fd: number | undefined
|
|
1734
1825
|
try {
|
|
1735
|
-
dir = mkdtempSync(path.join(os.tmpdir(), 'vx-git-'))
|
|
1826
|
+
const dir = mkdtempSync(path.join(os.tmpdir(), 'vx-git-'))
|
|
1827
|
+
try {
|
|
1828
|
+
fd = openSync(path.join(dir, 'out'), 'w+', 0o600)
|
|
1829
|
+
} finally {
|
|
1830
|
+
rmSync(dir, { recursive: true, force: true })
|
|
1831
|
+
}
|
|
1736
1832
|
} catch {
|
|
1737
|
-
|
|
1833
|
+
fd = undefined
|
|
1738
1834
|
}
|
|
1739
1835
|
try {
|
|
1740
1836
|
let stdout: Uint8Array
|
|
1741
1837
|
let stderr: string
|
|
1742
1838
|
let exitCode: number
|
|
1743
|
-
if (
|
|
1744
|
-
const
|
|
1745
|
-
const proc = spawn(Bun.file(out))
|
|
1839
|
+
if (fd !== undefined) {
|
|
1840
|
+
const proc = spawn(fd)
|
|
1746
1841
|
;[stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited])
|
|
1747
|
-
stdout =
|
|
1842
|
+
stdout = readWhole(fd)
|
|
1748
1843
|
} else {
|
|
1749
1844
|
const piped = spawn('pipe')
|
|
1750
1845
|
;[stdout, stderr, exitCode] = await Promise.all([
|
|
@@ -1758,7 +1853,7 @@ export async function startGitEnumeration(
|
|
|
1758
1853
|
} catch {
|
|
1759
1854
|
return null
|
|
1760
1855
|
} finally {
|
|
1761
|
-
if (
|
|
1856
|
+
if (fd !== undefined) closeSync(fd)
|
|
1762
1857
|
}
|
|
1763
1858
|
}
|
|
1764
1859
|
const startedAtMs = Date.now()
|
|
@@ -1834,7 +1929,9 @@ export async function startGitEnumeration(
|
|
|
1834
1929
|
bytes,
|
|
1835
1930
|
hash,
|
|
1836
1931
|
st,
|
|
1837
|
-
|
|
1932
|
+
// The mtime too: which entries git can see a same-second rewrite
|
|
1933
|
+
// of (`sameSecond`) turns on the second the index was written.
|
|
1934
|
+
key: `${hash}:${pathspecKey}:${Math.floor(st.mtimeMs / 1000)}`,
|
|
1838
1935
|
stamp: `${st.ino}:${st.ctimeMs}:${st.size}`,
|
|
1839
1936
|
}
|
|
1840
1937
|
} catch {
|
|
@@ -1863,10 +1960,16 @@ export async function startGitEnumeration(
|
|
|
1863
1960
|
index === undefined ? undefined : { GIT_INDEX_FILE: index },
|
|
1864
1961
|
)
|
|
1865
1962
|
// An index with no racy entry is read as it is: `status` spawns now.
|
|
1963
|
+
// The copy `status` read, when it read one: its stats are the ones the
|
|
1964
|
+
// dirty set came from, so the blob check lists it too.
|
|
1965
|
+
let statusCopy: string | undefined
|
|
1866
1966
|
const statusRun =
|
|
1867
1967
|
read === undefined || indexFile === undefined || !racyIndex(read, facts!.objectFormat)
|
|
1868
1968
|
? spawnStatus(undefined)
|
|
1869
|
-
: refreshedIndex(indexFile, read, spawnGit).then(
|
|
1969
|
+
: refreshedIndex(indexFile, read, spawnGit).then((copy) => {
|
|
1970
|
+
statusCopy = copy
|
|
1971
|
+
return spawnStatus(copy)
|
|
1972
|
+
})
|
|
1870
1973
|
const running = Promise.all([listing, statusRun, varsRun])
|
|
1871
1974
|
const gitPrefix = facts?.prefix ?? ''
|
|
1872
1975
|
const gitDir = facts?.commonDir ?? ''
|
|
@@ -1888,6 +1991,17 @@ export async function startGitEnumeration(
|
|
|
1888
1991
|
// the stamp moves with any write) and is still in place after it: a
|
|
1889
1992
|
// `git add` between the listing and the read paired one index's verdict
|
|
1890
1993
|
// with the other's entries. Otherwise this run asks for no verdict.
|
|
1994
|
+
// A verdict read from the copy holds for that copy file only: a copy
|
|
1995
|
+
// written again (a new inode) asks again.
|
|
1996
|
+
const copyKey = (key: string): string | undefined => {
|
|
1997
|
+
if (statusCopy === undefined) return key
|
|
1998
|
+
try {
|
|
1999
|
+
const st = lstatSync(statusCopy)
|
|
2000
|
+
return `${key}:${st.ino}:${st.mtimeMs}`
|
|
2001
|
+
} catch {
|
|
2002
|
+
return undefined
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
1891
2005
|
let indexKey: string | undefined
|
|
1892
2006
|
if (
|
|
1893
2007
|
read !== undefined &&
|
|
@@ -1895,7 +2009,7 @@ export async function startGitEnumeration(
|
|
|
1895
2009
|
) {
|
|
1896
2010
|
try {
|
|
1897
2011
|
const st = lstatSync(indexFile!)
|
|
1898
|
-
if (`${st.ino}:${st.ctimeMs}:${st.size}` === read.stamp) indexKey = read.key
|
|
2012
|
+
if (`${st.ino}:${st.ctimeMs}:${st.size}` === read.stamp) indexKey = copyKey(read.key)
|
|
1899
2013
|
} catch {
|
|
1900
2014
|
// gone: no key
|
|
1901
2015
|
}
|
|
@@ -1997,24 +2111,47 @@ export async function startGitEnumeration(
|
|
|
1997
2111
|
blobCheck: {
|
|
1998
2112
|
key: indexKey,
|
|
1999
2113
|
async blobs() {
|
|
2000
|
-
const
|
|
2114
|
+
const listed = statusCopy ?? indexFile
|
|
2115
|
+
let writtenSec = 0
|
|
2116
|
+
try {
|
|
2117
|
+
if (listed !== undefined) writtenSec = Math.floor(lstatSync(listed).mtimeMs / 1000)
|
|
2118
|
+
} catch {
|
|
2119
|
+
return null
|
|
2120
|
+
}
|
|
2121
|
+
const run = await spawnGit(
|
|
2122
|
+
['ls-files', '-s', '-v', '-z', '--debug', '--', ...pathspecs],
|
|
2123
|
+
undefined,
|
|
2124
|
+
statusCopy === undefined ? undefined : { GIT_INDEX_FILE: statusCopy },
|
|
2125
|
+
)
|
|
2001
2126
|
if (run === null || run.exitCode !== 0) return null
|
|
2002
2127
|
// Sizes from an index written since the listing judge entries the
|
|
2003
2128
|
// trusted OIDs did not come from: an add there rewrote a resized
|
|
2004
2129
|
// entry to one that fits, and the listed OID stood (A-60).
|
|
2005
2130
|
if (indexStamp() !== stampBefore) return null
|
|
2006
2131
|
const debug = splitLsDebug(run)
|
|
2007
|
-
const indexed = new Map<
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2132
|
+
const indexed = new Map<
|
|
2133
|
+
string,
|
|
2134
|
+
{ oid: string; size: number; ctime: number; mtime: number }
|
|
2135
|
+
>()
|
|
2136
|
+
parseLsFilesOutput(debug.records, debug.undecodable, {
|
|
2137
|
+
sizes: debug.sizes,
|
|
2138
|
+
ctimes: debug.ctimes,
|
|
2139
|
+
mtimes: debug.mtimes,
|
|
2140
|
+
indexed,
|
|
2141
|
+
})
|
|
2142
|
+
const blobs: IndexBlobs = { paths: [], oids: [], sizes: [], sameSecond: [] }
|
|
2143
|
+
for (const [rel, { oid, size, ctime, mtime }] of indexed) {
|
|
2011
2144
|
blobs.paths.push(rel)
|
|
2012
2145
|
blobs.oids.push(oid)
|
|
2013
2146
|
blobs.sizes.push(size)
|
|
2147
|
+
if (sameSecond(ctime, mtime, writtenSec)) blobs.sameSecond.push(rel)
|
|
2014
2148
|
}
|
|
2015
2149
|
return blobs
|
|
2016
2150
|
},
|
|
2017
|
-
rekey: async () =>
|
|
2151
|
+
rekey: async () => {
|
|
2152
|
+
const key = (await readIndex())?.key
|
|
2153
|
+
return key === undefined ? undefined : copyKey(key)
|
|
2154
|
+
},
|
|
2018
2155
|
},
|
|
2019
2156
|
// A missing blob is an answer ("missing"), never a fetch from a partial
|
|
2020
2157
|
// clone's promisor remote.
|
|
@@ -2083,7 +2220,7 @@ export async function applyGitEnumeration(
|
|
|
2083
2220
|
workspaceWide = false,
|
|
2084
2221
|
memo?: BlobSizeMemo,
|
|
2085
2222
|
): Promise<void> {
|
|
2086
|
-
await
|
|
2223
|
+
await checkBlobSizes(enumeration, memo)
|
|
2087
2224
|
const { all, trusted } = enumeration
|
|
2088
2225
|
cache.setWorktreeDirty(enumeration.dirty)
|
|
2089
2226
|
cache.setGitVars(enumeration.vars)
|
|
@@ -2218,3 +2355,15 @@ export async function applyGitEnumeration(
|
|
|
2218
2355
|
cache.setWorkspaceRoot(workspaceRoot)
|
|
2219
2356
|
}
|
|
2220
2357
|
}
|
|
2358
|
+
|
|
2359
|
+
/** The whole of a file held only by `fd`, from its start. */
|
|
2360
|
+
function readWhole(fd: number): Uint8Array {
|
|
2361
|
+
const bytes = new Uint8Array(fstatSync(fd).size)
|
|
2362
|
+
let at = 0
|
|
2363
|
+
while (at < bytes.length) {
|
|
2364
|
+
const n = readSync(fd, bytes, at, bytes.length - at, at)
|
|
2365
|
+
if (n === 0) break
|
|
2366
|
+
at += n
|
|
2367
|
+
}
|
|
2368
|
+
return at === bytes.length ? bytes : bytes.subarray(0, at)
|
|
2369
|
+
}
|
package/src/cache/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ export {
|
|
|
18
18
|
type CachePolicy,
|
|
19
19
|
cachesNothing,
|
|
20
20
|
EXECUTED_RUNS_SQL,
|
|
21
|
+
indexDb,
|
|
21
22
|
formatCachePolicy,
|
|
22
23
|
FULL_CACHE_POLICY,
|
|
23
24
|
type InvocationRecord,
|
|
@@ -80,6 +81,7 @@ export {
|
|
|
80
81
|
MAX_SCOPED_PATHSPECS,
|
|
81
82
|
repoRootOf,
|
|
82
83
|
startGitEnumeration,
|
|
84
|
+
checkBlobSizes,
|
|
83
85
|
lazyGitEnumeration,
|
|
84
86
|
type LazyGitEnumeration,
|
|
85
87
|
type GitEnumeration,
|
|
@@ -102,3 +104,13 @@ export {
|
|
|
102
104
|
} from './plugin-cache.js'
|
|
103
105
|
export { packRunOutputs, putRunEntry, runEntryName, writesRunStore } from './run-store.js'
|
|
104
106
|
export { branchSource } from './run-history.js'
|
|
107
|
+
export {
|
|
108
|
+
type KeptFailure,
|
|
109
|
+
keepRunNotes,
|
|
110
|
+
keptFailures,
|
|
111
|
+
keptSteps,
|
|
112
|
+
keptWarnings,
|
|
113
|
+
type RunNotes,
|
|
114
|
+
type RunStep,
|
|
115
|
+
} from './run-notes.js'
|
|
116
|
+
export { readLockfileMemo, writeLockfileMemo, type LockfileMemo } from './lockfile-memo.js'
|
package/src/cache/inputs.ts
CHANGED
|
@@ -101,6 +101,9 @@ const ALWAYS_IGNORE = [
|
|
|
101
101
|
// must not select every `**` reader under --affected.
|
|
102
102
|
'.vx/.gitattributes',
|
|
103
103
|
'.vx/.gitignore',
|
|
104
|
+
// `vx lock`'s record of the dependency edges and digests: the key folds
|
|
105
|
+
// the digests through `fingerprint` already.
|
|
106
|
+
'.vx/dependencies.lock.json',
|
|
104
107
|
// Where restores stage (archive.ts): vx's own, never an input.
|
|
105
108
|
'.vx/tmp/**',
|
|
106
109
|
]
|
package/src/cache/layer.ts
CHANGED
|
@@ -232,6 +232,15 @@ export interface RunRecord {
|
|
|
232
232
|
placementMemoryBytes?: number
|
|
233
233
|
/** The share of a core each runnable thread got there (`ExecuteResult.machine.cpuShare`, v49). */
|
|
234
234
|
cpuShare?: number
|
|
235
|
+
/** How long it waited ready for a slot (`TaskOutcome.queuedMs`, v51). */
|
|
236
|
+
queuedMs?: number
|
|
237
|
+
/** Its output files' bytes (`TaskOutcome.outputBytes`, v51). */
|
|
238
|
+
outputBytes?: number
|
|
239
|
+
/** The tasks and groups downstream of it in the run, the scheduler's count (v51). */
|
|
240
|
+
blocks?: number
|
|
241
|
+
/** The flaky record its row's note quoted (`TaskOutcome.flaky`, v51). */
|
|
242
|
+
flakyPasses?: number
|
|
243
|
+
flakyFailures?: number
|
|
235
244
|
}
|
|
236
245
|
|
|
237
246
|
/**
|
|
@@ -823,6 +832,8 @@ export interface CacheLayer {
|
|
|
823
832
|
outputsPath(hash: string): string
|
|
824
833
|
/** The stored artifact's compressed size, wherever it lives; undefined when none is stored. */
|
|
825
834
|
artifactSize?(hash: string): number | undefined
|
|
835
|
+
/** The output files' bytes, uncompressed, of the entry this process just saved; asked once. */
|
|
836
|
+
outputBytes?(hash: string): number | undefined
|
|
826
837
|
prune(options: PruneOptions): Promise<PruneResult>
|
|
827
838
|
close(): void
|
|
828
839
|
}
|
|
@@ -575,6 +575,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
575
575
|
return this.local.artifactSize(hash)
|
|
576
576
|
}
|
|
577
577
|
|
|
578
|
+
outputBytes(hash: string): number | undefined {
|
|
579
|
+
return this.local.outputBytes(hash)
|
|
580
|
+
}
|
|
581
|
+
|
|
578
582
|
hashFile(filePath: string): Promise<string> {
|
|
579
583
|
return this.local.hashFile(filePath)
|
|
580
584
|
}
|
|
@@ -71,6 +71,9 @@ export async function behindLink(
|
|
|
71
71
|
return NOTHING_BEHIND
|
|
72
72
|
}
|
|
73
73
|
if (seen.has(real)) return NOTHING_BEHIND
|
|
74
|
+
// A repository's own store, as a `.git/**` input names nothing: walked,
|
|
75
|
+
// every commit and index write re-keyed the task. Its target string keys.
|
|
76
|
+
if (real.split(path.sep).includes('.git')) return NOTHING_BEHIND
|
|
74
77
|
if (st.isFile()) return `file\0${await hashFile(real).catch(absentOr)}`
|
|
75
78
|
if (!st.isDirectory()) return 'special'
|
|
76
79
|
const inner = new Set(seen).add(real)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// A lockfile claimant's parse, kept in cache.db's `lockfile_memos` (v50; a
|
|
2
|
+
// `lockfile-claims/` file before): what orchestrator/lockfile-claim.ts reads
|
|
3
|
+
// so a warm run never parses the lockfile. Its readers run where no `Cache`
|
|
4
|
+
// is open yet — a `discover` hook, `--affected`'s selection, a verb that
|
|
5
|
+
// opens none — so each call opens the index on its own and closes it. Only
|
|
6
|
+
// an index that exists and holds this vx's schema is read or written: one
|
|
7
|
+
// another vx wrote is reset by the next run, and a directory with none is
|
|
8
|
+
// left for the run that makes it. A failure of any kind is a miss: the memo
|
|
9
|
+
// is a speed-up, never a requirement.
|
|
10
|
+
|
|
11
|
+
import { Database } from 'bun:sqlite'
|
|
12
|
+
import { existsSync } from 'node:fs'
|
|
13
|
+
import path from 'node:path'
|
|
14
|
+
import { closeDb, SCHEMA_VERSION, SQLITE_FCNTL_PERSIST_WAL } from './cache.js'
|
|
15
|
+
|
|
16
|
+
/** One claimant's parse of one lockfile's content. */
|
|
17
|
+
export interface LockfileMemo {
|
|
18
|
+
/** The claimant's own counter: bumped when it digests differently. */
|
|
19
|
+
version: number
|
|
20
|
+
/** xxh3 of the lockfile's bytes. */
|
|
21
|
+
lock: string
|
|
22
|
+
/** Each extra file the digests read → its content hash then. */
|
|
23
|
+
extras: Record<string, string>
|
|
24
|
+
/** Each importer's digest; null when the parse that wrote it left them out (a run's). */
|
|
25
|
+
importers: Record<string, string> | null
|
|
26
|
+
edges: Record<string, unknown>
|
|
27
|
+
/** Each importer's recorded dependency names; absent when the claimant reads none. */
|
|
28
|
+
recorded?: Record<string, readonly string[]>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function withIndex<T>(cacheDir: string, use: (db: Database) => T): T | undefined {
|
|
32
|
+
const file = path.join(cacheDir, 'cache.db')
|
|
33
|
+
if (!existsSync(file)) return undefined
|
|
34
|
+
let db: Database
|
|
35
|
+
try {
|
|
36
|
+
db = new Database(file, { readwrite: true })
|
|
37
|
+
} catch {
|
|
38
|
+
return undefined
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
// As `Cache` opens it: wait for another run's write, and keep `-wal`
|
|
42
|
+
// and `-shm` when this is the last connection to close (O-10).
|
|
43
|
+
db.exec('PRAGMA busy_timeout = 5000')
|
|
44
|
+
db.fileControl(SQLITE_FCNTL_PERSIST_WAL, 1)
|
|
45
|
+
db.exec('PRAGMA journal_size_limit = 67108864')
|
|
46
|
+
return use(db)
|
|
47
|
+
} catch {
|
|
48
|
+
return undefined
|
|
49
|
+
} finally {
|
|
50
|
+
closeDb(db)
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const CURRENT = "(SELECT value FROM schema_meta WHERE key = 'version') = ?"
|
|
55
|
+
|
|
56
|
+
/** The memo `claimant` kept of `file`; undefined when none or another content's. */
|
|
57
|
+
export function readLockfileMemo(
|
|
58
|
+
cacheDir: string,
|
|
59
|
+
file: string,
|
|
60
|
+
claimant: string,
|
|
61
|
+
): LockfileMemo | undefined {
|
|
62
|
+
return withIndex(cacheDir, (db) => {
|
|
63
|
+
const row = db
|
|
64
|
+
.query(
|
|
65
|
+
`SELECT version, lock, extras, importers, edges, recorded FROM lockfile_memos
|
|
66
|
+
WHERE file = ? AND claimant = ? AND ${CURRENT}`,
|
|
67
|
+
)
|
|
68
|
+
.get(file, claimant, SCHEMA_VERSION) as {
|
|
69
|
+
version: number
|
|
70
|
+
lock: string
|
|
71
|
+
extras: string
|
|
72
|
+
importers: string
|
|
73
|
+
edges: string
|
|
74
|
+
recorded: string | null
|
|
75
|
+
} | null
|
|
76
|
+
if (row === null) return undefined
|
|
77
|
+
return {
|
|
78
|
+
version: row.version,
|
|
79
|
+
lock: row.lock,
|
|
80
|
+
extras: JSON.parse(row.extras) as Record<string, string>,
|
|
81
|
+
importers: JSON.parse(row.importers) as Record<string, string> | null,
|
|
82
|
+
edges: JSON.parse(row.edges) as Record<string, unknown>,
|
|
83
|
+
...(row.recorded !== null
|
|
84
|
+
? { recorded: JSON.parse(row.recorded) as Record<string, readonly string[]> }
|
|
85
|
+
: {}),
|
|
86
|
+
}
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Keep `memo` as `claimant`'s of `file`, replacing the last; false when it was not kept. */
|
|
91
|
+
export function writeLockfileMemo(
|
|
92
|
+
cacheDir: string,
|
|
93
|
+
file: string,
|
|
94
|
+
claimant: string,
|
|
95
|
+
memo: LockfileMemo,
|
|
96
|
+
): boolean {
|
|
97
|
+
return (
|
|
98
|
+
withIndex(
|
|
99
|
+
cacheDir,
|
|
100
|
+
(db) =>
|
|
101
|
+
db
|
|
102
|
+
.query(
|
|
103
|
+
`INSERT OR REPLACE INTO lockfile_memos
|
|
104
|
+
(file, claimant, version, lock, extras, importers, edges, recorded)
|
|
105
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ? WHERE ${CURRENT}`,
|
|
106
|
+
)
|
|
107
|
+
.run(
|
|
108
|
+
file,
|
|
109
|
+
claimant,
|
|
110
|
+
memo.version,
|
|
111
|
+
memo.lock,
|
|
112
|
+
JSON.stringify(memo.extras),
|
|
113
|
+
JSON.stringify(memo.importers),
|
|
114
|
+
JSON.stringify(memo.edges),
|
|
115
|
+
memo.recorded === undefined ? null : JSON.stringify(memo.recorded),
|
|
116
|
+
SCHEMA_VERSION,
|
|
117
|
+
).changes > 0,
|
|
118
|
+
) ?? false
|
|
119
|
+
)
|
|
120
|
+
}
|
|
@@ -143,17 +143,20 @@ export class OutputIndex {
|
|
|
143
143
|
/**
|
|
144
144
|
* Replace an entry's file rows. Called INSIDE the store's save transaction,
|
|
145
145
|
* so an entry and its fingerprints land together (an UPDATE on the same
|
|
146
|
-
* hash refreshes rather than appends).
|
|
146
|
+
* hash refreshes rather than appends). Answers the files' bytes.
|
|
147
147
|
*/
|
|
148
|
-
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>):
|
|
148
|
+
replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): number {
|
|
149
149
|
// A stamp taken for the rows this replaces describes their files, not these.
|
|
150
150
|
this.held.delete(hash)
|
|
151
151
|
this.pendingStamps.delete(hash)
|
|
152
152
|
this.deleteOutputFiles.run(hash)
|
|
153
153
|
this.deleteStamps.run(hash)
|
|
154
|
+
let bytes = 0
|
|
154
155
|
for (const [rel, size, mode, mtime] of rows) {
|
|
155
156
|
this.insertOutputFile.run(hash, rel, size, mode, mtime)
|
|
157
|
+
bytes += size
|
|
156
158
|
}
|
|
159
|
+
return bytes
|
|
157
160
|
}
|
|
158
161
|
|
|
159
162
|
/** Hold a probe's rows for its restores (`held`), in place of the last probe's. */
|