@vzn/vx 0.0.650 → 0.0.651

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +5 -20
  2. package/package.json +5 -5
  3. package/schemas/last.json +15 -0
  4. package/src/bin.ts +9 -31
  5. package/src/cache/cache.ts +202 -25
  6. package/src/cache/file-hashes.ts +3 -0
  7. package/src/cache/git-inputs.ts +188 -39
  8. package/src/cache/index.ts +1 -0
  9. package/src/cache/inputs.ts +3 -0
  10. package/src/cache/layer.ts +11 -0
  11. package/src/cache/layered-cache.ts +4 -0
  12. package/src/cache/linked-inputs.ts +3 -0
  13. package/src/cache/lockfile-memo.ts +3 -2
  14. package/src/cache/output-index.ts +5 -2
  15. package/src/cache/remote-lane.ts +2 -1
  16. package/src/cache/run-history.ts +11 -1
  17. package/src/cache/schema.ts +11 -1
  18. package/src/cli/cache.ts +26 -16
  19. package/src/cli/completions.ts +4 -2
  20. package/src/cli/docs.ts +10 -11
  21. package/src/cli/help.ts +12 -23
  22. package/src/cli/index.ts +4 -9
  23. package/src/cli/info.ts +15 -7
  24. package/src/cli/init.ts +23 -15
  25. package/src/cli/last.ts +213 -203
  26. package/src/cli/lock.ts +55 -5
  27. package/src/cli/plan-format.ts +100 -118
  28. package/src/cli/plugin-commands.ts +8 -2
  29. package/src/cli/plugin-templates.ts +2 -2
  30. package/src/cli/run.ts +46 -27
  31. package/src/cli/select.ts +43 -5
  32. package/src/cli/show.ts +38 -42
  33. package/src/cli/task-verb.ts +22 -0
  34. package/src/cli/upgrade.ts +3 -1
  35. package/src/cli/watch-fs.ts +6 -1
  36. package/src/cli/watch-judge.ts +2 -1
  37. package/src/cli/watch.ts +83 -44
  38. package/src/cli/why.ts +123 -89
  39. package/src/cli/workspace-config.ts +2 -1
  40. package/src/exec/executor.ts +2 -1
  41. package/src/exec/kill-tree.ts +58 -5
  42. package/src/exec/local-executor.ts +1 -1
  43. package/src/exec/pipes.ts +152 -0
  44. package/src/exec/runner.ts +199 -182
  45. package/src/exec/sandbox-binds.ts +1 -1
  46. package/src/exec/sandbox-runtime.ts +11 -2
  47. package/src/exec/spawn-off.ts +154 -0
  48. package/src/exec/warm-shell.ts +146 -0
  49. package/src/graph/critical-path.ts +40 -28
  50. package/src/graph/priorities.ts +98 -44
  51. package/src/graph/scheduler.ts +14 -10
  52. package/src/graph/task-graph.ts +27 -1
  53. package/src/handoff.ts +10 -7
  54. package/src/history/history.ts +5 -0
  55. package/src/history/metrics.ts +9 -1
  56. package/src/index.ts +3 -1
  57. package/src/orchestrator/doctor.ts +1 -1
  58. package/src/orchestrator/execute-task.ts +16 -2
  59. package/src/orchestrator/graph-memo.ts +50 -1
  60. package/src/orchestrator/graph-stage.ts +6 -0
  61. package/src/orchestrator/index.ts +3 -0
  62. package/src/orchestrator/live-usage.ts +16 -11
  63. package/src/orchestrator/locked-digests.ts +63 -0
  64. package/src/orchestrator/lockfile-claim.ts +108 -29
  65. package/src/orchestrator/miss-save.ts +31 -3
  66. package/src/orchestrator/persistent-task.ts +3 -1
  67. package/src/orchestrator/plan.ts +5 -1
  68. package/src/orchestrator/plugin-host.ts +61 -0
  69. package/src/orchestrator/plugin.ts +47 -2
  70. package/src/orchestrator/prepare.ts +16 -1
  71. package/src/orchestrator/projects.ts +4 -1
  72. package/src/orchestrator/run-context.ts +29 -10
  73. package/src/orchestrator/run-failures.ts +1 -1
  74. package/src/orchestrator/run-records.ts +10 -0
  75. package/src/orchestrator/run.ts +53 -22
  76. package/src/orchestrator/sandbox-request.ts +2 -2
  77. package/src/orchestrator/signals.ts +8 -5
  78. package/src/orchestrator/task-hash.ts +17 -3
  79. package/src/report/events.ts +3 -11
  80. package/src/report/forecast.ts +10 -32
  81. package/src/report/framed-output.ts +272 -173
  82. package/src/report/index.ts +8 -3
  83. package/src/report/logger.ts +73 -38
  84. package/src/report/plain-output.ts +1 -1
  85. package/src/report/status-line.ts +49 -357
  86. package/src/report/summary.ts +100 -131
  87. package/src/ui/ansi.ts +48 -0
  88. package/src/ui/components.ts +506 -0
  89. package/src/ui/index.ts +79 -0
  90. package/src/ui/kit.ts +37 -0
  91. package/src/ui/live-region.ts +363 -0
  92. package/src/{util/refusal-notice.ts → ui/refusal.ts} +7 -5
  93. package/src/ui/terminal.ts +51 -0
  94. package/src/ui/theme.ts +224 -0
  95. package/src/ui/version-line.ts +28 -0
  96. package/src/util/config-frame.ts +65 -10
  97. package/src/util/edit-distance.ts +6 -1
  98. package/src/util/index.ts +2 -15
  99. package/src/util/paths.ts +20 -2
  100. package/src/util/secret-mask.ts +39 -3
  101. package/src/util/settle.ts +31 -0
  102. package/src/util/timing.ts +4 -16
  103. package/src/workspace/affected.ts +89 -30
  104. package/src/workspace/config-schema.ts +4 -2
  105. package/src/workspace/dependencies-lock.ts +191 -0
  106. package/src/workspace/filter.ts +28 -0
  107. package/src/workspace/index.ts +9 -0
  108. package/src/workspace/lockfile.ts +159 -6
  109. package/src/workspace/migration.ts +4 -3
  110. package/src/workspace/workspace.ts +3 -0
  111. package/src/util/colors.ts +0 -133
  112. package/src/util/report.ts +0 -44
  113. package/src/util/ui.ts +0 -258
  114. /package/src/{report/path-links.ts → ui/links.ts} +0 -0
  115. /package/src/{report → ui}/terminal-signals.ts +0 -0
@@ -8,11 +8,15 @@
8
8
  import os from 'node:os'
9
9
  import path from 'node:path'
10
10
  import {
11
+ closeSync,
11
12
  existsSync,
13
+ fstatSync,
12
14
  lstatSync,
13
15
  mkdtempSync,
16
+ openSync,
14
17
  readdirSync,
15
18
  readFileSync,
19
+ readSync,
16
20
  realpathSync,
17
21
  renameSync,
18
22
  rmSync,
@@ -892,7 +896,12 @@ export function fileIdentity(mode: string, oid: string): string {
892
896
  function parseLsFilesOutput(
893
897
  records: readonly string[],
894
898
  undecodableRecords: ReadonlySet<string>,
895
- debug?: { sizes: readonly number[]; indexed: Map<string, { oid: string; size: number }> },
899
+ debug?: {
900
+ sizes: readonly number[]
901
+ ctimes: readonly number[]
902
+ mtimes: readonly number[]
903
+ indexed: Map<string, { oid: string; size: number; ctime: number; mtime: number }>
904
+ },
896
905
  ): GitLsResult {
897
906
  const files: string[] = []
898
907
  const oids = new Map<string, string>()
@@ -914,7 +923,12 @@ function parseLsFilesOutput(
914
923
  if ((mode === '100644' || mode === '100755' || mode === '120000') && stage === '0') {
915
924
  oids.set(filePath, fileIdentity(mode, m[3]!))
916
925
  if (debug !== undefined && mode !== '120000') {
917
- debug.indexed.set(filePath, { oid: m[3]!, size: debug.sizes[i]! })
926
+ debug.indexed.set(filePath, {
927
+ oid: m[3]!,
928
+ size: debug.sizes[i]!,
929
+ ctime: debug.ctimes[i]!,
930
+ mtime: debug.mtimes[i]!,
931
+ })
918
932
  }
919
933
  } else if (mode === '160000') {
920
934
  gitlinks.add(filePath)
@@ -940,11 +954,15 @@ function parseLsFilesOutput(
940
954
  function splitLsDebug(run: GitRun): {
941
955
  records: string[]
942
956
  sizes: number[]
957
+ ctimes: number[]
958
+ mtimes: number[]
943
959
  undecodable: Set<string>
944
960
  } {
945
961
  const segs = run.stdout.split('\0')
946
962
  const records: string[] = []
947
963
  const sizes: number[] = []
964
+ const ctimes: number[] = []
965
+ const mtimes: number[] = []
948
966
  const undecodable = new Set<string>()
949
967
  let record = segs[0]!
950
968
  let recordRaw = record
@@ -952,12 +970,22 @@ function splitLsDebug(run: GitRun): {
952
970
  const seg = segs[i]
953
971
  let rest = ''
954
972
  let size = -1
973
+ let ctime = -1
974
+ let mtime = -1
955
975
  if (seg !== undefined) {
956
976
  let at = 0
957
977
  for (let k = 0; k < 5 && at >= 0; k++) {
958
978
  const nl = seg.indexOf('\n', at)
959
979
  if (nl < 0) at = -1
960
980
  else {
981
+ // ` ctime: <sec>:<nsec>`, then ` mtime: <sec>:<nsec>`.
982
+ if (k < 2) {
983
+ const n = parseInt(seg.slice(seg.indexOf(':', at) + 1, nl), 10)
984
+ if (!Number.isNaN(n)) {
985
+ if (k === 0) ctime = n
986
+ else mtime = n
987
+ }
988
+ }
961
989
  if (k === 4) {
962
990
  const field = seg.indexOf('size: ', at)
963
991
  if (field >= 0 && field < nl) {
@@ -973,12 +1001,14 @@ function splitLsDebug(run: GitRun): {
973
1001
  if (record.length > 0) {
974
1002
  records.push(record)
975
1003
  sizes.push(size)
1004
+ ctimes.push(ctime)
1005
+ mtimes.push(mtime)
976
1006
  if (run.undecodable.has(recordRaw)) undecodable.add(record)
977
1007
  }
978
1008
  record = rest
979
1009
  recordRaw = seg ?? ''
980
1010
  }
981
- return { records, sizes, undecodable }
1011
+ return { records, sizes, ctimes, mtimes, undecodable }
982
1012
  }
983
1013
 
984
1014
  /**
@@ -998,6 +1028,8 @@ export interface IndexBlobs {
998
1028
  paths: string[]
999
1029
  oids: string[]
1000
1030
  sizes: number[]
1031
+ /** Entries git cannot see a same-second rewrite of (`sameSecond`). */
1032
+ sameSecond: string[]
1001
1033
  }
1002
1034
 
1003
1035
  /** What the blob-size check reads, lazily: the verdict's key, and the listing a miss needs. */
@@ -1010,6 +1042,31 @@ export interface IndexBlobCheck {
1010
1042
  rekey(): Promise<string | undefined>
1011
1043
  }
1012
1044
 
1045
+ /** At most `n` contiguous parts of `items`, none under `CAT_FILE_MIN`: a spawn costs more than a few lookups. */
1046
+ function chunks<T>(items: readonly T[], n: number): T[][] {
1047
+ const parts = Math.min(n, Math.ceil(items.length / CAT_FILE_MIN))
1048
+ const size = Math.ceil(items.length / Math.max(1, parts))
1049
+ const out: T[][] = []
1050
+ for (let at = 0; at < items.length; at += size) out.push(items.slice(at, at + size))
1051
+ return out
1052
+ }
1053
+ const CAT_FILE_MIN = 256
1054
+
1055
+ const sizeChecks = new WeakMap<GitEnumeration, Promise<void>>()
1056
+ /**
1057
+ * `dropResizedOids` once per enumeration: a run starts it as soon as its
1058
+ * cache opens, so the check runs beside the config load and graph build
1059
+ * instead of after them, and `applyGitEnumeration` waits on the same one.
1060
+ */
1061
+ export function checkBlobSizes(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
1062
+ let check = sizeChecks.get(enumeration)
1063
+ if (check === undefined) {
1064
+ check = dropResizedOids(enumeration, memo)
1065
+ sizeChecks.set(enumeration, check)
1066
+ }
1067
+ return check
1068
+ }
1069
+
1013
1070
  /**
1014
1071
  * Drop each trusted OID whose blob is not the size the index recorded for
1015
1072
  * the worktree file (A-60). A clean filter's blob differs from the file it
@@ -1026,6 +1083,8 @@ export interface IndexBlobCheck {
1026
1083
  * (racy) entry records 0 and is hashed from disk. An answer that cannot be
1027
1084
  * read trusts nothing. A filter that keeps the size (`tr a-z A-Z`) is not
1028
1085
  * caught: only a read of every trusted file would be (caching.md, step 12).
1086
+ * The verdict also holds each entry git cannot see a same-second rewrite
1087
+ * of (`sameSecond`), read from the same listing.
1029
1088
  */
1030
1089
  async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
1031
1090
  const { trusted, blobCheck } = enumeration
@@ -1033,31 +1092,47 @@ async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo)
1033
1092
  const key = blobCheck.key
1034
1093
  let resized = key === undefined ? undefined : memo?.blobVerdict(key)
1035
1094
  if (resized === undefined) {
1036
- const blobs = await blobCheck.blobs()
1095
+ // The sizes are asked beside the listing: both read the index the
1096
+ // enumeration listed (the listing refuses one written since), so the
1097
+ // OIDs are the listing's. One after the other, a cold 9,603-task run
1098
+ // waited on the listing's 70-95 ms before the first spawn (2026-11-01).
1099
+ const listing = blobCheck.blobs()
1100
+ // Only what is still trusted: a path dropped already needs no size.
1101
+ // Built here, not with the listing: a run whose verdict is memoized
1102
+ // never needs it, and on 9,603 tasks it cost 17-35 ms (2026-11-01).
1103
+ // A non-regular mode rides its value (`100755:<oid>`); cat-file takes the OID.
1104
+ const oids = [...new Set(Array.from(trusted.values(), (v) => v.slice(v.lastIndexOf(':') + 1)))]
1105
+ const sizes = memo?.knownBlobSizes(oids) ?? new Map<string, number>()
1106
+ const unknown = oids.filter((oid) => !sizes.has(oid))
1107
+ // One cat-file per core: a fresh clone's objects are loose, and one
1108
+ // process opened and inflated 2,505 of them in 30 ms before the first
1109
+ // task of a cold 100-package run; four took 14 (2026-11-03).
1110
+ const asked = Promise.all(
1111
+ chunks(unknown, Math.max(1, navigator.hardwareConcurrency)).map((part) =>
1112
+ enumeration.catFile(part.join('\n') + '\n'),
1113
+ ),
1114
+ )
1115
+ const [blobs, runs] = await Promise.all([listing, asked])
1037
1116
  if (blobs === null) {
1038
1117
  trusted.clear()
1039
1118
  return
1040
1119
  }
1041
- const unique = [...new Set(blobs.oids)]
1042
- const sizes = memo?.knownBlobSizes(unique) ?? new Map<string, number>()
1043
- const unknown = unique.filter((oid) => !sizes.has(oid))
1044
- if (unknown.length > 0) {
1045
- const run = await enumeration.catFile(unknown.join('\n') + '\n')
1046
- if (run === null || run.exitCode !== 0) {
1047
- trusted.clear()
1048
- return
1049
- }
1050
- const learned = new Map<string, number>()
1051
- for (const line of run.stdout.split('\n')) {
1120
+ if (runs.some((run) => run === null || run.exitCode !== 0)) {
1121
+ trusted.clear()
1122
+ return
1123
+ }
1124
+ const learned = new Map<string, number>()
1125
+ for (const run of runs) {
1126
+ for (const line of run!.stdout.split('\n')) {
1052
1127
  const sp = line.indexOf(' ')
1053
1128
  const n = Number(line.slice(sp + 1))
1054
1129
  if (sp > 0 && line.slice(sp + 1) !== '' && Number.isInteger(n))
1055
1130
  learned.set(line.slice(0, sp), n)
1056
1131
  }
1057
- for (const [oid, n] of learned) sizes.set(oid, n)
1058
- memo?.rememberBlobSizes(learned)
1059
1132
  }
1060
- resized = []
1133
+ for (const [oid, n] of learned) sizes.set(oid, n)
1134
+ if (learned.size > 0) memo?.rememberBlobSizes(learned)
1135
+ resized = [...blobs.sameSecond]
1061
1136
  for (let i = 0; i < blobs.oids.length; i++) {
1062
1137
  const size = sizes.get(blobs.oids[i]!)
1063
1138
  // The index keeps the low 32 bits of a size.
@@ -1626,6 +1701,19 @@ function hasRacyEntry(index: Uint8Array, mtimeSec: number, oidBytes: number): bo
1626
1701
  return false
1627
1702
  }
1628
1703
 
1704
+ /**
1705
+ * git compares a file's ctime to the second only, and its racy rule reads
1706
+ * mtime alone: a same-size rewrite that keeps (or restores) an older mtime,
1707
+ * made in the second the index recorded the file, keeps every stat git
1708
+ * compares, and `status` calls it clean. An entry whose recorded ctime, not
1709
+ * its mtime, reaches the second its index was written can hide one; git
1710
+ * re-hashes an entry racy by mtime. Its OID is not trusted. The stamps are
1711
+ * git's own (`ls-files --debug`), judged once per index (the verdict).
1712
+ */
1713
+ function sameSecond(ctimeSec: number, mtimeSec: number, writtenSec: number): boolean {
1714
+ return ctimeSec < 0 || (ctimeSec >= writtenSec && mtimeSec < writtenSec)
1715
+ }
1716
+
1629
1717
  /** A private copy a run older than this may still have handed to `status`. */
1630
1718
  const STALE_INDEX_MS = 60_000
1631
1719
 
@@ -1711,7 +1799,7 @@ export async function startGitEnumeration(
1711
1799
  stdin?: string,
1712
1800
  env?: Record<string, string>,
1713
1801
  ): Promise<GitRun | null> => {
1714
- const spawn = <Out extends 'pipe' | Bun.BunFile>(stdout: Out) =>
1802
+ const spawn = <Out extends 'pipe' | number>(stdout: Out) =>
1715
1803
  Bun.spawn({
1716
1804
  // `status` refreshes the index when it can take `index.lock`, so
1717
1805
  // the user's own `git add` or `commit` failed on the lock while a
@@ -1728,23 +1816,30 @@ export async function startGitEnumeration(
1728
1816
  // loading configs while git lists, and a pipe held a 10 MB listing
1729
1817
  // until the thread came back to drain it 64 KiB at a time (~90 ms of
1730
1818
  // a warm 5,000-project run). In a directory of its own, so no link
1731
- // another user planted in a shared temp directory takes the bytes. A
1732
- // temp directory that takes none falls back to the pipe.
1733
- let dir: string | undefined
1819
+ // another user planted in a shared temp directory takes the bytes, and
1820
+ // the directory goes the moment the file is open: a run stopped
1821
+ // mid-listing (a Ctrl-C before the run's signal handlers, or a kill)
1822
+ // runs no `finally`, and each left a `vx-git-*` behind. A temp
1823
+ // directory that takes none falls back to the pipe.
1824
+ let fd: number | undefined
1734
1825
  try {
1735
- dir = mkdtempSync(path.join(os.tmpdir(), 'vx-git-'))
1826
+ const dir = mkdtempSync(path.join(os.tmpdir(), 'vx-git-'))
1827
+ try {
1828
+ fd = openSync(path.join(dir, 'out'), 'w+', 0o600)
1829
+ } finally {
1830
+ rmSync(dir, { recursive: true, force: true })
1831
+ }
1736
1832
  } catch {
1737
- dir = undefined
1833
+ fd = undefined
1738
1834
  }
1739
1835
  try {
1740
1836
  let stdout: Uint8Array
1741
1837
  let stderr: string
1742
1838
  let exitCode: number
1743
- if (dir !== undefined) {
1744
- const out = path.join(dir, 'out')
1745
- const proc = spawn(Bun.file(out))
1839
+ if (fd !== undefined) {
1840
+ const proc = spawn(fd)
1746
1841
  ;[stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited])
1747
- stdout = await Bun.file(out).bytes()
1842
+ stdout = readWhole(fd)
1748
1843
  } else {
1749
1844
  const piped = spawn('pipe')
1750
1845
  ;[stdout, stderr, exitCode] = await Promise.all([
@@ -1758,7 +1853,7 @@ export async function startGitEnumeration(
1758
1853
  } catch {
1759
1854
  return null
1760
1855
  } finally {
1761
- if (dir !== undefined) rmSync(dir, { recursive: true, force: true })
1856
+ if (fd !== undefined) closeSync(fd)
1762
1857
  }
1763
1858
  }
1764
1859
  const startedAtMs = Date.now()
@@ -1834,7 +1929,9 @@ export async function startGitEnumeration(
1834
1929
  bytes,
1835
1930
  hash,
1836
1931
  st,
1837
- key: `${hash}:${pathspecKey}`,
1932
+ // The mtime too: which entries git can see a same-second rewrite
1933
+ // of (`sameSecond`) turns on the second the index was written.
1934
+ key: `${hash}:${pathspecKey}:${Math.floor(st.mtimeMs / 1000)}`,
1838
1935
  stamp: `${st.ino}:${st.ctimeMs}:${st.size}`,
1839
1936
  }
1840
1937
  } catch {
@@ -1863,10 +1960,16 @@ export async function startGitEnumeration(
1863
1960
  index === undefined ? undefined : { GIT_INDEX_FILE: index },
1864
1961
  )
1865
1962
  // An index with no racy entry is read as it is: `status` spawns now.
1963
+ // The copy `status` read, when it read one: its stats are the ones the
1964
+ // dirty set came from, so the blob check lists it too.
1965
+ let statusCopy: string | undefined
1866
1966
  const statusRun =
1867
1967
  read === undefined || indexFile === undefined || !racyIndex(read, facts!.objectFormat)
1868
1968
  ? spawnStatus(undefined)
1869
- : refreshedIndex(indexFile, read, spawnGit).then(spawnStatus)
1969
+ : refreshedIndex(indexFile, read, spawnGit).then((copy) => {
1970
+ statusCopy = copy
1971
+ return spawnStatus(copy)
1972
+ })
1870
1973
  const running = Promise.all([listing, statusRun, varsRun])
1871
1974
  const gitPrefix = facts?.prefix ?? ''
1872
1975
  const gitDir = facts?.commonDir ?? ''
@@ -1888,6 +1991,17 @@ export async function startGitEnumeration(
1888
1991
  // the stamp moves with any write) and is still in place after it: a
1889
1992
  // `git add` between the listing and the read paired one index's verdict
1890
1993
  // with the other's entries. Otherwise this run asks for no verdict.
1994
+ // A verdict read from the copy holds for that copy file only: a copy
1995
+ // written again (a new inode) asks again.
1996
+ const copyKey = (key: string): string | undefined => {
1997
+ if (statusCopy === undefined) return key
1998
+ try {
1999
+ const st = lstatSync(statusCopy)
2000
+ return `${key}:${st.ino}:${st.mtimeMs}`
2001
+ } catch {
2002
+ return undefined
2003
+ }
2004
+ }
1891
2005
  let indexKey: string | undefined
1892
2006
  if (
1893
2007
  read !== undefined &&
@@ -1895,7 +2009,7 @@ export async function startGitEnumeration(
1895
2009
  ) {
1896
2010
  try {
1897
2011
  const st = lstatSync(indexFile!)
1898
- if (`${st.ino}:${st.ctimeMs}:${st.size}` === read.stamp) indexKey = read.key
2012
+ if (`${st.ino}:${st.ctimeMs}:${st.size}` === read.stamp) indexKey = copyKey(read.key)
1899
2013
  } catch {
1900
2014
  // gone: no key
1901
2015
  }
@@ -1997,24 +2111,47 @@ export async function startGitEnumeration(
1997
2111
  blobCheck: {
1998
2112
  key: indexKey,
1999
2113
  async blobs() {
2000
- const run = await spawnGit(['ls-files', '-s', '-v', '-z', '--debug', '--', ...pathspecs])
2114
+ const listed = statusCopy ?? indexFile
2115
+ let writtenSec = 0
2116
+ try {
2117
+ if (listed !== undefined) writtenSec = Math.floor(lstatSync(listed).mtimeMs / 1000)
2118
+ } catch {
2119
+ return null
2120
+ }
2121
+ const run = await spawnGit(
2122
+ ['ls-files', '-s', '-v', '-z', '--debug', '--', ...pathspecs],
2123
+ undefined,
2124
+ statusCopy === undefined ? undefined : { GIT_INDEX_FILE: statusCopy },
2125
+ )
2001
2126
  if (run === null || run.exitCode !== 0) return null
2002
2127
  // Sizes from an index written since the listing judge entries the
2003
2128
  // trusted OIDs did not come from: an add there rewrote a resized
2004
2129
  // entry to one that fits, and the listed OID stood (A-60).
2005
2130
  if (indexStamp() !== stampBefore) return null
2006
2131
  const debug = splitLsDebug(run)
2007
- const indexed = new Map<string, { oid: string; size: number }>()
2008
- parseLsFilesOutput(debug.records, debug.undecodable, { sizes: debug.sizes, indexed })
2009
- const blobs: IndexBlobs = { paths: [], oids: [], sizes: [] }
2010
- for (const [rel, { oid, size }] of indexed) {
2132
+ const indexed = new Map<
2133
+ string,
2134
+ { oid: string; size: number; ctime: number; mtime: number }
2135
+ >()
2136
+ parseLsFilesOutput(debug.records, debug.undecodable, {
2137
+ sizes: debug.sizes,
2138
+ ctimes: debug.ctimes,
2139
+ mtimes: debug.mtimes,
2140
+ indexed,
2141
+ })
2142
+ const blobs: IndexBlobs = { paths: [], oids: [], sizes: [], sameSecond: [] }
2143
+ for (const [rel, { oid, size, ctime, mtime }] of indexed) {
2011
2144
  blobs.paths.push(rel)
2012
2145
  blobs.oids.push(oid)
2013
2146
  blobs.sizes.push(size)
2147
+ if (sameSecond(ctime, mtime, writtenSec)) blobs.sameSecond.push(rel)
2014
2148
  }
2015
2149
  return blobs
2016
2150
  },
2017
- rekey: async () => (await readIndex())?.key,
2151
+ rekey: async () => {
2152
+ const key = (await readIndex())?.key
2153
+ return key === undefined ? undefined : copyKey(key)
2154
+ },
2018
2155
  },
2019
2156
  // A missing blob is an answer ("missing"), never a fetch from a partial
2020
2157
  // clone's promisor remote.
@@ -2083,7 +2220,7 @@ export async function applyGitEnumeration(
2083
2220
  workspaceWide = false,
2084
2221
  memo?: BlobSizeMemo,
2085
2222
  ): Promise<void> {
2086
- await dropResizedOids(enumeration, memo)
2223
+ await checkBlobSizes(enumeration, memo)
2087
2224
  const { all, trusted } = enumeration
2088
2225
  cache.setWorktreeDirty(enumeration.dirty)
2089
2226
  cache.setGitVars(enumeration.vars)
@@ -2218,3 +2355,15 @@ export async function applyGitEnumeration(
2218
2355
  cache.setWorkspaceRoot(workspaceRoot)
2219
2356
  }
2220
2357
  }
2358
+
2359
+ /** The whole of a file held only by `fd`, from its start. */
2360
+ function readWhole(fd: number): Uint8Array {
2361
+ const bytes = new Uint8Array(fstatSync(fd).size)
2362
+ let at = 0
2363
+ while (at < bytes.length) {
2364
+ const n = readSync(fd, bytes, at, bytes.length - at, at)
2365
+ if (n === 0) break
2366
+ at += n
2367
+ }
2368
+ return at === bytes.length ? bytes : bytes.subarray(0, at)
2369
+ }
@@ -81,6 +81,7 @@ export {
81
81
  MAX_SCOPED_PATHSPECS,
82
82
  repoRootOf,
83
83
  startGitEnumeration,
84
+ checkBlobSizes,
84
85
  lazyGitEnumeration,
85
86
  type LazyGitEnumeration,
86
87
  type GitEnumeration,
@@ -101,6 +101,9 @@ const ALWAYS_IGNORE = [
101
101
  // must not select every `**` reader under --affected.
102
102
  '.vx/.gitattributes',
103
103
  '.vx/.gitignore',
104
+ // `vx lock`'s record of the dependency edges and digests: the key folds
105
+ // the digests through `fingerprint` already.
106
+ '.vx/dependencies.lock.json',
104
107
  // Where restores stage (archive.ts): vx's own, never an input.
105
108
  '.vx/tmp/**',
106
109
  ]
@@ -232,6 +232,15 @@ export interface RunRecord {
232
232
  placementMemoryBytes?: number
233
233
  /** The share of a core each runnable thread got there (`ExecuteResult.machine.cpuShare`, v49). */
234
234
  cpuShare?: number
235
+ /** How long it waited ready for a slot (`TaskOutcome.queuedMs`, v51). */
236
+ queuedMs?: number
237
+ /** Its output files' bytes (`TaskOutcome.outputBytes`, v51). */
238
+ outputBytes?: number
239
+ /** The tasks and groups downstream of it in the run, the scheduler's count (v51). */
240
+ blocks?: number
241
+ /** The flaky record its row's note quoted (`TaskOutcome.flaky`, v51). */
242
+ flakyPasses?: number
243
+ flakyFailures?: number
235
244
  }
236
245
 
237
246
  /**
@@ -823,6 +832,8 @@ export interface CacheLayer {
823
832
  outputsPath(hash: string): string
824
833
  /** The stored artifact's compressed size, wherever it lives; undefined when none is stored. */
825
834
  artifactSize?(hash: string): number | undefined
835
+ /** The output files' bytes, uncompressed, of the entry this process just saved; asked once. */
836
+ outputBytes?(hash: string): number | undefined
826
837
  prune(options: PruneOptions): Promise<PruneResult>
827
838
  close(): void
828
839
  }
@@ -575,6 +575,10 @@ export class LayeredCache implements CacheLayer {
575
575
  return this.local.artifactSize(hash)
576
576
  }
577
577
 
578
+ outputBytes(hash: string): number | undefined {
579
+ return this.local.outputBytes(hash)
580
+ }
581
+
578
582
  hashFile(filePath: string): Promise<string> {
579
583
  return this.local.hashFile(filePath)
580
584
  }
@@ -71,6 +71,9 @@ export async function behindLink(
71
71
  return NOTHING_BEHIND
72
72
  }
73
73
  if (seen.has(real)) return NOTHING_BEHIND
74
+ // A repository's own store, as a `.git/**` input names nothing: walked,
75
+ // every commit and index write re-keyed the task. Its target string keys.
76
+ if (real.split(path.sep).includes('.git')) return NOTHING_BEHIND
74
77
  if (st.isFile()) return `file\0${await hashFile(real).catch(absentOr)}`
75
78
  if (!st.isDirectory()) return 'special'
76
79
  const inner = new Set(seen).add(real)
@@ -21,7 +21,8 @@ export interface LockfileMemo {
21
21
  lock: string
22
22
  /** Each extra file the digests read → its content hash then. */
23
23
  extras: Record<string, string>
24
- importers: Record<string, string>
24
+ /** Each importer's digest; null when the parse that wrote it left them out (a run's). */
25
+ importers: Record<string, string> | null
25
26
  edges: Record<string, unknown>
26
27
  /** Each importer's recorded dependency names; absent when the claimant reads none. */
27
28
  recorded?: Record<string, readonly string[]>
@@ -77,7 +78,7 @@ export function readLockfileMemo(
77
78
  version: row.version,
78
79
  lock: row.lock,
79
80
  extras: JSON.parse(row.extras) as Record<string, string>,
80
- importers: JSON.parse(row.importers) as Record<string, string>,
81
+ importers: JSON.parse(row.importers) as Record<string, string> | null,
81
82
  edges: JSON.parse(row.edges) as Record<string, unknown>,
82
83
  ...(row.recorded !== null
83
84
  ? { recorded: JSON.parse(row.recorded) as Record<string, readonly string[]> }
@@ -143,17 +143,20 @@ export class OutputIndex {
143
143
  /**
144
144
  * Replace an entry's file rows. Called INSIDE the store's save transaction,
145
145
  * so an entry and its fingerprints land together (an UPDATE on the same
146
- * hash refreshes rather than appends).
146
+ * hash refreshes rather than appends). Answers the files' bytes.
147
147
  */
148
- replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): void {
148
+ replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): number {
149
149
  // A stamp taken for the rows this replaces describes their files, not these.
150
150
  this.held.delete(hash)
151
151
  this.pendingStamps.delete(hash)
152
152
  this.deleteOutputFiles.run(hash)
153
153
  this.deleteStamps.run(hash)
154
+ let bytes = 0
154
155
  for (const [rel, size, mode, mtime] of rows) {
155
156
  this.insertOutputFile.run(hash, rel, size, mode, mtime)
157
+ bytes += size
156
158
  }
159
+ return bytes
157
160
  }
158
161
 
159
162
  /** Hold a probe's rows for its restores (`held`), in place of the last probe's. */
@@ -4,7 +4,8 @@
4
4
  // class with the endpoint named. The order across remotes, the local store
5
5
  // and provenance are `LayeredCache`'s; nothing here knows another remote.
6
6
 
7
- import { machineParallelism, thrownMessage, ui } from '../util/index.js'
7
+ import { machineParallelism, thrownMessage } from '../util/index.js'
8
+ import * as ui from '../ui/index.js'
8
9
  import type { RemoteKeyTraffic, RemoteTraffic } from './cache.js'
9
10
  import type { ArtifactInfo, RemoteCacheLayer } from './layered-cache.js'
10
11
 
@@ -41,6 +41,11 @@ const RUN_COLUMNS = [
41
41
  'keyless_by',
42
42
  'def',
43
43
  'cpu_share',
44
+ 'queued_ms',
45
+ 'output_bytes',
46
+ 'blocks',
47
+ 'flaky_passes',
48
+ 'flaky_failures',
44
49
  ]
45
50
 
46
51
  const INSERT_RUNS = `INSERT INTO runs(${RUN_COLUMNS.join(', ')}) VALUES `
@@ -282,7 +287,7 @@ export class RunHistory {
282
287
  }
283
288
  }
284
289
 
285
- /** Bind a RunRecord to one `RUN_TUPLE`'s 29 positional parameters. */
290
+ /** Bind a RunRecord to one `RUN_TUPLE`'s positional parameters, in `RUN_COLUMNS` order. */
286
291
  function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBindings[] {
287
292
  return [
288
293
  // The ONE place the no-key sentinel is applied, so the column's
@@ -321,6 +326,11 @@ function bindRun(run: RunRecord, digestValue: (v: string) => string): SQLQueryBi
321
326
  run.keylessBy ?? null,
322
327
  run.def ?? null,
323
328
  run.cpuShare ?? null,
329
+ run.queuedMs ?? null,
330
+ run.outputBytes ?? null,
331
+ run.blocks ?? null,
332
+ run.flakyPasses ?? null,
333
+ run.flakyFailures ?? null,
324
334
  ]
325
335
  }
326
336
 
@@ -119,7 +119,17 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
119
119
  def TEXT,
120
120
  -- v49: the share of a core each runnable thread got where it ran
121
121
  -- (ExecuteResult.machine.cpuShare); NULL = all it wanted.
122
- cpu_share REAL
122
+ cpu_share REAL,
123
+ -- v51: the rest of the results row. How long the task waited ready
124
+ -- for a slot (NULL: it started at once); its output files' bytes,
125
+ -- uncompressed (a hit's restored, a miss's saved; NULL: no entry);
126
+ -- the tasks downstream of it in the run; the flaky record its note
127
+ -- quoted (NULL: not proven flaky).
128
+ queued_ms INTEGER,
129
+ output_bytes INTEGER,
130
+ blocks INTEGER,
131
+ flaky_passes INTEGER,
132
+ flaky_failures INTEGER
123
133
  );
124
134
  -- Two whole-table indexes only, and both APPEND: every row of a run carries the
125
135
  -- same run_id and a started_at newer than everything before it, so
package/src/cli/cache.ts CHANGED
@@ -11,8 +11,8 @@ import {
11
11
  parseSize,
12
12
  relPosix,
13
13
  treeBytes,
14
- ui,
15
14
  } from '../util/index.js'
15
+ import * as ui from '../ui/index.js'
16
16
  import { acquireRunLock } from '../orchestrator/index.js'
17
17
  import { findWorkspaceRoot } from '../workspace/index.js'
18
18
  import {
@@ -270,9 +270,7 @@ function printPruned(
270
270
  ): void {
271
271
  if (json) {
272
272
  const { evicted, bytesFreed, orphans, orphanBytes } = result
273
- process.stdout.write(
274
- `${JSON.stringify({ dry: dry, evicted, bytesFreed, orphans, orphanBytes })}\n`,
275
- )
273
+ ui.data('out', `${JSON.stringify({ dry: dry, evicted, bytesFreed, orphans, orphanBytes })}\n`)
276
274
  return
277
275
  }
278
276
  const plural = (n: number, one: string, many: string): string => `${n} ${n === 1 ? one : many}`
@@ -321,6 +319,25 @@ async function clear(
321
319
  storeRoot: string | null,
322
320
  parsed: PruneArgs,
323
321
  ): Promise<number> {
322
+ // A clear beside a run deletes what the run is restoring or saving; it
323
+ // waits for the run's lock, as a trim does, and only then reads what to
324
+ // delete: read first, what the run saved meanwhile went uncounted.
325
+ const release =
326
+ parsed.dry === true ? async () => {} : await acquireRunLock(root, { log: warnToStderr })
327
+ try {
328
+ return clearHeld(args, root, dir, storeRoot, parsed)
329
+ } finally {
330
+ await release()
331
+ }
332
+ }
333
+
334
+ function clearHeld(
335
+ args: readonly string[],
336
+ root: string,
337
+ dir: string,
338
+ storeRoot: string | null,
339
+ parsed: PruneArgs,
340
+ ): number {
324
341
  // The store the index recorded and today's (a changed HOME, an origin
325
342
  // added) both go; a directory with no index is not a vx cache to delete.
326
343
  const targets: string[] = []
@@ -376,23 +393,16 @@ async function clear(
376
393
  ? readdirSync(t).map((e) => path.join(t, e))
377
394
  : [t]
378
395
  const bytes = doomed.reduce((n, t) => n + entries(t).reduce((m, e) => m + treeBytes(e), 0), 0)
379
- if (!dry && doomed.length > 0) {
380
- // A clear beside a run deletes what the run is restoring or saving; it
381
- // waits for the run's lock, as a trim does.
382
- const release = await acquireRunLock(root, { log: warnToStderr })
383
- try {
384
- for (const t of doomed) {
385
- for (const e of entries(t)) rmSync(e, { recursive: true, force: true })
386
- if (t === named && readdirSync(t).length === 0) rmSync(t, { recursive: true })
387
- }
388
- } finally {
389
- await release()
396
+ if (!dry) {
397
+ for (const t of doomed) {
398
+ for (const e of entries(t)) rmSync(e, { recursive: true, force: true })
399
+ if (t === named && readdirSync(t).length === 0) rmSync(t, { recursive: true })
390
400
  }
391
401
  }
392
402
  const shown = doomed.map((t) => (path.relative(root, t).startsWith('..') ? t : relPosix(root, t)))
393
403
  if (parsed.format === 'json') {
394
404
  const result: ClearResult = { dry: dry, cleared: shown, bytesFreed: bytes }
395
- process.stdout.write(`${JSON.stringify(result)}\n`)
405
+ ui.data('out', `${JSON.stringify(result)}\n`)
396
406
  } else
397
407
  ui.print(
398
408
  'out',