@vzn/vx 0.0.595 → 0.0.597
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/launcher.cjs +7 -3
- package/package.json +5 -5
- package/src/cache/archive.ts +35 -6
- package/src/cache/cache.ts +170 -99
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/git-inputs.ts +23 -0
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +99 -44
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +9 -6
- package/src/cache/policy.ts +5 -0
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +12 -6
- package/src/cli/foreign-flags.ts +57 -3
- package/src/cli/info.ts +1 -1
- package/src/cli/init.ts +2 -2
- package/src/cli/last.ts +10 -4
- package/src/cli/lock.ts +5 -3
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/run.ts +73 -30
- package/src/cli/select.ts +6 -3
- package/src/cli/show.ts +8 -2
- package/src/cli/upgrade.ts +1 -1
- package/src/cli/watch-filter.ts +63 -12
- package/src/cli/watch-fs.ts +11 -3
- package/src/cli/watch-judge.ts +15 -8
- package/src/cli/watch.ts +18 -11
- package/src/cli/workspace-config.ts +3 -1
- package/src/config.ts +42 -21
- package/src/exec/executor.ts +1 -1
- package/src/exec/kill-tree.ts +89 -18
- package/src/exec/proc-sample.ts +1 -1
- package/src/exec/runner.ts +74 -36
- package/src/exec/sandbox-paths.ts +12 -1
- package/src/exec/sandbox-runtime.ts +283 -88
- package/src/exec/sandbox-violations.ts +115 -15
- package/src/graph/scheduler.ts +30 -14
- package/src/graph/task-graph.ts +265 -44
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/affected-tasks.ts +6 -2
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/doctor.ts +2 -2
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/excluded-keys.ts +2 -13
- package/src/orchestrator/execute-task.ts +85 -39
- package/src/orchestrator/failure-mode.ts +43 -28
- package/src/orchestrator/framed-output.ts +8 -8
- package/src/orchestrator/index.ts +0 -1
- package/src/orchestrator/keyed-projects.ts +6 -3
- package/src/orchestrator/logger.ts +21 -9
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +9 -13
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +7 -23
- package/src/orchestrator/plugin.ts +37 -16
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +4 -3
- package/src/orchestrator/run-lock.ts +25 -24
- package/src/orchestrator/run.ts +22 -5
- package/src/orchestrator/sandbox-request.ts +96 -18
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/signals.ts +22 -4
- package/src/orchestrator/stable-keys.ts +3 -11
- package/src/orchestrator/summary.ts +7 -13
- package/src/orchestrator/task-hash.ts +13 -3
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/orchestrator/upstream.ts +11 -0
- package/src/util/hangup.ts +42 -0
- package/src/util/index.ts +2 -0
- package/src/util/paths.ts +24 -4
- package/src/workspace/affected.ts +79 -42
- package/src/workspace/config-eval.ts +27 -6
- package/src/workspace/config-imports.ts +8 -53
- package/src/workspace/config-schema.ts +56 -9
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/lockfile.ts +2 -1
- package/src/workspace/migrate-scripts.ts +5 -3
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/project-loader.ts +19 -4
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +53 -24
package/README.md
CHANGED
|
@@ -98,7 +98,7 @@ More: [Quickstart](https://vznjs.github.io/vx/quickstart/) ·
|
|
|
98
98
|
Beside `turbo.json` or `nx.json`, `vx init` writes only a
|
|
99
99
|
`vx.workspace.ts` that maps the old config until then: a temporary
|
|
100
100
|
start, not a way to run the repo. The numbers above are native vx
|
|
101
|
-
config.
|
|
101
|
+
config.
|
|
102
102
|
[Migration guide](https://vznjs.github.io/vx/guides/migrate/).
|
|
103
103
|
|
|
104
104
|
## Why it is faster
|
package/launcher.cjs
CHANGED
|
@@ -18,12 +18,16 @@
|
|
|
18
18
|
// CommonJS, not ESM: Node's ESM loader cost every `vx` call ~8 ms of a
|
|
19
19
|
// ~65 ms launch (min of 21, interleaved, Node 22, 2026-10-01).
|
|
20
20
|
const { execFileSync, spawn, spawnSync } = require('node:child_process')
|
|
21
|
-
const { existsSync, readFileSync } = require('node:fs')
|
|
21
|
+
const { existsSync, readFileSync, realpathSync } = require('node:fs')
|
|
22
22
|
const { constants: osConstants } = require('node:os')
|
|
23
23
|
const { dirname, join } = require('node:path')
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
// The real directory, not `__dirname`: npm's `node_modules/.bin/vx` is a
|
|
26
|
+
// symlink to this file, and under `--preserve-symlinks-main` (NODE_OPTIONS)
|
|
27
|
+
// `__dirname` is `.bin`, where `./package.json` does not resolve and
|
|
28
|
+
// the launcher died MODULE_NOT_FOUND before vx started.
|
|
29
|
+
const here = dirname(realpathSync(__filename))
|
|
30
|
+
const pkg = require(join(here, 'package.json'))
|
|
27
31
|
|
|
28
32
|
// Derive the platform-package prefix + binary basename from this package's
|
|
29
33
|
// name. `base` is the unscoped name — the command AND the binary filename
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.597",
|
|
4
4
|
"description": "The fastest task runner and build cache for JavaScript monorepos.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.597",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.597",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.597",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.597"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/cache/archive.ts
CHANGED
|
@@ -233,7 +233,7 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
233
233
|
const few = args.outputs.size <= ON_THREAD_STATS
|
|
234
234
|
const lstatOf = few ? async (p: string) => lstatSync(p) : lstat
|
|
235
235
|
const statOf = few ? async (p: string) => statSync(p) : stat
|
|
236
|
-
const realpathOf =
|
|
236
|
+
const realpathOf = (p: string) => realOf(p, few)
|
|
237
237
|
let withinReal: Promise<string> | undefined
|
|
238
238
|
// Where this artifact's own non-link outputs really are: a link to one of
|
|
239
239
|
// them packs bytes the task wrote itself. A root-anchored output tree is
|
|
@@ -262,6 +262,16 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
262
262
|
: name.startsWith(WORKSPACE_OUTPUT_PREFIX)
|
|
263
263
|
? `workspaceFiles output ${name.slice(WORKSPACE_OUTPUT_PREFIX.length)}`
|
|
264
264
|
: `output ${name}`
|
|
265
|
+
// The index stores a project row as its bare rel and reads one under
|
|
266
|
+
// `workspace-outputs/` as a workspace output, so every hit of this
|
|
267
|
+
// entry was "missing a recorded output", dropped and run again. The
|
|
268
|
+
// loader refuses a glob written under the name; one that reaches it
|
|
269
|
+
// otherwise (`**/*.js`, `w*/**`) is refused here, where names are known.
|
|
270
|
+
if (name.startsWith(`outputs/${WORKSPACE_OUTPUT_PREFIX}`)) {
|
|
271
|
+
throw new UserError(
|
|
272
|
+
`${shown} is under workspace-outputs/, a name vx's artifacts reserve for outputs.workspaceFiles — write the task's files to another directory, or take them back with a '!' entry`,
|
|
273
|
+
)
|
|
274
|
+
}
|
|
265
275
|
const dangling = (err: NodeJS.ErrnoException): never => {
|
|
266
276
|
if (err.code === 'ENOENT') {
|
|
267
277
|
throw new UserError(
|
|
@@ -293,12 +303,12 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
293
303
|
)
|
|
294
304
|
}
|
|
295
305
|
const mode = st.mode & 0o777
|
|
296
|
-
meta.files[name] = [mode, Math.floor(st.mtimeMs)]
|
|
297
306
|
return {
|
|
298
307
|
name,
|
|
299
308
|
abs: src,
|
|
300
309
|
size: st.size,
|
|
301
310
|
mode,
|
|
311
|
+
mtimeMs: Math.floor(st.mtimeMs),
|
|
302
312
|
// ustar's octal field holds no sign: an mtime before 1970 made the
|
|
303
313
|
// header unreadable and every save of it a "corrupt artifact". The
|
|
304
314
|
// sidecar above carries the real value.
|
|
@@ -306,6 +316,8 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
306
316
|
}
|
|
307
317
|
}),
|
|
308
318
|
)
|
|
319
|
+
// In the outputs' order, not as each stat settled: one tree, one artifact.
|
|
320
|
+
for (const f of files) meta.files[f.name] = [f.mode, f.mtimeMs]
|
|
309
321
|
const stdout = new TextEncoder().encode(args.stdout)
|
|
310
322
|
const metaBytes = new TextEncoder().encode(JSON.stringify(meta))
|
|
311
323
|
const inputs: TarInput[] = [
|
|
@@ -624,11 +636,11 @@ class Extractor {
|
|
|
624
636
|
// deepest EXISTING ancestor and re-append the rest: the answer is the
|
|
625
637
|
// same before and after the base is created, which is what makes the
|
|
626
638
|
// memo safe.
|
|
627
|
-
r = await
|
|
639
|
+
r = await realOf(key).catch(async () => {
|
|
628
640
|
let probe = path.dirname(key)
|
|
629
641
|
const tail: string[] = [path.basename(key)]
|
|
630
642
|
while (probe !== path.dirname(probe)) {
|
|
631
|
-
const real = await
|
|
643
|
+
const real = await realOf(probe).then(
|
|
632
644
|
(v) => v,
|
|
633
645
|
() => null,
|
|
634
646
|
)
|
|
@@ -674,7 +686,7 @@ class Extractor {
|
|
|
674
686
|
// resolve was a round trip spent on a comparison never made (item 627).
|
|
675
687
|
let probe = path.dirname(targetResolved)
|
|
676
688
|
while (probe.startsWith(baseResolved + path.sep)) {
|
|
677
|
-
const real = await
|
|
689
|
+
const real = await realOf(probe).then(
|
|
678
690
|
(r) => r,
|
|
679
691
|
() => null,
|
|
680
692
|
)
|
|
@@ -854,7 +866,7 @@ class Extractor {
|
|
|
854
866
|
const top = path.resolve(s.created)
|
|
855
867
|
// Created at a dangling link's resolved target (`mkdirThroughLink`),
|
|
856
868
|
// so the chain is pruned from where the link leads.
|
|
857
|
-
if (!dir.startsWith(top)) dir = await
|
|
869
|
+
if (!dir.startsWith(top)) dir = await realOf(dir).catch(() => dir)
|
|
858
870
|
while (dir.startsWith(top)) {
|
|
859
871
|
if (
|
|
860
872
|
!(await rmdir(dir).then(
|
|
@@ -903,6 +915,23 @@ async function linkOutError(base: string, probe: string, realBase: string): Prom
|
|
|
903
915
|
)
|
|
904
916
|
}
|
|
905
917
|
|
|
918
|
+
/**
|
|
919
|
+
* `realpath`, for a path holding a backslash too: Bun's answers ENOENT for
|
|
920
|
+
* any such path (1.4.2), so the containment walk took a link named
|
|
921
|
+
* `dist\x` for absent and wrote through it, and a save called a linked
|
|
922
|
+
* output under one dangling. Such a path's links are read one by one.
|
|
923
|
+
*/
|
|
924
|
+
async function realOf(p: string, sync = false): Promise<string> {
|
|
925
|
+
try {
|
|
926
|
+
return sync ? realpathSync(p) : await realpath(p)
|
|
927
|
+
} catch (err) {
|
|
928
|
+
if (!p.includes('\\')) throw err
|
|
929
|
+
const real = await resolveThrough(p)
|
|
930
|
+
await stat(real)
|
|
931
|
+
return real
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
906
935
|
/**
|
|
907
936
|
* Where `p` leads, every link on the way followed, a dangling one included:
|
|
908
937
|
* `realpath` for the part that exists, and past the first missing name the
|
package/src/cache/cache.ts
CHANGED
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
openSync,
|
|
45
45
|
readdirSync,
|
|
46
46
|
renameSync,
|
|
47
|
+
statSync,
|
|
47
48
|
utimesSync,
|
|
48
49
|
writeFileSync,
|
|
49
50
|
} from 'node:fs'
|
|
@@ -111,12 +112,13 @@ import { RunHistory } from './run-history.js'
|
|
|
111
112
|
import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
112
113
|
|
|
113
114
|
/**
|
|
114
|
-
* An artifact or temp file without an `entries` row is reaped
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
115
|
+
* An artifact or temp file without an `entries` row is never reaped
|
|
116
|
+
* younger than this: a crashed save's temp is stale long before it. A
|
|
117
|
+
* row-less artifact may still be in use (another vx version's open dropped
|
|
118
|
+
* its row), so past this it is judged by the retention policy, its file
|
|
119
|
+
* time standing for `accessed_at`. A hit renews a file time older than
|
|
120
|
+
* this, so the file time trails the last use by at most this much, and
|
|
121
|
+
* the policy reads the file time plus this as the last use.
|
|
120
122
|
*/
|
|
121
123
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
122
124
|
|
|
@@ -271,6 +273,62 @@ function indexOfAnotherSchema(dbFile: string): boolean {
|
|
|
271
273
|
const SELECT_ENTRY =
|
|
272
274
|
"SELECT e.*, COALESCE(s.stdout, '') AS stdout FROM entries e LEFT JOIN entry_stdout s ON s.hash = e.hash"
|
|
273
275
|
|
|
276
|
+
/** A file the orphan sweep may judge: a row-less artifact, or a temp. */
|
|
277
|
+
interface RowlessFile {
|
|
278
|
+
file: string
|
|
279
|
+
size: number
|
|
280
|
+
mtimeMs: number
|
|
281
|
+
temp: boolean
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* What a prune takes under `policy`: index rows by `accessed_at` and
|
|
286
|
+
* row-less artifacts by file time, under one rule (a temp always). The
|
|
287
|
+
* rows are the non-phantom ones; the files are past the in-flight grace.
|
|
288
|
+
*/
|
|
289
|
+
function pickVictims(
|
|
290
|
+
rows: ReadonlyArray<{ hash: string; size_bytes: number; accessed_at: number }>,
|
|
291
|
+
files: readonly RowlessFile[],
|
|
292
|
+
policy: { olderThanMs?: number; maxBytes?: number },
|
|
293
|
+
): { hashes: Set<string>; bytesFreed: number; files: RowlessFile[] } {
|
|
294
|
+
const { olderThanMs, maxBytes } = policy
|
|
295
|
+
const hashes = new Set<string>()
|
|
296
|
+
let bytesFreed = 0
|
|
297
|
+
const taken: RowlessFile[] = []
|
|
298
|
+
const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
|
|
299
|
+
let remaining = 0
|
|
300
|
+
for (const r of rows) {
|
|
301
|
+
if (olderThanMs !== undefined && r.accessed_at < olderThanMs) {
|
|
302
|
+
hashes.add(r.hash)
|
|
303
|
+
bytesFreed += r.size_bytes
|
|
304
|
+
} else {
|
|
305
|
+
kept.push({ at: r.accessed_at, size: r.size_bytes, row: r.hash })
|
|
306
|
+
remaining += r.size_bytes
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
for (const f of files) {
|
|
310
|
+
const at = f.mtimeMs + ORPHAN_GRACE_MS
|
|
311
|
+
if (f.temp || (olderThanMs !== undefined && at < olderThanMs)) taken.push(f)
|
|
312
|
+
else {
|
|
313
|
+
kept.push({ at, size: f.size, file: f })
|
|
314
|
+
remaining += f.size
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
if (maxBytes !== undefined && remaining > maxBytes) {
|
|
318
|
+
kept.sort((a, b) => a.at - b.at)
|
|
319
|
+
for (const k of kept) {
|
|
320
|
+
if (remaining <= maxBytes) break
|
|
321
|
+
remaining -= k.size
|
|
322
|
+
if (k.file !== undefined) taken.push(k.file)
|
|
323
|
+
else {
|
|
324
|
+
hashes.add(k.row!)
|
|
325
|
+
bytesFreed += k.size
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { hashes, bytesFreed, files: taken }
|
|
330
|
+
}
|
|
331
|
+
|
|
274
332
|
/**
|
|
275
333
|
* Artifacts `indexed` does not hold, and temps, past the in-flight grace
|
|
276
334
|
* window: one readdir, one stat per candidate. `indexed` is asked only
|
|
@@ -281,7 +339,7 @@ const SELECT_ENTRY =
|
|
|
281
339
|
async function scanOrphanFiles(
|
|
282
340
|
cacheDir: string,
|
|
283
341
|
indexedHashes: () => ReadonlySet<string>,
|
|
284
|
-
): Promise<
|
|
342
|
+
): Promise<RowlessFile[]> {
|
|
285
343
|
let names: string[]
|
|
286
344
|
try {
|
|
287
345
|
names = await readdir(cacheDir)
|
|
@@ -290,19 +348,22 @@ async function scanOrphanFiles(
|
|
|
290
348
|
}
|
|
291
349
|
const indexed = indexedHashes()
|
|
292
350
|
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
293
|
-
const candidates: string
|
|
351
|
+
const candidates: Array<{ name: string; temp: boolean }> = []
|
|
294
352
|
for (const name of names) {
|
|
295
353
|
const m = VX_ARTIFACT_NAME.exec(name)
|
|
296
354
|
if (m === null) continue
|
|
297
|
-
|
|
355
|
+
const temp = m[2] !== undefined
|
|
356
|
+
if (temp || !indexed.has(m[1]!)) candidates.push({ name, temp })
|
|
298
357
|
}
|
|
299
|
-
const found:
|
|
358
|
+
const found: RowlessFile[] = []
|
|
300
359
|
await Promise.all(
|
|
301
|
-
candidates.map(async (name) => {
|
|
360
|
+
candidates.map(async ({ name, temp }) => {
|
|
302
361
|
const file = path.join(cacheDir, name)
|
|
303
362
|
try {
|
|
304
363
|
const st = await stat(file)
|
|
305
|
-
if (st.isFile() && st.mtimeMs <= cutoff)
|
|
364
|
+
if (st.isFile() && st.mtimeMs <= cutoff) {
|
|
365
|
+
found.push({ file, size: st.size, mtimeMs: st.mtimeMs, temp })
|
|
366
|
+
}
|
|
306
367
|
} catch {
|
|
307
368
|
// Gone between readdir and stat: not an orphan any more.
|
|
308
369
|
}
|
|
@@ -570,6 +631,8 @@ export class Cache implements CacheLayer {
|
|
|
570
631
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
571
632
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
572
633
|
private readonly touched = new Set<string>()
|
|
634
|
+
/** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
|
|
635
|
+
private readonly staleTimes = new Set<string>()
|
|
573
636
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
574
637
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
575
638
|
private readonly files: FileHashStore
|
|
@@ -599,7 +662,7 @@ export class Cache implements CacheLayer {
|
|
|
599
662
|
|
|
600
663
|
/**
|
|
601
664
|
* The store this open was asked for and could not use, with why: the
|
|
602
|
-
* entries went to a store inside `cacheDir` instead.
|
|
665
|
+
* entries went to a store inside `cacheDir` instead. Never printed.
|
|
603
666
|
*/
|
|
604
667
|
readonly storeFallback: string | null = null
|
|
605
668
|
|
|
@@ -786,24 +849,33 @@ export class Cache implements CacheLayer {
|
|
|
786
849
|
this.schemaReset = this.db
|
|
787
850
|
.transaction((): SchemaReset | null => {
|
|
788
851
|
const found = readVersion()
|
|
789
|
-
if (found !== undefined && found !== SCHEMA_VERSION) refuseUnreadable(found)
|
|
790
|
-
if (found === undefined) {
|
|
791
|
-
this.db
|
|
792
|
-
.prepare("INSERT INTO schema_meta(key, value) VALUES ('version', ?)")
|
|
793
|
-
.run(SCHEMA_VERSION)
|
|
794
|
-
return null
|
|
795
|
-
}
|
|
796
852
|
if (found === SCHEMA_VERSION) return null
|
|
853
|
+
if (found !== undefined) refuseUnreadable(found)
|
|
797
854
|
// `main.` on every name: unqualified, a table the index lacks
|
|
798
855
|
// resolves to an attached store's (none is attached yet; this
|
|
799
856
|
// keeps it so).
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
857
|
+
if (found !== undefined) {
|
|
858
|
+
this.db.exec(
|
|
859
|
+
'DROP TABLE IF EXISTS main.entries; DROP TABLE IF EXISTS main.runs; DROP TABLE IF EXISTS main.file_hashes; DROP TABLE IF EXISTS main.blob_sizes; DROP TABLE IF EXISTS main.blob_verdicts; DROP TABLE IF EXISTS main.output_files; DROP TABLE IF EXISTS main.output_stamps; DROP TABLE IF EXISTS main.invocations; DROP TABLE IF EXISTS main.run_task_inputs; DROP TABLE IF EXISTS main.entry_inputs; DROP TABLE IF EXISTS main.config_evals; DROP TABLE IF EXISTS main.config_closures; DROP TABLE IF EXISTS main.output_dirs; DROP TABLE IF EXISTS main.entry_stdout; DROP TABLE IF EXISTS main.store_meta;',
|
|
860
|
+
)
|
|
861
|
+
}
|
|
862
|
+
// The tables land under the same write lock as the stamp: made
|
|
863
|
+
// after it, another vx's reset could stamp its own schema in
|
|
864
|
+
// between and keep these tables under it. An index beside a
|
|
865
|
+
// store keeps no entry tables (they would shadow the store's).
|
|
866
|
+
createTables(this.db, 'main')
|
|
867
|
+
const recordedStore = this.db
|
|
868
|
+
.prepare("SELECT 1 FROM schema_meta WHERE key = 'store_dir'")
|
|
869
|
+
.get()
|
|
870
|
+
if (storeRoot === undefined ? recordedStore != null : storeRoot !== null) {
|
|
871
|
+
for (const t of STORE_TABLES) this.db.exec(`DROP TABLE main.${t}`)
|
|
872
|
+
}
|
|
803
873
|
this.db
|
|
804
|
-
.prepare(
|
|
874
|
+
.prepare(
|
|
875
|
+
"INSERT INTO schema_meta(key, value) VALUES ('version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
876
|
+
)
|
|
805
877
|
.run(SCHEMA_VERSION)
|
|
806
|
-
return { from: found, to: SCHEMA_VERSION }
|
|
878
|
+
return found === undefined ? null : { from: found, to: SCHEMA_VERSION }
|
|
807
879
|
})
|
|
808
880
|
.immediate()
|
|
809
881
|
} catch (err) {
|
|
@@ -841,7 +913,8 @@ export class Cache implements CacheLayer {
|
|
|
841
913
|
// the entries stay in this workspace rather than fail the run.
|
|
842
914
|
const fallback = cacheDir
|
|
843
915
|
if (fallback !== storeDir) {
|
|
844
|
-
//
|
|
916
|
+
// Set by the open that falls back, never printed; the ones after it
|
|
917
|
+
// find it recorded.
|
|
845
918
|
if (recorded !== fallback) this.storeFallback = `${storeDir} (${blocked})`
|
|
846
919
|
storeDir = fallback
|
|
847
920
|
openCacheDir(fallback)
|
|
@@ -1268,7 +1341,9 @@ export class Cache implements CacheLayer {
|
|
|
1268
1341
|
|
|
1269
1342
|
// Verify the tar artifact actually exists. The DB and the
|
|
1270
1343
|
// filesystem can drift if someone manually deletes the cache dir.
|
|
1271
|
-
|
|
1344
|
+
const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
|
|
1345
|
+
if (st === undefined) return null
|
|
1346
|
+
if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
|
|
1272
1347
|
|
|
1273
1348
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1274
1349
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
@@ -1301,7 +1376,7 @@ export class Cache implements CacheLayer {
|
|
|
1301
1376
|
): Promise<Map<string, CacheEntry>> {
|
|
1302
1377
|
return this.guard(async () => {
|
|
1303
1378
|
const out = await this.getManyEntries(hashes)
|
|
1304
|
-
if (ctx === undefined || !this.write || out.size === hashes.length) return out
|
|
1379
|
+
if (ctx === undefined || !this.read || !this.write || out.size === hashes.length) return out
|
|
1305
1380
|
// An artifact the index does not know is still a hit (`adopt`).
|
|
1306
1381
|
for (const hash of hashes) {
|
|
1307
1382
|
if (out.has(hash) || !(await this.adopt(hash, ctx(hash)))) continue
|
|
@@ -1318,8 +1393,12 @@ export class Cache implements CacheLayer {
|
|
|
1318
1393
|
const { test, params } = inHashes(hashes)
|
|
1319
1394
|
const rows = this.db.query(`${SELECT_ENTRY} WHERE e.hash ${test}`).all(...params) as EntryRow[]
|
|
1320
1395
|
if (rows.length === 0) return out
|
|
1321
|
-
const
|
|
1322
|
-
const live = rows.filter((
|
|
1396
|
+
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1397
|
+
const live = rows.filter((r) => {
|
|
1398
|
+
const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
|
|
1399
|
+
if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1400
|
+
return st !== undefined
|
|
1401
|
+
})
|
|
1323
1402
|
const liveHashes = live.map((r) => r.hash)
|
|
1324
1403
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1325
1404
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
@@ -1700,6 +1779,9 @@ export class Cache implements CacheLayer {
|
|
|
1700
1779
|
hash,
|
|
1701
1780
|
`remote body runs past ${cap} bytes (the artifact ceiling's bound)`,
|
|
1702
1781
|
)
|
|
1782
|
+
// Every refusal releases the remote body: a held response pins its
|
|
1783
|
+
// connection, and a layer that settles on the body's end never hears.
|
|
1784
|
+
let counted: ReadableStream<Uint8Array> | undefined
|
|
1703
1785
|
try {
|
|
1704
1786
|
if (body instanceof Blob) {
|
|
1705
1787
|
if (body.size > cap) throw past()
|
|
@@ -1708,7 +1790,7 @@ export class Cache implements CacheLayer {
|
|
|
1708
1790
|
throw past()
|
|
1709
1791
|
} else {
|
|
1710
1792
|
let n = 0
|
|
1711
|
-
|
|
1793
|
+
counted = body.body?.pipeThrough(
|
|
1712
1794
|
new TransformStream<Uint8Array, Uint8Array>({
|
|
1713
1795
|
transform(chunk, controller) {
|
|
1714
1796
|
n += chunk.byteLength
|
|
@@ -1720,6 +1802,9 @@ export class Cache implements CacheLayer {
|
|
|
1720
1802
|
await Bun.write(tmpPath, new Response(counted ?? null))
|
|
1721
1803
|
}
|
|
1722
1804
|
} catch (err) {
|
|
1805
|
+
// A failed write leaves the pipe's end unlocked; cancelling it
|
|
1806
|
+
// cancels the source. A body refused by its length is cancelled itself.
|
|
1807
|
+
if (!(body instanceof Blob)) void (counted ?? body.body)?.cancel(err).catch(() => undefined)
|
|
1723
1808
|
await unlink(tmpPath).catch(() => undefined)
|
|
1724
1809
|
throw err
|
|
1725
1810
|
}
|
|
@@ -2047,6 +2132,17 @@ export class Cache implements CacheLayer {
|
|
|
2047
2132
|
const hashes = [...this.touched]
|
|
2048
2133
|
this.touched.clear()
|
|
2049
2134
|
const now = Date.now()
|
|
2135
|
+
// The file time is the last use another vx version's sweep can see:
|
|
2136
|
+
// its open dropped this index's rows (`pickVictims`).
|
|
2137
|
+
const at = new Date(now)
|
|
2138
|
+
for (const hash of this.staleTimes) {
|
|
2139
|
+
try {
|
|
2140
|
+
utimesSync(this.tarPath(hash), at, at)
|
|
2141
|
+
} catch {
|
|
2142
|
+
// Gone, or another user's file: a sweep judges its old time.
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
this.staleTimes.clear()
|
|
2050
2146
|
// LRU bookkeeping: on a full disk the bumps are dropped, never the prune
|
|
2051
2147
|
// or the stats that asked for them (A-14).
|
|
2052
2148
|
try {
|
|
@@ -2161,21 +2257,26 @@ export class Cache implements CacheLayer {
|
|
|
2161
2257
|
// above never sees them (upstream survey, nx#35483: 9 MiB of them sat
|
|
2162
2258
|
// under a 1 MB limit). Listing the directory to find them costs 0.5 ms
|
|
2163
2259
|
// per 1,000 entries (4.9 ms at 10,000), every run; the sweep's own
|
|
2164
|
-
// clock in `schema_meta` costs one indexed read.
|
|
2165
|
-
//
|
|
2166
|
-
// it within two.
|
|
2260
|
+
// clock in `schema_meta` costs one indexed read. The sweep is the
|
|
2261
|
+
// policy's own prune, with the row-less files counted.
|
|
2167
2262
|
const swept = this.db.prepare('SELECT value FROM schema_meta WHERE key = ?').get(SWEPT_AT) as {
|
|
2168
2263
|
value: string
|
|
2169
2264
|
} | null
|
|
2170
2265
|
if (swept !== null && now - Number(swept.value) < ORPHAN_GRACE_MS) return null
|
|
2171
|
-
return
|
|
2266
|
+
return this.pruneIndex(
|
|
2267
|
+
{
|
|
2268
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2269
|
+
...(policy.maxBytes !== undefined ? { maxBytes: policy.maxBytes } : {}),
|
|
2270
|
+
},
|
|
2271
|
+
now,
|
|
2272
|
+
)
|
|
2172
2273
|
}
|
|
2173
2274
|
|
|
2174
2275
|
async prune(options: PruneOptions): Promise<PruneResult> {
|
|
2175
2276
|
return this.guard(() => this.pruneIndex(options))
|
|
2176
2277
|
}
|
|
2177
2278
|
|
|
2178
|
-
private async pruneIndex(options: PruneOptions): Promise<PruneResult> {
|
|
2279
|
+
private async pruneIndex(options: PruneOptions, now: number = Date.now()): Promise<PruneResult> {
|
|
2179
2280
|
this.flushAccessed()
|
|
2180
2281
|
// Before any entry is deleted, so a kept entry's pending snapshot is on
|
|
2181
2282
|
// disk whatever the eviction does next. A pruned hash's snapshot cannot
|
|
@@ -2187,50 +2288,24 @@ export class Cache implements CacheLayer {
|
|
|
2187
2288
|
throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
|
|
2188
2289
|
}
|
|
2189
2290
|
|
|
2190
|
-
const victims = new Set<string>()
|
|
2191
|
-
let bytesFreed = 0
|
|
2192
2291
|
// Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
|
|
2193
2292
|
// never a hit, and their bytes are on no disk, but `--max-size` counted
|
|
2194
2293
|
// them and evicted real entries to make room for them (item 975). None
|
|
2195
2294
|
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
2196
2295
|
// grace window are dropped.
|
|
2197
|
-
const { phantoms, stale } = await this.phantomRows()
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
if (maxBytes !== undefined) {
|
|
2211
|
-
const totalRow = this.db
|
|
2212
|
-
.prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
|
|
2213
|
-
.get() as { bytes: number }
|
|
2214
|
-
let phantomBytes = 0
|
|
2215
|
-
for (const b of phantoms.values()) phantomBytes += b
|
|
2216
|
-
let remaining = totalRow.bytes - phantomBytes - bytesFreed
|
|
2217
|
-
if (remaining > maxBytes) {
|
|
2218
|
-
// Exclude already-picked victims in JS, not via a SQL NOT-IN —
|
|
2219
|
-
// an IN-list over tens of thousands of TTL victims would blow
|
|
2220
|
-
// SQLite's bound-parameter ceiling (see flushAccessed's 900 cap).
|
|
2221
|
-
const candidates = (
|
|
2222
|
-
this.db
|
|
2223
|
-
.prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
|
|
2224
|
-
.all() as Array<{ hash: string; size_bytes: number }>
|
|
2225
|
-
).filter((row) => !victims.has(row.hash) && !phantoms.has(row.hash))
|
|
2226
|
-
for (const row of candidates) {
|
|
2227
|
-
if (remaining <= maxBytes) break
|
|
2228
|
-
victims.add(row.hash)
|
|
2229
|
-
bytesFreed += row.size_bytes
|
|
2230
|
-
remaining -= row.size_bytes
|
|
2231
|
-
}
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2296
|
+
const { phantoms, stale, rows } = await this.phantomRows()
|
|
2297
|
+
const picked = pickVictims(
|
|
2298
|
+
rows.filter((r) => !phantoms.has(r.hash)),
|
|
2299
|
+
await this.scanOrphans(),
|
|
2300
|
+
{
|
|
2301
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2302
|
+
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
2303
|
+
},
|
|
2304
|
+
)
|
|
2305
|
+
const victims = picked.hashes
|
|
2306
|
+
const { bytesFreed } = picked
|
|
2307
|
+
let orphanBytes = 0
|
|
2308
|
+
for (const o of picked.files) orphanBytes += o.size
|
|
2234
2309
|
|
|
2235
2310
|
// Delete DB rows in a single transaction (one fsync; ON DELETE
|
|
2236
2311
|
// CASCADE clears `output_files`) and unlink artifacts in parallel.
|
|
@@ -2239,13 +2314,7 @@ export class Cache implements CacheLayer {
|
|
|
2239
2314
|
// flushAccessed so a huge eviction stays under any build's
|
|
2240
2315
|
// bound-parameter ceiling.
|
|
2241
2316
|
if (dryRun) {
|
|
2242
|
-
|
|
2243
|
-
return {
|
|
2244
|
-
evicted: victims.size,
|
|
2245
|
-
bytesFreed,
|
|
2246
|
-
orphans: orphans.orphans,
|
|
2247
|
-
orphanBytes: orphans.orphanBytes,
|
|
2248
|
-
}
|
|
2317
|
+
return { evicted: victims.size, bytesFreed, orphans: picked.files.length, orphanBytes }
|
|
2249
2318
|
}
|
|
2250
2319
|
if (victims.size > 0 || stale.length > 0) {
|
|
2251
2320
|
const hashes = [...victims]
|
|
@@ -2275,23 +2344,21 @@ export class Cache implements CacheLayer {
|
|
|
2275
2344
|
} catch (err) {
|
|
2276
2345
|
if (!isIndexFull(err)) throw err
|
|
2277
2346
|
}
|
|
2278
|
-
const orphans = await this.reapOrphans()
|
|
2347
|
+
const orphans = await this.reapOrphans(picked.files, now)
|
|
2279
2348
|
return { evicted: victims.size, bytesFreed, ...orphans }
|
|
2280
2349
|
}
|
|
2281
2350
|
|
|
2282
2351
|
/**
|
|
2283
|
-
* Unlink
|
|
2284
|
-
* `entries` row (
|
|
2285
|
-
* and a `<hash>.tar.zst.tmp-*` a
|
|
2286
|
-
* else reaps them — a lookup starts
|
|
2287
|
-
* key renames over the file, so a key
|
|
2288
|
-
*
|
|
2289
|
-
* renames the artifact into place BEFORE its row commits, and its temp
|
|
2290
|
-
* exists while the bytes are still being written, so a fresh file
|
|
2291
|
-
* without a row is a save in flight, not an orphan.
|
|
2352
|
+
* Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
|
|
2353
|
+
* no `entries` row (another vx version's open dropped it, a `cache.db`
|
|
2354
|
+
* deleted by hand) past the policy, and a `<hash>.tar.zst.tmp-*` a
|
|
2355
|
+
* crashed save never renamed. Nothing else reaps them — a lookup starts
|
|
2356
|
+
* at the row, and a save of the same key renames over the file, so a key
|
|
2357
|
+
* that never recurs leaks its bytes forever.
|
|
2292
2358
|
*/
|
|
2293
2359
|
private async reapOrphans(
|
|
2294
|
-
|
|
2360
|
+
taken: readonly RowlessFile[],
|
|
2361
|
+
now: number,
|
|
2295
2362
|
): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2296
2363
|
try {
|
|
2297
2364
|
this.db
|
|
@@ -2304,7 +2371,7 @@ export class Cache implements CacheLayer {
|
|
|
2304
2371
|
let orphans = 0
|
|
2305
2372
|
let orphanBytes = 0
|
|
2306
2373
|
await Promise.all(
|
|
2307
|
-
|
|
2374
|
+
taken.map(async (o) => {
|
|
2308
2375
|
try {
|
|
2309
2376
|
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
2310
2377
|
// concurrent prune took first must not be counted as ours.
|
|
@@ -2328,7 +2395,11 @@ export class Cache implements CacheLayer {
|
|
|
2328
2395
|
* dropped — a row is never deleted on one listing's word while it may be
|
|
2329
2396
|
* in use. An unreadable directory judges nothing.
|
|
2330
2397
|
*/
|
|
2331
|
-
private async phantomRows(): Promise<{
|
|
2398
|
+
private async phantomRows(): Promise<{
|
|
2399
|
+
phantoms: Map<string, number>
|
|
2400
|
+
stale: string[]
|
|
2401
|
+
rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
|
|
2402
|
+
}> {
|
|
2332
2403
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
2333
2404
|
const rows = this.db
|
|
2334
2405
|
.prepare('SELECT hash, size_bytes, accessed_at FROM entries')
|
|
@@ -2341,7 +2412,7 @@ export class Cache implements CacheLayer {
|
|
|
2341
2412
|
try {
|
|
2342
2413
|
names = await readdir(this.artifactDir)
|
|
2343
2414
|
} catch {
|
|
2344
|
-
return { phantoms: new Map(), stale: [] }
|
|
2415
|
+
return { phantoms: new Map(), stale: [], rows }
|
|
2345
2416
|
}
|
|
2346
2417
|
const present = new Set(names)
|
|
2347
2418
|
const phantoms = new Map<string, number>()
|
|
@@ -2351,10 +2422,10 @@ export class Cache implements CacheLayer {
|
|
|
2351
2422
|
phantoms.set(r.hash, r.size_bytes)
|
|
2352
2423
|
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
2353
2424
|
}
|
|
2354
|
-
return { phantoms, stale }
|
|
2425
|
+
return { phantoms, stale, rows }
|
|
2355
2426
|
}
|
|
2356
2427
|
|
|
2357
|
-
/**
|
|
2428
|
+
/** Row-less files past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
|
|
2358
2429
|
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2359
2430
|
const found = await this.scanOrphans()
|
|
2360
2431
|
let orphanBytes = 0
|
|
@@ -2363,7 +2434,7 @@ export class Cache implements CacheLayer {
|
|
|
2363
2434
|
}
|
|
2364
2435
|
|
|
2365
2436
|
/** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
|
|
2366
|
-
private async scanOrphans(): Promise<
|
|
2437
|
+
private async scanOrphans(): Promise<RowlessFile[]> {
|
|
2367
2438
|
const found = await scanOrphanFiles(
|
|
2368
2439
|
this.artifactDir,
|
|
2369
2440
|
() =>
|