@vzn/vx 0.0.594 → 0.0.596
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +5 -5
- package/src/bin.ts +26 -11
- package/src/cache/archive.ts +2 -2
- package/src/cache/cache.ts +235 -174
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/config-evals.ts +19 -4
- package/src/cache/git-inputs.ts +19 -8
- package/src/cache/index.ts +1 -0
- package/src/cache/inputs.ts +122 -49
- package/src/cache/layer.ts +7 -5
- package/src/cache/layered-cache.ts +25 -2
- package/src/cache/policy.ts +5 -0
- package/src/cache/tar-stream.ts +1 -1
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +11 -15
- package/src/cli/completions.ts +42 -6
- package/src/cli/foreign-flags.ts +26 -5
- package/src/cli/help.ts +5 -3
- package/src/cli/init.ts +1 -1
- package/src/cli/last.ts +4 -1
- package/src/cli/plan-format.ts +9 -0
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/run.ts +60 -42
- package/src/cli/select.ts +30 -3
- package/src/cli/show.ts +42 -3
- package/src/cli/watch-filter.ts +77 -13
- package/src/cli/watch-fs.ts +15 -7
- package/src/cli/watch-judge.ts +16 -9
- package/src/cli/watch.ts +22 -15
- package/src/config.ts +26 -26
- package/src/exec/env.ts +5 -0
- package/src/exec/index.ts +1 -0
- package/src/exec/proc-sample.ts +6 -2
- package/src/exec/runner.ts +98 -40
- package/src/exec/sandbox-paths.ts +12 -6
- package/src/exec/sandbox-runtime.ts +136 -44
- package/src/exec/sandbox-violations.ts +58 -11
- package/src/graph/index.ts +1 -0
- package/src/graph/scheduler.ts +29 -13
- package/src/graph/task-graph.ts +134 -25
- package/src/index.ts +6 -0
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/execute-task.ts +65 -20
- package/src/orchestrator/failure-mode.ts +8 -2
- package/src/orchestrator/framed-output.ts +3 -3
- package/src/orchestrator/hit-restore.ts +6 -3
- package/src/orchestrator/local-shortcircuit.ts +4 -1
- package/src/orchestrator/lockfile-claim.ts +43 -46
- package/src/orchestrator/logger.ts +11 -6
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +2 -2
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +28 -4
- package/src/orchestrator/plugin-host.ts +25 -3
- package/src/orchestrator/plugin.ts +39 -17
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +2 -1
- package/src/orchestrator/run.ts +72 -12
- package/src/orchestrator/sandbox-request.ts +65 -9
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/signals.ts +5 -2
- package/src/orchestrator/summary.ts +16 -5
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/util/index.ts +7 -1
- package/src/util/paths.ts +23 -3
- package/src/util/secret-mask.ts +36 -0
- package/src/util/settle.ts +15 -0
- package/src/workspace/affected.ts +30 -8
- package/src/workspace/config-cache.ts +25 -4
- package/src/workspace/config-imports.ts +8 -6
- package/src/workspace/config-schema.ts +74 -9
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/migrate-scripts.ts +5 -3
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/package-graph.ts +15 -3
- package/src/workspace/project-loader.ts +76 -2
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +98 -20
package/src/cache/cache.ts
CHANGED
|
@@ -44,6 +44,8 @@ import {
|
|
|
44
44
|
openSync,
|
|
45
45
|
readdirSync,
|
|
46
46
|
renameSync,
|
|
47
|
+
statSync,
|
|
48
|
+
utimesSync,
|
|
47
49
|
writeFileSync,
|
|
48
50
|
} from 'node:fs'
|
|
49
51
|
import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
@@ -110,12 +112,13 @@ import { RunHistory } from './run-history.js'
|
|
|
110
112
|
import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
111
113
|
|
|
112
114
|
/**
|
|
113
|
-
* An artifact or temp file without an `entries` row is reaped
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
115
|
+
* An artifact or temp file without an `entries` row is never reaped
|
|
116
|
+
* younger than this: a crashed save's temp is stale long before it. A
|
|
117
|
+
* row-less artifact may still be in use (another vx version's open dropped
|
|
118
|
+
* its row), so past this it is judged by the retention policy, its file
|
|
119
|
+
* time standing for `accessed_at`. A hit renews a file time older than
|
|
120
|
+
* this, so the file time trails the last use by at most this much, and
|
|
121
|
+
* the policy reads the file time plus this as the last use.
|
|
119
122
|
*/
|
|
120
123
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
121
124
|
|
|
@@ -242,10 +245,90 @@ export const SCHEMA_VERSION = 'v32'
|
|
|
242
245
|
/** The tables a store holds: dropped from a workspace index that held them itself. */
|
|
243
246
|
const STORE_TABLES = ['entry_inputs', 'output_files', 'entry_stdout', 'store_meta', 'entries']
|
|
244
247
|
|
|
248
|
+
/**
|
|
249
|
+
* An index that records a schema other than this vx's: the one an opener
|
|
250
|
+
* that writes resets. Reads the recorded version alone and touches
|
|
251
|
+
* nothing; an unreadable file is left to the open to refuse.
|
|
252
|
+
*/
|
|
253
|
+
function indexOfAnotherSchema(dbFile: string): boolean {
|
|
254
|
+
let found: string | undefined
|
|
255
|
+
try {
|
|
256
|
+
const db = new Database(dbFile, { readonly: true })
|
|
257
|
+
try {
|
|
258
|
+
found = (
|
|
259
|
+
db.query("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
|
|
260
|
+
value: string
|
|
261
|
+
} | null
|
|
262
|
+
)?.value
|
|
263
|
+
} finally {
|
|
264
|
+
closeDb(db)
|
|
265
|
+
}
|
|
266
|
+
} catch {
|
|
267
|
+
return false
|
|
268
|
+
}
|
|
269
|
+
return found !== undefined && found !== SCHEMA_VERSION
|
|
270
|
+
}
|
|
271
|
+
|
|
245
272
|
/** An entry row with its stdout, which lives apart (v29); none stored reads as ''. */
|
|
246
273
|
const SELECT_ENTRY =
|
|
247
274
|
"SELECT e.*, COALESCE(s.stdout, '') AS stdout FROM entries e LEFT JOIN entry_stdout s ON s.hash = e.hash"
|
|
248
275
|
|
|
276
|
+
/** A file the orphan sweep may judge: a row-less artifact, or a temp. */
|
|
277
|
+
interface RowlessFile {
|
|
278
|
+
file: string
|
|
279
|
+
size: number
|
|
280
|
+
mtimeMs: number
|
|
281
|
+
temp: boolean
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* What a prune takes under `policy`: index rows by `accessed_at` and
|
|
286
|
+
* row-less artifacts by file time, under one rule (a temp always). The
|
|
287
|
+
* rows are the non-phantom ones; the files are past the in-flight grace.
|
|
288
|
+
*/
|
|
289
|
+
function pickVictims(
|
|
290
|
+
rows: ReadonlyArray<{ hash: string; size_bytes: number; accessed_at: number }>,
|
|
291
|
+
files: readonly RowlessFile[],
|
|
292
|
+
policy: { olderThanMs?: number; maxBytes?: number },
|
|
293
|
+
): { hashes: Set<string>; bytesFreed: number; files: RowlessFile[] } {
|
|
294
|
+
const { olderThanMs, maxBytes } = policy
|
|
295
|
+
const hashes = new Set<string>()
|
|
296
|
+
let bytesFreed = 0
|
|
297
|
+
const taken: RowlessFile[] = []
|
|
298
|
+
const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
|
|
299
|
+
let remaining = 0
|
|
300
|
+
for (const r of rows) {
|
|
301
|
+
if (olderThanMs !== undefined && r.accessed_at < olderThanMs) {
|
|
302
|
+
hashes.add(r.hash)
|
|
303
|
+
bytesFreed += r.size_bytes
|
|
304
|
+
} else {
|
|
305
|
+
kept.push({ at: r.accessed_at, size: r.size_bytes, row: r.hash })
|
|
306
|
+
remaining += r.size_bytes
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
for (const f of files) {
|
|
310
|
+
const at = f.mtimeMs + ORPHAN_GRACE_MS
|
|
311
|
+
if (f.temp || (olderThanMs !== undefined && at < olderThanMs)) taken.push(f)
|
|
312
|
+
else {
|
|
313
|
+
kept.push({ at, size: f.size, file: f })
|
|
314
|
+
remaining += f.size
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
if (maxBytes !== undefined && remaining > maxBytes) {
|
|
318
|
+
kept.sort((a, b) => a.at - b.at)
|
|
319
|
+
for (const k of kept) {
|
|
320
|
+
if (remaining <= maxBytes) break
|
|
321
|
+
remaining -= k.size
|
|
322
|
+
if (k.file !== undefined) taken.push(k.file)
|
|
323
|
+
else {
|
|
324
|
+
hashes.add(k.row!)
|
|
325
|
+
bytesFreed += k.size
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { hashes, bytesFreed, files: taken }
|
|
330
|
+
}
|
|
331
|
+
|
|
249
332
|
/**
|
|
250
333
|
* Artifacts `indexed` does not hold, and temps, past the in-flight grace
|
|
251
334
|
* window: one readdir, one stat per candidate. `indexed` is asked only
|
|
@@ -256,7 +339,7 @@ const SELECT_ENTRY =
|
|
|
256
339
|
async function scanOrphanFiles(
|
|
257
340
|
cacheDir: string,
|
|
258
341
|
indexedHashes: () => ReadonlySet<string>,
|
|
259
|
-
): Promise<
|
|
342
|
+
): Promise<RowlessFile[]> {
|
|
260
343
|
let names: string[]
|
|
261
344
|
try {
|
|
262
345
|
names = await readdir(cacheDir)
|
|
@@ -265,19 +348,22 @@ async function scanOrphanFiles(
|
|
|
265
348
|
}
|
|
266
349
|
const indexed = indexedHashes()
|
|
267
350
|
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
268
|
-
const candidates: string
|
|
351
|
+
const candidates: Array<{ name: string; temp: boolean }> = []
|
|
269
352
|
for (const name of names) {
|
|
270
353
|
const m = VX_ARTIFACT_NAME.exec(name)
|
|
271
354
|
if (m === null) continue
|
|
272
|
-
|
|
355
|
+
const temp = m[2] !== undefined
|
|
356
|
+
if (temp || !indexed.has(m[1]!)) candidates.push({ name, temp })
|
|
273
357
|
}
|
|
274
|
-
const found:
|
|
358
|
+
const found: RowlessFile[] = []
|
|
275
359
|
await Promise.all(
|
|
276
|
-
candidates.map(async (name) => {
|
|
360
|
+
candidates.map(async ({ name, temp }) => {
|
|
277
361
|
const file = path.join(cacheDir, name)
|
|
278
362
|
try {
|
|
279
363
|
const st = await stat(file)
|
|
280
|
-
if (st.isFile() && st.mtimeMs <= cutoff)
|
|
364
|
+
if (st.isFile() && st.mtimeMs <= cutoff) {
|
|
365
|
+
found.push({ file, size: st.size, mtimeMs: st.mtimeMs, temp })
|
|
366
|
+
}
|
|
281
367
|
} catch {
|
|
282
368
|
// Gone between readdir and stat: not an orphan any more.
|
|
283
369
|
}
|
|
@@ -545,6 +631,8 @@ export class Cache implements CacheLayer {
|
|
|
545
631
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
546
632
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
547
633
|
private readonly touched = new Set<string>()
|
|
634
|
+
/** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
|
|
635
|
+
private readonly staleTimes = new Set<string>()
|
|
548
636
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
549
637
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
550
638
|
private readonly files: FileHashStore
|
|
@@ -608,41 +696,6 @@ export class Cache implements CacheLayer {
|
|
|
608
696
|
return new Cache(cacheDir, undefined, undefined, undefined, 'inspect')
|
|
609
697
|
}
|
|
610
698
|
|
|
611
|
-
/**
|
|
612
|
-
* What a real prune reaps from an index of an EARLIER schema, which it
|
|
613
|
-
* resets first, leaving every aged artifact row-less: a dry run that
|
|
614
|
-
* refused it could not preview the biggest prune there is, the one
|
|
615
|
-
* after an upgrade (item 1083). Reads the recorded version alone and
|
|
616
|
-
* touches nothing. Null for an absent, current, newer or unreadable
|
|
617
|
-
* index; `Cache.inspect` answers those.
|
|
618
|
-
*/
|
|
619
|
-
static async orphansBeforeReset(
|
|
620
|
-
cacheDir: string,
|
|
621
|
-
): Promise<{ found: string; orphans: number; orphanBytes: number } | null> {
|
|
622
|
-
const dbFile = path.join(cacheDir, 'cache.db')
|
|
623
|
-
if (!existsSync(dbFile)) return null
|
|
624
|
-
let found: string | undefined
|
|
625
|
-
try {
|
|
626
|
-
const db = new Database(dbFile, { readonly: true })
|
|
627
|
-
try {
|
|
628
|
-
found = (
|
|
629
|
-
db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
|
|
630
|
-
value: string
|
|
631
|
-
} | null
|
|
632
|
-
)?.value
|
|
633
|
-
} finally {
|
|
634
|
-
closeDb(db)
|
|
635
|
-
}
|
|
636
|
-
} catch {
|
|
637
|
-
return null
|
|
638
|
-
}
|
|
639
|
-
if (found === undefined || found === SCHEMA_VERSION) return null
|
|
640
|
-
const aged = await scanOrphanFiles(cacheDir, () => new Set())
|
|
641
|
-
let orphanBytes = 0
|
|
642
|
-
for (const o of aged) orphanBytes += o.size
|
|
643
|
-
return { found, orphans: aged.length, orphanBytes }
|
|
644
|
-
}
|
|
645
|
-
|
|
646
699
|
constructor(
|
|
647
700
|
private readonly cacheDir: string,
|
|
648
701
|
localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
|
|
@@ -655,23 +708,25 @@ export class Cache implements CacheLayer {
|
|
|
655
708
|
*/
|
|
656
709
|
private readonly artifactCeiling: number = MAX_DECOMPRESSED_ARTIFACT_BYTES,
|
|
657
710
|
/**
|
|
658
|
-
* `'inspect'`: a reading verb (`why`, `last`, `info
|
|
659
|
-
*
|
|
660
|
-
*
|
|
711
|
+
* `'inspect'`: a reading verb (`why`, `last`, `info`). It never resets
|
|
712
|
+
* the index: a schema it cannot read is refused, named, and left as it
|
|
713
|
+
* was. `'preview'`: a dry prune, which reads such an index as the reset
|
|
714
|
+
* the real prune does first leaves it, empty, with the store still
|
|
715
|
+
* reached, so it names what that prune takes (item 1083).
|
|
661
716
|
*/
|
|
662
|
-
mode: 'open' | 'inspect' = 'open',
|
|
717
|
+
mode: 'open' | 'inspect' | 'preview' = 'open',
|
|
663
718
|
/**
|
|
664
719
|
* The shared store's directory, unversioned: every key is seeded with
|
|
665
720
|
* `CACHE_VERSION`, so two vx versions never read each other's entries,
|
|
666
721
|
* and the store's own schema is `store_meta.schema` (`matchStoreSchema`). Or
|
|
667
722
|
* `null` for an index that holds its entries itself (a `cacheDir`).
|
|
668
723
|
* Undefined follows the layout the index records: a reading verb, a
|
|
669
|
-
* plugin's handle.
|
|
724
|
+
* plugin's handle. A reading open reads it only where the index
|
|
670
725
|
* records none (deleted, or never written beside a run's store).
|
|
671
726
|
*/
|
|
672
727
|
storeRoot?: string | null,
|
|
673
728
|
) {
|
|
674
|
-
this.inspecting = mode
|
|
729
|
+
this.inspecting = mode !== 'open'
|
|
675
730
|
this.read = localPolicy.read
|
|
676
731
|
// The directory exists before the DB opens — bun:sqlite won't create
|
|
677
732
|
// parent dirs for us. A directory this user cannot write into is a
|
|
@@ -687,7 +742,9 @@ export class Cache implements CacheLayer {
|
|
|
687
742
|
// said "no recorded runs yet" (item 900).
|
|
688
743
|
const dbFile = path.join(cacheDir, 'cache.db')
|
|
689
744
|
this.dbFile = dbFile
|
|
690
|
-
const absent =
|
|
745
|
+
const absent =
|
|
746
|
+
mode !== 'open' &&
|
|
747
|
+
(!existsSync(dbFile) || (mode === 'preview' && indexOfAnotherSchema(dbFile)))
|
|
691
748
|
this.writeBlocked = absent ? 'no index there yet' : openCacheDir(cacheDir, repoDir)
|
|
692
749
|
this.write = localPolicy.write && this.writeBlocked === null
|
|
693
750
|
try {
|
|
@@ -779,7 +836,7 @@ export class Cache implements CacheLayer {
|
|
|
779
836
|
// 2026-10-06; a newer one used to be refused, item 896). A reading verb
|
|
780
837
|
// leaves it as it was.
|
|
781
838
|
const refuseUnreadable = (found: string): void => {
|
|
782
|
-
if (mode
|
|
839
|
+
if (mode !== 'open') {
|
|
783
840
|
throw new UserError(
|
|
784
841
|
`the cache at ${cacheDir} holds index schema ${found} from another vx version; this vx reads ${SCHEMA_VERSION}, so nothing in it is readable here, and this verb leaves it untouched. The next \`vx run\` resets it`,
|
|
785
842
|
)
|
|
@@ -833,7 +890,7 @@ export class Cache implements CacheLayer {
|
|
|
833
890
|
)?.value,
|
|
834
891
|
)
|
|
835
892
|
let storeDir =
|
|
836
|
-
mode
|
|
893
|
+
mode !== 'open'
|
|
837
894
|
? (recorded ?? (typeof storeRoot === 'string' ? storeRoot : undefined))
|
|
838
895
|
: storeRoot === undefined
|
|
839
896
|
? recorded
|
|
@@ -877,17 +934,10 @@ export class Cache implements CacheLayer {
|
|
|
877
934
|
this.storeDir = storeDir
|
|
878
935
|
this.artifactDir = storeDir ?? cacheDir
|
|
879
936
|
if (storeDir !== undefined) {
|
|
880
|
-
this.attachStore(storeDir, mode
|
|
937
|
+
this.attachStore(storeDir, mode !== 'open')
|
|
881
938
|
this.storeReset = this.matchStoreSchema(mode === 'open' && this.writeBlocked === null)
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
createTables(this.db, storeDir === undefined ? 'main' : 'store')
|
|
885
|
-
if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
|
|
886
|
-
this.db
|
|
887
|
-
.prepare(
|
|
888
|
-
"INSERT INTO store.store_meta(key, value) VALUES ('schema', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
889
|
-
)
|
|
890
|
-
.run(SCHEMA_VERSION)
|
|
939
|
+
} else {
|
|
940
|
+
createTables(this.db, 'main')
|
|
891
941
|
}
|
|
892
942
|
if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
|
|
893
943
|
if (storeDir === undefined)
|
|
@@ -1018,50 +1068,63 @@ export class Cache implements CacheLayer {
|
|
|
1018
1068
|
* reads it: a store another `SCHEMA_VERSION` wrote has its tables dropped
|
|
1019
1069
|
* (owner, 2026-10-06), the artifacts kept. Each is indexed again from its
|
|
1020
1070
|
* own bytes on its next hit (`adopt`). A reading verb reads such a store
|
|
1021
|
-
* as empty and changes nothing.
|
|
1071
|
+
* as empty and changes nothing. Creates the store's tables either way.
|
|
1022
1072
|
*/
|
|
1023
1073
|
private matchStoreSchema(writable: boolean): SchemaReset | null {
|
|
1024
|
-
const
|
|
1025
|
-
(
|
|
1026
|
-
|
|
1027
|
-
.
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1074
|
+
const read = (): { has: boolean; found: string | undefined } => {
|
|
1075
|
+
const tables = new Set(
|
|
1076
|
+
(
|
|
1077
|
+
this.db
|
|
1078
|
+
.prepare("SELECT name FROM store.sqlite_master WHERE type = 'table'")
|
|
1079
|
+
.all() as Array<{
|
|
1080
|
+
name: string
|
|
1081
|
+
}>
|
|
1082
|
+
).map((r) => r.name),
|
|
1083
|
+
)
|
|
1084
|
+
const found = tables.has('store_meta')
|
|
1085
|
+
? (
|
|
1086
|
+
this.db.prepare("SELECT value FROM store.store_meta WHERE key = 'schema'").get() as {
|
|
1087
|
+
value: string
|
|
1088
|
+
} | null
|
|
1089
|
+
)?.value
|
|
1090
|
+
: undefined
|
|
1091
|
+
return { has: tables.has('entries'), found }
|
|
1092
|
+
}
|
|
1093
|
+
const seen = read()
|
|
1094
|
+
if (seen.has && seen.found === SCHEMA_VERSION) {
|
|
1095
|
+
createTables(this.db, 'store')
|
|
1096
|
+
return null
|
|
1097
|
+
}
|
|
1042
1098
|
if (!writable) {
|
|
1043
|
-
|
|
1044
|
-
|
|
1099
|
+
if (seen.has) {
|
|
1100
|
+
this.db.exec('DETACH DATABASE store')
|
|
1101
|
+
this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
|
|
1102
|
+
}
|
|
1103
|
+
createTables(this.db, 'store')
|
|
1045
1104
|
return null
|
|
1046
1105
|
}
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
const now = (
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
if (t !== 'store_meta') this.db.exec(`DROP TABLE IF EXISTS store.${t}`)
|
|
1106
|
+
// Re-read, drop, create and stamp under one write lock: as separate
|
|
1107
|
+
// transactions, another vx's open landing after the drop made its own
|
|
1108
|
+
// tables, and this one's `CREATE IF NOT EXISTS` kept them under its stamp.
|
|
1109
|
+
return this.db
|
|
1110
|
+
.transaction((): SchemaReset | null => {
|
|
1111
|
+
const now = read()
|
|
1112
|
+
let reset: SchemaReset | null = null
|
|
1113
|
+
if (now.has && now.found !== SCHEMA_VERSION) {
|
|
1114
|
+
for (const t of STORE_TABLES) {
|
|
1115
|
+
if (t !== 'store_meta') this.db.exec(`DROP TABLE IF EXISTS store.${t}`)
|
|
1116
|
+
}
|
|
1117
|
+
reset = { from: now.found ?? 'an earlier schema', to: SCHEMA_VERSION }
|
|
1060
1118
|
}
|
|
1061
|
-
|
|
1119
|
+
createTables(this.db, 'store')
|
|
1120
|
+
this.db
|
|
1121
|
+
.prepare(
|
|
1122
|
+
"INSERT INTO store.store_meta(key, value) VALUES ('schema', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
1123
|
+
)
|
|
1124
|
+
.run(SCHEMA_VERSION)
|
|
1125
|
+
return reset
|
|
1062
1126
|
})
|
|
1063
1127
|
.immediate()
|
|
1064
|
-
return reset
|
|
1065
1128
|
}
|
|
1066
1129
|
|
|
1067
1130
|
// --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
|
|
@@ -1222,6 +1285,12 @@ export class Cache implements CacheLayer {
|
|
|
1222
1285
|
// A second name for the same bytes: the index step renames it over
|
|
1223
1286
|
// the artifact inside its write transaction, as for a save.
|
|
1224
1287
|
linkSync(finalPath, tmpPath)
|
|
1288
|
+
// The link shares the artifact's inode and so its mtime, which an
|
|
1289
|
+
// artifact that needs adopting has had for hours: another process's
|
|
1290
|
+
// orphan sweep read the temp as a crashed save's and took it, and the
|
|
1291
|
+
// artifact with it, while this adopt was scanning them.
|
|
1292
|
+
const now = new Date()
|
|
1293
|
+
utimesSync(tmpPath, now, now)
|
|
1225
1294
|
} catch {
|
|
1226
1295
|
return false
|
|
1227
1296
|
}
|
|
@@ -1262,7 +1331,9 @@ export class Cache implements CacheLayer {
|
|
|
1262
1331
|
|
|
1263
1332
|
// Verify the tar artifact actually exists. The DB and the
|
|
1264
1333
|
// filesystem can drift if someone manually deletes the cache dir.
|
|
1265
|
-
|
|
1334
|
+
const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
|
|
1335
|
+
if (st === undefined) return null
|
|
1336
|
+
if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
|
|
1266
1337
|
|
|
1267
1338
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1268
1339
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
@@ -1295,7 +1366,7 @@ export class Cache implements CacheLayer {
|
|
|
1295
1366
|
): Promise<Map<string, CacheEntry>> {
|
|
1296
1367
|
return this.guard(async () => {
|
|
1297
1368
|
const out = await this.getManyEntries(hashes)
|
|
1298
|
-
if (ctx === undefined || !this.write || out.size === hashes.length) return out
|
|
1369
|
+
if (ctx === undefined || !this.read || !this.write || out.size === hashes.length) return out
|
|
1299
1370
|
// An artifact the index does not know is still a hit (`adopt`).
|
|
1300
1371
|
for (const hash of hashes) {
|
|
1301
1372
|
if (out.has(hash) || !(await this.adopt(hash, ctx(hash)))) continue
|
|
@@ -1312,8 +1383,12 @@ export class Cache implements CacheLayer {
|
|
|
1312
1383
|
const { test, params } = inHashes(hashes)
|
|
1313
1384
|
const rows = this.db.query(`${SELECT_ENTRY} WHERE e.hash ${test}`).all(...params) as EntryRow[]
|
|
1314
1385
|
if (rows.length === 0) return out
|
|
1315
|
-
const
|
|
1316
|
-
const live = rows.filter((
|
|
1386
|
+
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1387
|
+
const live = rows.filter((r) => {
|
|
1388
|
+
const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
|
|
1389
|
+
if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1390
|
+
return st !== undefined
|
|
1391
|
+
})
|
|
1317
1392
|
const liveHashes = live.map((r) => r.hash)
|
|
1318
1393
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1319
1394
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
@@ -2041,6 +2116,17 @@ export class Cache implements CacheLayer {
|
|
|
2041
2116
|
const hashes = [...this.touched]
|
|
2042
2117
|
this.touched.clear()
|
|
2043
2118
|
const now = Date.now()
|
|
2119
|
+
// The file time is the last use another vx version's sweep can see:
|
|
2120
|
+
// its open dropped this index's rows (`pickVictims`).
|
|
2121
|
+
const at = new Date(now)
|
|
2122
|
+
for (const hash of this.staleTimes) {
|
|
2123
|
+
try {
|
|
2124
|
+
utimesSync(this.tarPath(hash), at, at)
|
|
2125
|
+
} catch {
|
|
2126
|
+
// Gone, or another user's file: a sweep judges its old time.
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
this.staleTimes.clear()
|
|
2044
2130
|
// LRU bookkeeping: on a full disk the bumps are dropped, never the prune
|
|
2045
2131
|
// or the stats that asked for them (A-14).
|
|
2046
2132
|
try {
|
|
@@ -2155,21 +2241,26 @@ export class Cache implements CacheLayer {
|
|
|
2155
2241
|
// above never sees them (upstream survey, nx#35483: 9 MiB of them sat
|
|
2156
2242
|
// under a 1 MB limit). Listing the directory to find them costs 0.5 ms
|
|
2157
2243
|
// per 1,000 entries (4.9 ms at 10,000), every run; the sweep's own
|
|
2158
|
-
// clock in `schema_meta` costs one indexed read.
|
|
2159
|
-
//
|
|
2160
|
-
// it within two.
|
|
2244
|
+
// clock in `schema_meta` costs one indexed read. The sweep is the
|
|
2245
|
+
// policy's own prune, with the row-less files counted.
|
|
2161
2246
|
const swept = this.db.prepare('SELECT value FROM schema_meta WHERE key = ?').get(SWEPT_AT) as {
|
|
2162
2247
|
value: string
|
|
2163
2248
|
} | null
|
|
2164
2249
|
if (swept !== null && now - Number(swept.value) < ORPHAN_GRACE_MS) return null
|
|
2165
|
-
return
|
|
2250
|
+
return this.pruneIndex(
|
|
2251
|
+
{
|
|
2252
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2253
|
+
...(policy.maxBytes !== undefined ? { maxBytes: policy.maxBytes } : {}),
|
|
2254
|
+
},
|
|
2255
|
+
now,
|
|
2256
|
+
)
|
|
2166
2257
|
}
|
|
2167
2258
|
|
|
2168
2259
|
async prune(options: PruneOptions): Promise<PruneResult> {
|
|
2169
2260
|
return this.guard(() => this.pruneIndex(options))
|
|
2170
2261
|
}
|
|
2171
2262
|
|
|
2172
|
-
private async pruneIndex(options: PruneOptions): Promise<PruneResult> {
|
|
2263
|
+
private async pruneIndex(options: PruneOptions, now: number = Date.now()): Promise<PruneResult> {
|
|
2173
2264
|
this.flushAccessed()
|
|
2174
2265
|
// Before any entry is deleted, so a kept entry's pending snapshot is on
|
|
2175
2266
|
// disk whatever the eviction does next. A pruned hash's snapshot cannot
|
|
@@ -2181,50 +2272,24 @@ export class Cache implements CacheLayer {
|
|
|
2181
2272
|
throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
|
|
2182
2273
|
}
|
|
2183
2274
|
|
|
2184
|
-
const victims = new Set<string>()
|
|
2185
|
-
let bytesFreed = 0
|
|
2186
2275
|
// Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
|
|
2187
2276
|
// never a hit, and their bytes are on no disk, but `--max-size` counted
|
|
2188
2277
|
// them and evicted real entries to make room for them (item 975). None
|
|
2189
2278
|
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
2190
2279
|
// grace window are dropped.
|
|
2191
|
-
const { phantoms, stale } = await this.phantomRows()
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
if (maxBytes !== undefined) {
|
|
2205
|
-
const totalRow = this.db
|
|
2206
|
-
.prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
|
|
2207
|
-
.get() as { bytes: number }
|
|
2208
|
-
let phantomBytes = 0
|
|
2209
|
-
for (const b of phantoms.values()) phantomBytes += b
|
|
2210
|
-
let remaining = totalRow.bytes - phantomBytes - bytesFreed
|
|
2211
|
-
if (remaining > maxBytes) {
|
|
2212
|
-
// Exclude already-picked victims in JS, not via a SQL NOT-IN —
|
|
2213
|
-
// an IN-list over tens of thousands of TTL victims would blow
|
|
2214
|
-
// SQLite's bound-parameter ceiling (see flushAccessed's 900 cap).
|
|
2215
|
-
const candidates = (
|
|
2216
|
-
this.db
|
|
2217
|
-
.prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
|
|
2218
|
-
.all() as Array<{ hash: string; size_bytes: number }>
|
|
2219
|
-
).filter((row) => !victims.has(row.hash) && !phantoms.has(row.hash))
|
|
2220
|
-
for (const row of candidates) {
|
|
2221
|
-
if (remaining <= maxBytes) break
|
|
2222
|
-
victims.add(row.hash)
|
|
2223
|
-
bytesFreed += row.size_bytes
|
|
2224
|
-
remaining -= row.size_bytes
|
|
2225
|
-
}
|
|
2226
|
-
}
|
|
2227
|
-
}
|
|
2280
|
+
const { phantoms, stale, rows } = await this.phantomRows()
|
|
2281
|
+
const picked = pickVictims(
|
|
2282
|
+
rows.filter((r) => !phantoms.has(r.hash)),
|
|
2283
|
+
await this.scanOrphans(),
|
|
2284
|
+
{
|
|
2285
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2286
|
+
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
2287
|
+
},
|
|
2288
|
+
)
|
|
2289
|
+
const victims = picked.hashes
|
|
2290
|
+
const { bytesFreed } = picked
|
|
2291
|
+
let orphanBytes = 0
|
|
2292
|
+
for (const o of picked.files) orphanBytes += o.size
|
|
2228
2293
|
|
|
2229
2294
|
// Delete DB rows in a single transaction (one fsync; ON DELETE
|
|
2230
2295
|
// CASCADE clears `output_files`) and unlink artifacts in parallel.
|
|
@@ -2233,13 +2298,7 @@ export class Cache implements CacheLayer {
|
|
|
2233
2298
|
// flushAccessed so a huge eviction stays under any build's
|
|
2234
2299
|
// bound-parameter ceiling.
|
|
2235
2300
|
if (dryRun) {
|
|
2236
|
-
|
|
2237
|
-
return {
|
|
2238
|
-
evicted: victims.size,
|
|
2239
|
-
bytesFreed,
|
|
2240
|
-
orphans: orphans.orphans,
|
|
2241
|
-
orphanBytes: orphans.orphanBytes,
|
|
2242
|
-
}
|
|
2301
|
+
return { evicted: victims.size, bytesFreed, orphans: picked.files.length, orphanBytes }
|
|
2243
2302
|
}
|
|
2244
2303
|
if (victims.size > 0 || stale.length > 0) {
|
|
2245
2304
|
const hashes = [...victims]
|
|
@@ -2269,23 +2328,21 @@ export class Cache implements CacheLayer {
|
|
|
2269
2328
|
} catch (err) {
|
|
2270
2329
|
if (!isIndexFull(err)) throw err
|
|
2271
2330
|
}
|
|
2272
|
-
const orphans = await this.reapOrphans()
|
|
2331
|
+
const orphans = await this.reapOrphans(picked.files, now)
|
|
2273
2332
|
return { evicted: victims.size, bytesFreed, ...orphans }
|
|
2274
2333
|
}
|
|
2275
2334
|
|
|
2276
2335
|
/**
|
|
2277
|
-
* Unlink
|
|
2278
|
-
* `entries` row (
|
|
2279
|
-
* and a `<hash>.tar.zst.tmp-*` a
|
|
2280
|
-
* else reaps them — a lookup starts
|
|
2281
|
-
* key renames over the file, so a key
|
|
2282
|
-
*
|
|
2283
|
-
* renames the artifact into place BEFORE its row commits, and its temp
|
|
2284
|
-
* exists while the bytes are still being written, so a fresh file
|
|
2285
|
-
* without a row is a save in flight, not an orphan.
|
|
2336
|
+
* Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
|
|
2337
|
+
* no `entries` row (another vx version's open dropped it, a `cache.db`
|
|
2338
|
+
* deleted by hand) past the policy, and a `<hash>.tar.zst.tmp-*` a
|
|
2339
|
+
* crashed save never renamed. Nothing else reaps them — a lookup starts
|
|
2340
|
+
* at the row, and a save of the same key renames over the file, so a key
|
|
2341
|
+
* that never recurs leaks its bytes forever.
|
|
2286
2342
|
*/
|
|
2287
2343
|
private async reapOrphans(
|
|
2288
|
-
|
|
2344
|
+
taken: readonly RowlessFile[],
|
|
2345
|
+
now: number,
|
|
2289
2346
|
): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2290
2347
|
try {
|
|
2291
2348
|
this.db
|
|
@@ -2298,7 +2355,7 @@ export class Cache implements CacheLayer {
|
|
|
2298
2355
|
let orphans = 0
|
|
2299
2356
|
let orphanBytes = 0
|
|
2300
2357
|
await Promise.all(
|
|
2301
|
-
|
|
2358
|
+
taken.map(async (o) => {
|
|
2302
2359
|
try {
|
|
2303
2360
|
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
2304
2361
|
// concurrent prune took first must not be counted as ours.
|
|
@@ -2322,7 +2379,11 @@ export class Cache implements CacheLayer {
|
|
|
2322
2379
|
* dropped — a row is never deleted on one listing's word while it may be
|
|
2323
2380
|
* in use. An unreadable directory judges nothing.
|
|
2324
2381
|
*/
|
|
2325
|
-
private async phantomRows(): Promise<{
|
|
2382
|
+
private async phantomRows(): Promise<{
|
|
2383
|
+
phantoms: Map<string, number>
|
|
2384
|
+
stale: string[]
|
|
2385
|
+
rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
|
|
2386
|
+
}> {
|
|
2326
2387
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
2327
2388
|
const rows = this.db
|
|
2328
2389
|
.prepare('SELECT hash, size_bytes, accessed_at FROM entries')
|
|
@@ -2335,7 +2396,7 @@ export class Cache implements CacheLayer {
|
|
|
2335
2396
|
try {
|
|
2336
2397
|
names = await readdir(this.artifactDir)
|
|
2337
2398
|
} catch {
|
|
2338
|
-
return { phantoms: new Map(), stale: [] }
|
|
2399
|
+
return { phantoms: new Map(), stale: [], rows }
|
|
2339
2400
|
}
|
|
2340
2401
|
const present = new Set(names)
|
|
2341
2402
|
const phantoms = new Map<string, number>()
|
|
@@ -2345,10 +2406,10 @@ export class Cache implements CacheLayer {
|
|
|
2345
2406
|
phantoms.set(r.hash, r.size_bytes)
|
|
2346
2407
|
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
2347
2408
|
}
|
|
2348
|
-
return { phantoms, stale }
|
|
2409
|
+
return { phantoms, stale, rows }
|
|
2349
2410
|
}
|
|
2350
2411
|
|
|
2351
|
-
/**
|
|
2412
|
+
/** Row-less files past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
|
|
2352
2413
|
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2353
2414
|
const found = await this.scanOrphans()
|
|
2354
2415
|
let orphanBytes = 0
|
|
@@ -2357,7 +2418,7 @@ export class Cache implements CacheLayer {
|
|
|
2357
2418
|
}
|
|
2358
2419
|
|
|
2359
2420
|
/** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
|
|
2360
|
-
private async scanOrphans(): Promise<
|
|
2421
|
+
private async scanOrphans(): Promise<RowlessFile[]> {
|
|
2361
2422
|
const found = await scanOrphanFiles(
|
|
2362
2423
|
this.artifactDir,
|
|
2363
2424
|
() =>
|