@vzn/vx 0.0.595 → 0.0.596
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +5 -5
- package/src/cache/cache.ts +139 -84
- package/src/cache/chained-cache.ts +25 -7
- package/src/cache/index.ts +1 -0
- package/src/cache/inputs.ts +12 -12
- package/src/cache/layer.ts +7 -5
- package/src/cache/policy.ts +5 -0
- package/src/cache/zstd.ts +10 -4
- package/src/cli/cache.ts +10 -4
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/last.ts +4 -1
- package/src/cli/run.ts +31 -15
- package/src/cli/watch-filter.ts +63 -12
- package/src/cli/watch-judge.ts +15 -8
- package/src/cli/watch.ts +4 -4
- package/src/config.ts +21 -21
- package/src/exec/proc-sample.ts +1 -1
- package/src/exec/sandbox-paths.ts +1 -1
- package/src/exec/sandbox-runtime.ts +116 -40
- package/src/exec/sandbox-violations.ts +58 -11
- package/src/graph/scheduler.ts +28 -12
- package/src/graph/task-graph.ts +134 -25
- package/src/orchestrator/admission.ts +12 -3
- package/src/orchestrator/events.ts +10 -1
- package/src/orchestrator/execute-task.ts +36 -13
- package/src/orchestrator/failure-mode.ts +8 -2
- package/src/orchestrator/framed-output.ts +3 -3
- package/src/orchestrator/logger.ts +11 -6
- package/src/orchestrator/metrics.ts +19 -8
- package/src/orchestrator/miss-save.ts +2 -2
- package/src/orchestrator/options.ts +7 -0
- package/src/orchestrator/plan.ts +3 -3
- package/src/orchestrator/plugin.ts +37 -16
- package/src/orchestrator/prepare.ts +13 -4
- package/src/orchestrator/run-artifacts.ts +2 -1
- package/src/orchestrator/run.ts +20 -4
- package/src/orchestrator/sandbox-request.ts +65 -9
- package/src/orchestrator/save-lane.ts +5 -5
- package/src/orchestrator/summary.ts +2 -4
- package/src/orchestrator/telemetry.ts +16 -7
- package/src/util/paths.ts +23 -3
- package/src/workspace/config-imports.ts +7 -5
- package/src/workspace/config-schema.ts +26 -3
- package/src/workspace/filter.ts +19 -0
- package/src/workspace/migrate-scripts.ts +4 -2
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/project-loader.ts +4 -2
- package/src/workspace/repo-id.ts +42 -4
- package/src/workspace/workspace.ts +17 -10
package/README.md
CHANGED
|
@@ -98,7 +98,7 @@ More: [Quickstart](https://vznjs.github.io/vx/quickstart/) ·
|
|
|
98
98
|
Beside `turbo.json` or `nx.json`, `vx init` writes only a
|
|
99
99
|
`vx.workspace.ts` that maps the old config until then: a temporary
|
|
100
100
|
start, not a way to run the repo. The numbers above are native vx
|
|
101
|
-
config.
|
|
101
|
+
config.
|
|
102
102
|
[Migration guide](https://vznjs.github.io/vx/guides/migrate/).
|
|
103
103
|
|
|
104
104
|
## Why it is faster
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.596",
|
|
4
4
|
"description": "The fastest task runner and build cache for JavaScript monorepos.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.596",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.596",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.596",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.596"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/cache/cache.ts
CHANGED
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
openSync,
|
|
45
45
|
readdirSync,
|
|
46
46
|
renameSync,
|
|
47
|
+
statSync,
|
|
47
48
|
utimesSync,
|
|
48
49
|
writeFileSync,
|
|
49
50
|
} from 'node:fs'
|
|
@@ -111,12 +112,13 @@ import { RunHistory } from './run-history.js'
|
|
|
111
112
|
import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
112
113
|
|
|
113
114
|
/**
|
|
114
|
-
* An artifact or temp file without an `entries` row is reaped
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
115
|
+
* An artifact or temp file without an `entries` row is never reaped
|
|
116
|
+
* younger than this: a crashed save's temp is stale long before it. A
|
|
117
|
+
* row-less artifact may still be in use (another vx version's open dropped
|
|
118
|
+
* its row), so past this it is judged by the retention policy, its file
|
|
119
|
+
* time standing for `accessed_at`. A hit renews a file time older than
|
|
120
|
+
* this, so the file time trails the last use by at most this much, and
|
|
121
|
+
* the policy reads the file time plus this as the last use.
|
|
120
122
|
*/
|
|
121
123
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
122
124
|
|
|
@@ -271,6 +273,62 @@ function indexOfAnotherSchema(dbFile: string): boolean {
|
|
|
271
273
|
const SELECT_ENTRY =
|
|
272
274
|
"SELECT e.*, COALESCE(s.stdout, '') AS stdout FROM entries e LEFT JOIN entry_stdout s ON s.hash = e.hash"
|
|
273
275
|
|
|
276
|
+
/** A file the orphan sweep may judge: a row-less artifact, or a temp. */
|
|
277
|
+
interface RowlessFile {
|
|
278
|
+
file: string
|
|
279
|
+
size: number
|
|
280
|
+
mtimeMs: number
|
|
281
|
+
temp: boolean
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* What a prune takes under `policy`: index rows by `accessed_at` and
|
|
286
|
+
* row-less artifacts by file time, under one rule (a temp always). The
|
|
287
|
+
* rows are the non-phantom ones; the files are past the in-flight grace.
|
|
288
|
+
*/
|
|
289
|
+
function pickVictims(
|
|
290
|
+
rows: ReadonlyArray<{ hash: string; size_bytes: number; accessed_at: number }>,
|
|
291
|
+
files: readonly RowlessFile[],
|
|
292
|
+
policy: { olderThanMs?: number; maxBytes?: number },
|
|
293
|
+
): { hashes: Set<string>; bytesFreed: number; files: RowlessFile[] } {
|
|
294
|
+
const { olderThanMs, maxBytes } = policy
|
|
295
|
+
const hashes = new Set<string>()
|
|
296
|
+
let bytesFreed = 0
|
|
297
|
+
const taken: RowlessFile[] = []
|
|
298
|
+
const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
|
|
299
|
+
let remaining = 0
|
|
300
|
+
for (const r of rows) {
|
|
301
|
+
if (olderThanMs !== undefined && r.accessed_at < olderThanMs) {
|
|
302
|
+
hashes.add(r.hash)
|
|
303
|
+
bytesFreed += r.size_bytes
|
|
304
|
+
} else {
|
|
305
|
+
kept.push({ at: r.accessed_at, size: r.size_bytes, row: r.hash })
|
|
306
|
+
remaining += r.size_bytes
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
for (const f of files) {
|
|
310
|
+
const at = f.mtimeMs + ORPHAN_GRACE_MS
|
|
311
|
+
if (f.temp || (olderThanMs !== undefined && at < olderThanMs)) taken.push(f)
|
|
312
|
+
else {
|
|
313
|
+
kept.push({ at, size: f.size, file: f })
|
|
314
|
+
remaining += f.size
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
if (maxBytes !== undefined && remaining > maxBytes) {
|
|
318
|
+
kept.sort((a, b) => a.at - b.at)
|
|
319
|
+
for (const k of kept) {
|
|
320
|
+
if (remaining <= maxBytes) break
|
|
321
|
+
remaining -= k.size
|
|
322
|
+
if (k.file !== undefined) taken.push(k.file)
|
|
323
|
+
else {
|
|
324
|
+
hashes.add(k.row!)
|
|
325
|
+
bytesFreed += k.size
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return { hashes, bytesFreed, files: taken }
|
|
330
|
+
}
|
|
331
|
+
|
|
274
332
|
/**
|
|
275
333
|
* Artifacts `indexed` does not hold, and temps, past the in-flight grace
|
|
276
334
|
* window: one readdir, one stat per candidate. `indexed` is asked only
|
|
@@ -281,7 +339,7 @@ const SELECT_ENTRY =
|
|
|
281
339
|
async function scanOrphanFiles(
|
|
282
340
|
cacheDir: string,
|
|
283
341
|
indexedHashes: () => ReadonlySet<string>,
|
|
284
|
-
): Promise<
|
|
342
|
+
): Promise<RowlessFile[]> {
|
|
285
343
|
let names: string[]
|
|
286
344
|
try {
|
|
287
345
|
names = await readdir(cacheDir)
|
|
@@ -290,19 +348,22 @@ async function scanOrphanFiles(
|
|
|
290
348
|
}
|
|
291
349
|
const indexed = indexedHashes()
|
|
292
350
|
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
293
|
-
const candidates: string
|
|
351
|
+
const candidates: Array<{ name: string; temp: boolean }> = []
|
|
294
352
|
for (const name of names) {
|
|
295
353
|
const m = VX_ARTIFACT_NAME.exec(name)
|
|
296
354
|
if (m === null) continue
|
|
297
|
-
|
|
355
|
+
const temp = m[2] !== undefined
|
|
356
|
+
if (temp || !indexed.has(m[1]!)) candidates.push({ name, temp })
|
|
298
357
|
}
|
|
299
|
-
const found:
|
|
358
|
+
const found: RowlessFile[] = []
|
|
300
359
|
await Promise.all(
|
|
301
|
-
candidates.map(async (name) => {
|
|
360
|
+
candidates.map(async ({ name, temp }) => {
|
|
302
361
|
const file = path.join(cacheDir, name)
|
|
303
362
|
try {
|
|
304
363
|
const st = await stat(file)
|
|
305
|
-
if (st.isFile() && st.mtimeMs <= cutoff)
|
|
364
|
+
if (st.isFile() && st.mtimeMs <= cutoff) {
|
|
365
|
+
found.push({ file, size: st.size, mtimeMs: st.mtimeMs, temp })
|
|
366
|
+
}
|
|
306
367
|
} catch {
|
|
307
368
|
// Gone between readdir and stat: not an orphan any more.
|
|
308
369
|
}
|
|
@@ -570,6 +631,8 @@ export class Cache implements CacheLayer {
|
|
|
570
631
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
571
632
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
572
633
|
private readonly touched = new Set<string>()
|
|
634
|
+
/** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
|
|
635
|
+
private readonly staleTimes = new Set<string>()
|
|
573
636
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
574
637
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
575
638
|
private readonly files: FileHashStore
|
|
@@ -1268,7 +1331,9 @@ export class Cache implements CacheLayer {
|
|
|
1268
1331
|
|
|
1269
1332
|
// Verify the tar artifact actually exists. The DB and the
|
|
1270
1333
|
// filesystem can drift if someone manually deletes the cache dir.
|
|
1271
|
-
|
|
1334
|
+
const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
|
|
1335
|
+
if (st === undefined) return null
|
|
1336
|
+
if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
|
|
1272
1337
|
|
|
1273
1338
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1274
1339
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
@@ -1301,7 +1366,7 @@ export class Cache implements CacheLayer {
|
|
|
1301
1366
|
): Promise<Map<string, CacheEntry>> {
|
|
1302
1367
|
return this.guard(async () => {
|
|
1303
1368
|
const out = await this.getManyEntries(hashes)
|
|
1304
|
-
if (ctx === undefined || !this.write || out.size === hashes.length) return out
|
|
1369
|
+
if (ctx === undefined || !this.read || !this.write || out.size === hashes.length) return out
|
|
1305
1370
|
// An artifact the index does not know is still a hit (`adopt`).
|
|
1306
1371
|
for (const hash of hashes) {
|
|
1307
1372
|
if (out.has(hash) || !(await this.adopt(hash, ctx(hash)))) continue
|
|
@@ -1318,8 +1383,12 @@ export class Cache implements CacheLayer {
|
|
|
1318
1383
|
const { test, params } = inHashes(hashes)
|
|
1319
1384
|
const rows = this.db.query(`${SELECT_ENTRY} WHERE e.hash ${test}`).all(...params) as EntryRow[]
|
|
1320
1385
|
if (rows.length === 0) return out
|
|
1321
|
-
const
|
|
1322
|
-
const live = rows.filter((
|
|
1386
|
+
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1387
|
+
const live = rows.filter((r) => {
|
|
1388
|
+
const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
|
|
1389
|
+
if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1390
|
+
return st !== undefined
|
|
1391
|
+
})
|
|
1323
1392
|
const liveHashes = live.map((r) => r.hash)
|
|
1324
1393
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1325
1394
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
@@ -2047,6 +2116,17 @@ export class Cache implements CacheLayer {
|
|
|
2047
2116
|
const hashes = [...this.touched]
|
|
2048
2117
|
this.touched.clear()
|
|
2049
2118
|
const now = Date.now()
|
|
2119
|
+
// The file time is the last use another vx version's sweep can see:
|
|
2120
|
+
// its open dropped this index's rows (`pickVictims`).
|
|
2121
|
+
const at = new Date(now)
|
|
2122
|
+
for (const hash of this.staleTimes) {
|
|
2123
|
+
try {
|
|
2124
|
+
utimesSync(this.tarPath(hash), at, at)
|
|
2125
|
+
} catch {
|
|
2126
|
+
// Gone, or another user's file: a sweep judges its old time.
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
this.staleTimes.clear()
|
|
2050
2130
|
// LRU bookkeeping: on a full disk the bumps are dropped, never the prune
|
|
2051
2131
|
// or the stats that asked for them (A-14).
|
|
2052
2132
|
try {
|
|
@@ -2161,21 +2241,26 @@ export class Cache implements CacheLayer {
|
|
|
2161
2241
|
// above never sees them (upstream survey, nx#35483: 9 MiB of them sat
|
|
2162
2242
|
// under a 1 MB limit). Listing the directory to find them costs 0.5 ms
|
|
2163
2243
|
// per 1,000 entries (4.9 ms at 10,000), every run; the sweep's own
|
|
2164
|
-
// clock in `schema_meta` costs one indexed read.
|
|
2165
|
-
//
|
|
2166
|
-
// it within two.
|
|
2244
|
+
// clock in `schema_meta` costs one indexed read. The sweep is the
|
|
2245
|
+
// policy's own prune, with the row-less files counted.
|
|
2167
2246
|
const swept = this.db.prepare('SELECT value FROM schema_meta WHERE key = ?').get(SWEPT_AT) as {
|
|
2168
2247
|
value: string
|
|
2169
2248
|
} | null
|
|
2170
2249
|
if (swept !== null && now - Number(swept.value) < ORPHAN_GRACE_MS) return null
|
|
2171
|
-
return
|
|
2250
|
+
return this.pruneIndex(
|
|
2251
|
+
{
|
|
2252
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2253
|
+
...(policy.maxBytes !== undefined ? { maxBytes: policy.maxBytes } : {}),
|
|
2254
|
+
},
|
|
2255
|
+
now,
|
|
2256
|
+
)
|
|
2172
2257
|
}
|
|
2173
2258
|
|
|
2174
2259
|
async prune(options: PruneOptions): Promise<PruneResult> {
|
|
2175
2260
|
return this.guard(() => this.pruneIndex(options))
|
|
2176
2261
|
}
|
|
2177
2262
|
|
|
2178
|
-
private async pruneIndex(options: PruneOptions): Promise<PruneResult> {
|
|
2263
|
+
private async pruneIndex(options: PruneOptions, now: number = Date.now()): Promise<PruneResult> {
|
|
2179
2264
|
this.flushAccessed()
|
|
2180
2265
|
// Before any entry is deleted, so a kept entry's pending snapshot is on
|
|
2181
2266
|
// disk whatever the eviction does next. A pruned hash's snapshot cannot
|
|
@@ -2187,50 +2272,24 @@ export class Cache implements CacheLayer {
|
|
|
2187
2272
|
throw new Error('prune: pass at least one of `olderThanMs` or `maxBytes`')
|
|
2188
2273
|
}
|
|
2189
2274
|
|
|
2190
|
-
const victims = new Set<string>()
|
|
2191
|
-
let bytesFreed = 0
|
|
2192
2275
|
// Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
|
|
2193
2276
|
// never a hit, and their bytes are on no disk, but `--max-size` counted
|
|
2194
2277
|
// them and evicted real entries to make room for them (item 975). None
|
|
2195
2278
|
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
2196
2279
|
// grace window are dropped.
|
|
2197
|
-
const { phantoms, stale } = await this.phantomRows()
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
if (maxBytes !== undefined) {
|
|
2211
|
-
const totalRow = this.db
|
|
2212
|
-
.prepare('SELECT COALESCE(SUM(size_bytes), 0) AS bytes FROM entries')
|
|
2213
|
-
.get() as { bytes: number }
|
|
2214
|
-
let phantomBytes = 0
|
|
2215
|
-
for (const b of phantoms.values()) phantomBytes += b
|
|
2216
|
-
let remaining = totalRow.bytes - phantomBytes - bytesFreed
|
|
2217
|
-
if (remaining > maxBytes) {
|
|
2218
|
-
// Exclude already-picked victims in JS, not via a SQL NOT-IN —
|
|
2219
|
-
// an IN-list over tens of thousands of TTL victims would blow
|
|
2220
|
-
// SQLite's bound-parameter ceiling (see flushAccessed's 900 cap).
|
|
2221
|
-
const candidates = (
|
|
2222
|
-
this.db
|
|
2223
|
-
.prepare('SELECT hash, size_bytes FROM entries ORDER BY accessed_at ASC')
|
|
2224
|
-
.all() as Array<{ hash: string; size_bytes: number }>
|
|
2225
|
-
).filter((row) => !victims.has(row.hash) && !phantoms.has(row.hash))
|
|
2226
|
-
for (const row of candidates) {
|
|
2227
|
-
if (remaining <= maxBytes) break
|
|
2228
|
-
victims.add(row.hash)
|
|
2229
|
-
bytesFreed += row.size_bytes
|
|
2230
|
-
remaining -= row.size_bytes
|
|
2231
|
-
}
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2280
|
+
const { phantoms, stale, rows } = await this.phantomRows()
|
|
2281
|
+
const picked = pickVictims(
|
|
2282
|
+
rows.filter((r) => !phantoms.has(r.hash)),
|
|
2283
|
+
await this.scanOrphans(),
|
|
2284
|
+
{
|
|
2285
|
+
...(olderThanMs !== undefined ? { olderThanMs } : {}),
|
|
2286
|
+
...(maxBytes !== undefined ? { maxBytes } : {}),
|
|
2287
|
+
},
|
|
2288
|
+
)
|
|
2289
|
+
const victims = picked.hashes
|
|
2290
|
+
const { bytesFreed } = picked
|
|
2291
|
+
let orphanBytes = 0
|
|
2292
|
+
for (const o of picked.files) orphanBytes += o.size
|
|
2234
2293
|
|
|
2235
2294
|
// Delete DB rows in a single transaction (one fsync; ON DELETE
|
|
2236
2295
|
// CASCADE clears `output_files`) and unlink artifacts in parallel.
|
|
@@ -2239,13 +2298,7 @@ export class Cache implements CacheLayer {
|
|
|
2239
2298
|
// flushAccessed so a huge eviction stays under any build's
|
|
2240
2299
|
// bound-parameter ceiling.
|
|
2241
2300
|
if (dryRun) {
|
|
2242
|
-
|
|
2243
|
-
return {
|
|
2244
|
-
evicted: victims.size,
|
|
2245
|
-
bytesFreed,
|
|
2246
|
-
orphans: orphans.orphans,
|
|
2247
|
-
orphanBytes: orphans.orphanBytes,
|
|
2248
|
-
}
|
|
2301
|
+
return { evicted: victims.size, bytesFreed, orphans: picked.files.length, orphanBytes }
|
|
2249
2302
|
}
|
|
2250
2303
|
if (victims.size > 0 || stale.length > 0) {
|
|
2251
2304
|
const hashes = [...victims]
|
|
@@ -2275,23 +2328,21 @@ export class Cache implements CacheLayer {
|
|
|
2275
2328
|
} catch (err) {
|
|
2276
2329
|
if (!isIndexFull(err)) throw err
|
|
2277
2330
|
}
|
|
2278
|
-
const orphans = await this.reapOrphans()
|
|
2331
|
+
const orphans = await this.reapOrphans(picked.files, now)
|
|
2279
2332
|
return { evicted: victims.size, bytesFreed, ...orphans }
|
|
2280
2333
|
}
|
|
2281
2334
|
|
|
2282
2335
|
/**
|
|
2283
|
-
* Unlink
|
|
2284
|
-
* `entries` row (
|
|
2285
|
-
* and a `<hash>.tar.zst.tmp-*` a
|
|
2286
|
-
* else reaps them — a lookup starts
|
|
2287
|
-
* key renames over the file, so a key
|
|
2288
|
-
*
|
|
2289
|
-
* renames the artifact into place BEFORE its row commits, and its temp
|
|
2290
|
-
* exists while the bytes are still being written, so a fresh file
|
|
2291
|
-
* without a row is a save in flight, not an orphan.
|
|
2336
|
+
* Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
|
|
2337
|
+
* no `entries` row (another vx version's open dropped it, a `cache.db`
|
|
2338
|
+
* deleted by hand) past the policy, and a `<hash>.tar.zst.tmp-*` a
|
|
2339
|
+
* crashed save never renamed. Nothing else reaps them — a lookup starts
|
|
2340
|
+
* at the row, and a save of the same key renames over the file, so a key
|
|
2341
|
+
* that never recurs leaks its bytes forever.
|
|
2292
2342
|
*/
|
|
2293
2343
|
private async reapOrphans(
|
|
2294
|
-
|
|
2344
|
+
taken: readonly RowlessFile[],
|
|
2345
|
+
now: number,
|
|
2295
2346
|
): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2296
2347
|
try {
|
|
2297
2348
|
this.db
|
|
@@ -2304,7 +2355,7 @@ export class Cache implements CacheLayer {
|
|
|
2304
2355
|
let orphans = 0
|
|
2305
2356
|
let orphanBytes = 0
|
|
2306
2357
|
await Promise.all(
|
|
2307
|
-
|
|
2358
|
+
taken.map(async (o) => {
|
|
2308
2359
|
try {
|
|
2309
2360
|
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
2310
2361
|
// concurrent prune took first must not be counted as ours.
|
|
@@ -2328,7 +2379,11 @@ export class Cache implements CacheLayer {
|
|
|
2328
2379
|
* dropped — a row is never deleted on one listing's word while it may be
|
|
2329
2380
|
* in use. An unreadable directory judges nothing.
|
|
2330
2381
|
*/
|
|
2331
|
-
private async phantomRows(): Promise<{
|
|
2382
|
+
private async phantomRows(): Promise<{
|
|
2383
|
+
phantoms: Map<string, number>
|
|
2384
|
+
stale: string[]
|
|
2385
|
+
rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
|
|
2386
|
+
}> {
|
|
2332
2387
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
2333
2388
|
const rows = this.db
|
|
2334
2389
|
.prepare('SELECT hash, size_bytes, accessed_at FROM entries')
|
|
@@ -2341,7 +2396,7 @@ export class Cache implements CacheLayer {
|
|
|
2341
2396
|
try {
|
|
2342
2397
|
names = await readdir(this.artifactDir)
|
|
2343
2398
|
} catch {
|
|
2344
|
-
return { phantoms: new Map(), stale: [] }
|
|
2399
|
+
return { phantoms: new Map(), stale: [], rows }
|
|
2345
2400
|
}
|
|
2346
2401
|
const present = new Set(names)
|
|
2347
2402
|
const phantoms = new Map<string, number>()
|
|
@@ -2351,10 +2406,10 @@ export class Cache implements CacheLayer {
|
|
|
2351
2406
|
phantoms.set(r.hash, r.size_bytes)
|
|
2352
2407
|
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
2353
2408
|
}
|
|
2354
|
-
return { phantoms, stale }
|
|
2409
|
+
return { phantoms, stale, rows }
|
|
2355
2410
|
}
|
|
2356
2411
|
|
|
2357
|
-
/**
|
|
2412
|
+
/** Row-less files past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
|
|
2358
2413
|
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2359
2414
|
const found = await this.scanOrphans()
|
|
2360
2415
|
let orphanBytes = 0
|
|
@@ -2363,7 +2418,7 @@ export class Cache implements CacheLayer {
|
|
|
2363
2418
|
}
|
|
2364
2419
|
|
|
2365
2420
|
/** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
|
|
2366
|
-
private async scanOrphans(): Promise<
|
|
2421
|
+
private async scanOrphans(): Promise<RowlessFile[]> {
|
|
2367
2422
|
const found = await scanOrphanFiles(
|
|
2368
2423
|
this.artifactDir,
|
|
2369
2424
|
() =>
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// once. Restore goes to the layer that produced the hit — remembered per
|
|
5
5
|
// hash — because an entry's artifact lives wherever it was found.
|
|
6
6
|
//
|
|
7
|
-
// A layer that throws in a lookup
|
|
7
|
+
// A layer that throws in a lookup (a batch row load or a has-many included)
|
|
8
|
+
// is a miss there and the walk goes on; a throwing drain is passed; one
|
|
8
9
|
// that throws in a save is skipped and the rest still save. Unisolated, a
|
|
9
10
|
// plugin layer's throw ended the walk before the local floor under it: its
|
|
10
11
|
// `get` failed the task, its `save` kept the entry from every later layer
|
|
@@ -122,14 +123,22 @@ export class ChainedCache implements CacheLayer {
|
|
|
122
123
|
const found = await this.ask(
|
|
123
124
|
layer,
|
|
124
125
|
'remoteHasMany',
|
|
125
|
-
|
|
126
|
+
// The shape check and the mark sit inside the ask: a layer that
|
|
127
|
+
// answers a non-Set or throws marking gave no answer.
|
|
128
|
+
async () => {
|
|
129
|
+
const got: unknown = (await layer.remoteHasMany?.(hashes)) ?? null
|
|
130
|
+
if (got === null) return null
|
|
131
|
+
if (!(got instanceof Set)) throw new TypeError('remoteHasMany answered a non-Set')
|
|
132
|
+
const set = got as Set<string>
|
|
133
|
+
layer.markRemoteAbsent?.(hashes.filter((h) => !set.has(h)))
|
|
134
|
+
return set
|
|
135
|
+
},
|
|
126
136
|
null,
|
|
127
137
|
)
|
|
128
138
|
if (found === null) {
|
|
129
139
|
complete = false
|
|
130
140
|
continue
|
|
131
141
|
}
|
|
132
|
-
layer.markRemoteAbsent?.(hashes.filter((h) => !found.has(h)))
|
|
133
142
|
out ??= new Set()
|
|
134
143
|
for (const h of found) out.add(h)
|
|
135
144
|
}
|
|
@@ -142,15 +151,24 @@ export class ChainedCache implements CacheLayer {
|
|
|
142
151
|
}
|
|
143
152
|
|
|
144
153
|
async drainUploads(): Promise<void> {
|
|
145
|
-
await Promise.all(
|
|
154
|
+
await Promise.all(
|
|
155
|
+
this.layers.map((l) =>
|
|
156
|
+
this.ask(l, 'drainUploads', async () => l.drainUploads?.(), undefined),
|
|
157
|
+
),
|
|
158
|
+
)
|
|
146
159
|
}
|
|
147
160
|
|
|
148
161
|
loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
|
|
149
162
|
const out = new Map<string, OutputFileRow[]>()
|
|
150
|
-
for (const layer of this.layers) {
|
|
151
|
-
|
|
152
|
-
|
|
163
|
+
for (const [i, layer] of this.layers.entries()) {
|
|
164
|
+
let rows: Map<string, OutputFileRow[]>
|
|
165
|
+
try {
|
|
166
|
+
rows = layer.loadOutputFilesBatch(hashes)
|
|
167
|
+
} catch (err) {
|
|
168
|
+
this.onLayerError(i, 'loadOutputFilesBatch', err)
|
|
169
|
+
continue
|
|
153
170
|
}
|
|
171
|
+
for (const [h, r] of rows) if (!out.has(h)) out.set(h, r)
|
|
154
172
|
}
|
|
155
173
|
return out
|
|
156
174
|
}
|
package/src/cache/index.ts
CHANGED
package/src/cache/inputs.ts
CHANGED
|
@@ -109,10 +109,10 @@ export interface ResolveInputsArgs {
|
|
|
109
109
|
envSource: NodeJS.ProcessEnv
|
|
110
110
|
inputs: CacheInputs | undefined
|
|
111
111
|
/** Project-relative output globs to exclude from inputs. */
|
|
112
|
-
ownOutputs: string[]
|
|
112
|
+
ownOutputs: readonly string[]
|
|
113
113
|
/** Root-relative `outputs.workspaceFiles` globs to exclude from
|
|
114
114
|
* `inputs.workspaceFiles` (a task cannot invalidate itself). */
|
|
115
|
-
ownWorkspaceOutputs?: string[]
|
|
115
|
+
ownWorkspaceOutputs?: readonly string[]
|
|
116
116
|
/** Absolute dirs of nested projects (cross-boundary isolation). */
|
|
117
117
|
nestedProjectDirs: string[]
|
|
118
118
|
/**
|
|
@@ -646,7 +646,7 @@ function outputExcludes(outputs: readonly string[]): Bun.Glob[] {
|
|
|
646
646
|
/** Resolve declared output globs (project-relative) to actual produced files. */
|
|
647
647
|
export async function resolveOutputs(args: {
|
|
648
648
|
projectDir: string
|
|
649
|
-
outputs: string[]
|
|
649
|
+
outputs: readonly string[]
|
|
650
650
|
nestedProjectDirs: string[]
|
|
651
651
|
}): Promise<string[]> {
|
|
652
652
|
const { positive, negative } = splitNegations(args.outputs)
|
|
@@ -747,7 +747,7 @@ function isInside(dir: string, abs: string): boolean {
|
|
|
747
747
|
*/
|
|
748
748
|
export async function cleanOutputs(args: {
|
|
749
749
|
projectDir: string
|
|
750
|
-
outputs: string[]
|
|
750
|
+
outputs: readonly string[]
|
|
751
751
|
nestedProjectDirs: string[]
|
|
752
752
|
}): Promise<string[]> {
|
|
753
753
|
const files = await resolveOutputs(args)
|
|
@@ -801,7 +801,7 @@ export interface OutputStamp {
|
|
|
801
801
|
*/
|
|
802
802
|
export async function stampOutputs(args: {
|
|
803
803
|
projectDir: string
|
|
804
|
-
outputs: string[]
|
|
804
|
+
outputs: readonly string[]
|
|
805
805
|
nestedProjectDirs: string[]
|
|
806
806
|
}): Promise<Map<string, OutputStamp>> {
|
|
807
807
|
return stampFiles(await resolveOutputs(args))
|
|
@@ -810,7 +810,7 @@ export async function stampOutputs(args: {
|
|
|
810
810
|
/** `stampOutputs` for root-anchored `cache.outputs.workspaceFiles` (A-43). */
|
|
811
811
|
export async function stampWorkspaceOutputs(args: {
|
|
812
812
|
workspaceRoot: string
|
|
813
|
-
outputs: string[]
|
|
813
|
+
outputs: readonly string[]
|
|
814
814
|
}): Promise<Map<string, OutputStamp>> {
|
|
815
815
|
return stampFiles(await resolveWorkspaceOutputs(args))
|
|
816
816
|
}
|
|
@@ -841,7 +841,7 @@ function stampFiles(files: readonly string[]): Map<string, OutputStamp> {
|
|
|
841
841
|
* file back, so no entry reproduces that run and none is saved.
|
|
842
842
|
*/
|
|
843
843
|
export async function ownOutputsSince(
|
|
844
|
-
args: { projectDir: string; outputs: string[]; nestedProjectDirs: string[] },
|
|
844
|
+
args: { projectDir: string; outputs: readonly string[]; nestedProjectDirs: string[] },
|
|
845
845
|
before: ReadonlyMap<string, OutputStamp>,
|
|
846
846
|
): Promise<string[] | undefined> {
|
|
847
847
|
return changedSince(await resolveOutputs(args), before)
|
|
@@ -849,7 +849,7 @@ export async function ownOutputsSince(
|
|
|
849
849
|
|
|
850
850
|
/** `ownOutputsSince` for root-anchored `cache.outputs.workspaceFiles` (A-43). */
|
|
851
851
|
export async function ownWorkspaceOutputsSince(
|
|
852
|
-
args: { workspaceRoot: string; outputs: string[] },
|
|
852
|
+
args: { workspaceRoot: string; outputs: readonly string[] },
|
|
853
853
|
before: ReadonlyMap<string, OutputStamp>,
|
|
854
854
|
): Promise<string[] | undefined> {
|
|
855
855
|
return changedSince(await resolveWorkspaceOutputs(args), before)
|
|
@@ -1067,7 +1067,7 @@ function inScope(dir: string, scope: PruneScope): boolean {
|
|
|
1067
1067
|
*/
|
|
1068
1068
|
export async function resolveWorkspaceOutputs(args: {
|
|
1069
1069
|
workspaceRoot: string
|
|
1070
|
-
outputs: string[]
|
|
1070
|
+
outputs: readonly string[]
|
|
1071
1071
|
}): Promise<string[]> {
|
|
1072
1072
|
const { positive, negative } = splitNegations(args.outputs)
|
|
1073
1073
|
if (positive.length === 0) return []
|
|
@@ -1090,7 +1090,7 @@ export async function resolveWorkspaceOutputs(args: {
|
|
|
1090
1090
|
*/
|
|
1091
1091
|
export async function cleanWorkspaceOutputs(args: {
|
|
1092
1092
|
workspaceRoot: string
|
|
1093
|
-
outputs: string[]
|
|
1093
|
+
outputs: readonly string[]
|
|
1094
1094
|
}): Promise<string[]> {
|
|
1095
1095
|
const files = await resolveWorkspaceOutputs(args)
|
|
1096
1096
|
await pruneEmptiedDirs(
|
|
@@ -1195,8 +1195,8 @@ async function assertNoInvisibleLiteralInputs(
|
|
|
1195
1195
|
interface ResolveFilesArgs {
|
|
1196
1196
|
projectDir: string
|
|
1197
1197
|
workspaceRoot: string
|
|
1198
|
-
files: string[] | undefined
|
|
1199
|
-
ownOutputs: string[]
|
|
1198
|
+
files: readonly string[] | undefined
|
|
1199
|
+
ownOutputs: readonly string[]
|
|
1200
1200
|
/** Root-relative `outputs.workspaceFiles`, and the project's root-relative directory. */
|
|
1201
1201
|
ownWorkspaceOutputs: readonly string[]
|
|
1202
1202
|
projectRel: string
|
package/src/cache/layer.ts
CHANGED
|
@@ -296,7 +296,8 @@ export interface PruneOptions {
|
|
|
296
296
|
olderThanMs?: number
|
|
297
297
|
/**
|
|
298
298
|
* After applying olderThanMs, if the cache still exceeds this size in
|
|
299
|
-
* bytes, evict LRU (smallest `accessed_at` first
|
|
299
|
+
* bytes, evict LRU (smallest `accessed_at` first, a row-less artifact
|
|
300
|
+
* by its file time) until under it.
|
|
300
301
|
*/
|
|
301
302
|
maxBytes?: number
|
|
302
303
|
/**
|
|
@@ -312,10 +313,11 @@ export interface PruneResult {
|
|
|
312
313
|
/** Bytes the evicted entries occupied, per the index. */
|
|
313
314
|
bytesFreed: number
|
|
314
315
|
/**
|
|
315
|
-
* Artifacts and temp files with no index row, unlinked by the
|
|
316
|
-
*
|
|
317
|
-
* `cache.db`, a save that crashed before its rename).
|
|
318
|
-
*
|
|
316
|
+
* Artifacts and temp files with no index row, unlinked by the same
|
|
317
|
+
* policy (another vx version's open dropped the row, a deleted
|
|
318
|
+
* `cache.db`, a save that crashed before its rename). An artifact's file
|
|
319
|
+
* time plus an hour stands for its last use; a temp goes once an hour
|
|
320
|
+
* old. Files younger than one hour are never counted.
|
|
319
321
|
*/
|
|
320
322
|
orphans: number
|
|
321
323
|
/** Bytes the reaped orphans occupied on disk. */
|
package/src/cache/policy.ts
CHANGED
|
@@ -36,6 +36,11 @@ export const FULL_CACHE_POLICY: CachePolicy = {
|
|
|
36
36
|
remoteWrite: true,
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/** No axis on (`--no-cache`, `--cache=local:`): no cache answers for any task. */
|
|
40
|
+
export function cachesNothing(p: CachePolicy): boolean {
|
|
41
|
+
return !(p.localRead || p.localWrite || p.remoteRead || p.remoteWrite)
|
|
42
|
+
}
|
|
43
|
+
|
|
39
44
|
/**
|
|
40
45
|
* Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
|
|
41
46
|
* base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
|
package/src/cache/zstd.ts
CHANGED
|
@@ -194,15 +194,21 @@ export async function decodedTar(
|
|
|
194
194
|
}
|
|
195
195
|
return oneChunk(await zstdDecompressBounded(source, hash, cap))
|
|
196
196
|
}
|
|
197
|
+
// A BunFile reads no further than the size its first stat cached, and a
|
|
198
|
+
// save or ingest of the same key may rename a longer copy over the path
|
|
199
|
+
// after that stat: the read came back cut short, the restore called the
|
|
200
|
+
// new artifact corrupt and dropped it. The size only picks the decode;
|
|
201
|
+
// each read goes through a handle that has not stat the path yet.
|
|
202
|
+
const fresh = (): Bun.BunFile => Bun.file(source.name!)
|
|
197
203
|
if (source.size <= STREAM_DECODE_FROM) {
|
|
198
|
-
const bytes = await
|
|
204
|
+
const bytes = await fresh().bytes()
|
|
199
205
|
if (assertDeclaredSize(bytes, hash, cap) === null || !isOneFrame(bytes)) {
|
|
200
|
-
return zstdDecodeStream(
|
|
206
|
+
return zstdDecodeStream(new Blob([bytes]), hash, cap)
|
|
201
207
|
}
|
|
202
208
|
return oneChunk(await zstdDecompressBounded(bytes, hash, cap))
|
|
203
209
|
}
|
|
204
|
-
assertDeclaredSize(await
|
|
205
|
-
return zstdDecodeStream(
|
|
210
|
+
assertDeclaredSize(await fresh().slice(0, 32).bytes(), hash, cap)
|
|
211
|
+
return zstdDecodeStream(fresh(), hash, cap)
|
|
206
212
|
}
|
|
207
213
|
|
|
208
214
|
/**
|