@vzn/vx 0.0.50 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -31
- package/package.json +5 -5
- package/src/bin.ts +1 -1
- package/src/cache/archive.ts +18 -6
- package/src/cache/cache.ts +150 -67
- package/src/cache/chained-cache.ts +9 -2
- package/src/cache/file-hashes.ts +32 -3
- package/src/cache/git-inputs.ts +96 -11
- package/src/cache/index.ts +1 -0
- package/src/cache/layer.ts +28 -3
- package/src/cache/layered-cache.ts +7 -2
- package/src/cache/output-index.ts +16 -7
- package/src/cache/run-history.ts +5 -1
- package/src/cache/zstd.ts +40 -8
- package/src/cli/cache.ts +35 -9
- package/src/cli/completions.ts +10 -7
- package/src/cli/help.ts +48 -4
- package/src/cli/index.ts +5 -3
- package/src/cli/info.ts +1 -1
- package/src/cli/plugin-commands.ts +10 -2
- package/src/cli/run.ts +1 -0
- package/src/cli/select.ts +29 -2
- package/src/cli/upgrade.ts +110 -22
- package/src/cli/watch.ts +35 -18
- package/src/config.ts +18 -5
- package/src/exec/executor.ts +3 -2
- package/src/exec/index.ts +1 -1
- package/src/exec/runner.ts +102 -24
- package/src/exec/sandbox-binds.ts +1 -1
- package/src/exec/sandbox-runtime.ts +149 -40
- package/src/exec/sandbox-violations.ts +42 -10
- package/src/graph/scheduler.ts +11 -5
- package/src/graph/task-graph.ts +96 -12
- package/src/orchestrator/admission.ts +72 -17
- package/src/orchestrator/execute-task.ts +39 -25
- package/src/orchestrator/hit-restore.ts +7 -1
- package/src/orchestrator/miss-save.ts +29 -1
- package/src/orchestrator/placement.ts +23 -3
- package/src/orchestrator/plugin-host.ts +1 -1
- package/src/orchestrator/plugin.ts +9 -1
- package/src/orchestrator/prepare.ts +1 -0
- package/src/orchestrator/run.ts +75 -12
- package/src/orchestrator/sandbox-request.ts +19 -10
- package/src/orchestrator/summary.ts +26 -15
- package/src/orchestrator/task-hash.ts +7 -2
- package/src/orchestrator/telemetry.ts +14 -4
- package/src/util/paths.ts +16 -9
- package/src/workspace/affected.ts +201 -38
- package/src/workspace/config-eval.ts +4 -1
- package/src/workspace/config-schema.ts +70 -10
- package/src/workspace/fingerprint.ts +6 -0
- package/src/workspace/migration.ts +1 -1
- package/src/workspace/project-loader.ts +53 -1
- package/src/workspace/workspace.ts +54 -14
package/README.md
CHANGED
|
@@ -101,10 +101,12 @@ with the invariant that keeps it valid —
|
|
|
101
101
|
|
|
102
102
|
## Built for trust
|
|
103
103
|
|
|
104
|
-
- **Corruption can't go live.** A remote artifact is
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
104
|
+
- **Corruption can't go live.** A remote artifact is validated before
|
|
105
|
+
it enters the store: a zstd bomb, bytes that are not a vx archive,
|
|
106
|
+
and an archive recording another key than the one it was fetched
|
|
107
|
+
under are refused, and `@vzn/vx-reapi` re-hashes every blob against
|
|
108
|
+
its content digest; bad bytes degrade to a cache miss, never a wrong
|
|
109
|
+
hit and never a crash.
|
|
108
110
|
- **Clean exits.** SIGINT/SIGTERM reap every child process — no
|
|
109
111
|
orphaned dev servers in CI.
|
|
110
112
|
- **Readiness you can bound.** Persistent tasks gate downstream work
|
|
@@ -129,13 +131,15 @@ vx lock # evaluate everything once, write vx-lock.json
|
|
|
129
131
|
vx lock --check && vx run ci --frozen # CI: audit, then run EXACTLY that graph
|
|
130
132
|
```
|
|
131
133
|
|
|
132
|
-
| Command | Evaluates configs | Uses lock
|
|
133
|
-
| ----------------- | ----------------- |
|
|
134
|
-
| `vx run` | always, live | never — local truth has no asterisks
|
|
135
|
-
| `vx run --frozen` | never | yes; refuses
|
|
136
|
-
| `vx lock --check` | full graph | compares —
|
|
134
|
+
| Command | Evaluates configs | Uses lock |
|
|
135
|
+
| ----------------- | ----------------- | ---------------------------------------------------------------- |
|
|
136
|
+
| `vx run` | always, live | never — local truth has no asterisks |
|
|
137
|
+
| `vx run --frozen` | never | yes, as written; refuses only a missing lock or project entry |
|
|
138
|
+
| `vx lock --check` | full graph | compares — a changed file, and env or import drift it cannot see |
|
|
137
139
|
|
|
138
|
-
|
|
140
|
+
`--frozen` checks nothing about staleness itself: an edited config runs
|
|
141
|
+
its locked form until `vx lock` re-writes it, and `vx lock --check` is
|
|
142
|
+
the guard that fails first. Env values read at lock time are frozen by design — cache keys become
|
|
139
143
|
reproducible across machines. Take it for that, not for speed: measured
|
|
140
144
|
on the 1,000-project bench, a plain warm run and a frozen one are a tie
|
|
141
145
|
(177 ms against 165, 2026-09-12), because the config-evaluation cache
|
|
@@ -162,17 +166,20 @@ task graph, derive keys, schedule, execute, cache, observe. Plugins
|
|
|
162
166
|
declared in `vx.workspace.ts` hook each stage, Vite-style, on one
|
|
163
167
|
`VxPlugin` object:
|
|
164
168
|
|
|
165
|
-
| Stage | Hook | A plugin can…
|
|
166
|
-
| --------- | -------------------------- |
|
|
167
|
-
| workspace | `config(ws, ctx)` | edit the workspace config before it is used
|
|
168
|
-
| project | `project(config, ctx)` | add, remove or rewrite a project's tasks (keyed like yours)
|
|
169
|
-
| graph | `graph(nodes, ctx)` | add or drop edges, mark tasks requested
|
|
170
|
-
| key | `key(task, ctx)` | fold extra material into the cache key (named in `vx why`)
|
|
171
|
-
|
|
|
172
|
-
|
|
|
173
|
-
|
|
|
174
|
-
|
|
|
175
|
-
|
|
|
169
|
+
| Stage | Hook | A plugin can… |
|
|
170
|
+
| --------- | -------------------------- | ------------------------------------------------------------ |
|
|
171
|
+
| workspace | `config(ws, ctx)` | edit the workspace config before it is used |
|
|
172
|
+
| project | `project(config, ctx)` | add, remove or rewrite a project's tasks (keyed like yours) |
|
|
173
|
+
| graph | `graph(nodes, ctx)` | add or drop edges, mark tasks requested |
|
|
174
|
+
| key | `key(task, ctx)` | fold extra material into the cache key (named in `vx why`) |
|
|
175
|
+
| key | `fingerprint` (a claim) | claim a root file (a lockfile) and key each task on its part |
|
|
176
|
+
| schedule | `schedule(nodes, ctx)` | decide which ready task runs first |
|
|
177
|
+
| schedule | `admit(task, ctx)` | hold a ready task until something running here finishes |
|
|
178
|
+
| execute | `executor(ctx)` | decide WHERE a task's command runs (local, a REAPI worker) |
|
|
179
|
+
| store | `cache(ctx)` | decide where artifacts live (local, a shared remote) |
|
|
180
|
+
| observe | `telemetry(ctx)` / `setup` | receive every run record, or the raw event bus |
|
|
181
|
+
| run | `setup` / `teardown` | acquire before the run, release after it |
|
|
182
|
+
| cli | `commands` | add verbs to `vx` |
|
|
176
183
|
|
|
177
184
|
Core applies **no plugin by default** and names none. Running here and
|
|
178
185
|
caching here are its floor — the tail of every executor list and cache
|
|
@@ -194,10 +201,9 @@ Context Protocol server for AI coding agents, no SDK),
|
|
|
194
201
|
artifacts are vx's),
|
|
195
202
|
[`@vzn/vx-lockfile`](packages/vx-lockfile) (`pnpm()`, `bun()`, `npm()`,
|
|
196
203
|
`yarn()`: the lockfile keyed per project, so one install re-keys only the
|
|
197
|
-
projects it reaches, and `--affected` follows)
|
|
198
|
-
(order by learned critical path
|
|
199
|
-
|
|
200
|
-
core's migration seam). Core ships no plugin and reads no other runner's
|
|
204
|
+
projects it reaches, and `--affected` follows) and [`@vzn/vx-schedule-history`](packages/vx-schedule-history)
|
|
205
|
+
(order by learned critical path, and pack admission by what past runs
|
|
206
|
+
used). Core ships no plugin and reads no other runner's
|
|
201
207
|
format; nothing
|
|
202
208
|
distributed ships in this repo; the seams are how you build it.
|
|
203
209
|
|
|
@@ -205,7 +211,7 @@ distributed ships in this repo; the seams are how you build it.
|
|
|
205
211
|
|
|
206
212
|
| | vx | Turborepo | Nx |
|
|
207
213
|
| ------------------------- | ---------------------------------------------------------------- | ------------------------------ | ---------------- |
|
|
208
|
-
| Fully cached,
|
|
214
|
+
| Fully cached, 1,090 pkgs¹ | **510 ms** | 760 ms | 3.59 s |
|
|
209
215
|
| Config | TypeScript, evaluated into the cache key | JSON (static) | JSON (static) |
|
|
210
216
|
| Output ownership | **Strict** — wiped before exec AND restore | Additive (stale files survive) | Additive |
|
|
211
217
|
| Clean-tree hashing | **Zero reads** (git index OIDs) | git OIDs | re-hash / daemon |
|
|
@@ -215,9 +221,10 @@ distributed ships in this repo; the seams are how you build it.
|
|
|
215
221
|
| OTel CI/CD spans | **Yes** — `otel()` plugin, zero OTel-SDK deps | No | Paid |
|
|
216
222
|
| Install | **Single binary** — npm, or one release file; no Node/Bun needed | npm + Node | npm + Node |
|
|
217
223
|
|
|
218
|
-
¹
|
|
219
|
-
|
|
220
|
-
|
|
224
|
+
¹ Warm, nothing to rebuild, on the 3,270-task graph quoted at the top:
|
|
225
|
+
wall-clock, direct binaries, same machine and workspace — methodology
|
|
226
|
+
and more scenarios in
|
|
227
|
+
[`packages/vx/docs/benchmarks.md`](packages/vx/docs/benchmarks.md) § A real monorepo.
|
|
221
228
|
|
|
222
229
|
## Switching from another runner
|
|
223
230
|
|
|
@@ -309,7 +316,7 @@ maintainers' handoff is [`packages/vx/docs/STATUS.md`](packages/vx/docs/STATUS.m
|
|
|
309
316
|
|
|
310
317
|
**Pre-alpha.** The schema is settling; we bump `CACHE_VERSION` rather
|
|
311
318
|
than maintain back-compat. What 1.0 will promise is in
|
|
312
|
-
[`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md).
|
|
319
|
+
[`versioning-1.0.md`](packages/vx/docs/design/versioning-1.0.md). **over 4,000 core tests plus the package suites; CI green on every commit**;
|
|
313
320
|
the project dogfoods itself (`vx run ci`). Published on npm:
|
|
314
321
|
[`@vzn/vx`](https://www.npmjs.com/package/@vzn/vx) (a prebuilt standalone
|
|
315
322
|
binary).
|
|
@@ -321,7 +328,7 @@ build.
|
|
|
321
328
|
|
|
322
329
|
| Surface | Maturity | Notes |
|
|
323
330
|
| -------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
324
|
-
| Core task runner + caching | **production-ready** | dogfooded continuously;
|
|
331
|
+
| Core task runner + caching | **production-ready** | dogfooded continuously; over 4,000 core tests + the package suites, green |
|
|
325
332
|
| Plugin pipeline (13 hooks, `commands` included) | **shippable** | crash-isolated, re-validated; the local executor + cache are the floor under every plugin |
|
|
326
333
|
| `vx init` / `@vzn/vx-migrate` (scripts; Turbo, Nx) | **shippable** | one config per package, TODOs where a source cannot say |
|
|
327
334
|
| REAPI remote cache + execution (`@vzn/vx-reapi`) | **shippable** | Bazel AC + CAS + Execute; NativeLink / BuildBuddy / Buildbarn / bazel-remote |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.51",
|
|
4
4
|
"description": "An open, extensible monorepo task runner.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
"node": ">=18"
|
|
18
18
|
},
|
|
19
19
|
"optionalDependencies": {
|
|
20
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
21
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
22
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
23
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
20
|
+
"@vzn/vx-linux-x64": "0.0.51",
|
|
21
|
+
"@vzn/vx-linux-arm64": "0.0.51",
|
|
22
|
+
"@vzn/vx-darwin-x64": "0.0.51",
|
|
23
|
+
"@vzn/vx-darwin-arm64": "0.0.51"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/bin.ts
CHANGED
package/src/cache/archive.ts
CHANGED
|
@@ -163,7 +163,10 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
163
163
|
abs,
|
|
164
164
|
size: st.size,
|
|
165
165
|
mode: st.mode & 0o777,
|
|
166
|
-
|
|
166
|
+
// ustar's octal field holds no sign: an mtime before 1970 made the
|
|
167
|
+
// header unreadable and every save of it a "corrupt artifact". The
|
|
168
|
+
// sidecar above carries the real value.
|
|
169
|
+
mtime: Math.max(0, Math.floor(st.mtimeMs / 1000)),
|
|
167
170
|
}
|
|
168
171
|
}),
|
|
169
172
|
)
|
|
@@ -333,9 +336,14 @@ export async function extractArtifactStream(
|
|
|
333
336
|
)
|
|
334
337
|
}
|
|
335
338
|
verify?.(provided)
|
|
339
|
+
// The sidecar's stat is the truth, a mode of 000 and an mtime at or
|
|
340
|
+
// before the epoch included (`SOURCE_DATE_EPOCH=0`). Without one, a
|
|
341
|
+
// header mtime of 0 is a producer that wrote none: unknown.
|
|
336
342
|
await x.commit((name) => {
|
|
337
343
|
const m = meta[name]
|
|
338
|
-
return [m
|
|
344
|
+
if (m !== undefined) return [m[0], m[1]]
|
|
345
|
+
const header = headerMtime.get(name)
|
|
346
|
+
return [0o644, header !== undefined && header > 0 ? header : undefined]
|
|
339
347
|
})
|
|
340
348
|
} catch (err) {
|
|
341
349
|
await x.abort()
|
|
@@ -623,16 +631,20 @@ class Extractor {
|
|
|
623
631
|
* restores' round trips flowing. The chmod is skipped when the mode is
|
|
624
632
|
* the one the temp file was created with — the common case.
|
|
625
633
|
*/
|
|
626
|
-
async commit(
|
|
634
|
+
async commit(
|
|
635
|
+
metaFor: (name: string) => [mode: number, mtimeMs: number | undefined],
|
|
636
|
+
): Promise<void> {
|
|
627
637
|
await this.drain()
|
|
628
638
|
const first = this.staged[0]
|
|
629
639
|
const createdMode = first === undefined ? -1 : statSync(first.tmp).mode & 0o777
|
|
630
640
|
let n = 0
|
|
631
641
|
for (const s of this.staged) {
|
|
632
642
|
const [mode, mtimeMs] = metaFor(s.name)
|
|
633
|
-
if (
|
|
634
|
-
if (mtimeMs
|
|
635
|
-
|
|
643
|
+
if ((mode & 0o777) !== createdMode) chmodSync(s.tmp, mode & 0o777)
|
|
644
|
+
if (mtimeMs !== undefined) {
|
|
645
|
+
// A Date, not seconds: Bun reads a negative number of seconds as
|
|
646
|
+
// "now", and a Date before 1970 as itself (Bun 1.4.2).
|
|
647
|
+
const t = new Date(mtimeMs)
|
|
636
648
|
utimesSync(s.tmp, t, t)
|
|
637
649
|
}
|
|
638
650
|
renameSync(s.tmp, s.target)
|
package/src/cache/cache.ts
CHANGED
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
// layer speaks is `CacheLayer` in layer.ts; `plugin-host.ts` enforces it.
|
|
27
27
|
|
|
28
28
|
import { Database, type SQLQueryBindings } from 'bun:sqlite'
|
|
29
|
-
import { accessSync, constants, existsSync, mkdirSync, writeFileSync } from 'node:fs'
|
|
30
|
-
import { readdir,
|
|
29
|
+
import { accessSync, constants, existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs'
|
|
30
|
+
import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
31
31
|
import path from 'node:path'
|
|
32
32
|
import {
|
|
33
33
|
UserError,
|
|
@@ -188,6 +188,46 @@ function schemaOrdinal(version: string): number {
|
|
|
188
188
|
return m === null ? -1 : Number(m[1])
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
/**
|
|
192
|
+
* Artifacts `indexed` does not hold, and temps, past the in-flight grace
|
|
193
|
+
* window: one readdir, one stat per candidate. `indexed` is asked only
|
|
194
|
+
* once the readdir succeeded: with the directory gone, macOS answers a
|
|
195
|
+
* read of the index SQLITE_IOERR_VNODE, and an unreadable directory
|
|
196
|
+
* reports nothing.
|
|
197
|
+
*/
|
|
198
|
+
async function scanOrphanFiles(
|
|
199
|
+
cacheDir: string,
|
|
200
|
+
indexedHashes: () => ReadonlySet<string>,
|
|
201
|
+
): Promise<Array<{ file: string; size: number }>> {
|
|
202
|
+
let names: string[]
|
|
203
|
+
try {
|
|
204
|
+
names = await readdir(cacheDir)
|
|
205
|
+
} catch {
|
|
206
|
+
return []
|
|
207
|
+
}
|
|
208
|
+
const indexed = indexedHashes()
|
|
209
|
+
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
210
|
+
const candidates: string[] = []
|
|
211
|
+
for (const name of names) {
|
|
212
|
+
const m = VX_ARTIFACT_NAME.exec(name)
|
|
213
|
+
if (m === null) continue
|
|
214
|
+
if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
|
|
215
|
+
}
|
|
216
|
+
const found: Array<{ file: string; size: number }> = []
|
|
217
|
+
await Promise.all(
|
|
218
|
+
candidates.map(async (name) => {
|
|
219
|
+
const file = path.join(cacheDir, name)
|
|
220
|
+
try {
|
|
221
|
+
const st = await stat(file)
|
|
222
|
+
if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
|
|
223
|
+
} catch {
|
|
224
|
+
// Gone between readdir and stat: not an orphan any more.
|
|
225
|
+
}
|
|
226
|
+
}),
|
|
227
|
+
)
|
|
228
|
+
return found
|
|
229
|
+
}
|
|
230
|
+
|
|
191
231
|
/**
|
|
192
232
|
* SQL predicate selecting `runs` rows that record an EXECUTION.
|
|
193
233
|
*
|
|
@@ -400,6 +440,41 @@ export class Cache implements CacheLayer {
|
|
|
400
440
|
return new Cache(cacheDir, undefined, undefined, undefined, 'inspect')
|
|
401
441
|
}
|
|
402
442
|
|
|
443
|
+
/**
|
|
444
|
+
* What a real prune reaps from an index of an EARLIER schema, which it
|
|
445
|
+
* resets first, leaving every aged artifact row-less: a dry run that
|
|
446
|
+
* refused it could not preview the biggest prune there is, the one
|
|
447
|
+
* after an upgrade (item 1083). Reads the recorded version alone and
|
|
448
|
+
* touches nothing. Null for an absent, current, newer or unreadable
|
|
449
|
+
* index; `Cache.inspect` answers those.
|
|
450
|
+
*/
|
|
451
|
+
static async orphansBeforeReset(
|
|
452
|
+
cacheDir: string,
|
|
453
|
+
): Promise<{ found: string; orphans: number; orphanBytes: number } | null> {
|
|
454
|
+
const dbFile = path.join(cacheDir, 'cache.db')
|
|
455
|
+
if (!existsSync(dbFile)) return null
|
|
456
|
+
let found: string | undefined
|
|
457
|
+
try {
|
|
458
|
+
const db = new Database(dbFile, { readonly: true })
|
|
459
|
+
try {
|
|
460
|
+
found = (
|
|
461
|
+
db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
|
|
462
|
+
value: string
|
|
463
|
+
} | null
|
|
464
|
+
)?.value
|
|
465
|
+
} finally {
|
|
466
|
+
db.close()
|
|
467
|
+
}
|
|
468
|
+
} catch {
|
|
469
|
+
return null
|
|
470
|
+
}
|
|
471
|
+
if (found === undefined || schemaOrdinal(found) >= schemaOrdinal(SCHEMA_VERSION)) return null
|
|
472
|
+
const aged = await scanOrphanFiles(cacheDir, () => new Set())
|
|
473
|
+
let orphanBytes = 0
|
|
474
|
+
for (const o of aged) orphanBytes += o.size
|
|
475
|
+
return { found, orphans: aged.length, orphanBytes }
|
|
476
|
+
}
|
|
477
|
+
|
|
403
478
|
constructor(
|
|
404
479
|
private readonly cacheDir: string,
|
|
405
480
|
localPolicy: { read: boolean; write: boolean } = { read: true, write: true },
|
|
@@ -785,7 +860,7 @@ export class Cache implements CacheLayer {
|
|
|
785
860
|
const format = this.db
|
|
786
861
|
.prepare("SELECT value FROM schema_meta WHERE key = 'cache_version'")
|
|
787
862
|
.get() as { value: string } | null
|
|
788
|
-
if (format?.value !== CACHE_VERSION && this.writeBlocked === null) {
|
|
863
|
+
if (format?.value !== CACHE_VERSION && this.writeBlocked === null && !this.inspecting) {
|
|
789
864
|
const hasEntries = this.db.prepare('SELECT 1 FROM entries LIMIT 1').get() != null
|
|
790
865
|
if (this.schemaReset === null && (format !== null || hasEntries)) {
|
|
791
866
|
this.formatChange = { from: format?.value ?? 'an earlier format', to: CACHE_VERSION }
|
|
@@ -976,8 +1051,9 @@ export class Cache implements CacheLayer {
|
|
|
976
1051
|
hash: string,
|
|
977
1052
|
projectDir: string,
|
|
978
1053
|
prefixes: readonly string[],
|
|
1054
|
+
holds?: (files: readonly string[]) => boolean,
|
|
979
1055
|
): Promise<void> {
|
|
980
|
-
await this.outputs.recordOutputDirs(hash, projectDir, prefixes)
|
|
1056
|
+
await this.outputs.recordOutputDirs(hash, projectDir, prefixes, holds)
|
|
981
1057
|
}
|
|
982
1058
|
recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
|
|
983
1059
|
this.outputs.recordOutputStamps(hash, projectDir, workspaceRoot)
|
|
@@ -1070,8 +1146,11 @@ export class Cache implements CacheLayer {
|
|
|
1070
1146
|
}
|
|
1071
1147
|
// What is on disk, not what is in the archive: a directory standing
|
|
1072
1148
|
// where the entry holds a file, or a file where it needs a directory.
|
|
1073
|
-
// The clean removes
|
|
1074
|
-
//
|
|
1149
|
+
// The clean removes the FILES the output globs select and the
|
|
1150
|
+
// directories that removal emptied, so this is a path it left: one
|
|
1151
|
+
// the globs do not cover, or an empty directory where a file goes,
|
|
1152
|
+
// which `dist/**` covers and the clean does not take (item 1094).
|
|
1153
|
+
// Named as such, not as a corrupt artifact. A
|
|
1075
1154
|
// link on the way never lands here: the extractor writes through one
|
|
1076
1155
|
// that stays in the project, dangling or not, and names one that
|
|
1077
1156
|
// leaves it or loops (item 748).
|
|
@@ -1079,7 +1158,7 @@ export class Cache implements CacheLayer {
|
|
|
1079
1158
|
if (code === 'EISDIR' || code === 'ENOTDIR' || code === 'EEXIST' || code === 'ENOTEMPTY') {
|
|
1080
1159
|
throw new UserError(
|
|
1081
1160
|
`restore of ${hash} into ${projectDir} was blocked by what is on disk (${code}: ${(err as Error).message}). ` +
|
|
1082
|
-
`
|
|
1161
|
+
`The clean before a restore removes the files the output globs select, not a directory standing where the entry holds a file, nor a path the globs do not cover — remove it and re-run.`,
|
|
1083
1162
|
)
|
|
1084
1163
|
}
|
|
1085
1164
|
// A legal name under a destination deep enough that the two together
|
|
@@ -1405,18 +1484,8 @@ export class Cache implements CacheLayer {
|
|
|
1405
1484
|
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
1406
1485
|
}
|
|
1407
1486
|
const { entries } = scanned
|
|
1408
|
-
// POSIX rename atomically REPLACES the destination if it exists,
|
|
1409
|
-
// so we don't need a pre-rm. The pre-rm was actively harmful —
|
|
1410
|
-
// it opened a race window where writer B could delete writer A's
|
|
1411
|
-
// just-renamed file BEFORE A's subsequent stat, producing a
|
|
1412
|
-
// spurious ENOENT. The rename itself preserves the "either-or"
|
|
1413
|
-
// semantics for concurrent readers.
|
|
1414
|
-
const endRename = span('save: rename')
|
|
1415
|
-
await rename(tmpPath, finalPath)
|
|
1416
|
-
endRename()
|
|
1417
|
-
|
|
1418
1487
|
const totalBytes =
|
|
1419
|
-
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(
|
|
1488
|
+
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath).size
|
|
1420
1489
|
const outputFileRows: Array<[string, number, number, number]> = []
|
|
1421
1490
|
// Per-output-file fingerprint rows feed the skip-restore check.
|
|
1422
1491
|
// Row paths: project entries store the bare rel (`outputs/`
|
|
@@ -1455,7 +1524,23 @@ export class Cache implements CacheLayer {
|
|
|
1455
1524
|
const outputs = this.outputs
|
|
1456
1525
|
const insertEntryInput = this.insertEntryInput
|
|
1457
1526
|
const inputComponents = meta.inputComponents
|
|
1527
|
+
// The artifact goes live INSIDE the write transaction, taken
|
|
1528
|
+
// IMMEDIATE: the rename happens only once this writer holds the
|
|
1529
|
+
// database's write lock, and its rows commit before any other writer
|
|
1530
|
+
// can rename. Renamed first and indexed after, a commit that failed
|
|
1531
|
+
// (`SQLITE_BUSY` past the busy timeout, `SQLITE_FULL`) left these bytes
|
|
1532
|
+
// beside the previous save's rows, and two writers of one key (two
|
|
1533
|
+
// workspaces on one `--cache-dir`) could pair one's bytes with the
|
|
1534
|
+
// other's rows: every later hit failed the task on an artifact
|
|
1535
|
+
// "missing" a recorded output (A-3). POSIX rename replaces the
|
|
1536
|
+
// destination atomically, so a reader holding the old file reads it
|
|
1537
|
+
// whole. A commit that fails after the rename takes the artifact back
|
|
1538
|
+
// out: the old rows then name no artifact, which a probe reads as a
|
|
1539
|
+
// miss.
|
|
1540
|
+
let renamed = false
|
|
1458
1541
|
const tx = this.db.transaction(() => {
|
|
1542
|
+
renameSync(tmpPath, finalPath)
|
|
1543
|
+
renamed = true
|
|
1459
1544
|
insertEntry.run(
|
|
1460
1545
|
hash,
|
|
1461
1546
|
project,
|
|
@@ -1487,8 +1572,14 @@ export class Cache implements CacheLayer {
|
|
|
1487
1572
|
}
|
|
1488
1573
|
})
|
|
1489
1574
|
const endTx = span('save: index tx')
|
|
1490
|
-
|
|
1491
|
-
|
|
1575
|
+
try {
|
|
1576
|
+
tx.immediate()
|
|
1577
|
+
} catch (err) {
|
|
1578
|
+
await unlink(renamed ? finalPath : tmpPath).catch(() => undefined)
|
|
1579
|
+
throw err
|
|
1580
|
+
} finally {
|
|
1581
|
+
endTx()
|
|
1582
|
+
}
|
|
1492
1583
|
}
|
|
1493
1584
|
|
|
1494
1585
|
/** Apply the deferred accessed_at bumps in one statement. */
|
|
@@ -1618,9 +1709,10 @@ export class Cache implements CacheLayer {
|
|
|
1618
1709
|
let bytesFreed = 0
|
|
1619
1710
|
// Rows whose artifact is gone (deleted by hand, or by a disk cleaner):
|
|
1620
1711
|
// never a hit, and their bytes are on no disk, but `--max-size` counted
|
|
1621
|
-
// them and evicted real entries to make room for them (item 975).
|
|
1622
|
-
//
|
|
1623
|
-
|
|
1712
|
+
// them and evicted real entries to make room for them (item 975). None
|
|
1713
|
+
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
1714
|
+
// grace window are dropped.
|
|
1715
|
+
const { phantoms, stale } = await this.phantomRows()
|
|
1624
1716
|
|
|
1625
1717
|
if (olderThanMs !== undefined) {
|
|
1626
1718
|
const rows = this.db
|
|
@@ -1673,9 +1765,9 @@ export class Cache implements CacheLayer {
|
|
|
1673
1765
|
orphanBytes: orphans.orphanBytes,
|
|
1674
1766
|
}
|
|
1675
1767
|
}
|
|
1676
|
-
if (victims.size > 0 ||
|
|
1768
|
+
if (victims.size > 0 || stale.length > 0) {
|
|
1677
1769
|
const hashes = [...victims]
|
|
1678
|
-
const rows = [...hashes, ...
|
|
1770
|
+
const rows = [...hashes, ...stale]
|
|
1679
1771
|
this.db.transaction(() => {
|
|
1680
1772
|
for (let i = 0; i < rows.length; i += 900) {
|
|
1681
1773
|
const chunk = rows.slice(i, i + 900)
|
|
@@ -1728,25 +1820,37 @@ export class Cache implements CacheLayer {
|
|
|
1728
1820
|
|
|
1729
1821
|
/**
|
|
1730
1822
|
* Index rows whose artifact is not in the directory, with their recorded
|
|
1731
|
-
* sizes
|
|
1732
|
-
*
|
|
1733
|
-
*
|
|
1734
|
-
*
|
|
1823
|
+
* sizes, and the ones among them last used before the grace window. The
|
|
1824
|
+
* rows are read BEFORE the directory: a save renames its artifact in
|
|
1825
|
+
* before its row commits, so every row read has its artifact listed
|
|
1826
|
+
* unless it is gone, and the count is exact. Only `stale` rows are
|
|
1827
|
+
* dropped — a row is never deleted on one listing's word while it may be
|
|
1828
|
+
* in use. An unreadable directory judges nothing.
|
|
1735
1829
|
*/
|
|
1736
|
-
private async phantomRows(): Promise<Map<string, number
|
|
1830
|
+
private async phantomRows(): Promise<{ phantoms: Map<string, number>; stale: string[] }> {
|
|
1831
|
+
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1832
|
+
const rows = this.db
|
|
1833
|
+
.prepare('SELECT hash, size_bytes, accessed_at FROM entries')
|
|
1834
|
+
.all() as Array<{
|
|
1835
|
+
hash: string
|
|
1836
|
+
size_bytes: number
|
|
1837
|
+
accessed_at: number
|
|
1838
|
+
}>
|
|
1737
1839
|
let names: string[]
|
|
1738
1840
|
try {
|
|
1739
1841
|
names = await readdir(this.cacheDir)
|
|
1740
1842
|
} catch {
|
|
1741
|
-
return new Map()
|
|
1843
|
+
return { phantoms: new Map(), stale: [] }
|
|
1742
1844
|
}
|
|
1743
1845
|
const present = new Set(names)
|
|
1744
|
-
const
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1846
|
+
const phantoms = new Map<string, number>()
|
|
1847
|
+
const stale: string[] = []
|
|
1848
|
+
for (const r of rows) {
|
|
1849
|
+
if (present.has(`${r.hash}.tar.zst`)) continue
|
|
1850
|
+
phantoms.set(r.hash, r.size_bytes)
|
|
1851
|
+
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
1852
|
+
}
|
|
1853
|
+
return { phantoms, stale }
|
|
1750
1854
|
}
|
|
1751
1855
|
|
|
1752
1856
|
/** What `prune()` would reap right now, for `vx info` to say before anyone prunes. */
|
|
@@ -1759,37 +1863,15 @@ export class Cache implements CacheLayer {
|
|
|
1759
1863
|
|
|
1760
1864
|
/** Row-less artifacts and temps past the in-flight grace window: one readdir, one stat per candidate. */
|
|
1761
1865
|
private async scanOrphans(): Promise<Array<{ file: string; size: number }>> {
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
(r) => r.hash,
|
|
1771
|
-
),
|
|
1772
|
-
)
|
|
1773
|
-
const cutoff = Date.now() - ORPHAN_GRACE_MS
|
|
1774
|
-
const candidates: string[] = []
|
|
1775
|
-
for (const name of names) {
|
|
1776
|
-
const m = VX_ARTIFACT_NAME.exec(name)
|
|
1777
|
-
if (m === null) continue
|
|
1778
|
-
if (m[2] !== undefined || !indexed.has(m[1]!)) candidates.push(name)
|
|
1779
|
-
}
|
|
1780
|
-
const found: Array<{ file: string; size: number }> = []
|
|
1781
|
-
await Promise.all(
|
|
1782
|
-
candidates.map(async (name) => {
|
|
1783
|
-
const file = path.join(this.cacheDir, name)
|
|
1784
|
-
try {
|
|
1785
|
-
const st = await stat(file)
|
|
1786
|
-
if (st.isFile() && st.mtimeMs <= cutoff) found.push({ file, size: st.size })
|
|
1787
|
-
} catch {
|
|
1788
|
-
// Gone between readdir and stat: not an orphan any more.
|
|
1789
|
-
}
|
|
1790
|
-
}),
|
|
1866
|
+
return scanOrphanFiles(
|
|
1867
|
+
this.cacheDir,
|
|
1868
|
+
() =>
|
|
1869
|
+
new Set(
|
|
1870
|
+
(this.db.prepare('SELECT hash FROM entries').all() as Array<{ hash: string }>).map(
|
|
1871
|
+
(r) => r.hash,
|
|
1872
|
+
),
|
|
1873
|
+
),
|
|
1791
1874
|
)
|
|
1792
|
-
return found
|
|
1793
1875
|
}
|
|
1794
1876
|
|
|
1795
1877
|
close(): void {
|
|
@@ -1810,6 +1892,7 @@ export class Cache implements CacheLayer {
|
|
|
1810
1892
|
const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000
|
|
1811
1893
|
this.history.pruneOlderThan(cutoff)
|
|
1812
1894
|
this.configEvals.pruneOlderThan(cutoff)
|
|
1895
|
+
if (this.write) this.files.pruneOlderThan(cutoff)
|
|
1813
1896
|
}
|
|
1814
1897
|
} catch {
|
|
1815
1898
|
// Retention is best-effort; never block closing the handle.
|
|
@@ -161,8 +161,15 @@ export class ChainedCache implements CacheLayer {
|
|
|
161
161
|
|
|
162
162
|
// The directory short-circuit lives with the layer that owns the local
|
|
163
163
|
// rows — the first, like the file check.
|
|
164
|
-
recordOutputDirs(
|
|
165
|
-
|
|
164
|
+
recordOutputDirs(
|
|
165
|
+
hash: string,
|
|
166
|
+
projectDir: string,
|
|
167
|
+
prefixes: readonly string[],
|
|
168
|
+
holds?: (files: readonly string[]) => boolean,
|
|
169
|
+
): Promise<void> {
|
|
170
|
+
return (
|
|
171
|
+
this.layers[0]!.recordOutputDirs?.(hash, projectDir, prefixes, holds) ?? Promise.resolve()
|
|
172
|
+
)
|
|
166
173
|
}
|
|
167
174
|
recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
|
|
168
175
|
this.layers[0]!.recordOutputStamps?.(hash, projectDir, workspaceRoot)
|
package/src/cache/file-hashes.ts
CHANGED
|
@@ -10,7 +10,10 @@ import type { Database } from 'bun:sqlite'
|
|
|
10
10
|
import { lstatSync, readlinkSync } from 'node:fs'
|
|
11
11
|
import path from 'node:path'
|
|
12
12
|
import { fileIdentity, repoFacts } from './git-inputs.js'
|
|
13
|
-
import { FILE_HASH_RACY_MS } from './layer.js'
|
|
13
|
+
import { FILE_HASH_RACY_MS, racyWindowMs } from './layer.js'
|
|
14
|
+
|
|
15
|
+
const FILE_HASHES_SWEPT_AT = 'file_hashes_swept_at'
|
|
16
|
+
const DAY_MS = 24 * 60 * 60 * 1000
|
|
14
17
|
|
|
15
18
|
export class FileHashStore {
|
|
16
19
|
private readonly selectFileHash: ReturnType<Database['prepare']>
|
|
@@ -47,6 +50,29 @@ export class FileHashStore {
|
|
|
47
50
|
`)
|
|
48
51
|
}
|
|
49
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Drop the memo rows last written before `cutoff`: a path that is gone
|
|
55
|
+
* (a deleted generated input, a worktree removed, a CI checkout at a new
|
|
56
|
+
* path) keeps its row forever otherwise, and `cache.db` grows with every
|
|
57
|
+
* path ever hashed. A hit does not refresh `seen_at` (a write per file
|
|
58
|
+
* per run), so a file unchanged for the window loses its row too and is
|
|
59
|
+
* read once more; a dropped row is only a memo miss. The scan costs
|
|
60
|
+
* 8 ms at 100,000 rows, so it runs once a day, on its own clock in
|
|
61
|
+
* `schema_meta` (item 1082).
|
|
62
|
+
*/
|
|
63
|
+
pruneOlderThan(cutoff: number, now: number = Date.now()): void {
|
|
64
|
+
const last = this.db
|
|
65
|
+
.prepare('SELECT value FROM schema_meta WHERE key = ?')
|
|
66
|
+
.get(FILE_HASHES_SWEPT_AT) as { value: string } | null
|
|
67
|
+
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
68
|
+
this.db.transaction(() => {
|
|
69
|
+
this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
|
|
70
|
+
this.db
|
|
71
|
+
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
72
|
+
.run(FILE_HASHES_SWEPT_AT, String(now))
|
|
73
|
+
})()
|
|
74
|
+
}
|
|
75
|
+
|
|
50
76
|
/**
|
|
51
77
|
* Content-hash a file (as a git blob OID, v20) with an mtime+size
|
|
52
78
|
* fast path. If the `file_hashes` table has a row for `path` whose
|
|
@@ -136,7 +162,7 @@ export class FileHashStore {
|
|
|
136
162
|
// changed within the window is hashed again on its next call rather
|
|
137
163
|
// than memoised; the warm path never meets it (keys are derived long
|
|
138
164
|
// after the files were written).
|
|
139
|
-
if (this.write && Date.now() - ctimeMs >= FILE_HASH_RACY_MS) {
|
|
165
|
+
if (this.write && Date.now() - ctimeMs >= racyWindowMs(st.ctimeMs, FILE_HASH_RACY_MS)) {
|
|
140
166
|
this.upsertFileHash.run(filePath, mtimeMs, size, ctimeMs, ino, ch, Date.now())
|
|
141
167
|
}
|
|
142
168
|
return fileIdentity(mode, ch)
|
|
@@ -161,6 +187,8 @@ export class FileHashStore {
|
|
|
161
187
|
mtimeMs: number
|
|
162
188
|
size: number
|
|
163
189
|
ctimeMs: number
|
|
190
|
+
/** Unfloored, for the racy window: a whole second is a whole-second clock only here. */
|
|
191
|
+
ctimeRaw: number
|
|
164
192
|
ino: number
|
|
165
193
|
}
|
|
166
194
|
const stats = new Map<string, Stat>()
|
|
@@ -188,6 +216,7 @@ export class FileHashStore {
|
|
|
188
216
|
mtimeMs: Math.floor(st.mtimeMs),
|
|
189
217
|
size: st.size,
|
|
190
218
|
ctimeMs: Math.floor(st.ctimeMs),
|
|
219
|
+
ctimeRaw: st.ctimeMs,
|
|
191
220
|
ino: Number(st.ino),
|
|
192
221
|
})
|
|
193
222
|
} catch {
|
|
@@ -241,7 +270,7 @@ export class FileHashStore {
|
|
|
241
270
|
out.set(p, fileIdentity(st.mode, digest))
|
|
242
271
|
// The same racy-clean rule as `hashFile`: a stat taken within the
|
|
243
272
|
// window of the file's last change is not memoised.
|
|
244
|
-
if (this.write && now - st.ctimeMs >= FILE_HASH_RACY_MS) {
|
|
273
|
+
if (this.write && now - st.ctimeMs >= racyWindowMs(st.ctimeRaw, FILE_HASH_RACY_MS)) {
|
|
245
274
|
this.upsertFileHash.run(p, st.mtimeMs, st.size, st.ctimeMs, st.ino, digest, now)
|
|
246
275
|
}
|
|
247
276
|
}
|