@vzn/vx 0.0.651 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/launcher.sh +2 -1
- package/package.json +5 -5
- package/schemas/error.json +1 -1
- package/schemas/show.json +15 -20
- package/skills/vx/SKILL.md +3 -3
- package/src/bin.ts +37 -3
- package/src/cache/archive.ts +197 -32
- package/src/cache/cache.ts +882 -230
- package/src/cache/checkpointer.ts +102 -0
- package/src/cache/file-hashes.ts +2 -1
- package/src/cache/fold-memo.ts +24 -4
- package/src/cache/fs-clock.ts +26 -0
- package/src/cache/git-inputs.ts +188 -22
- package/src/cache/history-stream.ts +151 -0
- package/src/cache/history-writer.ts +76 -0
- package/src/cache/index.ts +12 -1
- package/src/cache/inputs.ts +616 -135
- package/src/cache/key-fold.ts +1 -1
- package/src/cache/layer.ts +29 -16
- package/src/cache/layered-cache.ts +28 -7
- package/src/cache/lockfile-memo.ts +2 -1
- package/src/cache/output-index.ts +106 -44
- package/src/cache/plugin-cache.ts +2 -2
- package/src/cache/policy.ts +6 -2
- package/src/cache/remote-lane.ts +3 -1
- package/src/cache/run-history.ts +94 -25
- package/src/cache/schema.ts +46 -10
- package/src/cli/cache.ts +149 -36
- package/src/cli/completions.ts +2 -2
- package/src/cli/core-alias.ts +9 -2
- package/src/cli/docs.ts +4 -2
- package/src/cli/help.ts +27 -4
- package/src/cli/index.ts +62 -7
- package/src/cli/info.ts +5 -4
- package/src/cli/init.ts +55 -20
- package/src/cli/last.ts +218 -43
- package/src/cli/lock.ts +187 -24
- package/src/cli/plan-format.ts +24 -21
- package/src/cli/plugin-commands.ts +6 -1
- package/src/cli/plugin-templates.ts +36 -45
- package/src/cli/run.ts +68 -34
- package/src/cli/select.ts +160 -64
- package/src/cli/show.ts +70 -28
- package/src/cli/task-verb.ts +12 -7
- package/src/cli/upgrade.ts +210 -76
- package/src/cli/watch-filter.ts +88 -15
- package/src/cli/watch-fs.ts +34 -47
- package/src/cli/watch-judge.ts +73 -9
- package/src/cli/watch-set.ts +10 -8
- package/src/cli/watch.ts +190 -33
- package/src/cli/why.ts +107 -33
- package/src/cli/workspace-config.ts +11 -1
- package/src/config.ts +56 -30
- package/src/exec/env.ts +3 -1
- package/src/exec/executor.ts +116 -16
- package/src/exec/index.ts +1 -0
- package/src/exec/kill-tree.ts +7 -7
- package/src/exec/runner.ts +270 -108
- package/src/exec/sandbox-binds.ts +109 -18
- package/src/exec/sandbox-paths.ts +3 -1
- package/src/exec/sandbox-runtime.ts +446 -64
- package/src/exec/sandbox-violations.ts +250 -46
- package/src/exec/spawn-off.ts +10 -5
- package/src/exec/task-proxy.ts +93 -0
- package/src/exec/warm-shell.ts +29 -7
- package/src/graph/definition.ts +16 -3
- package/src/graph/folds.ts +89 -16
- package/src/graph/index.ts +8 -0
- package/src/graph/priorities.ts +8 -5
- package/src/graph/scheduler.ts +176 -39
- package/src/graph/stand-ins.ts +36 -0
- package/src/graph/task-graph.ts +347 -160
- package/src/graph/task-status.ts +5 -1
- package/src/handoff.ts +45 -10
- package/src/history/failure-mode.ts +39 -6
- package/src/history/history.ts +103 -30
- package/src/history/index.ts +2 -0
- package/src/history/metrics.ts +193 -73
- package/src/index.ts +6 -3
- package/src/orchestrator/admission.ts +14 -10
- package/src/orchestrator/affected-tasks.ts +35 -13
- package/src/orchestrator/command-windows.ts +8 -5
- package/src/orchestrator/deferred-outputs.ts +42 -13
- package/src/orchestrator/doctor.ts +21 -9
- package/src/orchestrator/download-policy.ts +8 -10
- package/src/orchestrator/excluded-keys.ts +15 -3
- package/src/orchestrator/execute-args.ts +9 -2
- package/src/orchestrator/execute-task.ts +612 -138
- package/src/orchestrator/graph-memo.ts +26 -4
- package/src/orchestrator/graph-stage.ts +27 -4
- package/src/orchestrator/held-servers.ts +36 -14
- package/src/orchestrator/hit-restore.ts +141 -93
- package/src/orchestrator/index.ts +4 -1
- package/src/orchestrator/keyed-projects.ts +53 -0
- package/src/orchestrator/local-floor.ts +21 -3
- package/src/orchestrator/local-shortcircuit.ts +2 -2
- package/src/orchestrator/lockfile-claim.ts +30 -4
- package/src/orchestrator/miss-save.ts +60 -61
- package/src/orchestrator/options.ts +13 -0
- package/src/orchestrator/output-host.ts +16 -4
- package/src/orchestrator/output-log.ts +1 -54
- package/src/orchestrator/persistent-task.ts +32 -16
- package/src/orchestrator/persistent.ts +152 -15
- package/src/orchestrator/placement.ts +82 -16
- package/src/orchestrator/plan.ts +39 -17
- package/src/orchestrator/plugin-host.ts +238 -110
- package/src/orchestrator/plugin.ts +59 -27
- package/src/orchestrator/prepare.ts +194 -38
- package/src/orchestrator/projects.ts +89 -10
- package/src/orchestrator/read-only.ts +115 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-context.ts +6 -2
- package/src/orchestrator/run-failures.ts +101 -23
- package/src/orchestrator/run-id.ts +13 -2
- package/src/orchestrator/run-keys.ts +87 -28
- package/src/orchestrator/run-live.ts +16 -13
- package/src/orchestrator/run-lock.ts +27 -4
- package/src/orchestrator/run-records.ts +95 -74
- package/src/orchestrator/run-report.ts +3 -2
- package/src/orchestrator/run-stop.ts +17 -9
- package/src/orchestrator/run.ts +531 -122
- package/src/orchestrator/sandbox-fix.ts +40 -12
- package/src/orchestrator/sandbox-request.ts +314 -44
- package/src/orchestrator/save-lane.ts +24 -13
- package/src/orchestrator/shell-verdict.ts +29 -1
- package/src/orchestrator/signals.ts +103 -8
- package/src/orchestrator/stable-keys.ts +34 -27
- package/src/orchestrator/task-artifacts.ts +16 -3
- package/src/orchestrator/task-hash.ts +191 -34
- package/src/orchestrator/telemetry-host.ts +25 -6
- package/src/orchestrator/telemetry.ts +85 -13
- package/src/orchestrator/upstream.ts +3 -1
- package/src/report/events.ts +23 -6
- package/src/report/forecast.ts +103 -37
- package/src/report/framed-output.ts +179 -162
- package/src/report/index.ts +2 -1
- package/src/report/logger.ts +108 -31
- package/src/report/plain-output.ts +15 -9
- package/src/report/status-line.ts +28 -55
- package/src/report/summary.ts +61 -18
- package/src/settings.ts +50 -14
- package/src/ui/components.ts +36 -42
- package/src/ui/index.ts +3 -1
- package/src/ui/live-region.ts +67 -8
- package/src/ui/terminal.ts +12 -0
- package/src/ui/theme.ts +16 -1
- package/src/util/config-frame.ts +142 -6
- package/src/util/edit-distance.ts +17 -1
- package/src/util/errors.ts +23 -7
- package/src/util/foreign-cache-dir.ts +60 -0
- package/src/util/git-ignored.ts +59 -5
- package/src/util/index.ts +15 -1
- package/src/util/paths.ts +51 -7
- package/src/util/plain-text.ts +19 -0
- package/src/util/secret-mask.ts +57 -1
- package/src/util/settle.ts +18 -1
- package/src/util/size.ts +12 -2
- package/src/util/tail.ts +4 -1
- package/src/util/write-file.ts +86 -9
- package/src/workspace/affected.ts +225 -27
- package/src/workspace/bracket-literal.ts +2 -2
- package/src/workspace/config-bytecode.ts +23 -4
- package/src/workspace/config-eval.ts +90 -16
- package/src/workspace/config-imports.ts +96 -41
- package/src/workspace/config-schema.ts +192 -86
- package/src/workspace/dependencies-lock.ts +7 -14
- package/src/workspace/filter.ts +18 -2
- package/src/workspace/index.ts +3 -1
- package/src/workspace/json-data.ts +34 -8
- package/src/workspace/lockfile.ts +86 -37
- package/src/workspace/migration.ts +58 -31
- package/src/workspace/module-state.ts +62 -0
- package/src/workspace/nested-dirs.ts +8 -0
- package/src/workspace/project-loader.ts +276 -32
- package/src/workspace/repo-id.ts +29 -6
- package/src/workspace/workspace.ts +190 -30
package/src/cache/cache.ts
CHANGED
|
@@ -45,13 +45,31 @@ import {
|
|
|
45
45
|
readdirSync,
|
|
46
46
|
readFileSync,
|
|
47
47
|
renameSync,
|
|
48
|
+
type Stats,
|
|
48
49
|
statSync,
|
|
49
50
|
utimesSync,
|
|
50
51
|
writeFileSync,
|
|
51
52
|
} from 'node:fs'
|
|
52
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
chmod,
|
|
55
|
+
mkdir,
|
|
56
|
+
readdir,
|
|
57
|
+
rename,
|
|
58
|
+
rm,
|
|
59
|
+
stat,
|
|
60
|
+
unlink,
|
|
61
|
+
utimes,
|
|
62
|
+
writeFile,
|
|
63
|
+
} from 'node:fs/promises'
|
|
53
64
|
import path from 'node:path'
|
|
54
|
-
import {
|
|
65
|
+
import {
|
|
66
|
+
createArtifactTables,
|
|
67
|
+
createTables,
|
|
68
|
+
inHashes,
|
|
69
|
+
lazyStatement,
|
|
70
|
+
RUNS_VIEW,
|
|
71
|
+
SQLITE_FCNTL_PERSIST_WAL,
|
|
72
|
+
} from './schema.js'
|
|
55
73
|
import {
|
|
56
74
|
UserError,
|
|
57
75
|
DOT_VX_GITIGNORE,
|
|
@@ -61,6 +79,8 @@ import {
|
|
|
61
79
|
isFsRefusal,
|
|
62
80
|
isOutOfFds,
|
|
63
81
|
OUT_OF_FDS_HINT,
|
|
82
|
+
OWN_DIR_HINT,
|
|
83
|
+
refuseForeignCacheDir,
|
|
64
84
|
restorableOutput,
|
|
65
85
|
relPosix,
|
|
66
86
|
span,
|
|
@@ -71,6 +91,7 @@ import {
|
|
|
71
91
|
ArchiveSecurityError,
|
|
72
92
|
type ArtifactPlan,
|
|
73
93
|
type ExecUsage,
|
|
94
|
+
copyStaging,
|
|
74
95
|
extractArtifactStream,
|
|
75
96
|
scanArtifact,
|
|
76
97
|
scanTarBytes,
|
|
@@ -79,9 +100,10 @@ import {
|
|
|
79
100
|
planArtifact,
|
|
80
101
|
planOfTree,
|
|
81
102
|
sealTree,
|
|
103
|
+
stagesInto,
|
|
82
104
|
writeTree,
|
|
83
105
|
} from './archive.js'
|
|
84
|
-
import { readUmask, restoredMode } from './umask.js'
|
|
106
|
+
import { modeMatches, readUmask, restoredMode } from './umask.js'
|
|
85
107
|
import {
|
|
86
108
|
ArtifactVanishedError,
|
|
87
109
|
type CacheEntry,
|
|
@@ -120,6 +142,7 @@ import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
|
120
142
|
import { type FoldMemo, IndexFoldMemo } from './fold-memo.js'
|
|
121
143
|
import { IndexRemoteHits, type RemoteHits } from './remote-hits.js'
|
|
122
144
|
import { behindLink } from './linked-inputs.js'
|
|
145
|
+
import { WalCheckpointer } from './checkpointer.js'
|
|
123
146
|
|
|
124
147
|
/**
|
|
125
148
|
* An artifact or temp file without an `entries` row is never reaped
|
|
@@ -132,6 +155,10 @@ import { behindLink } from './linked-inputs.js'
|
|
|
132
155
|
*/
|
|
133
156
|
// About 1.3 ms of inserts per turn: a task's exit waits at most that.
|
|
134
157
|
const HELD_SLICE = 250
|
|
158
|
+
/** SQLite's own default, back on a connection whose checkpoint thread failed. */
|
|
159
|
+
const WAL_AUTOCHECKPOINT = 1000
|
|
160
|
+
/** Asks whether an open database file is still the one at its name (`attachStore`). */
|
|
161
|
+
const SQLITE_FCNTL_HAS_MOVED = 20
|
|
135
162
|
|
|
136
163
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
137
164
|
|
|
@@ -167,6 +194,7 @@ const VX_CACHE_NAMES = new Set([
|
|
|
167
194
|
'cache.db-wal',
|
|
168
195
|
'cache.db-shm',
|
|
169
196
|
'cache.db-journal',
|
|
197
|
+
'cache.lock',
|
|
170
198
|
// Every record vx keeps lives in cache.db, and the config bundle always
|
|
171
199
|
// in `.vx/cache/compiled`, so any directory here is the user's.
|
|
172
200
|
])
|
|
@@ -275,7 +303,7 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
|
|
|
275
303
|
// the memo simply stops answering wrongly, so an affected task's
|
|
276
304
|
// key moves from a WRONG value to the right one: it misses once,
|
|
277
305
|
// re-runs, re-caches. Self-healing, never a wrong hit.
|
|
278
|
-
// v25: runs.cached — whether the task declared
|
|
306
|
+
// v25: runs.cached — whether the task declared `cache: true`. An
|
|
279
307
|
// uncached task's row looked like any executed miss, so `vx why`
|
|
280
308
|
// headlined "cache key changed (inputs differ)" for a task that
|
|
281
309
|
// runs every time by design, and `vx last` could not mark it the
|
|
@@ -361,7 +389,23 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
|
|
|
361
389
|
// v51: runs.queued_ms, output_bytes, blocks, flaky_passes and
|
|
362
390
|
// flaky_failures — the rest of what a results row shows, so `vx
|
|
363
391
|
// last --sql` reads it. The cache KEY is unchanged.
|
|
364
|
-
|
|
392
|
+
// v52: entry_stdout.stdout is a zstd frame of the log's UTF-8, not its
|
|
393
|
+
// text: stored logs were half a store's bytes, more than every
|
|
394
|
+
// inline artifact. The cache KEY is unchanged.
|
|
395
|
+
// v53: a runs row names its run by its row in run_ids, not by the
|
|
396
|
+
// 36-char text; a TEMP view `runs` shows run_id as before. An
|
|
397
|
+
// all-hit run of 500 tasks added 115 KB to the index. The cache
|
|
398
|
+
// KEY is unchanged.
|
|
399
|
+
// v54: entries.run_id — the run that wrote the entry, so `vx last
|
|
400
|
+
// --log` tells a run's own output from another run's under the same
|
|
401
|
+
// key. The cache KEY is unchanged.
|
|
402
|
+
// v55: output_stamps.mtime_ms — the mtime the file carried when
|
|
403
|
+
// stamped. A restore dates each file it writes now, as the task's
|
|
404
|
+
// write would, not with the entry's saved mtime: a make-style
|
|
405
|
+
// dependant took a restored upstream as older than its own output
|
|
406
|
+
// and saved bytes built from another upstream. The cache KEY and
|
|
407
|
+
// the stored artifact are unchanged.
|
|
408
|
+
export const SCHEMA_VERSION = 'v55'
|
|
365
409
|
|
|
366
410
|
/**
|
|
367
411
|
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
@@ -413,38 +457,50 @@ const ARTIFACT_TABLES = ['artifacts', 'artifacts_meta']
|
|
|
413
457
|
const ARTIFACT_LAYOUT = 'a2'
|
|
414
458
|
|
|
415
459
|
/**
|
|
416
|
-
*
|
|
417
|
-
*
|
|
418
|
-
*
|
|
460
|
+
* A preview's index connection and the schema it records, which the open
|
|
461
|
+
* keeps and reuses. A version other than this vx's is the index an opener
|
|
462
|
+
* that writes resets; an unreadable file is left to the open to refuse.
|
|
419
463
|
*/
|
|
420
|
-
function
|
|
421
|
-
let
|
|
464
|
+
function previewIndex(dbFile: string): { db: Database; version: string | undefined } | undefined {
|
|
465
|
+
let db: Database
|
|
422
466
|
try {
|
|
423
|
-
|
|
424
|
-
try {
|
|
425
|
-
found = (
|
|
426
|
-
db.query("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
|
|
427
|
-
value: string
|
|
428
|
-
} | null
|
|
429
|
-
)?.value
|
|
430
|
-
} finally {
|
|
431
|
-
closeDb(db)
|
|
432
|
-
}
|
|
467
|
+
db = new Database(dbFile, { create: true })
|
|
433
468
|
} catch {
|
|
434
|
-
|
|
469
|
+
// The main open names why.
|
|
470
|
+
return undefined
|
|
471
|
+
}
|
|
472
|
+
try {
|
|
473
|
+
const version = (
|
|
474
|
+
db.prepare("SELECT value FROM schema_meta WHERE key = 'version'").get() as {
|
|
475
|
+
value: string
|
|
476
|
+
} | null
|
|
477
|
+
)?.value
|
|
478
|
+
return { db, version }
|
|
479
|
+
} catch {
|
|
480
|
+
// No stamp read (no table yet, busy, not a database): the main open
|
|
481
|
+
// reads it again and refuses what it must.
|
|
482
|
+
return { db, version: undefined }
|
|
435
483
|
}
|
|
436
|
-
return found !== undefined && found !== SCHEMA_VERSION
|
|
437
484
|
}
|
|
438
485
|
|
|
486
|
+
/** The largest compressed log a probe loads with its entry. */
|
|
487
|
+
const LOG_INLINE_BYTES = 4096
|
|
488
|
+
|
|
439
489
|
/**
|
|
440
|
-
* Each asked key's entry row
|
|
441
|
-
*
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
490
|
+
* Each asked key's entry row, its stored log while that is small, and its
|
|
491
|
+
* inline artifact's last use, null when the artifact is a file. A larger
|
|
492
|
+
* log is only flagged (`stdout_apart`), and read when its hit replays: a
|
|
493
|
+
* warm run probes every key in one batch and replays the hits as the
|
|
494
|
+
* scheduler reaches them, so every log joined here was held until its
|
|
495
|
+
* replay (500 hits printing 1 MiB each held 3.4 GB). A small one rides
|
|
496
|
+
* along, still compressed: a read apart per hit cost a warm run of 300
|
|
497
|
+
* short logs ~1 ms of 44. `length` reads a blob's size, not its pages.
|
|
498
|
+
* Driven by the keys, so a key with an inline artifact and no row (a reset
|
|
499
|
+
* kept it) answers too, `hash` null: the miss learns there is something to
|
|
500
|
+
* adopt in the statement that found no entry.
|
|
445
501
|
*/
|
|
446
502
|
const selectEntries = (keys: string): string =>
|
|
447
|
-
`SELECT q.value AS key, e.*,
|
|
503
|
+
`SELECT q.value AS key, e.*, CASE WHEN length(s.stdout) <= ${LOG_INLINE_BYTES} THEN s.stdout END AS stdout, length(s.stdout) > ${LOG_INLINE_BYTES} AS stdout_apart, a.at AS inline_at FROM ${keys} q LEFT JOIN entries e ON e.hash = q.value LEFT JOIN entry_stdout s ON s.hash = e.hash LEFT JOIN artifacts a ON a.hash = q.value WHERE e.hash IS NOT NULL OR a.at IS NOT NULL`
|
|
448
504
|
const SELECT_ENTRY = selectEntries('(SELECT ? AS value)')
|
|
449
505
|
const SELECT_ENTRIES = selectEntries('json_each(?)')
|
|
450
506
|
|
|
@@ -610,7 +666,10 @@ interface EntryRow {
|
|
|
610
666
|
exit_code: number
|
|
611
667
|
duration_ms: number
|
|
612
668
|
size_bytes: number
|
|
613
|
-
|
|
669
|
+
/** The compressed log when no more than `LOG_INLINE_BYTES`, else null. */
|
|
670
|
+
stdout: Uint8Array | null
|
|
671
|
+
/** 1 when the log is larger, and read apart; null when none was stored. */
|
|
672
|
+
stdout_apart: number | null
|
|
614
673
|
created_at: number
|
|
615
674
|
accessed_at: number
|
|
616
675
|
cpu_ms: number | null
|
|
@@ -618,24 +677,45 @@ interface EntryRow {
|
|
|
618
677
|
inline_at: number | null
|
|
619
678
|
}
|
|
620
679
|
|
|
621
|
-
|
|
622
|
-
|
|
680
|
+
/**
|
|
681
|
+
* The entry a row answers. Its log is a getter, decompressed on each access
|
|
682
|
+
* and, past `LOG_INLINE_BYTES`, read from the index then (`log`), so an
|
|
683
|
+
* entry held from the probe to its replay holds no log text and the replay
|
|
684
|
+
* drops it once printed. A reader takes it once.
|
|
685
|
+
*/
|
|
686
|
+
function entryOf(
|
|
687
|
+
row: LiveRow,
|
|
688
|
+
fileRows: OutputFileRow[],
|
|
689
|
+
log: (hash: string) => string,
|
|
690
|
+
): CacheEntry {
|
|
691
|
+
// Assigned, not spread, and the date formatted only when read: a warm
|
|
692
|
+
// run's probe builds one per hit and reads neither (~4 ms of 8,002
|
|
693
|
+
// entries, 2026-11-07).
|
|
694
|
+
const entry: CacheEntry = {
|
|
623
695
|
hash: row.hash,
|
|
624
696
|
taskId: `${row.project}#${row.task}`,
|
|
625
697
|
command: row.command,
|
|
626
698
|
exitCode: row.exit_code,
|
|
627
699
|
durationMs: row.duration_ms,
|
|
628
700
|
sizeBytes: row.size_bytes,
|
|
629
|
-
...(row.cpu_ms !== null ? { cpuMs: row.cpu_ms } : {}),
|
|
630
|
-
...(row.peak_rss_bytes !== null ? { peakRssBytes: row.peak_rss_bytes } : {}),
|
|
631
701
|
outputFiles: fileRows.map((r) => r.path),
|
|
632
702
|
outputRows: fileRows,
|
|
633
|
-
stdout
|
|
634
|
-
|
|
703
|
+
get stdout() {
|
|
704
|
+
return row.stdout_apart === 1 ? log(row.hash) : storedLog(row.stdout)
|
|
705
|
+
},
|
|
706
|
+
get storedAt() {
|
|
707
|
+
return new Date(row.created_at).toISOString()
|
|
708
|
+
},
|
|
635
709
|
source: 'local',
|
|
636
710
|
}
|
|
711
|
+
if (row.cpu_ms !== null) entry.cpuMs = row.cpu_ms
|
|
712
|
+
if (row.peak_rss_bytes !== null) entry.peakRssBytes = row.peak_rss_bytes
|
|
713
|
+
return entry
|
|
637
714
|
}
|
|
638
715
|
|
|
716
|
+
/** Why a reading handle (`'inspect'`, `'preview'`) writes nothing: no index to read was there. */
|
|
717
|
+
const NO_INDEX = 'no index there yet'
|
|
718
|
+
|
|
639
719
|
/** The sidecar's `exec` from a save's entry: only the axes the runner reported. */
|
|
640
720
|
function usageOfEntry(entry: { cpuMs?: number; peakRssBytes?: number }): ExecUsage | undefined {
|
|
641
721
|
if (entry.cpuMs === undefined && entry.peakRssBytes === undefined) return undefined
|
|
@@ -657,7 +737,8 @@ function usageOfEntry(entry: { cpuMs?: number; peakRssBytes?: number }): ExecUsa
|
|
|
657
737
|
* failure is another question's answer: `access` on the directory is also
|
|
658
738
|
* "does it exist" (made only on ENOENT), `access` on the database also "is
|
|
659
739
|
* it there", and the ignore file is created exclusively instead of probed
|
|
660
|
-
* first. An existing cache costs three calls
|
|
740
|
+
* first. An existing cache costs three calls, it cost six, and two more
|
|
741
|
+
* resolve it (`refuseForeignCacheDir`).
|
|
661
742
|
*/
|
|
662
743
|
function openCacheDir(cacheDir: string, workspaceRoot?: string, index = 'cache.db'): string | null {
|
|
663
744
|
const [ignore, ignoreText] = ignoreFileFor(cacheDir)
|
|
@@ -678,12 +759,17 @@ function openCacheDir(cacheDir: string, workspaceRoot?: string, index = 'cache.d
|
|
|
678
759
|
if (code !== 'ENOENT') return errorText(err)
|
|
679
760
|
refuseManifestDir(cacheDir, workspaceRoot)
|
|
680
761
|
}
|
|
762
|
+
if (index === 'cache.db') refuseForeignCacheDir(cacheDir, workspaceRoot)
|
|
681
763
|
writeIgnore(ignore, ignoreText)
|
|
682
764
|
return null
|
|
683
765
|
}
|
|
684
766
|
|
|
685
|
-
/**
|
|
686
|
-
|
|
767
|
+
/**
|
|
768
|
+
* `openCacheDir` for the shared store: why it cannot be used, or null. Never
|
|
769
|
+
* throws. `make` false asks the same question and makes or changes
|
|
770
|
+
* nothing, so a reading verb settles where the entries are as a run does.
|
|
771
|
+
*/
|
|
772
|
+
function storeBlocked(storeDir: string, storeRoot: string | null, make: boolean): string | null {
|
|
687
773
|
try {
|
|
688
774
|
if (storeRoot !== null) {
|
|
689
775
|
// `~/.vx`, `~/.vx/<id>`, `~/.vx/<id>/cache`: owner-only, each level
|
|
@@ -691,21 +777,53 @@ function storeBlocked(storeDir: string, storeRoot: string | null): string | null
|
|
|
691
777
|
// replayed into the workspace, so nobody else may write them.
|
|
692
778
|
const vxHome = path.dirname(path.dirname(storeRoot))
|
|
693
779
|
for (const dir of [vxHome, path.dirname(storeRoot), storeRoot]) {
|
|
694
|
-
|
|
780
|
+
// The levels below one not made yet are made by this user, 0700.
|
|
781
|
+
if (!make && lstatSync(dir, { throwIfNoEntry: false }) === undefined)
|
|
782
|
+
return mkdirBlocked(dir, false)
|
|
783
|
+
const refused = ensurePrivateDir(dir, make)
|
|
695
784
|
if (refused !== null) return refused
|
|
696
785
|
}
|
|
697
786
|
}
|
|
698
|
-
return openCacheDir(storeDir, undefined, 'store.db')
|
|
787
|
+
if (make) return openCacheDir(storeDir, undefined, 'store.db')
|
|
788
|
+
try {
|
|
789
|
+
accessSync(storeDir, constants.W_OK)
|
|
790
|
+
} catch (err) {
|
|
791
|
+
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') return errorText(err)
|
|
792
|
+
return mkdirBlocked(storeDir, true)
|
|
793
|
+
}
|
|
794
|
+
try {
|
|
795
|
+
accessSync(path.join(storeDir, 'store.db'), constants.W_OK)
|
|
796
|
+
} catch (err) {
|
|
797
|
+
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') return errorText(err)
|
|
798
|
+
refuseManifestDir(storeDir, undefined)
|
|
799
|
+
}
|
|
800
|
+
return null
|
|
699
801
|
} catch (err) {
|
|
700
802
|
return errorText(err)
|
|
701
803
|
}
|
|
702
804
|
}
|
|
703
805
|
|
|
704
|
-
|
|
806
|
+
/** Why `mkdir` (`-p` with `recursive`) of `dir` would fail, asked without making it. */
|
|
807
|
+
function mkdirBlocked(dir: string, recursive: boolean): string | null {
|
|
808
|
+
let up = path.dirname(dir)
|
|
809
|
+
if (recursive)
|
|
810
|
+
while (lstatSync(up, { throwIfNoEntry: false }) === undefined) up = path.dirname(up)
|
|
705
811
|
try {
|
|
706
|
-
|
|
812
|
+
if (!statSync(up).isDirectory()) return `${up} is not a directory`
|
|
813
|
+
accessSync(up, constants.W_OK)
|
|
814
|
+
return null
|
|
707
815
|
} catch (err) {
|
|
708
|
-
|
|
816
|
+
return errorText(err)
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function ensurePrivateDir(dir: string, make: boolean): string | null {
|
|
821
|
+
if (make) {
|
|
822
|
+
try {
|
|
823
|
+
mkdirSync(dir, { mode: 0o700 })
|
|
824
|
+
} catch (err) {
|
|
825
|
+
if ((err as NodeJS.ErrnoException).code !== 'EEXIST') return errorText(err)
|
|
826
|
+
}
|
|
709
827
|
}
|
|
710
828
|
const st = lstatSync(dir)
|
|
711
829
|
if (!st.isDirectory()) return `${dir} is not a directory`
|
|
@@ -717,7 +835,7 @@ function ensurePrivateDir(dir: string): string | null {
|
|
|
717
835
|
// The user's own directory open to others (a `~/.vx` made at the
|
|
718
836
|
// default umask, 0755) is vx's to close: refusing it left the store
|
|
719
837
|
// unused on a stock macOS home (2026-10-06: "you should own it").
|
|
720
|
-
if ((st.mode & 0o077) !== 0) {
|
|
838
|
+
if (make && (st.mode & 0o077) !== 0) {
|
|
721
839
|
try {
|
|
722
840
|
chmodSync(dir, 0o700)
|
|
723
841
|
} catch (err) {
|
|
@@ -813,8 +931,6 @@ const MANIFESTS = [
|
|
|
813
931
|
...['ts', 'mts', 'js', 'mjs', 'cts', 'cjs'].map((ext) => `vx.config.${ext}`),
|
|
814
932
|
...['ts', 'mts', 'js', 'mjs', 'cts', 'cjs'].map((ext) => `.vx/workspace.${ext}`),
|
|
815
933
|
]
|
|
816
|
-
const OWN_DIR_HINT =
|
|
817
|
-
'Point `cache.dir` in .vx/workspace.ts (or --cache-dir) at a directory of its own, such as .vx/cache.'
|
|
818
934
|
|
|
819
935
|
function makeCacheDir(cacheDir: string): void {
|
|
820
936
|
try {
|
|
@@ -908,6 +1024,43 @@ export interface RemoteTraffic {
|
|
|
908
1024
|
readonly keys: Map<string, RemoteKeyTraffic>
|
|
909
1025
|
}
|
|
910
1026
|
|
|
1027
|
+
/**
|
|
1028
|
+
* Core's own cache's `hashFile`, also told the ctime of the stat its digest
|
|
1029
|
+
* came from (`hashFileSeen`). Kept off `CacheLayer`, a public type: another
|
|
1030
|
+
* layer has none here.
|
|
1031
|
+
*/
|
|
1032
|
+
const seenHashers = new WeakMap<
|
|
1033
|
+
CacheLayer,
|
|
1034
|
+
{
|
|
1035
|
+
/** The layer's `hashFile` when it registered: one replaced since is not this. */
|
|
1036
|
+
method: CacheLayer['hashFile']
|
|
1037
|
+
hash: (file: string, seen: (ctimeMs: number) => void) => Promise<string>
|
|
1038
|
+
}
|
|
1039
|
+
>()
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* `layer.hashFile(file)`, and `seen` told the ctime of the stat the digest
|
|
1043
|
+
* came from when the layer is core's own cache (the re-check's tie rule,
|
|
1044
|
+
* `InputFact.ctimeMs`). Any other layer tells nothing, and so does one
|
|
1045
|
+
* whose `hashFile` was replaced since.
|
|
1046
|
+
*/
|
|
1047
|
+
export function hashFileSeen(
|
|
1048
|
+
layer: CacheLayer,
|
|
1049
|
+
file: string,
|
|
1050
|
+
seen: (ctimeMs: number) => void,
|
|
1051
|
+
): Promise<string> {
|
|
1052
|
+
const own = seenHashers.get(layer)
|
|
1053
|
+
return own === undefined || own.method !== layer.hashFile
|
|
1054
|
+
? layer.hashFile(file)
|
|
1055
|
+
: own.hash(file, seen)
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
/** `to` answers `hashFileSeen` as `from` does: a layer over core's cache. */
|
|
1059
|
+
export function shareSeenHasher(from: CacheLayer, to: CacheLayer): void {
|
|
1060
|
+
const own = seenHashers.get(from)
|
|
1061
|
+
if (own !== undefined) seenHashers.set(to, { method: to.hashFile, hash: own.hash })
|
|
1062
|
+
}
|
|
1063
|
+
|
|
911
1064
|
export class Cache implements CacheLayer {
|
|
912
1065
|
/** This IS the local layer — there is nothing slower behind it. */
|
|
913
1066
|
readonly hasRemote = false
|
|
@@ -928,6 +1081,8 @@ export class Cache implements CacheLayer {
|
|
|
928
1081
|
private readonly selectEntry: ReturnType<Database['prepare']>
|
|
929
1082
|
private readonly upsertStdout: ReturnType<Database['prepare']>
|
|
930
1083
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
1084
|
+
private readonly selectStdout: ReturnType<Database['prepare']>
|
|
1085
|
+
private readonly logOf = (hash: string): string => this.guard(() => this.stdoutText(hash))
|
|
931
1086
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
932
1087
|
private readonly selectBlob: ReturnType<Database['prepare']>
|
|
933
1088
|
private readonly upsertBlob: ReturnType<Database['prepare']>
|
|
@@ -942,6 +1097,12 @@ export class Cache implements CacheLayer {
|
|
|
942
1097
|
* serves those rows only while the blob still carries it.
|
|
943
1098
|
*/
|
|
944
1099
|
private readonly heldInlineAt = new Map<string, number>()
|
|
1100
|
+
/**
|
|
1101
|
+
* The mtime each file the last restore of a key left, by row path
|
|
1102
|
+
* (`extractArtifactStream`'s `dated`): its stamps take that, not the
|
|
1103
|
+
* row's saved one. Taken by `recordOutputStamps`.
|
|
1104
|
+
*/
|
|
1105
|
+
private readonly datedMtimes = new Map<string, Map<string, number>>()
|
|
945
1106
|
/** Keys the last lookups found an inline artifact for and no row: `adopt`'s to read. */
|
|
946
1107
|
private readonly rowless = new Set<string>()
|
|
947
1108
|
/** Row-less lookups that stat'ed their artifact path (`artifactFileMayExist`). */
|
|
@@ -952,7 +1113,14 @@ export class Cache implements CacheLayer {
|
|
|
952
1113
|
private heldTimer: ReturnType<typeof setTimeout> | undefined
|
|
953
1114
|
/** Saves' index writes waiting for this turn's one transaction (`indexInBatch`). */
|
|
954
1115
|
private indexQueue: Array<{ write: () => void; settle: (err?: unknown) => void }> = []
|
|
955
|
-
|
|
1116
|
+
/**
|
|
1117
|
+
* The held sizes being stored, `oid\0size` in OID order, and how many
|
|
1118
|
+
* are: fields, not a closure's, so the store lets them go.
|
|
1119
|
+
*/
|
|
1120
|
+
private storing: string[] | undefined
|
|
1121
|
+
private stored = 0
|
|
1122
|
+
/** The checkpoint thread: `armed` until a save commits (`checkpointOffThread`). */
|
|
1123
|
+
private checkpoints: 'armed' | WalCheckpointer | undefined
|
|
956
1124
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
957
1125
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
958
1126
|
private readonly files: FileHashStore
|
|
@@ -963,6 +1131,10 @@ export class Cache implements CacheLayer {
|
|
|
963
1131
|
* principle 9). Taken once, by `outputBytes`.
|
|
964
1132
|
*/
|
|
965
1133
|
private readonly savedBytes = new Map<string, number>()
|
|
1134
|
+
/** Each key a `--force` save found stored and kept: no save of this run's. Taken once, by `kept`. */
|
|
1135
|
+
private readonly keptHashes = new Set<string>()
|
|
1136
|
+
/** The run this cache saves for: each entry it writes records it (`entries.run_id`). */
|
|
1137
|
+
runId: string | undefined
|
|
966
1138
|
private readonly folds: IndexFoldMemo
|
|
967
1139
|
private readonly history: RunHistory
|
|
968
1140
|
/** What the remotes said they hold, across runs (remote-hits.ts). */
|
|
@@ -998,13 +1170,6 @@ export class Cache implements CacheLayer {
|
|
|
998
1170
|
*/
|
|
999
1171
|
readonly schemaReset: SchemaReset | null = null
|
|
1000
1172
|
|
|
1001
|
-
/**
|
|
1002
|
-
* Set when THIS open found entries written under another
|
|
1003
|
-
* `CACHE_VERSION`: the index survives, but no old key is derived again,
|
|
1004
|
-
* so every cached task misses once. Not printed, like `schemaReset`.
|
|
1005
|
-
*/
|
|
1006
|
-
readonly formatChange: SchemaReset | null = null
|
|
1007
|
-
|
|
1008
1173
|
/**
|
|
1009
1174
|
* Set when THIS open found the shared store written under another
|
|
1010
1175
|
* `SCHEMA_VERSION` and dropped its inventory; the artifacts stay and are
|
|
@@ -1033,9 +1198,10 @@ export class Cache implements CacheLayer {
|
|
|
1033
1198
|
/**
|
|
1034
1199
|
* `'inspect'`: a reading verb (`why`, `last`, `info`). It never resets
|
|
1035
1200
|
* the index: a schema it cannot read is refused, named, and left as it
|
|
1036
|
-
* was. `'preview'`: a dry prune, which reads such an
|
|
1037
|
-
* the real prune does first leaves it, empty,
|
|
1038
|
-
* reached, so it names what that prune takes
|
|
1201
|
+
* was. `'preview'`: a dry prune or a `--dry` plan, which reads such an
|
|
1202
|
+
* index as the reset the real prune or run does first leaves it, empty,
|
|
1203
|
+
* with the store still reached, so it names what that prune takes
|
|
1204
|
+
* (item 1083).
|
|
1039
1205
|
*/
|
|
1040
1206
|
mode: 'open' | 'inspect' | 'preview' = 'open',
|
|
1041
1207
|
/**
|
|
@@ -1044,8 +1210,7 @@ export class Cache implements CacheLayer {
|
|
|
1044
1210
|
* and the store's own schema is `store_meta.schema` (`matchStoreSchema`). Or
|
|
1045
1211
|
* `null` for an index that holds its entries itself (a `cacheDir`).
|
|
1046
1212
|
* Undefined follows the layout the index records: a reading verb, a
|
|
1047
|
-
* plugin's handle.
|
|
1048
|
-
* records none (deleted, or never written beside a run's store).
|
|
1213
|
+
* plugin's handle.
|
|
1049
1214
|
*/
|
|
1050
1215
|
storeRoot?: string | null,
|
|
1051
1216
|
private readonly store: StoreOptions = DEFAULT_STORE,
|
|
@@ -1065,10 +1230,20 @@ export class Cache implements CacheLayer {
|
|
|
1065
1230
|
// said "no recorded runs yet" (item 900).
|
|
1066
1231
|
const dbFile = path.join(cacheDir, 'cache.db')
|
|
1067
1232
|
this.dbFile = dbFile
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1233
|
+
let absent = mode !== 'open' && !existsSync(dbFile)
|
|
1234
|
+
// A preview reads an index of another schema as the empty one a reset
|
|
1235
|
+
// leaves. The version is read on the connection the index keeps, once:
|
|
1236
|
+
// a second connection for it cost a plan a statement (perf-guard).
|
|
1237
|
+
let early: { db: Database; version: string | undefined } | undefined
|
|
1238
|
+
if (mode === 'preview' && !absent) {
|
|
1239
|
+
early = previewIndex(dbFile)
|
|
1240
|
+
if (early !== undefined && early.version !== undefined && early.version !== SCHEMA_VERSION) {
|
|
1241
|
+
closeDb(early.db)
|
|
1242
|
+
early = undefined
|
|
1243
|
+
absent = true
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
this.writeBlocked = absent ? NO_INDEX : openCacheDir(cacheDir, repoDir)
|
|
1072
1247
|
this.write = localPolicy.write && this.writeBlocked === null
|
|
1073
1248
|
// A directory this user cannot write with no index in it yet: SQLite
|
|
1074
1249
|
// cannot make one, and its `unable to open database file` stack reached
|
|
@@ -1078,7 +1253,7 @@ export class Cache implements CacheLayer {
|
|
|
1078
1253
|
const memory = absent || (this.writeBlocked !== null && !existsSync(dbFile))
|
|
1079
1254
|
this.empty = memory
|
|
1080
1255
|
try {
|
|
1081
|
-
this.db = new Database(memory ? ':memory:' : dbFile, { create: true })
|
|
1256
|
+
this.db = early?.db ?? new Database(memory ? ':memory:' : dbFile, { create: true })
|
|
1082
1257
|
} catch (err) {
|
|
1083
1258
|
throw cantOpenIndex(dbFile, err) ?? err
|
|
1084
1259
|
}
|
|
@@ -1173,7 +1348,7 @@ export class Cache implements CacheLayer {
|
|
|
1173
1348
|
)
|
|
1174
1349
|
}
|
|
1175
1350
|
}
|
|
1176
|
-
const current = readable(readVersion)
|
|
1351
|
+
const current = early?.version ?? readable(readVersion)
|
|
1177
1352
|
if (current !== SCHEMA_VERSION) {
|
|
1178
1353
|
try {
|
|
1179
1354
|
if (current !== undefined) refuseUnreadable(current)
|
|
@@ -1187,7 +1362,7 @@ export class Cache implements CacheLayer {
|
|
|
1187
1362
|
// keeps it so).
|
|
1188
1363
|
if (found !== undefined) {
|
|
1189
1364
|
this.db.exec(
|
|
1190
|
-
'DROP TABLE IF EXISTS main.entries; DROP TABLE IF EXISTS main.runs; DROP TABLE IF EXISTS main.file_hashes; DROP TABLE IF EXISTS main.blob_sizes; DROP TABLE IF EXISTS main.blob_verdicts; DROP TABLE IF EXISTS main.output_files; DROP TABLE IF EXISTS main.output_stamps; DROP TABLE IF EXISTS main.invocations; DROP TABLE IF EXISTS main.run_task_inputs; DROP TABLE IF EXISTS main.entry_inputs; DROP TABLE IF EXISTS main.config_evals; DROP TABLE IF EXISTS main.config_closures; DROP TABLE IF EXISTS main.output_dirs; DROP TABLE IF EXISTS main.entry_stdout; DROP TABLE IF EXISTS main.task_durations; DROP TABLE IF EXISTS main.plugin_blobs; DROP TABLE IF EXISTS main.task_outcomes; DROP TABLE IF EXISTS main.store_meta; DROP TABLE IF EXISTS main.fold_memo; DROP TABLE IF EXISTS main.remote_hits; DROP TABLE IF EXISTS main.steps; DROP TABLE IF EXISTS main.warnings; DROP TABLE IF EXISTS main.failures; DROP TABLE IF EXISTS main.lockfile_memos;',
|
|
1365
|
+
'DROP TABLE IF EXISTS main.entries; DROP TABLE IF EXISTS main.runs; DROP TABLE IF EXISTS main.file_hashes; DROP TABLE IF EXISTS main.blob_sizes; DROP TABLE IF EXISTS main.blob_verdicts; DROP TABLE IF EXISTS main.output_files; DROP TABLE IF EXISTS main.output_stamps; DROP TABLE IF EXISTS main.invocations; DROP TABLE IF EXISTS main.run_task_inputs; DROP TABLE IF EXISTS main.entry_inputs; DROP TABLE IF EXISTS main.config_evals; DROP TABLE IF EXISTS main.config_closures; DROP TABLE IF EXISTS main.output_dirs; DROP TABLE IF EXISTS main.entry_stdout; DROP TABLE IF EXISTS main.task_durations; DROP TABLE IF EXISTS main.plugin_blobs; DROP TABLE IF EXISTS main.task_outcomes; DROP TABLE IF EXISTS main.store_meta; DROP TABLE IF EXISTS main.fold_memo; DROP TABLE IF EXISTS main.remote_hits; DROP TABLE IF EXISTS main.steps; DROP TABLE IF EXISTS main.warnings; DROP TABLE IF EXISTS main.failures; DROP TABLE IF EXISTS main.lockfile_memos; DROP TABLE IF EXISTS main.run_ids;',
|
|
1191
1366
|
)
|
|
1192
1367
|
}
|
|
1193
1368
|
// The tables land under the same write lock as the stamp: made
|
|
@@ -1230,25 +1405,22 @@ export class Cache implements CacheLayer {
|
|
|
1230
1405
|
} | null
|
|
1231
1406
|
)?.value,
|
|
1232
1407
|
)
|
|
1233
|
-
// A
|
|
1234
|
-
//
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
|
|
1244
|
-
const blocked = storeBlocked(storeDir, typeof storeRoot === 'string' ? storeRoot : null)
|
|
1245
|
-
if (blocked !== null) {
|
|
1408
|
+
// A store named by the caller is the one a run opens (today's), and
|
|
1409
|
+
// every mode reads it: a dry prune names what the real one takes, and
|
|
1410
|
+
// `vx info` the store the next run fills, not one a moved HOME left.
|
|
1411
|
+
let storeDir = storeRoot === undefined ? recorded : (storeRoot ?? undefined)
|
|
1412
|
+
// Every mode settles the store as a run does, the reading ones without
|
|
1413
|
+
// touching it: `vx info` and a dry prune read the entries runs use, not
|
|
1414
|
+
// a store another user owns (one `sudo vx run`).
|
|
1415
|
+
const root = typeof storeRoot === 'string' ? storeRoot : null
|
|
1416
|
+
if (storeDir !== undefined && (this.writeBlocked === null || absent)) {
|
|
1417
|
+
if (storeBlocked(storeDir, root, mode === 'open') !== null) {
|
|
1246
1418
|
// A home this user cannot write (a container, a read-only mount):
|
|
1247
1419
|
// the entries stay in this workspace rather than fail the run.
|
|
1248
1420
|
const fallback = cacheDir
|
|
1249
1421
|
if (fallback !== storeDir) {
|
|
1250
1422
|
storeDir = fallback
|
|
1251
|
-
openCacheDir(fallback)
|
|
1423
|
+
if (mode === 'open') openCacheDir(fallback)
|
|
1252
1424
|
}
|
|
1253
1425
|
}
|
|
1254
1426
|
}
|
|
@@ -1269,7 +1441,9 @@ export class Cache implements CacheLayer {
|
|
|
1269
1441
|
this.storeDir = storeDir
|
|
1270
1442
|
this.artifactDir = storeDir ?? cacheDir
|
|
1271
1443
|
if (storeDir !== undefined) {
|
|
1272
|
-
this.attachStore(storeDir, mode !== 'open')
|
|
1444
|
+
this.attachStore(storeDir, mode !== 'open', (dir) =>
|
|
1445
|
+
dir === cacheDir ? openCacheDir(dir) : storeBlocked(dir, root, true),
|
|
1446
|
+
)
|
|
1273
1447
|
this.storeReset = this.matchStoreSchema(mode === 'open' && this.writeBlocked === null)
|
|
1274
1448
|
} else {
|
|
1275
1449
|
createTables(this.db, 'main')
|
|
@@ -1295,8 +1469,8 @@ export class Cache implements CacheLayer {
|
|
|
1295
1469
|
this.insertEntry = lazyStatement(
|
|
1296
1470
|
this.db,
|
|
1297
1471
|
`
|
|
1298
|
-
INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, created_at, accessed_at, cpu_ms, peak_rss_bytes)
|
|
1299
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1472
|
+
INSERT INTO entries(hash, project, task, command, exit_code, duration_ms, size_bytes, created_at, accessed_at, cpu_ms, peak_rss_bytes, run_id)
|
|
1473
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1300
1474
|
ON CONFLICT(hash) DO UPDATE SET
|
|
1301
1475
|
project = excluded.project,
|
|
1302
1476
|
task = excluded.task,
|
|
@@ -1306,14 +1480,17 @@ export class Cache implements CacheLayer {
|
|
|
1306
1480
|
size_bytes = excluded.size_bytes,
|
|
1307
1481
|
accessed_at = excluded.accessed_at,
|
|
1308
1482
|
cpu_ms = excluded.cpu_ms,
|
|
1309
|
-
peak_rss_bytes = excluded.peak_rss_bytes
|
|
1483
|
+
peak_rss_bytes = excluded.peak_rss_bytes,
|
|
1484
|
+
created_at = excluded.created_at,
|
|
1485
|
+
run_id = excluded.run_id
|
|
1310
1486
|
`,
|
|
1311
1487
|
)
|
|
1312
1488
|
this.upsertStdout = lazyStatement(
|
|
1313
1489
|
this.db,
|
|
1314
|
-
'INSERT INTO entry_stdout(hash, stdout) VALUES (?,
|
|
1490
|
+
'INSERT INTO entry_stdout(hash, stdout) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
|
|
1315
1491
|
)
|
|
1316
1492
|
this.deleteStdout = lazyStatement(this.db, 'DELETE FROM entry_stdout WHERE hash = ?')
|
|
1493
|
+
this.selectStdout = lazyStatement(this.db, 'SELECT stdout FROM entry_stdout WHERE hash = ?')
|
|
1317
1494
|
this.selectEntry = lazyStatement(this.db, SELECT_ENTRY)
|
|
1318
1495
|
// `has` asks only whether the row is there and where its artifact is,
|
|
1319
1496
|
// with no stdout joined. A column off the index, so the table row is
|
|
@@ -1342,23 +1519,23 @@ export class Cache implements CacheLayer {
|
|
|
1342
1519
|
// The slices: each owns its statements over this handle and its table(s);
|
|
1343
1520
|
// the schema above is the one place every table is declared.
|
|
1344
1521
|
this.files = new FileHashStore(this.db, cacheDir, this.write, repoDir)
|
|
1522
|
+
seenHashers.set(this, {
|
|
1523
|
+
method: this.hashFile,
|
|
1524
|
+
hash: (file, seen) => this.guard(() => this.files.hashFile(file, seen)),
|
|
1525
|
+
})
|
|
1345
1526
|
this.outputs = new OutputIndex(this.db, this.writeBlocked === null && !this.inspecting)
|
|
1346
1527
|
this.folds = new IndexFoldMemo(this.db, this.write && !this.inspecting)
|
|
1347
1528
|
this.history = new RunHistory(this.db, (v) => this.digestValue(v))
|
|
1348
1529
|
this.remoteHits = new IndexRemoteHits(this.db, this.writeBlocked === null && !this.inspecting)
|
|
1349
1530
|
|
|
1350
1531
|
// A CACHE_VERSION bump keeps the index but moves every key, so the run
|
|
1351
|
-
// after an upgrade misses everything
|
|
1352
|
-
//
|
|
1353
|
-
//
|
|
1532
|
+
// after an upgrade misses everything, in silence: the cache is vx's to
|
|
1533
|
+
// keep (2026-10-06). The version is recorded for whoever reads the
|
|
1534
|
+
// index by hand; a reading verb writes nothing (item 1080).
|
|
1354
1535
|
const format = this.db
|
|
1355
1536
|
.prepare("SELECT value FROM schema_meta WHERE key = 'cache_version'")
|
|
1356
1537
|
.get() as { value: string } | null
|
|
1357
1538
|
if (format?.value !== CACHE_VERSION && this.writeBlocked === null && !this.inspecting) {
|
|
1358
|
-
const hasEntries = this.db.prepare('SELECT 1 FROM entries LIMIT 1').get() != null
|
|
1359
|
-
if (this.schemaReset === null && (format !== null || hasEntries)) {
|
|
1360
|
-
this.formatChange = { from: format?.value ?? 'an earlier format', to: CACHE_VERSION }
|
|
1361
|
-
}
|
|
1362
1539
|
this.db
|
|
1363
1540
|
.prepare(
|
|
1364
1541
|
"INSERT INTO schema_meta(key, value) VALUES ('cache_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
@@ -1472,25 +1649,59 @@ export class Cache implements CacheLayer {
|
|
|
1472
1649
|
* WAL for both, so a workspace's run and another's share the store as
|
|
1473
1650
|
* two runs on one `--cache-dir` share an index. A reading verb over a
|
|
1474
1651
|
* store with no file or no tables yet reads an empty one in memory.
|
|
1652
|
+
*
|
|
1653
|
+
* Every checkout of a repository shares its store, each with a run lock
|
|
1654
|
+
* of its own, and a `vx cache clean` in one holds the store alone
|
|
1655
|
+
* (`holdStore`) until it has deleted it. A writing open meeting that hold
|
|
1656
|
+
* answers SQLITE_BUSY within a second, for its caller to wait and open
|
|
1657
|
+
* again (prepare.ts). One let in once the clean let go opened the
|
|
1658
|
+
* store.db the clean unlinked, and saved its entry into it: it makes the
|
|
1659
|
+
* store again (`remake`, private and ignored) and opens that. Once in,
|
|
1660
|
+
* its lock keeps a clean out until it closes.
|
|
1475
1661
|
*/
|
|
1476
|
-
private attachStore(
|
|
1662
|
+
private attachStore(
|
|
1663
|
+
storeDir: string,
|
|
1664
|
+
inspecting: boolean,
|
|
1665
|
+
remake: (dir: string) => string | null,
|
|
1666
|
+
): void {
|
|
1477
1667
|
const storeFile = path.join(storeDir, 'store.db')
|
|
1478
1668
|
let absent = inspecting && !existsSync(storeFile)
|
|
1479
1669
|
try {
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1670
|
+
for (let retried = false; ;) {
|
|
1671
|
+
// The attach reads the store's schema: its first lock on it.
|
|
1672
|
+
if (!inspecting) this.db.exec('PRAGMA busy_timeout = 1000')
|
|
1673
|
+
try {
|
|
1674
|
+
this.db.prepare('ATTACH DATABASE ? AS store').run(absent ? ':memory:' : storeFile)
|
|
1675
|
+
} catch (err) {
|
|
1676
|
+
// A file whose directory went: SQLite cannot open its -wal there.
|
|
1677
|
+
const busy = (err as { code?: unknown }).code === 'SQLITE_BUSY'
|
|
1678
|
+
if (inspecting || busy || retried || remake(storeDir) !== null) throw err
|
|
1679
|
+
retried = true
|
|
1680
|
+
continue
|
|
1681
|
+
}
|
|
1682
|
+
// One with no tables yet is read empty in memory too: the journal
|
|
1683
|
+
// switch and the table creation below are writes, under no write
|
|
1684
|
+
// lock, racing an opener making that store.
|
|
1685
|
+
if (
|
|
1686
|
+
inspecting &&
|
|
1687
|
+
!absent &&
|
|
1688
|
+
this.db.prepare("SELECT 1 FROM store.sqlite_master WHERE name = 'entries'").get() == null
|
|
1689
|
+
) {
|
|
1690
|
+
this.db.exec('DETACH DATABASE store')
|
|
1691
|
+
this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
|
|
1692
|
+
absent = true
|
|
1693
|
+
}
|
|
1694
|
+
this.db.exec('PRAGMA store.journal_mode = WAL')
|
|
1695
|
+
if (inspecting) break
|
|
1696
|
+
this.db.exec('PRAGMA busy_timeout = 5000')
|
|
1697
|
+
// One stat, SQLite's: is the file open still the one at its name?
|
|
1698
|
+
const moved = new Int32Array(2)
|
|
1699
|
+
this.db.fileControl('store', SQLITE_FCNTL_HAS_MOVED, moved)
|
|
1700
|
+
if (moved[0] === 0) break
|
|
1489
1701
|
this.db.exec('DETACH DATABASE store')
|
|
1490
|
-
|
|
1491
|
-
|
|
1702
|
+
const refused = remake(storeDir)
|
|
1703
|
+
if (refused !== null) throw new Error(`${storeFile} was deleted, and ${refused}`)
|
|
1492
1704
|
}
|
|
1493
|
-
this.db.exec('PRAGMA store.journal_mode = WAL')
|
|
1494
1705
|
if (!absent) this.db.fileControl('store', SQLITE_FCNTL_PERSIST_WAL, 1)
|
|
1495
1706
|
this.db.exec('PRAGMA store.journal_size_limit = 67108864')
|
|
1496
1707
|
this.db.exec('PRAGMA store.synchronous = NORMAL')
|
|
@@ -1670,13 +1881,46 @@ export class Cache implements CacheLayer {
|
|
|
1670
1881
|
const held = this.heldBlobSizes
|
|
1671
1882
|
this.heldBlobSizes = undefined
|
|
1672
1883
|
if (held === undefined || held.length === 0) return
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1884
|
+
// In OID order, so each slice appends at the key index's right edge:
|
|
1885
|
+
// in git's path order every slice dirtied pages across the whole
|
|
1886
|
+
// index, and the 160 commits of a cold 9,603-task run cost its main
|
|
1887
|
+
// thread ~5x more (2026-11-06). Sorted as `oid\0size` strings with no
|
|
1888
|
+
// comparator: a JS comparator over 40,000 entries was 32 ms of a cold
|
|
1889
|
+
// 8,002-task run's main thread before its first task, the native
|
|
1890
|
+
// string sort 8 (2026-11-07). `\0` sorts below every OID character,
|
|
1891
|
+
// so the order is the OIDs'. No entry outlives this: the strings are
|
|
1892
|
+
// all the store keeps.
|
|
1893
|
+
const keys: string[] = []
|
|
1894
|
+
for (let i = 0; i < held.length; i++) keys.push(`${held[i]![0]}\0${held[i]![1]}`)
|
|
1895
|
+
keys.sort()
|
|
1896
|
+
this.storing = keys
|
|
1897
|
+
this.stored = 0
|
|
1898
|
+
this.heldTimer = setTimeout(this.storeHeldSlice, 0)
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
// A closure over the sizes kept all of them while anything held it (the
|
|
1902
|
+
// rest's getter, a fired timer's callback): a cold 9,603-task run's
|
|
1903
|
+
// 38,331 to its end, ~5 MB of heap (2026-11-06).
|
|
1904
|
+
private readonly storeHeldSlice = (): void => {
|
|
1905
|
+
const held = this.storing
|
|
1906
|
+
if (held === undefined) return
|
|
1907
|
+
const at = this.stored
|
|
1908
|
+
this.stored = at + HELD_SLICE
|
|
1909
|
+
this.guard(() => this.files.rememberBlobSizes(this.heldSizes(held, at, this.stored)))
|
|
1910
|
+
if (this.stored < held.length) this.heldTimer = setTimeout(this.storeHeldSlice, 0)
|
|
1911
|
+
else {
|
|
1912
|
+
this.heldTimer = undefined
|
|
1913
|
+
this.storing = undefined
|
|
1677
1914
|
}
|
|
1678
|
-
|
|
1679
|
-
|
|
1915
|
+
}
|
|
1916
|
+
private heldSizes(held: readonly string[], from: number, to: number): Map<string, number> {
|
|
1917
|
+
const out = new Map<string, number>()
|
|
1918
|
+
for (let i = from; i < Math.min(to, held.length); i++) {
|
|
1919
|
+
const k = held[i]!
|
|
1920
|
+
const at = k.indexOf('\0')
|
|
1921
|
+
out.set(k.slice(0, at), Number(k.slice(at + 1)))
|
|
1922
|
+
}
|
|
1923
|
+
return out
|
|
1680
1924
|
}
|
|
1681
1925
|
/** `BlobSizeMemo`: the paths an index with this digest distrusts, when known. */
|
|
1682
1926
|
blobVerdict(digest: string): string[] | undefined {
|
|
@@ -1908,7 +2152,7 @@ export class Cache implements CacheLayer {
|
|
|
1908
2152
|
// for four up-to-date hits on ~70 MB binaries). restoreOutputs
|
|
1909
2153
|
// reads the artifact itself, only when extraction actually runs.
|
|
1910
2154
|
const fileRows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
1911
|
-
const entry = entryOf(row as LiveRow, fileRows)
|
|
2155
|
+
const entry = entryOf(row as LiveRow, fileRows, this.logOf)
|
|
1912
2156
|
entry.outputDirRows = this.loadOutputDirsBatch([hash]).get(hash) ?? []
|
|
1913
2157
|
return entry
|
|
1914
2158
|
}
|
|
@@ -1967,32 +2211,40 @@ export class Cache implements CacheLayer {
|
|
|
1967
2211
|
? [this.selectEntry.get(hashes[0]!)].filter((r) => r !== null)
|
|
1968
2212
|
: this.db.query(SELECT_ENTRIES).all(JSON.stringify(hashes))
|
|
1969
2213
|
) as EntryRow[]
|
|
1970
|
-
const rows: LiveRow[] = []
|
|
1971
|
-
for (const r of found) {
|
|
1972
|
-
if (r.hash === null) this.rowless.add(r.key)
|
|
1973
|
-
else rows.push(r as LiveRow)
|
|
1974
|
-
}
|
|
1975
|
-
if (rows.length === 0) return out
|
|
1976
2214
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1977
2215
|
const refreshStart = Date.now() - ACCESS_REFRESH_MS
|
|
1978
|
-
|
|
2216
|
+
// A hash asked twice is one row here: the output queries join from the
|
|
2217
|
+
// live hashes as given (`loadOutputFilesBatch`'s `distinct`).
|
|
2218
|
+
const seen = new Set<string>()
|
|
2219
|
+
const live: LiveRow[] = []
|
|
2220
|
+
const liveHashes: string[] = []
|
|
2221
|
+
for (const r of found) {
|
|
2222
|
+
if (r.hash === null) {
|
|
2223
|
+
this.rowless.add(r.key)
|
|
2224
|
+
continue
|
|
2225
|
+
}
|
|
2226
|
+
if (seen.has(r.hash)) continue
|
|
2227
|
+
seen.add(r.hash)
|
|
1979
2228
|
if (r.inline_at !== null) {
|
|
1980
2229
|
if (r.inline_at < graceStart) this.staleInline.add(r.hash)
|
|
1981
|
-
|
|
2230
|
+
} else {
|
|
2231
|
+
const st = this.artifactStat(r.hash)
|
|
2232
|
+
if (st === undefined) continue
|
|
2233
|
+
if (st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1982
2234
|
}
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
const
|
|
1988
|
-
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
2235
|
+
live.push(r as LiveRow)
|
|
2236
|
+
liveHashes.push(r.hash)
|
|
2237
|
+
}
|
|
2238
|
+
if (seen.size === 0) return out
|
|
2239
|
+
const distinct = JSON.stringify(liveHashes)
|
|
2240
|
+
const fileRows = this.outputs.loadOutputFilesBatch(liveHashes, distinct)
|
|
1989
2241
|
this.outputs.hold(fileRows)
|
|
1990
2242
|
this.heldInlineAt.clear()
|
|
1991
2243
|
for (const r of live) if (r.inline_at !== null) this.heldInlineAt.set(r.hash, r.inline_at)
|
|
1992
|
-
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
2244
|
+
const dirRows = this.outputs.loadOutputDirsBatch(liveHashes, distinct)
|
|
1993
2245
|
for (const row of live) {
|
|
1994
2246
|
if (row.accessed_at < refreshStart) this.touched.add(row.hash)
|
|
1995
|
-
const entry = entryOf(row, fileRows.get(row.hash) ?? [])
|
|
2247
|
+
const entry = entryOf(row, fileRows.get(row.hash) ?? [], this.logOf)
|
|
1996
2248
|
entry.outputDirRows = dirRows.get(row.hash) ?? []
|
|
1997
2249
|
out.set(row.hash, entry)
|
|
1998
2250
|
}
|
|
@@ -2054,6 +2306,9 @@ export class Cache implements CacheLayer {
|
|
|
2054
2306
|
this.savedBytes.delete(hash)
|
|
2055
2307
|
return bytes
|
|
2056
2308
|
}
|
|
2309
|
+
kept(hash: string): boolean {
|
|
2310
|
+
return this.keptHashes.delete(hash)
|
|
2311
|
+
}
|
|
2057
2312
|
artifactSize(hash: string): number | undefined {
|
|
2058
2313
|
return this.guard(
|
|
2059
2314
|
() =>
|
|
@@ -2074,7 +2329,9 @@ export class Cache implements CacheLayer {
|
|
|
2074
2329
|
await this.outputs.recordOutputDirs(hash, projectDir, prefixes, holds)
|
|
2075
2330
|
}
|
|
2076
2331
|
recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
|
|
2077
|
-
this.
|
|
2332
|
+
const dated = this.datedMtimes.get(hash)
|
|
2333
|
+
this.datedMtimes.delete(hash)
|
|
2334
|
+
this.outputs.recordOutputStamps(hash, projectDir, workspaceRoot, dated)
|
|
2078
2335
|
}
|
|
2079
2336
|
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
2080
2337
|
return this.guard(() => this.outputs.loadOutputDirsBatch(hashes))
|
|
@@ -2088,23 +2345,52 @@ export class Cache implements CacheLayer {
|
|
|
2088
2345
|
workspaceRoot?: string,
|
|
2089
2346
|
outputs?: DeclaredOutputs,
|
|
2090
2347
|
): Promise<void> {
|
|
2348
|
+
const used: { rows: readonly OutputFileRow[] } = { rows: [] }
|
|
2091
2349
|
try {
|
|
2092
|
-
await this.restoreOutputsOnce(hash, projectDir, workspaceRoot, outputs)
|
|
2350
|
+
await this.restoreOutputsOnce(hash, projectDir, workspaceRoot, outputs, used, false)
|
|
2093
2351
|
} catch (err) {
|
|
2094
2352
|
if (!(err instanceof CorruptArtifactError)) throw err
|
|
2353
|
+
let failed = err
|
|
2354
|
+
// A file artifact has no `at` to say a re-save in another process
|
|
2355
|
+
// replaced it since the probe: the probe's rows then name another
|
|
2356
|
+
// build's files, and this one reads as corrupt. Rows that changed
|
|
2357
|
+
// since are that race, so the restore runs once more on the rows read
|
|
2358
|
+
// now, and an entry is dropped only while its rows are the ones that
|
|
2359
|
+
// failed: another process's fresh entry is never taken.
|
|
2360
|
+
if (this.entryChanged(hash, used.rows)) {
|
|
2361
|
+
try {
|
|
2362
|
+
await this.restoreOutputsOnce(hash, projectDir, workspaceRoot, outputs, used, true)
|
|
2363
|
+
return
|
|
2364
|
+
} catch (again) {
|
|
2365
|
+
if (!(again instanceof CorruptArtifactError)) throw again
|
|
2366
|
+
failed = again
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2095
2369
|
// Bad bytes under a good key failed the task on every run until
|
|
2096
2370
|
// `--force`, since nothing rewrote them (J's lead, nx#30338). The
|
|
2097
2371
|
// entry goes, and the caller runs the task, whose save stores it
|
|
2098
2372
|
// again; running is always correct, only slower than a hit.
|
|
2099
|
-
if (this.write
|
|
2373
|
+
if (this.write && !(await this.dropUnchanged(hash, used.rows)))
|
|
2374
|
+
throw new ArtifactVanishedError(hash)
|
|
2100
2375
|
throw new ArtifactVanishedError(
|
|
2101
2376
|
hash,
|
|
2102
|
-
`${
|
|
2103
|
-
|
|
2377
|
+
`${failed.message}; ${this.write ? 'dropped it' : 'left it (this cache is read-only)'}`,
|
|
2378
|
+
failed,
|
|
2104
2379
|
)
|
|
2105
2380
|
}
|
|
2106
2381
|
}
|
|
2107
2382
|
|
|
2383
|
+
/** Whether `hash`'s committed rows are no longer `rows` (path, size, mode, mtime). */
|
|
2384
|
+
private entryChanged(hash: string, rows: readonly OutputFileRow[]): boolean {
|
|
2385
|
+
const now = this.guard(() => this.outputs.loadOutputFilesBatch([hash]).get(hash) ?? [])
|
|
2386
|
+
if (now.length !== rows.length) return true
|
|
2387
|
+
const was = new Map(rows.map((r) => [r.path, r]))
|
|
2388
|
+
return now.some((r) => {
|
|
2389
|
+
const w = was.get(r.path)
|
|
2390
|
+
return w === undefined || w.size !== r.size || w.mode !== r.mode || w.mtimeMs !== r.mtimeMs
|
|
2391
|
+
})
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2108
2394
|
/**
|
|
2109
2395
|
* Restore a tree-form entry: its files copied (or cloned) out one by
|
|
2110
2396
|
* one where the outputs are absent, as on every restore after a clean. Anything else (a tree to
|
|
@@ -2114,11 +2400,13 @@ export class Cache implements CacheLayer {
|
|
|
2114
2400
|
private async restoreTree(
|
|
2115
2401
|
hash: string,
|
|
2116
2402
|
tree: string,
|
|
2403
|
+
treeIno: number,
|
|
2117
2404
|
rows: readonly OutputFileRow[],
|
|
2118
2405
|
projectDir: string,
|
|
2119
2406
|
workspaceRoot: string | undefined,
|
|
2120
2407
|
verify: (provided: ReadonlySet<string>) => void,
|
|
2121
2408
|
accept: ((name: string) => boolean) | undefined,
|
|
2409
|
+
dated: Map<string, number>,
|
|
2122
2410
|
): Promise<void> {
|
|
2123
2411
|
const files = rows.filter((r) => !r.path.endsWith('/') && r.mode !== LINK_MODE)
|
|
2124
2412
|
let bytes = 0
|
|
@@ -2129,7 +2417,20 @@ export class Cache implements CacheLayer {
|
|
|
2129
2417
|
`its outputs hold ${formatBytes(bytes)}, past the ${formatBytes(this.artifactCeiling)} artifact ceiling`,
|
|
2130
2418
|
)
|
|
2131
2419
|
}
|
|
2132
|
-
if (
|
|
2420
|
+
if (
|
|
2421
|
+
await this.copyTree(
|
|
2422
|
+
hash,
|
|
2423
|
+
tree,
|
|
2424
|
+
treeIno,
|
|
2425
|
+
rows,
|
|
2426
|
+
files,
|
|
2427
|
+
projectDir,
|
|
2428
|
+
workspaceRoot,
|
|
2429
|
+
accept,
|
|
2430
|
+
dated,
|
|
2431
|
+
)
|
|
2432
|
+
)
|
|
2433
|
+
return
|
|
2133
2434
|
await extractArtifactStream(
|
|
2134
2435
|
packArtifactStream(this.treePlan(hash, tree, rows)),
|
|
2135
2436
|
projectDir,
|
|
@@ -2137,6 +2438,8 @@ export class Cache implements CacheLayer {
|
|
|
2137
2438
|
verify,
|
|
2138
2439
|
accept,
|
|
2139
2440
|
workspaceRoot,
|
|
2441
|
+
undefined,
|
|
2442
|
+
dated,
|
|
2140
2443
|
)
|
|
2141
2444
|
}
|
|
2142
2445
|
|
|
@@ -2171,15 +2474,23 @@ export class Cache implements CacheLayer {
|
|
|
2171
2474
|
* and every directory on the way either is a real directory or is absent
|
|
2172
2475
|
* and made here. solid's build outputs `store/dist/**` beside the sources
|
|
2173
2476
|
* in `store/`; asking that each output's TOP be absent sent every such
|
|
2174
|
-
* restore through the tar extractor, ~25 ms a task (2026-10-22).
|
|
2477
|
+
* restore through the tar extractor, ~25 ms a task (2026-10-22). The
|
|
2478
|
+
* files of each absent top directory are copied into the workspace's
|
|
2479
|
+
* `.vx/tmp/` (`copyStaging`) and the directory renamed in once, so a
|
|
2480
|
+
* reader finds it complete or not at all; one staging cannot rename into
|
|
2481
|
+
* as a write in place would land (another filesystem, a setgid
|
|
2482
|
+
* directory) is the extractor's.
|
|
2175
2483
|
*/
|
|
2176
2484
|
private async copyTree(
|
|
2177
2485
|
hash: string,
|
|
2178
2486
|
tree: string,
|
|
2487
|
+
ino: number,
|
|
2179
2488
|
rows: readonly OutputFileRow[],
|
|
2180
2489
|
files: readonly OutputFileRow[],
|
|
2181
2490
|
projectDir: string,
|
|
2491
|
+
workspaceRoot: string | undefined,
|
|
2182
2492
|
accept: ((name: string) => boolean) | undefined,
|
|
2493
|
+
dated: Map<string, number>,
|
|
2183
2494
|
): Promise<boolean> {
|
|
2184
2495
|
for (const r of rows) {
|
|
2185
2496
|
if (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) || r.mode === LINK_MODE) return false
|
|
@@ -2188,14 +2499,19 @@ export class Cache implements CacheLayer {
|
|
|
2188
2499
|
}
|
|
2189
2500
|
// What stands at each path, asked once: a real directory, nothing, or
|
|
2190
2501
|
// anything else (a file, a link), which the extractor's rules decide.
|
|
2191
|
-
|
|
2192
|
-
|
|
2502
|
+
// Synchronous: the walk before this one looked these paths up, so each
|
|
2503
|
+
// answer is a cached dentry, and the async form cost a 1,600-restore run
|
|
2504
|
+
// ~40 ms of its main thread in promises (2026-11-07).
|
|
2505
|
+
const standing = new Map<string, 'dir' | 'absent' | 'other'>()
|
|
2506
|
+
const at = (rel: string): 'dir' | 'absent' | 'other' => {
|
|
2193
2507
|
let known = standing.get(rel)
|
|
2194
2508
|
if (known === undefined) {
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2509
|
+
try {
|
|
2510
|
+
const st = lstatSync(path.join(projectDir, rel), { throwIfNoEntry: false })
|
|
2511
|
+
known = st === undefined ? 'absent' : st.isDirectory() ? 'dir' : 'other'
|
|
2512
|
+
} catch {
|
|
2513
|
+
known = 'other'
|
|
2514
|
+
}
|
|
2199
2515
|
standing.set(rel, known)
|
|
2200
2516
|
}
|
|
2201
2517
|
return known
|
|
@@ -2204,24 +2520,22 @@ export class Cache implements CacheLayer {
|
|
|
2204
2520
|
// writes into a directory that stood: what a failure removes, and only that.
|
|
2205
2521
|
const made = new Set<string>()
|
|
2206
2522
|
const placed: string[] = []
|
|
2207
|
-
const fits =
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
if (
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
return true
|
|
2218
|
-
}
|
|
2219
|
-
if (state === 'other' || (isLeaf && !r.path.endsWith('/'))) return false
|
|
2523
|
+
const fits = rows.every((r) => {
|
|
2524
|
+
const segs = r.path.replace(/\/$/, '').split('/')
|
|
2525
|
+
for (let i = 1; i <= segs.length; i++) {
|
|
2526
|
+
const rel = segs.slice(0, i).join('/')
|
|
2527
|
+
const isLeaf = i === segs.length
|
|
2528
|
+
const state = at(rel)
|
|
2529
|
+
if (state === 'absent') {
|
|
2530
|
+
if (!isLeaf || r.path.endsWith('/')) made.add(rel)
|
|
2531
|
+
else placed.push(rel)
|
|
2532
|
+
return true
|
|
2220
2533
|
}
|
|
2221
|
-
return
|
|
2222
|
-
}
|
|
2223
|
-
|
|
2224
|
-
|
|
2534
|
+
if (state === 'other' || (isLeaf && !r.path.endsWith('/'))) return false
|
|
2535
|
+
}
|
|
2536
|
+
return true
|
|
2537
|
+
})
|
|
2538
|
+
if (!fits) return false
|
|
2225
2539
|
// A directory made under another one made here is the outer one's.
|
|
2226
2540
|
const roots = [...made].filter((d) => ![...made].some((o) => d.startsWith(`${o}/`)))
|
|
2227
2541
|
readUmask()
|
|
@@ -2229,8 +2543,8 @@ export class Cache implements CacheLayer {
|
|
|
2229
2543
|
// Another process's re-save of this key renames the tree aside and a
|
|
2230
2544
|
// new one in mid-copy: the files copied after came from another build.
|
|
2231
2545
|
// A changed inode is that race, a miss (the task runs), and the entry
|
|
2232
|
-
// the other process saved is kept.
|
|
2233
|
-
|
|
2546
|
+
// the other process saved is kept. `ino` is the tree's as the restore
|
|
2547
|
+
// found it, before this walk: a swap since is caught all the same.
|
|
2234
2548
|
const swapped = (): boolean => lstatSync(tree, { throwIfNoEntry: false })?.ino !== ino
|
|
2235
2549
|
// A standing directory of this user's that withholds its own write (a
|
|
2236
2550
|
// task's `chmod 555 dist/ro`) gets owner write and search for the copy
|
|
@@ -2246,6 +2560,46 @@ export class Cache implements CacheLayer {
|
|
|
2246
2560
|
chmodSync(dir, (st.mode & 0o7777) | 0o300)
|
|
2247
2561
|
return true
|
|
2248
2562
|
}
|
|
2563
|
+
// A missing top directory is built whole in staging and renamed in
|
|
2564
|
+
// once, as the extractor does: copied in place, a reader met its files
|
|
2565
|
+
// empty, then short. A file missing from a directory that stands is
|
|
2566
|
+
// copied straight to its place: a temp renamed in cost such a restore
|
|
2567
|
+
// a fifth to a third more (tmpfs, 3,000 files, 2026-11-06).
|
|
2568
|
+
if (roots.length === 0 && placed.length === 0) return true
|
|
2569
|
+
// A restore into a project directory not there yet makes it, as the
|
|
2570
|
+
// copy in place did.
|
|
2571
|
+
// Its stat also answers, below, what a rename into it keeps.
|
|
2572
|
+
let projectStat: Stats | undefined
|
|
2573
|
+
try {
|
|
2574
|
+
projectStat = statSync(projectDir, { throwIfNoEntry: false })
|
|
2575
|
+
} catch {
|
|
2576
|
+
projectStat = undefined
|
|
2577
|
+
}
|
|
2578
|
+
if (projectStat === undefined) await mkdir(projectDir, { recursive: true })
|
|
2579
|
+
const staging = roots.length > 0 ? await copyStaging(workspaceRoot) : undefined
|
|
2580
|
+
if (staging?.stats !== undefined) {
|
|
2581
|
+
const fits = new Map<string, boolean>()
|
|
2582
|
+
for (const rel of roots) {
|
|
2583
|
+
const dir = path.dirname(path.join(projectDir, rel))
|
|
2584
|
+
let fit = fits.get(dir)
|
|
2585
|
+
if (fit === undefined) {
|
|
2586
|
+
const top = !rel.includes('/') && projectStat !== undefined
|
|
2587
|
+
fit = stagesInto(
|
|
2588
|
+
staging.stats,
|
|
2589
|
+
top ? projectStat : statSync(dir, { throwIfNoEntry: false }),
|
|
2590
|
+
)
|
|
2591
|
+
fits.set(dir, fit)
|
|
2592
|
+
}
|
|
2593
|
+
if (!fit) return false
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
const temps = new Map(roots.map((rel) => [rel, staging!.temp(path.join(projectDir, rel))]))
|
|
2597
|
+
const target = (rel: string): string => {
|
|
2598
|
+
for (const d of roots) {
|
|
2599
|
+
if (rel === d || rel.startsWith(`${d}/`)) return temps.get(d)! + rel.slice(d.length)
|
|
2600
|
+
}
|
|
2601
|
+
return path.join(projectDir, rel)
|
|
2602
|
+
}
|
|
2249
2603
|
const copy = async (to: string, from: string): Promise<number> => {
|
|
2250
2604
|
try {
|
|
2251
2605
|
return await Bun.write(to, Bun.file(from), { createPath: false })
|
|
@@ -2254,29 +2608,49 @@ export class Cache implements CacheLayer {
|
|
|
2254
2608
|
return await Bun.write(to, Bun.file(from), { createPath: false })
|
|
2255
2609
|
}
|
|
2256
2610
|
}
|
|
2611
|
+
const landed: string[] = []
|
|
2612
|
+
let landing = false
|
|
2613
|
+
// Every file here is written, so each is dated now (see `dated`).
|
|
2614
|
+
const now = Date.now()
|
|
2615
|
+
const nowDate = new Date(now)
|
|
2257
2616
|
try {
|
|
2258
2617
|
const dirs = new Set<string>()
|
|
2259
2618
|
for (const r of rows) {
|
|
2260
|
-
dirs.add(
|
|
2619
|
+
dirs.add(target(r.path.endsWith('/') ? r.path.slice(0, -1) : path.posix.dirname(r.path)))
|
|
2261
2620
|
}
|
|
2262
2621
|
for (const d of dirs) await mkdir(d, { recursive: true })
|
|
2263
|
-
|
|
2622
|
+
// Every copy settles before a failure is handled: one still in flight
|
|
2623
|
+
// would write its file again after the cleanup removed it.
|
|
2624
|
+
const copied = await Promise.allSettled(
|
|
2264
2625
|
files.map(async (r) => {
|
|
2265
|
-
const to =
|
|
2626
|
+
const to = target(r.path)
|
|
2266
2627
|
const n = await copy(to, path.join(tree, treeName(r.path)))
|
|
2267
2628
|
if (n !== r.size)
|
|
2268
2629
|
throw new CorruptArtifactError(hash, `${r.path} holds ${n} bytes, recorded ${r.size}`)
|
|
2269
2630
|
const mode = restoredMode(r.mode)
|
|
2270
2631
|
if (mode !== plain) await chmod(to, mode)
|
|
2271
|
-
|
|
2272
|
-
|
|
2632
|
+
await utimes(to, nowDate, nowDate)
|
|
2633
|
+
dated.set(`outputs/${r.path}`, now)
|
|
2273
2634
|
}),
|
|
2274
2635
|
)
|
|
2636
|
+
const failed = copied.find((c) => c.status === 'rejected')
|
|
2637
|
+
if (failed !== undefined) throw failed.reason
|
|
2275
2638
|
if (swapped()) throw new ArtifactVanishedError(hash)
|
|
2639
|
+
landing = true
|
|
2640
|
+
for (const rel of roots) {
|
|
2641
|
+
await rename(temps.get(rel)!, path.join(projectDir, rel))
|
|
2642
|
+
landed.push(rel)
|
|
2643
|
+
}
|
|
2276
2644
|
} catch (err) {
|
|
2645
|
+
await Promise.all([...temps.values()].map((t) => rm(t, { recursive: true, force: true })))
|
|
2646
|
+
// Something made at a missing directory's place since the walk: what
|
|
2647
|
+
// landed holds the entry's bytes, and the extractor restores the rest.
|
|
2648
|
+
const code = landing ? (err as NodeJS.ErrnoException).code : undefined
|
|
2649
|
+
if (code === 'EEXIST' || code === 'ENOTEMPTY' || code === 'EISDIR' || code === 'ENOTDIR')
|
|
2650
|
+
return false
|
|
2277
2651
|
await Promise.all([
|
|
2278
|
-
...
|
|
2279
|
-
...placed.map((
|
|
2652
|
+
...landed.map((rel) => rm(path.join(projectDir, rel), { recursive: true, force: true })),
|
|
2653
|
+
...placed.map((rel) => rm(path.join(projectDir, rel), { force: true })),
|
|
2280
2654
|
])
|
|
2281
2655
|
if (!(err instanceof ArtifactVanishedError) && swapped())
|
|
2282
2656
|
throw new ArtifactVanishedError(hash)
|
|
@@ -2294,28 +2668,45 @@ export class Cache implements CacheLayer {
|
|
|
2294
2668
|
}
|
|
2295
2669
|
|
|
2296
2670
|
private stdoutText(hash: string): string {
|
|
2297
|
-
const row = this.
|
|
2298
|
-
|
|
2299
|
-
} | null
|
|
2300
|
-
if (row === null) return ''
|
|
2301
|
-
return typeof row.stdout === 'string' ? row.stdout : utf8Decode.decode(row.stdout)
|
|
2671
|
+
const row = this.selectStdout.get(hash) as { stdout: Uint8Array } | null
|
|
2672
|
+
return storedLog(row?.stdout ?? null)
|
|
2302
2673
|
}
|
|
2303
2674
|
|
|
2304
|
-
/**
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2675
|
+
/**
|
|
2676
|
+
* Remove one entry while its rows are still `rows`: false, touching
|
|
2677
|
+
* nothing, once another process re-saved the key. Checked and dropped
|
|
2678
|
+
* under the write lock a save renames its artifact in under, so no save
|
|
2679
|
+
* lands between; the artifact is moved aside inside, unlinked after.
|
|
2680
|
+
*/
|
|
2681
|
+
private async dropUnchanged(hash: string, rows: readonly OutputFileRow[]): Promise<boolean> {
|
|
2682
|
+
const aside: string[] = []
|
|
2308
2683
|
const drop = this.db.transaction(() => {
|
|
2684
|
+
if (this.entryChanged(hash, rows)) return false
|
|
2685
|
+
for (const [from, to] of [
|
|
2686
|
+
[this.tarPath(hash), this.tempPath(hash)],
|
|
2687
|
+
[this.treePath(hash), this.tempTreePath(hash)],
|
|
2688
|
+
] as const) {
|
|
2689
|
+
try {
|
|
2690
|
+
renameSync(from, to)
|
|
2691
|
+
aside.push(to)
|
|
2692
|
+
} catch (err) {
|
|
2693
|
+
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
|
|
2694
|
+
}
|
|
2695
|
+
}
|
|
2309
2696
|
this.deleteBlob.run(hash)
|
|
2310
2697
|
this.deleteEntryRow.run(hash)
|
|
2311
2698
|
this.outputs.forget([hash])
|
|
2699
|
+
return true
|
|
2312
2700
|
})
|
|
2701
|
+
let dropped: boolean
|
|
2313
2702
|
try {
|
|
2314
|
-
drop()
|
|
2703
|
+
dropped = drop.immediate()
|
|
2315
2704
|
} catch (err) {
|
|
2316
2705
|
if (!isIndexFull(err)) throw err
|
|
2317
|
-
drop()
|
|
2706
|
+
dropped = drop.immediate()
|
|
2318
2707
|
}
|
|
2708
|
+
for (const a of aside) await rm(a, { recursive: true, force: true })
|
|
2709
|
+
return dropped
|
|
2319
2710
|
}
|
|
2320
2711
|
|
|
2321
2712
|
private async restoreOutputsOnce(
|
|
@@ -2323,6 +2714,8 @@ export class Cache implements CacheLayer {
|
|
|
2323
2714
|
projectDir: string,
|
|
2324
2715
|
workspaceRoot: string | undefined,
|
|
2325
2716
|
outputs: DeclaredOutputs | undefined,
|
|
2717
|
+
used: { rows: readonly OutputFileRow[] },
|
|
2718
|
+
fresh: boolean,
|
|
2326
2719
|
): Promise<void> {
|
|
2327
2720
|
// In-process extraction — no fork+exec on the hot path
|
|
2328
2721
|
// (~5-10ms reclaimed per hit vs the prior subprocess `tar -xf`).
|
|
@@ -2366,24 +2759,56 @@ export class Cache implements CacheLayer {
|
|
|
2366
2759
|
// in one commit, and two reads could pair its rows with the old bytes.
|
|
2367
2760
|
// The probe's held rows (X-162) serve only while the blob carries the
|
|
2368
2761
|
// `at` the probe saw: a re-save or a renewal since moves it, and then
|
|
2369
|
-
// the rows are read again in the blob's snapshot.
|
|
2762
|
+
// the rows are read again in the blob's snapshot. A file artifact's
|
|
2763
|
+
// re-save is caught after, by its rows (`restoreOutputs`).
|
|
2370
2764
|
let rows: readonly OutputFileRow[] = []
|
|
2371
2765
|
let inline: Uint8Array | undefined
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2766
|
+
// With the probe's rows held, no transaction: a file artifact's tree,
|
|
2767
|
+
// standing now, is the live one, read with no statement (an inline save
|
|
2768
|
+
// moves the tree aside in the commit that adds its blob, and one moved
|
|
2769
|
+
// aside mid-copy is the race `copyTree` catches); an inline one is the
|
|
2770
|
+
// blob read alone, its own snapshot. The blob read and its transaction
|
|
2771
|
+
// cost a 1,600-restore run ~70 ms of its main thread (2026-11-07).
|
|
2772
|
+
const held = fresh ? undefined : this.outputs.heldRowsOf(hash)
|
|
2773
|
+
const heldAt = this.heldInlineAt.get(hash)
|
|
2774
|
+
let treeIno: number | undefined
|
|
2775
|
+
let treeAsked = false
|
|
2776
|
+
if (held !== undefined && heldAt === undefined) {
|
|
2777
|
+
treeIno = lstatSync(this.treePath(hash), { throwIfNoEntry: false })?.ino
|
|
2778
|
+
treeAsked = true
|
|
2779
|
+
if (treeIno !== undefined) rows = held
|
|
2780
|
+
} else if (held !== undefined) {
|
|
2781
|
+
const blob = this.guard(() => this.selectBlob.get(hash)) as {
|
|
2782
|
+
at: number
|
|
2783
|
+
bytes: Uint8Array
|
|
2784
|
+
} | null
|
|
2785
|
+
if (blob?.at === heldAt) {
|
|
2786
|
+
inline = blob!.bytes
|
|
2787
|
+
rows = held
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
if (treeIno === undefined && inline === undefined) {
|
|
2791
|
+
this.guard(() =>
|
|
2792
|
+
this.db.transaction(() => {
|
|
2793
|
+
const blob = this.selectBlob.get(hash) as { at: number; bytes: Uint8Array } | null
|
|
2794
|
+
inline = blob?.bytes
|
|
2795
|
+
rows =
|
|
2796
|
+
!fresh && (blob === null || blob.at === this.heldInlineAt.get(hash))
|
|
2797
|
+
? this.outputs.rowsOf(hash)
|
|
2798
|
+
: (this.outputs.loadOutputFilesBatch([hash]).get(hash) ?? [])
|
|
2799
|
+
})(),
|
|
2800
|
+
)
|
|
2801
|
+
// A tree asked of above and absent: the artifact is the blob or the
|
|
2802
|
+
// tar, and a tree saved since is a miss at worst (the tar is gone).
|
|
2803
|
+
if (inline === undefined && !treeAsked)
|
|
2804
|
+
treeIno = lstatSync(this.treePath(hash), { throwIfNoEntry: false })?.ino
|
|
2805
|
+
}
|
|
2806
|
+
used.rows = rows
|
|
2382
2807
|
const expected = rows
|
|
2383
2808
|
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
2384
2809
|
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
|
2385
2810
|
endRows()
|
|
2386
|
-
const moded =
|
|
2811
|
+
const moded = modedDirs(rows, projectDir, workspaceRoot)
|
|
2387
2812
|
const verify = (provided: ReadonlySet<string>): void => {
|
|
2388
2813
|
const missing = expected.filter((name) => !provided.has(name))
|
|
2389
2814
|
if (missing.length > 0) {
|
|
@@ -2403,8 +2828,8 @@ export class Cache implements CacheLayer {
|
|
|
2403
2828
|
// when every artifact is a one-file `dist/` (measured: 390 vs 355 ms
|
|
2404
2829
|
// per 1 000). The local artifact was validated at ingest, so a missing
|
|
2405
2830
|
// declared size is allowed; the output ceiling applies to both.
|
|
2406
|
-
const tree =
|
|
2407
|
-
|
|
2831
|
+
const tree = treeIno === undefined ? undefined : this.treePath(hash)
|
|
2832
|
+
const written = new Map<string, number>()
|
|
2408
2833
|
const endExtract = span('restore: extract')
|
|
2409
2834
|
try {
|
|
2410
2835
|
// Every step here is an async filesystem call on purpose: a
|
|
@@ -2414,11 +2839,38 @@ export class Cache implements CacheLayer {
|
|
|
2414
2839
|
// the other three (1,000-project restore row 1.1–1.2 s → 1.5 s).
|
|
2415
2840
|
const accept = outputs === undefined ? undefined : declaredName(outputs)
|
|
2416
2841
|
if (tree !== undefined) {
|
|
2417
|
-
await this.restoreTree(
|
|
2842
|
+
await this.restoreTree(
|
|
2843
|
+
hash,
|
|
2844
|
+
tree,
|
|
2845
|
+
treeIno!,
|
|
2846
|
+
rows,
|
|
2847
|
+
projectDir,
|
|
2848
|
+
workspaceRoot,
|
|
2849
|
+
verify,
|
|
2850
|
+
accept,
|
|
2851
|
+
written,
|
|
2852
|
+
)
|
|
2418
2853
|
} else {
|
|
2419
2854
|
const tar = await decodedTar(inline ?? Bun.file(src), hash, this.artifactCeiling)
|
|
2420
|
-
await extractArtifactStream(
|
|
2855
|
+
await extractArtifactStream(
|
|
2856
|
+
tar,
|
|
2857
|
+
projectDir,
|
|
2858
|
+
workspaceRoot,
|
|
2859
|
+
verify,
|
|
2860
|
+
accept,
|
|
2861
|
+
workspaceRoot,
|
|
2862
|
+
undefined,
|
|
2863
|
+
written,
|
|
2864
|
+
)
|
|
2421
2865
|
}
|
|
2866
|
+
// Keyed by row path: a project output's row drops the `outputs/`.
|
|
2867
|
+
const byRow = new Map<string, number>()
|
|
2868
|
+
for (const [name, t] of written)
|
|
2869
|
+
byRow.set(
|
|
2870
|
+
name.startsWith(WORKSPACE_OUTPUT_PREFIX) ? name : name.slice('outputs/'.length),
|
|
2871
|
+
t,
|
|
2872
|
+
)
|
|
2873
|
+
this.datedMtimes.set(hash, byRow)
|
|
2422
2874
|
for (const [dir, mode] of moded) await chmod(dir, restoredMode(mode))
|
|
2423
2875
|
} catch (err) {
|
|
2424
2876
|
// A UserError here is the extractor naming the tree's fault (an
|
|
@@ -2457,7 +2909,7 @@ export class Cache implements CacheLayer {
|
|
|
2457
2909
|
? 'a directory stands where the entry holds a file'
|
|
2458
2910
|
: 'a file stands where the entry needs a directory'
|
|
2459
2911
|
throw new UserError(
|
|
2460
|
-
`restore of ${hash} into ${projectDir} was blocked by what is on disk (${
|
|
2912
|
+
`restore of ${hash} into ${projectDir} was blocked by what is on disk (${codeFirst(code, msg)}): ${what}. ` +
|
|
2461
2913
|
`A restore replaces what the output globs select, not this — remove it and re-run.`,
|
|
2462
2914
|
)
|
|
2463
2915
|
}
|
|
@@ -2483,7 +2935,7 @@ export class Cache implements CacheLayer {
|
|
|
2483
2935
|
(err.path === src || (tree !== undefined && inTree(err.path, tree)))
|
|
2484
2936
|
) {
|
|
2485
2937
|
throw new UserError(
|
|
2486
|
-
`restore of ${hash} could not read its artifact (${code
|
|
2938
|
+
`restore of ${hash} could not read its artifact (${codeFirst(code, err.message)}). ` +
|
|
2487
2939
|
`Make the cache directory readable by this user, or point cacheDir / --cache-dir at one that is.`,
|
|
2488
2940
|
)
|
|
2489
2941
|
}
|
|
@@ -2492,7 +2944,7 @@ export class Cache implements CacheLayer {
|
|
|
2492
2944
|
? 'Free space on that disk and re-run.'
|
|
2493
2945
|
: 'Make the declared output paths writable by this user, or stop declaring them as outputs.'
|
|
2494
2946
|
throw new UserError(
|
|
2495
|
-
`restore of ${hash} into ${projectDir} could not write its outputs (${code
|
|
2947
|
+
`restore of ${hash} into ${projectDir} could not write its outputs (${codeFirst(code, err.message)}). ${remedy}`,
|
|
2496
2948
|
)
|
|
2497
2949
|
}
|
|
2498
2950
|
// The process is out of descriptors: the artifact is fine, and
|
|
@@ -2507,14 +2959,20 @@ export class Cache implements CacheLayer {
|
|
|
2507
2959
|
if (code === 'ENOENT' && (err as NodeJS.ErrnoException).path === src) {
|
|
2508
2960
|
throw new ArtifactVanishedError(hash)
|
|
2509
2961
|
}
|
|
2510
|
-
// So is a tree
|
|
2511
|
-
//
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
) {
|
|
2517
|
-
|
|
2962
|
+
// So is a tree a prune or another process's re-save moved aside
|
|
2963
|
+
// mid-copy. The tree the restore found, still in place, lacking a
|
|
2964
|
+
// file is damage at rest (a hand-deleted file): called vanished "(a
|
|
2965
|
+
// vx cache clean)" that never ran, and kept, so the demoted task's
|
|
2966
|
+
// own probe hit it again and said so twice.
|
|
2967
|
+
const missing = (err as NodeJS.ErrnoException).path
|
|
2968
|
+
if (code === 'ENOENT' && tree !== undefined && inTree(missing, tree)) {
|
|
2969
|
+
if (lstatSync(tree, { throwIfNoEntry: false })?.ino !== treeIno)
|
|
2970
|
+
throw new ArtifactVanishedError(hash)
|
|
2971
|
+
const rel = path.relative(tree, missing!).split(path.sep).join('/')
|
|
2972
|
+
throw new CorruptArtifactError(
|
|
2973
|
+
hash,
|
|
2974
|
+
`${rel.startsWith('outputs/') ? rel.slice('outputs/'.length) : rel} is missing from its stored tree`,
|
|
2975
|
+
)
|
|
2518
2976
|
}
|
|
2519
2977
|
// A staged file the restore itself just wrote is gone before its
|
|
2520
2978
|
// commit: another process cleaned the same outputs under us (two runs
|
|
@@ -2556,7 +3014,8 @@ export class Cache implements CacheLayer {
|
|
|
2556
3014
|
* returns says so; an empty one has nothing to serve and refuses.
|
|
2557
3015
|
*/
|
|
2558
3016
|
readOnlyNotice(): string | null {
|
|
2559
|
-
|
|
3017
|
+
// A reading handle over no index: nothing was refused it.
|
|
3018
|
+
if (this.writeBlocked === null || this.writeBlocked === NO_INDEX) return null
|
|
2560
3019
|
if (this.empty) this.assertWritable()
|
|
2561
3020
|
return (
|
|
2562
3021
|
`cache directory ${this.cacheDir} is not writable (${this.writeBlocked}) — serving hits ` +
|
|
@@ -2605,6 +3064,7 @@ export class Cache implements CacheLayer {
|
|
|
2605
3064
|
args.hash,
|
|
2606
3065
|
rows.reduce((n, r) => n + r.size, 0),
|
|
2607
3066
|
)
|
|
3067
|
+
this.keptHashes.add(args.hash)
|
|
2608
3068
|
return
|
|
2609
3069
|
}
|
|
2610
3070
|
// The cache directory is the constructor's (`mkdirSync`, with a
|
|
@@ -2851,7 +3311,7 @@ export class Cache implements CacheLayer {
|
|
|
2851
3311
|
if (plan.size > this.artifactCeiling) {
|
|
2852
3312
|
throw new Error(
|
|
2853
3313
|
`${args.entry.taskId} is not cached: its outputs pack to ${formatBytes(plan.size)}, past the ` +
|
|
2854
|
-
`${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow
|
|
3314
|
+
`${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow outputs.files`,
|
|
2855
3315
|
)
|
|
2856
3316
|
}
|
|
2857
3317
|
return plan
|
|
@@ -3154,7 +3614,10 @@ export class Cache implements CacheLayer {
|
|
|
3154
3614
|
Bun.hash.crc32(bytes),
|
|
3155
3615
|
])
|
|
3156
3616
|
}
|
|
3157
|
-
|
|
3617
|
+
// Compressed before the write lock is taken: logs are most of what the
|
|
3618
|
+
// index held as text (v52).
|
|
3619
|
+
const stdoutBlob =
|
|
3620
|
+
scanned.stdout === '' ? null : await zstdCompress(utf8.encode(scanned.stdout))
|
|
3158
3621
|
|
|
3159
3622
|
const [project, task] = splitTaskId(meta.taskId)
|
|
3160
3623
|
const now = Date.now()
|
|
@@ -3251,15 +3714,10 @@ export class Cache implements CacheLayer {
|
|
|
3251
3714
|
// packed, an ingest what the producing machine packed.
|
|
3252
3715
|
scanned.exec?.cpuMs ?? null,
|
|
3253
3716
|
scanned.exec?.peakRssBytes ?? null,
|
|
3717
|
+
this.runId ?? null,
|
|
3254
3718
|
)
|
|
3255
|
-
if (
|
|
3256
|
-
|
|
3257
|
-
// bytes, cast to TEXT, keep it.
|
|
3258
|
-
else
|
|
3259
|
-
upsertStdout.run(
|
|
3260
|
-
hash,
|
|
3261
|
-
stdoutText.startsWith('\uFEFF') ? utf8.encode(stdoutText) : stdoutText,
|
|
3262
|
-
)
|
|
3719
|
+
if (stdoutBlob === null) deleteStdout.run(hash)
|
|
3720
|
+
else upsertStdout.run(hash, stdoutBlob)
|
|
3263
3721
|
outputBytes = outputs.replaceFileRows(hash, outputFileRows)
|
|
3264
3722
|
// INSERT OR IGNORE: identical inputs derive this same hash, so a
|
|
3265
3723
|
// re-save's row is identical — keep the first, skip the rest.
|
|
@@ -3332,9 +3790,36 @@ export class Cache implements CacheLayer {
|
|
|
3332
3790
|
for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i) ?? err)
|
|
3333
3791
|
return
|
|
3334
3792
|
}
|
|
3793
|
+
if (this.checkpoints === 'armed') this.startCheckpointer()
|
|
3335
3794
|
for (let i = 0; i < batch.length; i++) batch[i]!.settle(failed.get(i))
|
|
3336
3795
|
}
|
|
3337
3796
|
|
|
3797
|
+
/**
|
|
3798
|
+
* A run's saves from now on leave the WAL's checkpoints to a thread of
|
|
3799
|
+
* their own (checkpointer.ts), started by the first save's commit, so a
|
|
3800
|
+
* run that saves nothing starts none. `stopCheckpoints` ends it.
|
|
3801
|
+
*/
|
|
3802
|
+
checkpointOffThread(): void {
|
|
3803
|
+
if (this.checkpoints === undefined && historyFiles(this) !== undefined)
|
|
3804
|
+
this.checkpoints = 'armed'
|
|
3805
|
+
}
|
|
3806
|
+
|
|
3807
|
+
private startCheckpointer(): void {
|
|
3808
|
+
const files = historyFiles(this)!
|
|
3809
|
+
this.db.exec('PRAGMA wal_autocheckpoint = 0')
|
|
3810
|
+
this.checkpoints = new WalCheckpointer(files, () => {
|
|
3811
|
+
// No thread: the run's connection checkpoints as it did.
|
|
3812
|
+
this.db.exec(`PRAGMA wal_autocheckpoint = ${WAL_AUTOCHECKPOINT}`)
|
|
3813
|
+
})
|
|
3814
|
+
}
|
|
3815
|
+
|
|
3816
|
+
/** Ends the checkpoint thread before `close`, so this connection's close is the last. */
|
|
3817
|
+
async stopCheckpoints(): Promise<void> {
|
|
3818
|
+
const c = this.checkpoints
|
|
3819
|
+
this.checkpoints = undefined
|
|
3820
|
+
if (c instanceof WalCheckpointer) await c.stop()
|
|
3821
|
+
}
|
|
3822
|
+
|
|
3338
3823
|
private tempTreePath(hash: string): string {
|
|
3339
3824
|
return `${this.treePath(hash)}.tmp-${process.pid}-${process.hrtime.bigint()}-${Math.random().toString(36).slice(2, 10)}`
|
|
3340
3825
|
}
|
|
@@ -3386,9 +3871,12 @@ export class Cache implements CacheLayer {
|
|
|
3386
3871
|
if (this.writeBlocked !== null) return
|
|
3387
3872
|
return this.guard(() => this.history.recordRun(run))
|
|
3388
3873
|
}
|
|
3389
|
-
recordRuns(
|
|
3874
|
+
recordRuns(
|
|
3875
|
+
runs: readonly RunRecord[],
|
|
3876
|
+
head?: Pick<InvocationRecord, 'branch' | 'commitSha'>,
|
|
3877
|
+
): void {
|
|
3390
3878
|
if (this.writeBlocked !== null) return
|
|
3391
|
-
return this.guard(() => this.history.recordRuns(runs))
|
|
3879
|
+
return this.guard(() => this.history.recordRuns(runs, head))
|
|
3392
3880
|
}
|
|
3393
3881
|
recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
|
|
3394
3882
|
if (this.writeBlocked !== null) return
|
|
@@ -3418,7 +3906,7 @@ export class Cache implements CacheLayer {
|
|
|
3418
3906
|
const since = Date.now() - 24 * 60 * 60 * 1000
|
|
3419
3907
|
const runs = this.db
|
|
3420
3908
|
.prepare(
|
|
3421
|
-
`SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN status IN ('cache-hit', 'cache-hit-remote') THEN 1 ELSE 0 END), 0) AS hits, COALESCE(SUM(restored = 1), 0) AS restored FROM runs WHERE started_at >= ? AND ${EXECUTED_RUNS_SQL}${
|
|
3909
|
+
`SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN status IN ('cache-hit', 'cache-hit-remote') THEN 1 ELSE 0 END), 0) AS hits, COALESCE(SUM(restored = 1), 0) AS restored FROM main.runs WHERE started_at >= ? AND ${EXECUTED_RUNS_SQL}${
|
|
3422
3910
|
scoped ? ' AND project = ?' : ''
|
|
3423
3911
|
}`,
|
|
3424
3912
|
)
|
|
@@ -3536,7 +4024,12 @@ export class Cache implements CacheLayer {
|
|
|
3536
4024
|
freelist_count: number
|
|
3537
4025
|
}
|
|
3538
4026
|
if (free === 0) return
|
|
3539
|
-
this.db.exec(
|
|
4027
|
+
this.db.exec('DROP VIEW IF EXISTS temp.runs')
|
|
4028
|
+
try {
|
|
4029
|
+
this.db.exec(`VACUUM ${schema}`)
|
|
4030
|
+
} finally {
|
|
4031
|
+
this.db.exec(RUNS_VIEW)
|
|
4032
|
+
}
|
|
3540
4033
|
this.db.exec(`PRAGMA ${schema}.wal_checkpoint(TRUNCATE)`)
|
|
3541
4034
|
} catch {
|
|
3542
4035
|
// Another process holds the database, or the disk has no room for the copy.
|
|
@@ -3688,10 +4181,16 @@ export class Cache implements CacheLayer {
|
|
|
3688
4181
|
try {
|
|
3689
4182
|
this.flushIndex()
|
|
3690
4183
|
clearTimeout(this.heldTimer)
|
|
3691
|
-
const held =
|
|
4184
|
+
const held =
|
|
4185
|
+
this.heldBlobSizes !== undefined
|
|
4186
|
+
? new Map(this.heldBlobSizes)
|
|
4187
|
+
: this.storing !== undefined
|
|
4188
|
+
? this.heldSizes(this.storing, this.stored, this.storing.length)
|
|
4189
|
+
: undefined
|
|
3692
4190
|
this.heldBlobSizes = undefined
|
|
4191
|
+
this.storing = undefined
|
|
3693
4192
|
this.heldTimer = undefined
|
|
3694
|
-
if (held.
|
|
4193
|
+
if (held !== undefined && held.size > 0) this.files.rememberBlobSizes(held)
|
|
3695
4194
|
this.flushAccessed()
|
|
3696
4195
|
this.outputs.flushOutputDirs()
|
|
3697
4196
|
this.folds.flush()
|
|
@@ -3704,6 +4203,8 @@ export class Cache implements CacheLayer {
|
|
|
3704
4203
|
// `rm -rf .vx/cache`): macOS answers SQLITE_IOERR_VNODE on a
|
|
3705
4204
|
// write to an unlinked file where Linux happily writes on.
|
|
3706
4205
|
}
|
|
4206
|
+
if (this.checkpoints instanceof WalCheckpointer) this.checkpoints.terminate()
|
|
4207
|
+
this.checkpoints = undefined
|
|
3707
4208
|
closeDb(this.db)
|
|
3708
4209
|
}
|
|
3709
4210
|
|
|
@@ -3727,8 +4228,16 @@ export class Cache implements CacheLayer {
|
|
|
3727
4228
|
}
|
|
3728
4229
|
}
|
|
3729
4230
|
|
|
3730
|
-
/**
|
|
3731
|
-
|
|
4231
|
+
/**
|
|
4232
|
+
* The files a second connection writes a run's hits through
|
|
4233
|
+
* (history-stream.ts); undefined when the store keeps no history on disk.
|
|
4234
|
+
* A function, as `indexDb` is: no plugin writes the history.
|
|
4235
|
+
*/
|
|
4236
|
+
export function historyFiles(cache: Cache): { index: string; store?: string } | undefined {
|
|
4237
|
+
if (cache['empty'] || cache['writeBlocked'] !== null || cache['inspecting']) return undefined
|
|
4238
|
+
if (cache.storeDir === undefined) return { index: cache['dbFile'] }
|
|
4239
|
+
return { index: cache['dbFile'], store: path.join(cache.storeDir, 'store.db') }
|
|
4240
|
+
}
|
|
3732
4241
|
|
|
3733
4242
|
/**
|
|
3734
4243
|
* The cache's `bun:sqlite` handle, for core's own readers (`vx last`, `vx
|
|
@@ -3759,6 +4268,131 @@ export function closeDb(db: Database): void {
|
|
|
3759
4268
|
}
|
|
3760
4269
|
}
|
|
3761
4270
|
|
|
4271
|
+
/**
|
|
4272
|
+
* Wait until no other connection has the shared store in `storeDir` open,
|
|
4273
|
+
* then hold it alone until the release. Every checkout of a repository
|
|
4274
|
+
* shares its store and each has a run lock of its own, so a clean in one
|
|
4275
|
+
* deleted store.db under a run in another. A run holds the store open from
|
|
4276
|
+
* its run lock to its end (never while it waits for that lock, which a
|
|
4277
|
+
* clean holds while it waits here), and in WAL mode every open connection keeps a
|
|
4278
|
+
* shared lock on the file: SQLite grants an exclusive one only to a sole
|
|
4279
|
+
* holder, and a holder's lock dies with its process. The run path pays
|
|
4280
|
+
* nothing. Holds nothing where the store has no readable file.
|
|
4281
|
+
*/
|
|
4282
|
+
export async function holdStore(
|
|
4283
|
+
storeDir: string,
|
|
4284
|
+
log: (line: string) => void,
|
|
4285
|
+
): Promise<() => void> {
|
|
4286
|
+
const file = path.join(storeDir, 'store.db')
|
|
4287
|
+
if (!existsSync(file)) return () => {}
|
|
4288
|
+
let db: Database
|
|
4289
|
+
try {
|
|
4290
|
+
db = new Database(file)
|
|
4291
|
+
db.exec('PRAGMA busy_timeout = 0')
|
|
4292
|
+
db.exec('PRAGMA locking_mode = EXCLUSIVE')
|
|
4293
|
+
} catch {
|
|
4294
|
+
return () => {}
|
|
4295
|
+
}
|
|
4296
|
+
const started = performance.now()
|
|
4297
|
+
let said = false
|
|
4298
|
+
for (;;) {
|
|
4299
|
+
try {
|
|
4300
|
+
db.exec('BEGIN EXCLUSIVE')
|
|
4301
|
+
break
|
|
4302
|
+
} catch (err) {
|
|
4303
|
+
if ((err as { code?: string }).code !== 'SQLITE_BUSY') {
|
|
4304
|
+
// Not a store SQLite reads: nothing opens it to wait for.
|
|
4305
|
+
closeDb(db)
|
|
4306
|
+
return () => {}
|
|
4307
|
+
}
|
|
4308
|
+
}
|
|
4309
|
+
if (!said && performance.now() - started >= 1_000) {
|
|
4310
|
+
said = true
|
|
4311
|
+
log(`[vx] waiting for another vx process using the cache store ${storeDir} to finish…`)
|
|
4312
|
+
}
|
|
4313
|
+
await Bun.sleep(50)
|
|
4314
|
+
}
|
|
4315
|
+
return () => closeDb(db)
|
|
4316
|
+
}
|
|
4317
|
+
|
|
4318
|
+
/** The file in a cache directory with no store that its runs hold shared and a clean alone (`holdCacheDir`). */
|
|
4319
|
+
const DIR_LOCK = 'cache.lock'
|
|
4320
|
+
|
|
4321
|
+
/**
|
|
4322
|
+
* Hold a cache directory with no store — `--cache-dir`, `cache.dir`,
|
|
4323
|
+
* `VX_CACHE_DIR`, which workspaces may share — `shared` for a run, from
|
|
4324
|
+
* before its cache opens to its close, or `exclusive` for `vx cache clean`,
|
|
4325
|
+
* waiting for the other side. Each workspace's run lock is its own, and
|
|
4326
|
+
* `holdStore` waits only on a store.db, so a clean in one workspace deleted
|
|
4327
|
+
* the directory under a run of another opening it (ENOENT on its
|
|
4328
|
+
* `.gitignore`, SQLite's disk I/O error). The lock is SQLite's on
|
|
4329
|
+
* `cache.lock` in the directory: shared and exclusive, any spelling of the
|
|
4330
|
+
* directory meets it, and a holder's lock dies with its process. A clean
|
|
4331
|
+
* deletes the file with the rest while it holds it, so a taker checks the
|
|
4332
|
+
* name still is the file it locked. A directory with no index yet, or one
|
|
4333
|
+
* this user cannot write, holds nothing: there is nothing a clean deletes.
|
|
4334
|
+
* A default directory beside its store pays nothing.
|
|
4335
|
+
*/
|
|
4336
|
+
export async function holdCacheDir(
|
|
4337
|
+
dir: string,
|
|
4338
|
+
mode: 'shared' | 'exclusive',
|
|
4339
|
+
log: (line: string) => void,
|
|
4340
|
+
signal?: AbortSignal,
|
|
4341
|
+
): Promise<() => void> {
|
|
4342
|
+
const file = path.join(dir, DIR_LOCK)
|
|
4343
|
+
const started = performance.now()
|
|
4344
|
+
let said = false
|
|
4345
|
+
for (;;) {
|
|
4346
|
+
if (signal?.aborted === true) return () => {}
|
|
4347
|
+
let db: Database
|
|
4348
|
+
let before: Stats
|
|
4349
|
+
try {
|
|
4350
|
+
let seen = statSync(file, { throwIfNoEntry: false })
|
|
4351
|
+
if (seen === undefined) {
|
|
4352
|
+
// A directory with no index has nothing a clean deletes, and may be
|
|
4353
|
+
// one a first index is refused in (a project's own): nothing is
|
|
4354
|
+
// written there.
|
|
4355
|
+
if (!existsSync(path.join(dir, 'cache.db'))) return () => {}
|
|
4356
|
+
// Made before it is stat'ed: a file Database creates is not the one
|
|
4357
|
+
// a stat before it saw.
|
|
4358
|
+
writeFileSync(file, '', { flag: 'a' })
|
|
4359
|
+
seen = statSync(file)
|
|
4360
|
+
}
|
|
4361
|
+
before = seen
|
|
4362
|
+
db = new Database(file)
|
|
4363
|
+
db.exec('PRAGMA busy_timeout = 0')
|
|
4364
|
+
} catch {
|
|
4365
|
+
return () => {}
|
|
4366
|
+
}
|
|
4367
|
+
try {
|
|
4368
|
+
if (mode === 'exclusive') db.exec('BEGIN EXCLUSIVE')
|
|
4369
|
+
else {
|
|
4370
|
+
db.exec('BEGIN')
|
|
4371
|
+
db.query('SELECT count(*) FROM sqlite_master').get()
|
|
4372
|
+
}
|
|
4373
|
+
const now = statSync(file, { throwIfNoEntry: false })
|
|
4374
|
+
if (now !== undefined && now.ino === before.ino && now.dev === before.dev) {
|
|
4375
|
+
return () => closeDb(db)
|
|
4376
|
+
}
|
|
4377
|
+
// A clean deleted it while this waited: take the one there now.
|
|
4378
|
+
closeDb(db)
|
|
4379
|
+
continue
|
|
4380
|
+
} catch (err) {
|
|
4381
|
+
closeDb(db)
|
|
4382
|
+
if ((err as { code?: string }).code !== 'SQLITE_BUSY') return () => {}
|
|
4383
|
+
}
|
|
4384
|
+
if (!said && performance.now() - started >= 1_000) {
|
|
4385
|
+
said = true
|
|
4386
|
+
log(
|
|
4387
|
+
mode === 'exclusive'
|
|
4388
|
+
? `[vx] waiting for another vx process using the cache directory ${dir} to finish…`
|
|
4389
|
+
: `[vx] waiting for a vx cache clean of ${dir} to finish…`,
|
|
4390
|
+
)
|
|
4391
|
+
}
|
|
4392
|
+
await Bun.sleep(50)
|
|
4393
|
+
}
|
|
4394
|
+
}
|
|
4395
|
+
|
|
3762
4396
|
/**
|
|
3763
4397
|
* Refuse an artifact whose file names no restore can materialise as one
|
|
3764
4398
|
* tree, or (given the task's declared outputs) that names a file outside
|
|
@@ -3805,12 +4439,14 @@ function assertArtifactNames(
|
|
|
3805
4439
|
|
|
3806
4440
|
/**
|
|
3807
4441
|
* The directories with a mode to restore (`DIR_MODE` rows) that are absent
|
|
3808
|
-
* before the restore,
|
|
3809
|
-
* default, and each is then given
|
|
3810
|
-
*
|
|
3811
|
-
*
|
|
4442
|
+
* before the restore, or stand with another mode, deepest first: the
|
|
4443
|
+
* restore makes a missing one with mkdir's default, and each is then given
|
|
4444
|
+
* the mode the miss left, minus the umask as a file's is. A standing one
|
|
4445
|
+
* kept its own mode once, so a branch switch A → B → A left B's 0755 on
|
|
4446
|
+
* A's `chmod 700 dist/keys`. A symlink is never followed. A mode only ever
|
|
4447
|
+
* narrows mkdir's default, whatever an artifact says.
|
|
3812
4448
|
*/
|
|
3813
|
-
function
|
|
4449
|
+
function modedDirs(
|
|
3814
4450
|
rows: readonly OutputFileRow[],
|
|
3815
4451
|
projectDir: string,
|
|
3816
4452
|
workspaceRoot: string | undefined,
|
|
@@ -3824,7 +4460,9 @@ function absentModedDirs(
|
|
|
3824
4460
|
const dir = ws
|
|
3825
4461
|
? path.join(workspaceRoot!, rel.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
3826
4462
|
: path.join(projectDir, rel)
|
|
3827
|
-
|
|
4463
|
+
const st = lstatSync(dir, { throwIfNoEntry: false })
|
|
4464
|
+
if (st === undefined || (st.isDirectory() && !modeMatches(st.mode, r.mode)))
|
|
4465
|
+
out.push([dir, r.mode & 0o777])
|
|
3828
4466
|
}
|
|
3829
4467
|
return out.sort((a, b) => b[0].length - a[0].length)
|
|
3830
4468
|
}
|
|
@@ -3834,12 +4472,26 @@ function treeName(rowPath: string): string {
|
|
|
3834
4472
|
return rowPath.startsWith(WORKSPACE_OUTPUT_PREFIX) ? rowPath : `outputs/${rowPath}`
|
|
3835
4473
|
}
|
|
3836
4474
|
|
|
4475
|
+
/** `code: message`, where Bun's message does not already open with the code. */
|
|
4476
|
+
function codeFirst(code: string | undefined, message: string): string {
|
|
4477
|
+
return code === undefined || message.startsWith(code) ? message : `${code}: ${message}`
|
|
4478
|
+
}
|
|
4479
|
+
|
|
3837
4480
|
function inTree(p: string | undefined, tree: string): boolean {
|
|
3838
4481
|
return p !== undefined && (p === tree || p.startsWith(`${tree}/`))
|
|
3839
4482
|
}
|
|
3840
4483
|
|
|
3841
4484
|
const utf8Decode = new TextDecoder('utf-8', { ignoreBOM: true })
|
|
3842
4485
|
|
|
4486
|
+
/**
|
|
4487
|
+
* An `entry_stdout` row's log (v52): a zstd frame of its UTF-8, which keeps
|
|
4488
|
+
* a leading U+FEFF a bound string loses. Null, no row, reads as ''. The
|
|
4489
|
+
* index is vx's own, so the frame is trusted, as a tree's files are.
|
|
4490
|
+
*/
|
|
4491
|
+
export function storedLog(blob: Uint8Array | null): string {
|
|
4492
|
+
return blob === null ? '' : utf8Decode.decode(Bun.zstdDecompressSync(blob))
|
|
4493
|
+
}
|
|
4494
|
+
|
|
3843
4495
|
/**
|
|
3844
4496
|
* Whether an artifact entry names one of the task's declared outputs, as
|
|
3845
4497
|
* the save could have packed it (`restorableOutput`: a `!` take-back, vx's
|