@vzn/vx 0.0.631 → 0.0.633
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -18
- package/docs/caching.md +2247 -0
- package/docs/cli.md +3214 -0
- package/docs/execution.md +708 -0
- package/docs/features.md +172 -0
- package/docs/patterns.md +219 -0
- package/docs/schema.md +2137 -0
- package/docs/security.md +209 -0
- package/package.json +14 -5
- package/schemas/cache.json +24 -0
- package/schemas/docs.json +28 -0
- package/schemas/error.json +23 -0
- package/schemas/info.json +147 -0
- package/schemas/init.json +57 -0
- package/schemas/last.json +188 -0
- package/schemas/lock.json +21 -0
- package/schemas/plan.json +88 -0
- package/schemas/show.json +124 -0
- package/schemas/summary.json +133 -0
- package/schemas/why.json +168 -0
- package/skills/vx/SKILL.md +89 -0
- package/src/bin.ts +9 -0
- package/src/cache/archive.ts +153 -51
- package/src/cache/cache.ts +542 -137
- package/src/cache/chained-cache.ts +9 -0
- package/src/cache/inputs.ts +5 -0
- package/src/cache/layer.ts +10 -5
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/schema.ts +44 -17
- package/src/cache/tar-stream.ts +204 -66
- package/src/cache/zstd.ts +1 -1
- package/src/cli/cache.ts +14 -14
- package/src/cli/docs-corpus.ts +21 -0
- package/src/cli/docs-text.d.ts +5 -0
- package/src/cli/docs.ts +167 -0
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/help.ts +42 -7
- package/src/cli/index.ts +25 -11
- package/src/cli/info.ts +38 -9
- package/src/cli/init.ts +131 -24
- package/src/cli/last.ts +54 -3
- package/src/cli/lock.ts +43 -15
- package/src/cli/plan-format.ts +23 -1
- package/src/cli/plugin-commands.ts +3 -0
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +40 -21
- package/src/cli/show.ts +10 -7
- package/src/cli/upgrade.ts +19 -7
- package/src/cli/watch.ts +7 -2
- package/src/cli/why.ts +14 -40
- package/src/exec/runner.ts +27 -10
- package/src/graph/scheduler.ts +36 -0
- package/src/graph/task-graph.ts +4 -4
- package/src/index.ts +22 -5
- package/src/orchestrator/admission.ts +29 -1
- package/src/orchestrator/affected-tasks.ts +87 -18
- package/src/orchestrator/doctor.ts +14 -1
- package/src/orchestrator/execute-task.ts +149 -64
- package/src/orchestrator/failure-mode.ts +5 -3
- package/src/orchestrator/framed-output.ts +17 -3
- package/src/orchestrator/index.ts +11 -0
- package/src/orchestrator/logger.ts +57 -6
- package/src/orchestrator/metrics.ts +76 -4
- package/src/orchestrator/miss-reason.ts +3 -5
- package/src/orchestrator/miss-save.ts +6 -5
- package/src/orchestrator/options.ts +12 -0
- package/src/orchestrator/output-log.ts +14 -1
- package/src/orchestrator/path-links.ts +79 -0
- package/src/orchestrator/plan.ts +20 -3
- package/src/orchestrator/plugin.ts +7 -0
- package/src/orchestrator/prepare.ts +14 -4
- package/src/orchestrator/projects.ts +2 -2
- package/src/orchestrator/run-artifacts.ts +18 -2
- package/src/orchestrator/run-failures.ts +192 -0
- package/src/orchestrator/run-records.ts +10 -8
- package/src/orchestrator/run.ts +87 -22
- package/src/orchestrator/task-hash.ts +4 -4
- package/src/orchestrator/terminal-signals.ts +16 -0
- package/src/orchestrator/upstream.ts +65 -10
- package/src/util/errors.ts +55 -1
- package/src/util/index.ts +3 -0
- package/src/util/secret-mask.ts +13 -1
- package/src/util/verbs.ts +1 -0
- package/src/workspace/config-schema.ts +10 -8
- package/src/workspace/migration.ts +30 -3
- package/src/workspace/workspace.ts +1 -0
package/src/cache/cache.ts
CHANGED
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
} from 'node:fs'
|
|
51
51
|
import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
52
52
|
import path from 'node:path'
|
|
53
|
-
import { createTables, inHashes, lazyStatement } from './schema.js'
|
|
53
|
+
import { createArtifactTables, createTables, inHashes, lazyStatement } from './schema.js'
|
|
54
54
|
import {
|
|
55
55
|
UserError,
|
|
56
56
|
formatBytes,
|
|
@@ -70,6 +70,7 @@ import {
|
|
|
70
70
|
type ExecUsage,
|
|
71
71
|
extractArtifactStream,
|
|
72
72
|
scanArtifact,
|
|
73
|
+
scanTarBytes,
|
|
73
74
|
packArtifactBytes,
|
|
74
75
|
packArtifactStream,
|
|
75
76
|
planArtifact,
|
|
@@ -100,7 +101,6 @@ import {
|
|
|
100
101
|
bytesOf,
|
|
101
102
|
decodedTar,
|
|
102
103
|
MAX_DECOMPRESSED_ARTIFACT_BYTES,
|
|
103
|
-
oneChunk,
|
|
104
104
|
STREAM_DECODE_FROM,
|
|
105
105
|
zstdCompress,
|
|
106
106
|
zstdEncoder,
|
|
@@ -122,6 +122,16 @@ import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
|
122
122
|
*/
|
|
123
123
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
124
124
|
|
|
125
|
+
/**
|
|
126
|
+
* A hit renews an entry's `accessed_at` only once it is older than this:
|
|
127
|
+
* renewing every hit rewrote 2,180 rows at a warm 1,090-package run's
|
|
128
|
+
* close, ~30 ms of it. So `accessed_at` trails the last use by at most
|
|
129
|
+
* this much, and the retention policy reads it plus this as the last use:
|
|
130
|
+
* an entry used within the limit is never pruned, and one may stay up to
|
|
131
|
+
* this long past it (owner, 2026-10-08).
|
|
132
|
+
*/
|
|
133
|
+
const ACCESS_REFRESH_MS = 60 * 60 * 1000
|
|
134
|
+
|
|
125
135
|
/**
|
|
126
136
|
* The only names the orphan sweep may unlink: an artifact under a key
|
|
127
137
|
* `foldKey` prints (16 lowercase hex) and the temp `tempPath` makes of
|
|
@@ -240,11 +250,61 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
|
|
|
240
250
|
// output_files' ino + ctime_ms moved to the workspace's own
|
|
241
251
|
// output_stamps, output_dirs lost its foreign key, and the value
|
|
242
252
|
// salt moved to store_meta. The cache KEY is unchanged.
|
|
243
|
-
|
|
253
|
+
// v33: an artifact of at most INLINE_MAX compressed bytes lives in the
|
|
254
|
+
// store's `artifacts` table, written in its rows' transaction, and
|
|
255
|
+
// every entry read joins it. The artifact bytes and the cache KEY are
|
|
256
|
+
// unchanged (CACHE_VERSION not bumped).
|
|
257
|
+
// v34: entry_inputs holds one row per entry, its key components as one
|
|
258
|
+
// JSON array, not a row per component. The cache KEY is unchanged.
|
|
259
|
+
export const SCHEMA_VERSION = 'v34'
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
263
|
+
* `[kind, name, hash]` triples in (kind, name) code-unit order, the first
|
|
264
|
+
* of a repeated (kind, name) kept, as the per-component primary key did.
|
|
265
|
+
*/
|
|
266
|
+
function encodeEntryInputs(
|
|
267
|
+
components: readonly Pick<TaskInputRow, 'kind' | 'name' | 'hash'>[],
|
|
268
|
+
): string {
|
|
269
|
+
const byId = new Map<string, Pick<TaskInputRow, 'kind' | 'name' | 'hash'>>()
|
|
270
|
+
for (const c of components) {
|
|
271
|
+
const id = `${c.kind}\0${c.name}`
|
|
272
|
+
if (!byId.has(id)) byId.set(id, c)
|
|
273
|
+
}
|
|
274
|
+
const ids = [...byId.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
|
275
|
+
return JSON.stringify(
|
|
276
|
+
ids.map((id) => {
|
|
277
|
+
const c = byId.get(id)!
|
|
278
|
+
return [c.kind, c.name, c.hash]
|
|
279
|
+
}),
|
|
280
|
+
)
|
|
281
|
+
}
|
|
244
282
|
|
|
245
283
|
/** The tables a store holds: dropped from a workspace index that held them itself. */
|
|
246
284
|
const STORE_TABLES = ['entry_inputs', 'output_files', 'entry_stdout', 'store_meta', 'entries']
|
|
247
285
|
|
|
286
|
+
/**
|
|
287
|
+
* The inline artifacts' tables (schema.ts): dropped from a workspace index
|
|
288
|
+
* whose entries moved to a store, where an unqualified name would find them
|
|
289
|
+
* first, and never by a schema reset.
|
|
290
|
+
*/
|
|
291
|
+
const ARTIFACT_TABLES = ['artifacts', 'artifacts_meta']
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* The `artifacts` table's own layout (`artifacts_meta.layout`): another value
|
|
295
|
+
* drops the table. It is read only where the schema was not this vx's (a
|
|
296
|
+
* reset, a new index or store, a store moved), so a change to it bumps
|
|
297
|
+
* `SCHEMA_VERSION` too: a warm open reads no statement for it.
|
|
298
|
+
*/
|
|
299
|
+
const ARTIFACT_LAYOUT = 'a1'
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* The largest compressed artifact kept in the index rather than in a file
|
|
303
|
+
* of its own: it is written in its rows' transaction, with no temp and no
|
|
304
|
+
* rename (docs/design/cache-save-cpu-2026-10.md).
|
|
305
|
+
*/
|
|
306
|
+
const INLINE_MAX = 32 * 1024
|
|
307
|
+
|
|
248
308
|
/**
|
|
249
309
|
* An index that records a schema other than this vx's: the one an opener
|
|
250
310
|
* that writes resets. Reads the recorded version alone and touches
|
|
@@ -269,13 +329,26 @@ function indexOfAnotherSchema(dbFile: string): boolean {
|
|
|
269
329
|
return found !== undefined && found !== SCHEMA_VERSION
|
|
270
330
|
}
|
|
271
331
|
|
|
272
|
-
/**
|
|
273
|
-
|
|
274
|
-
|
|
332
|
+
/**
|
|
333
|
+
* Each asked key's entry row with its stdout, which lives apart (v29), none
|
|
334
|
+
* stored reading as '', and its inline artifact's last use, null when the
|
|
335
|
+
* artifact is a file. Driven by the keys, so a key with an inline artifact
|
|
336
|
+
* and no row (a reset kept it) answers too, `hash` null: the miss learns
|
|
337
|
+
* there is something to adopt in the statement that found no entry.
|
|
338
|
+
*/
|
|
339
|
+
const selectEntries = (keys: string): string =>
|
|
340
|
+
`SELECT q.value AS key, e.*, COALESCE(s.stdout, '') AS stdout, a.at AS inline_at FROM ${keys} q LEFT JOIN entries e ON e.hash = q.value LEFT JOIN entry_stdout s ON s.hash = e.hash LEFT JOIN artifacts a ON a.hash = q.value WHERE e.hash IS NOT NULL OR a.at IS NOT NULL`
|
|
341
|
+
const SELECT_ENTRY = selectEntries('(SELECT ? AS value)')
|
|
342
|
+
const SELECT_ENTRIES = selectEntries('json_each(?)')
|
|
275
343
|
|
|
276
|
-
/**
|
|
344
|
+
/**
|
|
345
|
+
* What the orphan sweep may judge: a row-less artifact file or inline
|
|
346
|
+
* artifact (`blob`, its hash), or a dead file — a temp, or a file an
|
|
347
|
+
* inline artifact of its key shadows — which goes whatever the policy.
|
|
348
|
+
*/
|
|
277
349
|
interface RowlessFile {
|
|
278
|
-
file
|
|
350
|
+
file?: string
|
|
351
|
+
blob?: string
|
|
279
352
|
size: number
|
|
280
353
|
mtimeMs: number
|
|
281
354
|
temp: boolean
|
|
@@ -298,11 +371,12 @@ function pickVictims(
|
|
|
298
371
|
const kept: Array<{ at: number; size: number; row?: string; file?: RowlessFile }> = []
|
|
299
372
|
let remaining = 0
|
|
300
373
|
for (const r of rows) {
|
|
301
|
-
|
|
374
|
+
const at = r.accessed_at + ACCESS_REFRESH_MS
|
|
375
|
+
if (olderThanMs !== undefined && at < olderThanMs) {
|
|
302
376
|
hashes.add(r.hash)
|
|
303
377
|
bytesFreed += r.size_bytes
|
|
304
378
|
} else {
|
|
305
|
-
kept.push({ at
|
|
379
|
+
kept.push({ at, size: r.size_bytes, row: r.hash })
|
|
306
380
|
remaining += r.size_bytes
|
|
307
381
|
}
|
|
308
382
|
}
|
|
@@ -338,7 +412,7 @@ function pickVictims(
|
|
|
338
412
|
*/
|
|
339
413
|
async function scanOrphanFiles(
|
|
340
414
|
cacheDir: string,
|
|
341
|
-
indexedHashes: () => ReadonlySet<string
|
|
415
|
+
indexedHashes: () => { files: ReadonlySet<string>; inline: ReadonlySet<string> },
|
|
342
416
|
): Promise<RowlessFile[]> {
|
|
343
417
|
let names: string[]
|
|
344
418
|
try {
|
|
@@ -352,8 +426,10 @@ async function scanOrphanFiles(
|
|
|
352
426
|
for (const name of names) {
|
|
353
427
|
const m = VX_ARTIFACT_NAME.exec(name)
|
|
354
428
|
if (m === null) continue
|
|
355
|
-
|
|
356
|
-
|
|
429
|
+
// A file whose key has an inline artifact is never read: readers take
|
|
430
|
+
// the blob first. It is what an inline save of a re-built key left.
|
|
431
|
+
const temp = m[2] !== undefined || indexed.inline.has(m[1]!)
|
|
432
|
+
if (temp || !indexed.files.has(m[1]!)) candidates.push({ name, temp })
|
|
357
433
|
}
|
|
358
434
|
const found: RowlessFile[] = []
|
|
359
435
|
await Promise.all(
|
|
@@ -404,8 +480,12 @@ export * from './layer.js'
|
|
|
404
480
|
export * from './policy.js'
|
|
405
481
|
export { zstdContentSize } from './zstd.js'
|
|
406
482
|
|
|
483
|
+
type LiveRow = EntryRow & { hash: string }
|
|
484
|
+
|
|
407
485
|
interface EntryRow {
|
|
408
|
-
|
|
486
|
+
key: string
|
|
487
|
+
/** Null for a row-less inline artifact. */
|
|
488
|
+
hash: string | null
|
|
409
489
|
project: string
|
|
410
490
|
task: string
|
|
411
491
|
command: string
|
|
@@ -417,9 +497,10 @@ interface EntryRow {
|
|
|
417
497
|
accessed_at: number
|
|
418
498
|
cpu_ms: number | null
|
|
419
499
|
peak_rss_bytes: number | null
|
|
500
|
+
inline_at: number | null
|
|
420
501
|
}
|
|
421
502
|
|
|
422
|
-
function entryOf(row:
|
|
503
|
+
function entryOf(row: LiveRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
423
504
|
return {
|
|
424
505
|
hash: row.hash,
|
|
425
506
|
taskId: `${row.project}#${row.task}`,
|
|
@@ -516,6 +597,9 @@ function ensurePrivateDir(dir: string): string | null {
|
|
|
516
597
|
if (!st.isDirectory()) return `${dir} is not a directory`
|
|
517
598
|
const uid = process.getuid?.()
|
|
518
599
|
if (uid !== undefined && st.uid !== uid) return `${dir} belongs to another user`
|
|
600
|
+
// One others could write may already hold what they planted (a link, an
|
|
601
|
+
// entry of their own), and closing it now would keep those: not used.
|
|
602
|
+
if ((st.mode & 0o022) !== 0) return `${dir} is writable by other users`
|
|
519
603
|
// The user's own directory open to others (a `~/.vx` made at the
|
|
520
604
|
// default umask, 0755) is vx's to close: refusing it left the store
|
|
521
605
|
// unused on a stock macOS home (owner, 2026-10-06: "you should own it").
|
|
@@ -536,6 +620,7 @@ function ensurePrivateDir(dir: string): string | null {
|
|
|
536
620
|
// enumeration on the bench workspace before its generator ignored `.vx`. An
|
|
537
621
|
// ignored directory is skipped by the walk entirely. Only written when
|
|
538
622
|
// absent, so a user's own file wins.
|
|
623
|
+
const utf8 = new TextEncoder()
|
|
539
624
|
const IGNORE_ALL = '*\n'
|
|
540
625
|
|
|
541
626
|
/**
|
|
@@ -630,9 +715,21 @@ export class Cache implements CacheLayer {
|
|
|
630
715
|
private readonly upsertStdout: ReturnType<Database['prepare']>
|
|
631
716
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
632
717
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
718
|
+
private readonly selectBlob: ReturnType<Database['prepare']>
|
|
719
|
+
private readonly upsertBlob: ReturnType<Database['prepare']>
|
|
720
|
+
private readonly deleteBlob: ReturnType<Database['prepare']>
|
|
633
721
|
private readonly touched = new Set<string>()
|
|
634
722
|
/** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
|
|
635
723
|
private readonly staleTimes = new Set<string>()
|
|
724
|
+
/** The same for inline artifacts, whose `at` stands for the file time. */
|
|
725
|
+
private readonly staleInline = new Set<string>()
|
|
726
|
+
/**
|
|
727
|
+
* The `at` of each inline artifact the last probe held rows for: a restore
|
|
728
|
+
* serves those rows only while the blob still carries it.
|
|
729
|
+
*/
|
|
730
|
+
private readonly heldInlineAt = new Map<string, number>()
|
|
731
|
+
/** Keys the last lookups found an inline artifact for and no row: `adopt`'s to read. */
|
|
732
|
+
private readonly rowless = new Set<string>()
|
|
636
733
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
637
734
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
638
735
|
private readonly files: FileHashStore
|
|
@@ -681,6 +778,8 @@ export class Cache implements CacheLayer {
|
|
|
681
778
|
* indexed again as they hit. Never printed: the cache is vx's to keep.
|
|
682
779
|
*/
|
|
683
780
|
readonly storeReset: SchemaReset | null = null
|
|
781
|
+
/** This open stamped the store's schema: it was new or another vx's. */
|
|
782
|
+
private storeStamped = false
|
|
684
783
|
|
|
685
784
|
/** A reading verb's open: it never resets the index (the constructor's `mode`). */
|
|
686
785
|
static inspect(cacheDir: string): Cache {
|
|
@@ -830,6 +929,7 @@ export class Cache implements CacheLayer {
|
|
|
830
929
|
if (mode !== 'open') {
|
|
831
930
|
throw new UserError(
|
|
832
931
|
`the cache at ${cacheDir} holds index schema ${found} from another vx version; this vx reads ${SCHEMA_VERSION}, so nothing in it is readable here, and this verb leaves it untouched. The next \`vx run\` resets it`,
|
|
932
|
+
'VX_E_CACHE_VERSION',
|
|
833
933
|
)
|
|
834
934
|
}
|
|
835
935
|
}
|
|
@@ -859,7 +959,8 @@ export class Cache implements CacheLayer {
|
|
|
859
959
|
.prepare("SELECT 1 FROM schema_meta WHERE key = 'store_dir'")
|
|
860
960
|
.get()
|
|
861
961
|
if (storeRoot === undefined ? recordedStore != null : storeRoot !== null) {
|
|
862
|
-
for (const t of STORE_TABLES
|
|
962
|
+
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES])
|
|
963
|
+
this.db.exec(`DROP TABLE main.${t}`)
|
|
863
964
|
}
|
|
864
965
|
this.db
|
|
865
966
|
.prepare(
|
|
@@ -912,17 +1013,22 @@ export class Cache implements CacheLayer {
|
|
|
912
1013
|
if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
|
|
913
1014
|
// An index that held its entries itself would shadow the store's:
|
|
914
1015
|
// SQLite resolves an unqualified name in `main` first. Its history
|
|
915
|
-
// and memos stay; its entries'
|
|
1016
|
+
// and memos stay; its entries' artifact files are orphans `prune`
|
|
1017
|
+
// reaps, and its inline artifacts go with their table.
|
|
916
1018
|
const holds = readable(
|
|
917
1019
|
() =>
|
|
918
1020
|
this.db
|
|
919
|
-
.prepare(
|
|
1021
|
+
.prepare(
|
|
1022
|
+
"SELECT 1 FROM main.sqlite_master WHERE type = 'table' AND name IN ('entries', 'artifacts')",
|
|
1023
|
+
)
|
|
920
1024
|
.get() != null,
|
|
921
1025
|
)
|
|
922
1026
|
if (holds) {
|
|
923
1027
|
this.db
|
|
924
1028
|
.transaction(() => {
|
|
925
|
-
for (const t of STORE_TABLES)
|
|
1029
|
+
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
|
|
1030
|
+
this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
|
|
1031
|
+
}
|
|
926
1032
|
})
|
|
927
1033
|
.immediate()
|
|
928
1034
|
}
|
|
@@ -934,6 +1040,11 @@ export class Cache implements CacheLayer {
|
|
|
934
1040
|
this.storeReset = this.matchStoreSchema(mode === 'open' && this.writeBlocked === null)
|
|
935
1041
|
} else {
|
|
936
1042
|
createTables(this.db, 'main')
|
|
1043
|
+
if (absent && existsSync(dbFile)) this.keepArtifacts(dbFile)
|
|
1044
|
+
}
|
|
1045
|
+
const foreign = storeDir === undefined ? current !== SCHEMA_VERSION : this.storeStamped
|
|
1046
|
+
if (mode === 'open' && this.writeBlocked === null && (foreign || storeDir !== recorded)) {
|
|
1047
|
+
readable(() => this.matchArtifactLayout(storeDir === undefined ? 'main' : 'store'))
|
|
937
1048
|
}
|
|
938
1049
|
if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
|
|
939
1050
|
if (storeDir === undefined)
|
|
@@ -966,23 +1077,32 @@ export class Cache implements CacheLayer {
|
|
|
966
1077
|
)
|
|
967
1078
|
this.upsertStdout = lazyStatement(
|
|
968
1079
|
this.db,
|
|
969
|
-
'INSERT INTO entry_stdout(hash, stdout) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
|
|
1080
|
+
'INSERT INTO entry_stdout(hash, stdout) VALUES (?, CAST(? AS TEXT)) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
|
|
970
1081
|
)
|
|
971
1082
|
this.deleteStdout = lazyStatement(this.db, 'DELETE FROM entry_stdout WHERE hash = ?')
|
|
972
|
-
this.selectEntry = lazyStatement(this.db,
|
|
973
|
-
// `has` asks only whether the row is there
|
|
974
|
-
// column off the index, so the table row is
|
|
975
|
-
// table refuses here as it does on `get`
|
|
976
|
-
// index alone).
|
|
977
|
-
this.entryExists = lazyStatement(
|
|
1083
|
+
this.selectEntry = lazyStatement(this.db, SELECT_ENTRY)
|
|
1084
|
+
// `has` asks only whether the row is there and where its artifact is,
|
|
1085
|
+
// with no stdout joined. A column off the index, so the table row is
|
|
1086
|
+
// still read and a corrupt table refuses here as it does on `get`
|
|
1087
|
+
// (`SELECT 1` answers from the index alone).
|
|
1088
|
+
this.entryExists = lazyStatement(
|
|
1089
|
+
this.db,
|
|
1090
|
+
'SELECT e.exit_code, a.at AS inline_at FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash WHERE e.hash = ?',
|
|
1091
|
+
)
|
|
1092
|
+
this.selectBlob = lazyStatement(this.db, 'SELECT at, bytes FROM artifacts WHERE hash = ?')
|
|
1093
|
+
this.upsertBlob = lazyStatement(
|
|
1094
|
+
this.db,
|
|
1095
|
+
'INSERT INTO artifacts(hash, at, bytes) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET at = excluded.at, bytes = excluded.bytes',
|
|
1096
|
+
)
|
|
1097
|
+
this.deleteBlob = lazyStatement(this.db, 'DELETE FROM artifacts WHERE hash = ?')
|
|
978
1098
|
// INSERT OR IGNORE: re-saving the same hash (idempotent ingest /
|
|
979
1099
|
// overlapping concurrent saves) leaves the existing rows untouched —
|
|
980
1100
|
// identical inputs derive the identical hash, so the rows are too.
|
|
981
1101
|
this.insertEntryInput = lazyStatement(
|
|
982
1102
|
this.db,
|
|
983
1103
|
`
|
|
984
|
-
INSERT OR IGNORE INTO entry_inputs(entry_hash,
|
|
985
|
-
VALUES (?,
|
|
1104
|
+
INSERT OR IGNORE INTO entry_inputs(entry_hash, components)
|
|
1105
|
+
VALUES (?, ?)
|
|
986
1106
|
`,
|
|
987
1107
|
)
|
|
988
1108
|
// The slices: each owns its statements over this handle and its table(s);
|
|
@@ -1104,11 +1224,17 @@ export class Cache implements CacheLayer {
|
|
|
1104
1224
|
return null
|
|
1105
1225
|
}
|
|
1106
1226
|
if (!writable) {
|
|
1227
|
+
const file = this.db
|
|
1228
|
+
.prepare("SELECT file FROM pragma_database_list WHERE name = 'store'")
|
|
1229
|
+
.get() as {
|
|
1230
|
+
file: string
|
|
1231
|
+
}
|
|
1107
1232
|
if (seen.has) {
|
|
1108
1233
|
this.db.exec('DETACH DATABASE store')
|
|
1109
1234
|
this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
|
|
1110
1235
|
}
|
|
1111
1236
|
createTables(this.db, 'store')
|
|
1237
|
+
if (seen.has && file.file !== '') this.keepArtifacts(file.file)
|
|
1112
1238
|
return null
|
|
1113
1239
|
}
|
|
1114
1240
|
// Re-read, drop, create and stamp under one write lock: as separate
|
|
@@ -1116,6 +1242,7 @@ export class Cache implements CacheLayer {
|
|
|
1116
1242
|
// tables, and this one's `CREATE IF NOT EXISTS` kept them under its stamp.
|
|
1117
1243
|
return this.db
|
|
1118
1244
|
.transaction((): SchemaReset | null => {
|
|
1245
|
+
this.storeStamped = true
|
|
1119
1246
|
const now = read()
|
|
1120
1247
|
let reset: SchemaReset | null = null
|
|
1121
1248
|
if (now.has && now.found !== SCHEMA_VERSION) {
|
|
@@ -1135,6 +1262,64 @@ export class Cache implements CacheLayer {
|
|
|
1135
1262
|
.immediate()
|
|
1136
1263
|
}
|
|
1137
1264
|
|
|
1265
|
+
/**
|
|
1266
|
+
* A reading open over an index or store of another schema reads it as
|
|
1267
|
+
* the reset a writing open makes leaves it: empty, but for the inline
|
|
1268
|
+
* artifacts, which a reset keeps. They are read from the file in place,
|
|
1269
|
+
* through a view that shadows the empty table (a TEMP name is found
|
|
1270
|
+
* first), so a dry prune names the row-less ones the real prune reaps.
|
|
1271
|
+
*/
|
|
1272
|
+
private keepArtifacts(file: string): void {
|
|
1273
|
+
this.db.prepare('ATTACH DATABASE ? AS kept').run(file)
|
|
1274
|
+
const layout = this.db
|
|
1275
|
+
.prepare("SELECT 1 FROM kept.sqlite_master WHERE type = 'table' AND name = 'artifacts_meta'")
|
|
1276
|
+
.get()
|
|
1277
|
+
? (
|
|
1278
|
+
this.db.prepare("SELECT value FROM kept.artifacts_meta WHERE key = 'layout'").get() as {
|
|
1279
|
+
value: string
|
|
1280
|
+
} | null
|
|
1281
|
+
)?.value
|
|
1282
|
+
: undefined
|
|
1283
|
+
if (layout === ARTIFACT_LAYOUT) {
|
|
1284
|
+
this.db.exec('CREATE TEMP VIEW artifacts AS SELECT hash, at, bytes FROM kept.artifacts')
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/**
|
|
1289
|
+
* `artifacts` holds the layout this vx writes, or is dropped and made
|
|
1290
|
+
* again: its version is its own (`ARTIFACT_LAYOUT`), apart from the
|
|
1291
|
+
* schema's, so a schema reset keeps it. Silent, as a reset is. Asked
|
|
1292
|
+
* only where the schema was not this vx's (`ARTIFACT_LAYOUT`).
|
|
1293
|
+
*/
|
|
1294
|
+
private matchArtifactLayout(schema: 'main' | 'store'): void {
|
|
1295
|
+
const read = (): string | undefined =>
|
|
1296
|
+
(
|
|
1297
|
+
this.db
|
|
1298
|
+
.prepare(`SELECT value FROM ${schema}.artifacts_meta WHERE key = 'layout'`)
|
|
1299
|
+
.get() as {
|
|
1300
|
+
value: string
|
|
1301
|
+
} | null
|
|
1302
|
+
)?.value
|
|
1303
|
+
// Read first: a workspace's first open of a current store asks too,
|
|
1304
|
+
// and must not wait on the store's write lock to learn it holds `a1`.
|
|
1305
|
+
if (read() === ARTIFACT_LAYOUT) return
|
|
1306
|
+
this.db
|
|
1307
|
+
.transaction(() => {
|
|
1308
|
+
const found = read()
|
|
1309
|
+
if (found === ARTIFACT_LAYOUT) return
|
|
1310
|
+
if (found !== undefined) {
|
|
1311
|
+
this.db.exec(`DROP TABLE ${schema}.artifacts`)
|
|
1312
|
+
createArtifactTables(this.db, schema)
|
|
1313
|
+
}
|
|
1314
|
+
this.db
|
|
1315
|
+
.prepare(
|
|
1316
|
+
`INSERT INTO ${schema}.artifacts_meta(key, value) VALUES ('layout', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
1317
|
+
)
|
|
1318
|
+
.run(ARTIFACT_LAYOUT)
|
|
1319
|
+
})
|
|
1320
|
+
.immediate()
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1138
1323
|
// --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
|
|
1139
1324
|
getConfigEval(key: string): string | null {
|
|
1140
1325
|
return this.guard(() => this.configEvals.getConfigEval(key))
|
|
@@ -1283,9 +1468,22 @@ export class Cache implements CacheLayer {
|
|
|
1283
1468
|
* `store.db` deleted) loses nothing a task asks for again. The artifact
|
|
1284
1469
|
* is checked as an ingest checks a remote's (its recorded key, its names
|
|
1285
1470
|
* against the declared outputs); one that fails is a miss, and the run
|
|
1286
|
-
* that follows saves over it.
|
|
1471
|
+
* that follows saves over it. An inline artifact is read first, as every
|
|
1472
|
+
* reader reads it, and indexed in one transaction with its bytes.
|
|
1287
1473
|
*/
|
|
1288
1474
|
private async adopt(hash: string, ctx: CacheGetContext): Promise<boolean> {
|
|
1475
|
+
const meta: IngestMeta = {
|
|
1476
|
+
taskId: ctx.taskId,
|
|
1477
|
+
command: ctx.command,
|
|
1478
|
+
durationMs: 0,
|
|
1479
|
+
...(ctx.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
1480
|
+
}
|
|
1481
|
+
// Only a key whose lookup saw a row-less inline artifact asks for it: a
|
|
1482
|
+
// statement per miss otherwise, on every cold save.
|
|
1483
|
+
const blob = this.rowless.delete(hash)
|
|
1484
|
+
? (this.selectBlob.get(hash) as { bytes: Uint8Array } | null)
|
|
1485
|
+
: null
|
|
1486
|
+
if (blob !== null) return this.adopted(this.writeArtifactAndIndex(hash, blob.bytes, meta))
|
|
1289
1487
|
const finalPath = this.tarPath(hash)
|
|
1290
1488
|
if (!existsSync(finalPath)) return false
|
|
1291
1489
|
const tmpPath = this.tempPath(hash)
|
|
@@ -1302,17 +1500,13 @@ export class Cache implements CacheLayer {
|
|
|
1302
1500
|
} catch {
|
|
1303
1501
|
return false
|
|
1304
1502
|
}
|
|
1503
|
+
return this.adopted(this.writeArtifactAndIndex(hash, { tmpPath }, meta))
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
/** An adoption's verdict: an artifact that fails the checks is a miss. */
|
|
1507
|
+
private async adopted(indexing: Promise<void>): Promise<boolean> {
|
|
1305
1508
|
try {
|
|
1306
|
-
await
|
|
1307
|
-
hash,
|
|
1308
|
-
{ tmpPath },
|
|
1309
|
-
{
|
|
1310
|
-
taskId: ctx.taskId,
|
|
1311
|
-
command: ctx.command,
|
|
1312
|
-
durationMs: 0,
|
|
1313
|
-
...(ctx.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
1314
|
-
},
|
|
1315
|
-
)
|
|
1509
|
+
await indexing
|
|
1316
1510
|
return true
|
|
1317
1511
|
} catch (err) {
|
|
1318
1512
|
if (err instanceof CorruptArtifactError || err instanceof ArchiveSecurityError) return false
|
|
@@ -1334,20 +1528,28 @@ export class Cache implements CacheLayer {
|
|
|
1334
1528
|
}
|
|
1335
1529
|
|
|
1336
1530
|
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
1337
|
-
const row = this.selectEntry.get(hash) as EntryRow |
|
|
1338
|
-
if (
|
|
1531
|
+
const row = this.selectEntry.get(hash) as EntryRow | null
|
|
1532
|
+
if (row === null) return null
|
|
1533
|
+
if (row.hash === null) {
|
|
1534
|
+
this.rowless.add(hash)
|
|
1535
|
+
return null
|
|
1536
|
+
}
|
|
1339
1537
|
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1538
|
+
if (row.inline_at !== null) {
|
|
1539
|
+
if (row.inline_at < Date.now() - ORPHAN_GRACE_MS) this.staleInline.add(hash)
|
|
1540
|
+
} else {
|
|
1541
|
+
// Verify the artifact file actually exists. The DB and the
|
|
1542
|
+
// filesystem can drift if someone manually deletes the cache dir.
|
|
1543
|
+
const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
|
|
1544
|
+
if (st === undefined) return null
|
|
1545
|
+
if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
|
|
1546
|
+
}
|
|
1345
1547
|
|
|
1346
1548
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1347
1549
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
1348
1550
|
// UPDATE by flushAccessed() (called from prune/stats/close), which
|
|
1349
1551
|
// is when accessed_at is actually read.
|
|
1350
|
-
this.touched.add(hash)
|
|
1552
|
+
if (row.accessed_at < Date.now() - ACCESS_REFRESH_MS) this.touched.add(hash)
|
|
1351
1553
|
|
|
1352
1554
|
// Pure SQL: outputFiles come from the output_files rows and
|
|
1353
1555
|
// stdout from the entries row. The artifact is NOT touched here —
|
|
@@ -1355,7 +1557,7 @@ export class Cache implements CacheLayer {
|
|
|
1355
1557
|
// for four up-to-date hits on ~70 MB binaries). restoreOutputs
|
|
1356
1558
|
// reads the artifact itself, only when extraction actually runs.
|
|
1357
1559
|
const fileRows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
1358
|
-
const entry = entryOf(row, fileRows)
|
|
1560
|
+
const entry = entryOf(row as LiveRow, fileRows)
|
|
1359
1561
|
entry.outputDirRows = this.loadOutputDirsBatch([hash]).get(hash) ?? []
|
|
1360
1562
|
return entry
|
|
1361
1563
|
}
|
|
@@ -1388,11 +1590,24 @@ export class Cache implements CacheLayer {
|
|
|
1388
1590
|
private async getManyEntries(hashes: readonly string[]): Promise<Map<string, CacheEntry>> {
|
|
1389
1591
|
const out = new Map<string, CacheEntry>()
|
|
1390
1592
|
if (!this.read || hashes.length === 0) return out
|
|
1391
|
-
const
|
|
1392
|
-
|
|
1593
|
+
const found = (
|
|
1594
|
+
hashes.length === 1
|
|
1595
|
+
? [this.selectEntry.get(hashes[0]!)].filter((r) => r !== null)
|
|
1596
|
+
: this.db.query(SELECT_ENTRIES).all(JSON.stringify(hashes))
|
|
1597
|
+
) as EntryRow[]
|
|
1598
|
+
const rows: LiveRow[] = []
|
|
1599
|
+
for (const r of found) {
|
|
1600
|
+
if (r.hash === null) this.rowless.add(r.key)
|
|
1601
|
+
else rows.push(r as LiveRow)
|
|
1602
|
+
}
|
|
1393
1603
|
if (rows.length === 0) return out
|
|
1394
1604
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1605
|
+
const refreshStart = Date.now() - ACCESS_REFRESH_MS
|
|
1395
1606
|
const live = rows.filter((r) => {
|
|
1607
|
+
if (r.inline_at !== null) {
|
|
1608
|
+
if (r.inline_at < graceStart) this.staleInline.add(r.hash)
|
|
1609
|
+
return true
|
|
1610
|
+
}
|
|
1396
1611
|
const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
|
|
1397
1612
|
if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1398
1613
|
return st !== undefined
|
|
@@ -1400,9 +1615,11 @@ export class Cache implements CacheLayer {
|
|
|
1400
1615
|
const liveHashes = live.map((r) => r.hash)
|
|
1401
1616
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1402
1617
|
this.outputs.hold(fileRows)
|
|
1618
|
+
this.heldInlineAt.clear()
|
|
1619
|
+
for (const r of live) if (r.inline_at !== null) this.heldInlineAt.set(r.hash, r.inline_at)
|
|
1403
1620
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
1404
1621
|
for (const row of live) {
|
|
1405
|
-
this.touched.add(row.hash)
|
|
1622
|
+
if (row.accessed_at < refreshStart) this.touched.add(row.hash)
|
|
1406
1623
|
const entry = entryOf(row, fileRows.get(row.hash) ?? [])
|
|
1407
1624
|
entry.outputDirRows = dirRows.get(row.hash) ?? []
|
|
1408
1625
|
out.set(row.hash, entry)
|
|
@@ -1410,16 +1627,18 @@ export class Cache implements CacheLayer {
|
|
|
1410
1627
|
return out
|
|
1411
1628
|
}
|
|
1412
1629
|
|
|
1413
|
-
// Existence probe: SQL row + artifact
|
|
1414
|
-
// and no accessed_at bump (the plan path
|
|
1630
|
+
// Existence probe: SQL row + where its artifact is (an inline one, or a
|
|
1631
|
+
// file on disk), no byte reads and no accessed_at bump (the plan path
|
|
1632
|
+
// must stay read-only).
|
|
1415
1633
|
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
1416
1634
|
return this.guard(() => this.hasEntry(hash))
|
|
1417
1635
|
}
|
|
1418
1636
|
|
|
1419
1637
|
private async hasEntry(hash: string): Promise<'local' | 'remote' | null> {
|
|
1420
1638
|
if (!this.read) return null
|
|
1421
|
-
|
|
1422
|
-
|
|
1639
|
+
const row = this.entryExists.get(hash) as { inline_at: number | null } | null
|
|
1640
|
+
if (row === null) return null
|
|
1641
|
+
return row.inline_at !== null || existsSync(this.tarPath(hash)) ? 'local' : null
|
|
1423
1642
|
}
|
|
1424
1643
|
|
|
1425
1644
|
// Local cache has no slower layer to warm from — prefetch is a no-op.
|
|
@@ -1443,6 +1662,16 @@ export class Cache implements CacheLayer {
|
|
|
1443
1662
|
outputsPath(hash: string): string {
|
|
1444
1663
|
return this.tarPath(hash)
|
|
1445
1664
|
}
|
|
1665
|
+
artifactSize(hash: string): number | undefined {
|
|
1666
|
+
return this.guard(
|
|
1667
|
+
() =>
|
|
1668
|
+
(
|
|
1669
|
+
this.db.query('SELECT size_bytes FROM entries WHERE hash = ?').get(hash) as {
|
|
1670
|
+
size_bytes: number
|
|
1671
|
+
} | null
|
|
1672
|
+
)?.size_bytes,
|
|
1673
|
+
)
|
|
1674
|
+
}
|
|
1446
1675
|
|
|
1447
1676
|
async recordOutputDirs(
|
|
1448
1677
|
hash: string,
|
|
@@ -1483,6 +1712,7 @@ export class Cache implements CacheLayer {
|
|
|
1483
1712
|
private async dropEntry(hash: string): Promise<void> {
|
|
1484
1713
|
await rm(this.tarPath(hash), { force: true })
|
|
1485
1714
|
const drop = this.db.transaction(() => {
|
|
1715
|
+
this.deleteBlob.run(hash)
|
|
1486
1716
|
this.deleteEntryRow.run(hash)
|
|
1487
1717
|
this.outputs.forget([hash])
|
|
1488
1718
|
})
|
|
@@ -1535,7 +1765,25 @@ export class Cache implements CacheLayer {
|
|
|
1535
1765
|
// same truncated expectation against the same truncated tree and agrees
|
|
1536
1766
|
// forever. Refuse instead of silently under-restoring — checked before
|
|
1537
1767
|
// anything is renamed into place.
|
|
1538
|
-
|
|
1768
|
+
//
|
|
1769
|
+
// An inline artifact and its rows are read in one read transaction, so
|
|
1770
|
+
// one snapshot: a re-save of the key in another process replaces both
|
|
1771
|
+
// in one commit, and two reads could pair its rows with the old bytes.
|
|
1772
|
+
// The probe's held rows (X-162) serve only while the blob carries the
|
|
1773
|
+
// `at` the probe saw: a re-save or a renewal since moves it, and then
|
|
1774
|
+
// the rows are read again in the blob's snapshot.
|
|
1775
|
+
let rows: readonly OutputFileRow[] = []
|
|
1776
|
+
let inline: Uint8Array | undefined
|
|
1777
|
+
this.guard(() =>
|
|
1778
|
+
this.db.transaction(() => {
|
|
1779
|
+
const blob = this.selectBlob.get(hash) as { at: number; bytes: Uint8Array } | null
|
|
1780
|
+
inline = blob?.bytes
|
|
1781
|
+
rows =
|
|
1782
|
+
blob === null || blob.at === this.heldInlineAt.get(hash)
|
|
1783
|
+
? this.outputs.rowsOf(hash)
|
|
1784
|
+
: (this.outputs.loadOutputFilesBatch([hash]).get(hash) ?? [])
|
|
1785
|
+
})(),
|
|
1786
|
+
)
|
|
1539
1787
|
const expected = rows
|
|
1540
1788
|
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
1541
1789
|
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
|
@@ -1566,7 +1814,7 @@ export class Cache implements CacheLayer {
|
|
|
1566
1814
|
// 0.62 ms sequential, 2026-09-10) and 30% slower in the run, where
|
|
1567
1815
|
// four workers overlap their round trips and a blocking one stalls
|
|
1568
1816
|
// the other three (1,000-project restore row 1.1–1.2 s → 1.5 s).
|
|
1569
|
-
const tar = await decodedTar(Bun.file(src), hash, this.artifactCeiling)
|
|
1817
|
+
const tar = await decodedTar(inline ?? Bun.file(src), hash, this.artifactCeiling)
|
|
1570
1818
|
await extractArtifactStream(tar, projectDir, workspaceRoot, verify)
|
|
1571
1819
|
} catch (err) {
|
|
1572
1820
|
// A UserError here is the extractor naming the tree's fault (an
|
|
@@ -1767,9 +2015,12 @@ export class Cache implements CacheLayer {
|
|
|
1767
2015
|
* opened by path read those mid-upload (a digest of one artifact over
|
|
1768
2016
|
* the bytes of another). A `Bun.file` over an fd is no answer — its
|
|
1769
2017
|
* second read starts where the first ended. Throws when the artifact is
|
|
1770
|
-
* gone; `release` unlinks the name.
|
|
2018
|
+
* gone; `release` unlinks the name. An inline artifact is read once into
|
|
2019
|
+
* the body, which no re-save can change.
|
|
1771
2020
|
*/
|
|
1772
2021
|
pinArtifact(hash: string): { body: Blob; release: () => Promise<void> } {
|
|
2022
|
+
const blob = this.guard(() => this.selectBlob.get(hash) as { bytes: Uint8Array } | null)
|
|
2023
|
+
if (blob !== null) return { body: new Blob([blob.bytes]), release: async () => {} }
|
|
1773
2024
|
const pinned = this.tempPath(hash)
|
|
1774
2025
|
linkSync(this.tarPath(hash), pinned)
|
|
1775
2026
|
return { body: Bun.file(pinned), release: () => unlink(pinned).catch(() => undefined) }
|
|
@@ -1780,7 +2031,8 @@ export class Cache implements CacheLayer {
|
|
|
1780
2031
|
* `Response` and copies a file `Blob` without collecting either, so a
|
|
1781
2032
|
* pull never holds the artifact. A body that fails mid-stream (a dropped
|
|
1782
2033
|
* socket) leaves a partial temp behind it, removed here; validation
|
|
1783
|
-
* removes its own.
|
|
2034
|
+
* removes its own. A body of at most `INLINE_MAX` bytes — by its size,
|
|
2035
|
+
* or counted as it streams — is read to memory and stored inline.
|
|
1784
2036
|
*/
|
|
1785
2037
|
async ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {
|
|
1786
2038
|
const tmpPath = this.tempPath(hash)
|
|
@@ -1797,33 +2049,51 @@ export class Cache implements CacheLayer {
|
|
|
1797
2049
|
// Every refusal releases the remote body: a held response pins its
|
|
1798
2050
|
// connection, and a layer that settles on the body's end never hears.
|
|
1799
2051
|
let counted: ReadableStream<Uint8Array> | undefined
|
|
2052
|
+
let reader: ChunkReader | undefined
|
|
2053
|
+
let small: Uint8Array | undefined
|
|
1800
2054
|
try {
|
|
2055
|
+
const length =
|
|
2056
|
+
body instanceof Blob ? body.size : Number(body.headers.get('content-length') ?? 0)
|
|
2057
|
+
if (length > cap) throw past()
|
|
1801
2058
|
if (body instanceof Blob) {
|
|
1802
|
-
if (
|
|
1803
|
-
await Bun.write(tmpPath, body)
|
|
1804
|
-
} else if (Number(body.headers.get('content-length') ?? 0) > cap) {
|
|
1805
|
-
throw past()
|
|
2059
|
+
if (length <= INLINE_MAX) small = await body.bytes()
|
|
2060
|
+
else await Bun.write(tmpPath, body)
|
|
1806
2061
|
} else {
|
|
1807
|
-
let
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
2062
|
+
let source = body.body
|
|
2063
|
+
if (source === null) small = new Uint8Array(0)
|
|
2064
|
+
else if (length <= INLINE_MAX) {
|
|
2065
|
+
// Counted as it streams: a length the server did not send, or
|
|
2066
|
+
// one it sent short, cannot make a large body inline.
|
|
2067
|
+
const r: ChunkReader = source.getReader()
|
|
2068
|
+
reader = r
|
|
2069
|
+
const head = await readUpTo(r, INLINE_MAX)
|
|
2070
|
+
if (head.done) small = head.bytes
|
|
2071
|
+
else source = prepended(head.bytes, r)
|
|
2072
|
+
}
|
|
2073
|
+
if (small === undefined) {
|
|
2074
|
+
let n = 0
|
|
2075
|
+
counted = source?.pipeThrough(
|
|
2076
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
2077
|
+
transform(chunk, controller) {
|
|
2078
|
+
n += chunk.byteLength
|
|
2079
|
+
if (n > cap) controller.error(past())
|
|
2080
|
+
else controller.enqueue(chunk)
|
|
2081
|
+
},
|
|
2082
|
+
}),
|
|
2083
|
+
)
|
|
2084
|
+
await Bun.write(tmpPath, new Response(counted ?? null))
|
|
2085
|
+
}
|
|
1818
2086
|
}
|
|
1819
2087
|
} catch (err) {
|
|
1820
2088
|
// A failed write leaves the pipe's end unlocked; cancelling it
|
|
1821
2089
|
// cancels the source. A body refused by its length is cancelled itself.
|
|
1822
|
-
if (!(body instanceof Blob))
|
|
2090
|
+
if (!(body instanceof Blob)) {
|
|
2091
|
+
void (counted ?? reader ?? body.body)?.cancel(err).catch(() => undefined)
|
|
2092
|
+
}
|
|
1823
2093
|
await unlink(tmpPath).catch(() => undefined)
|
|
1824
2094
|
throw err
|
|
1825
2095
|
}
|
|
1826
|
-
await this.guard(() => this.writeArtifactAndIndex(hash, { tmpPath }, meta))
|
|
2096
|
+
await this.guard(() => this.writeArtifactAndIndex(hash, small ?? { tmpPath }, meta))
|
|
1827
2097
|
}
|
|
1828
2098
|
|
|
1829
2099
|
/** Archive name → absolute source path for every declared output. */
|
|
@@ -1940,7 +2210,8 @@ export class Cache implements CacheLayer {
|
|
|
1940
2210
|
}
|
|
1941
2211
|
|
|
1942
2212
|
/**
|
|
1943
|
-
* Atomically write `compressed` to `<hash>.tar.zst
|
|
2213
|
+
* Atomically write `compressed` to `<hash>.tar.zst`, or inline into
|
|
2214
|
+
* `artifacts` when it is at most `INLINE_MAX` bytes, and (re)build the
|
|
1944
2215
|
* entries + output_files SQL rows from the archive itself. Shared by
|
|
1945
2216
|
* `save()` (we just packed the bytes) and `ingest()` (it streamed them
|
|
1946
2217
|
* from the remote layer into a temp) — both index the identical values, because both
|
|
@@ -1965,9 +2236,14 @@ export class Cache implements CacheLayer {
|
|
|
1965
2236
|
// restore, so the size/mode/mtime fingerprint we store matches
|
|
1966
2237
|
// what isOutputsCurrent will compare against post-restore.
|
|
1967
2238
|
const finalPath = this.tarPath(hash)
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
2239
|
+
// An inline artifact is written in the rows' transaction below: no
|
|
2240
|
+
// temp, no rename.
|
|
2241
|
+
const inline =
|
|
2242
|
+
compressed instanceof Uint8Array && compressed.byteLength <= INLINE_MAX ? compressed : null
|
|
2243
|
+
let tmpPath: string | undefined
|
|
2244
|
+
if (inline === null && compressed instanceof Uint8Array) {
|
|
2245
|
+
const temp = this.tempPath(hash)
|
|
2246
|
+
tmpPath = temp
|
|
1971
2247
|
// The temp is written BEFORE validation so a large artifact can be
|
|
1972
2248
|
// scanned from the file as it decodes — a file stream reads in
|
|
1973
2249
|
// bounded pieces; the bytes in memory would not (see `decodedTar`).
|
|
@@ -1979,28 +2255,31 @@ export class Cache implements CacheLayer {
|
|
|
1979
2255
|
// and a fresh one is an orphan only an hour on (A-14).
|
|
1980
2256
|
// Off this thread even when small: a file's creation on the main
|
|
1981
2257
|
// thread cost more CPU and wall than the round trip (A/B, 2026-10-04).
|
|
1982
|
-
await writeFile(
|
|
1983
|
-
await unlink(
|
|
2258
|
+
await writeFile(temp, compressed).catch(async (err: unknown) => {
|
|
2259
|
+
await unlink(temp).catch(() => undefined)
|
|
1984
2260
|
throw err
|
|
1985
2261
|
})
|
|
1986
2262
|
endWrite()
|
|
1987
|
-
} else {
|
|
2263
|
+
} else if (!(compressed instanceof Uint8Array)) {
|
|
1988
2264
|
// `save` or `ingest` already streamed the artifact into its temp.
|
|
1989
2265
|
tmpPath = compressed.tmpPath
|
|
1990
2266
|
}
|
|
1991
2267
|
let scanned: Awaited<ReturnType<typeof scanArtifact>>
|
|
1992
2268
|
try {
|
|
1993
|
-
// ingest() is the UNTRUSTED boundary — its temp holds bytes just
|
|
1994
|
-
// pulled from a remote. Refuse a bomb (declared or sizeless) before it
|
|
1995
|
-
// can expand into memory.
|
|
1996
|
-
const source =
|
|
1997
|
-
compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
|
|
1998
|
-
? compressed
|
|
1999
|
-
: Bun.file(tmpPath)
|
|
2000
2269
|
const endScan = span('save: scan')
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2270
|
+
if (tar !== undefined) {
|
|
2271
|
+
// Our own tar, in memory: read in place, on this thread.
|
|
2272
|
+
scanned = scanTarBytes(tar)
|
|
2273
|
+
} else {
|
|
2274
|
+
// ingest() is the UNTRUSTED boundary — its temp holds bytes just
|
|
2275
|
+
// pulled from a remote. Refuse a bomb (declared or sizeless) before it
|
|
2276
|
+
// can expand into memory.
|
|
2277
|
+
const source =
|
|
2278
|
+
compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
|
|
2279
|
+
? compressed
|
|
2280
|
+
: Bun.file(tmpPath!)
|
|
2281
|
+
scanned = await scanArtifact(await decodedTar(source, hash, this.artifactCeiling))
|
|
2282
|
+
}
|
|
2004
2283
|
endScan()
|
|
2005
2284
|
// v17 invariant: every artifact carries a `stdout` entry. Its
|
|
2006
2285
|
// absence means the bytes decompressed but aren't a vx artifact.
|
|
@@ -2017,13 +2296,13 @@ export class Cache implements CacheLayer {
|
|
|
2017
2296
|
}
|
|
2018
2297
|
assertArtifactNames(hash, scanned.entries, meta.outputs)
|
|
2019
2298
|
} catch (err) {
|
|
2020
|
-
await unlink(tmpPath).catch(() => undefined)
|
|
2299
|
+
if (tmpPath !== undefined) await unlink(tmpPath).catch(() => undefined)
|
|
2021
2300
|
if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
|
|
2022
2301
|
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
2023
2302
|
}
|
|
2024
2303
|
const { entries } = scanned
|
|
2025
2304
|
const totalBytes =
|
|
2026
|
-
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath).size
|
|
2305
|
+
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath!).size
|
|
2027
2306
|
const outputFileRows: Array<[string, number, number, number]> = []
|
|
2028
2307
|
// Per-output-file fingerprint rows feed the skip-restore check.
|
|
2029
2308
|
// Row paths: project entries store the bare rel (`outputs/`
|
|
@@ -2085,18 +2364,36 @@ export class Cache implements CacheLayer {
|
|
|
2085
2364
|
// renames finds no artifact, a miss (`ArtifactVanishedError`); one
|
|
2086
2365
|
// holding the old file still reads it whole. The aside name is a temp
|
|
2087
2366
|
// name, so a crash before its unlink leaves an orphan the sweep takes.
|
|
2088
|
-
|
|
2089
|
-
|
|
2367
|
+
//
|
|
2368
|
+
// An inline artifact is upserted in the same transaction, and a file
|
|
2369
|
+
// save deletes the key's inline one: as of every commit a key's live
|
|
2370
|
+
// bytes are in one place. An inline save moves aside whatever file the
|
|
2371
|
+
// key has, as a file save does: one rename that finds nothing on a
|
|
2372
|
+
// first save, where asking the index first was a statement per save.
|
|
2373
|
+
let aside: string | undefined
|
|
2090
2374
|
let renamed = false
|
|
2091
|
-
const
|
|
2375
|
+
const upsertBlob = this.upsertBlob
|
|
2376
|
+
const deleteBlob = this.deleteBlob
|
|
2377
|
+
const moveAside = (): string | undefined => {
|
|
2378
|
+
const to = this.tempPath(hash)
|
|
2092
2379
|
try {
|
|
2093
|
-
renameSync(finalPath,
|
|
2094
|
-
|
|
2380
|
+
renameSync(finalPath, to)
|
|
2381
|
+
return to
|
|
2095
2382
|
} catch (err) {
|
|
2096
2383
|
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
|
|
2384
|
+
return undefined
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
const tx = this.db.transaction(() => {
|
|
2388
|
+
if (inline !== null) {
|
|
2389
|
+
aside = moveAside()
|
|
2390
|
+
upsertBlob.run(hash, now, inline)
|
|
2391
|
+
} else {
|
|
2392
|
+
aside = moveAside()
|
|
2393
|
+
renameSync(tmpPath!, finalPath)
|
|
2394
|
+
renamed = true
|
|
2395
|
+
deleteBlob.run(hash)
|
|
2097
2396
|
}
|
|
2098
|
-
renameSync(tmpPath, finalPath)
|
|
2099
|
-
renamed = true
|
|
2100
2397
|
insertEntry.run(
|
|
2101
2398
|
hash,
|
|
2102
2399
|
project,
|
|
@@ -2118,14 +2415,18 @@ export class Cache implements CacheLayer {
|
|
|
2118
2415
|
scanned.exec?.peakRssBytes ?? null,
|
|
2119
2416
|
)
|
|
2120
2417
|
if (stdoutText === '') deleteStdout.run(hash)
|
|
2121
|
-
|
|
2418
|
+
// bun:sqlite drops a leading U+FEFF from a bound string; its UTF-8
|
|
2419
|
+
// bytes, cast to TEXT, keep it.
|
|
2420
|
+
else
|
|
2421
|
+
upsertStdout.run(
|
|
2422
|
+
hash,
|
|
2423
|
+
stdoutText.startsWith('\uFEFF') ? utf8.encode(stdoutText) : stdoutText,
|
|
2424
|
+
)
|
|
2122
2425
|
outputs.replaceFileRows(hash, outputFileRows)
|
|
2123
2426
|
// INSERT OR IGNORE: identical inputs derive this same hash, so a
|
|
2124
|
-
// re-save's
|
|
2125
|
-
if (inputComponents !== undefined) {
|
|
2126
|
-
|
|
2127
|
-
insertEntryInput.run(hash, c.kind, c.name, c.hash)
|
|
2128
|
-
}
|
|
2427
|
+
// re-save's row is identical — keep the first, skip the rest.
|
|
2428
|
+
if (inputComponents !== undefined && inputComponents.length > 0) {
|
|
2429
|
+
insertEntryInput.run(hash, encodeEntryInputs(inputComponents))
|
|
2129
2430
|
}
|
|
2130
2431
|
})
|
|
2131
2432
|
const endTx = span('save: index tx')
|
|
@@ -2133,17 +2434,21 @@ export class Cache implements CacheLayer {
|
|
|
2133
2434
|
tx.immediate()
|
|
2134
2435
|
} catch (err) {
|
|
2135
2436
|
endTx()
|
|
2136
|
-
|
|
2137
|
-
if (
|
|
2437
|
+
const left = renamed ? finalPath : tmpPath
|
|
2438
|
+
if (left !== undefined) await unlink(left).catch(() => undefined)
|
|
2439
|
+
if (aside !== undefined) await unlink(aside).catch(() => undefined)
|
|
2138
2440
|
throw err
|
|
2139
2441
|
}
|
|
2140
2442
|
endTx()
|
|
2141
|
-
if (
|
|
2443
|
+
if (aside !== undefined) await unlink(aside).catch(() => undefined)
|
|
2142
2444
|
}
|
|
2143
2445
|
|
|
2144
2446
|
/** Apply the deferred accessed_at bumps in one statement. */
|
|
2145
2447
|
private flushAccessed(): void {
|
|
2146
|
-
|
|
2448
|
+
// Apart: a hit whose row is fresh still renews an old file time.
|
|
2449
|
+
if (this.touched.size === 0 && this.staleTimes.size === 0 && this.staleInline.size === 0) {
|
|
2450
|
+
return
|
|
2451
|
+
}
|
|
2147
2452
|
const hashes = [...this.touched]
|
|
2148
2453
|
this.touched.clear()
|
|
2149
2454
|
const now = Date.now()
|
|
@@ -2158,10 +2463,16 @@ export class Cache implements CacheLayer {
|
|
|
2158
2463
|
}
|
|
2159
2464
|
}
|
|
2160
2465
|
this.staleTimes.clear()
|
|
2466
|
+
const inline = [...this.staleInline]
|
|
2467
|
+
this.staleInline.clear()
|
|
2161
2468
|
// LRU bookkeeping: on a full disk the bumps are dropped, never the prune
|
|
2162
2469
|
// or the stats that asked for them (A-14).
|
|
2163
2470
|
try {
|
|
2164
|
-
|
|
2471
|
+
if (inline.length > 0) {
|
|
2472
|
+
const { test, params } = inHashes(inline)
|
|
2473
|
+
this.db.prepare(`UPDATE artifacts SET at = ? WHERE hash ${test}`).run(now, ...params)
|
|
2474
|
+
}
|
|
2475
|
+
if (hashes.length > 0) this.writeAccessed(hashes, now)
|
|
2165
2476
|
} catch (err) {
|
|
2166
2477
|
if (!isIndexFull(err)) throw err
|
|
2167
2478
|
}
|
|
@@ -2260,7 +2571,8 @@ export class Cache implements CacheLayer {
|
|
|
2260
2571
|
'SELECT MIN(accessed_at) AS oldest, COALESCE(SUM(size_bytes), 0) AS bytes FROM entries',
|
|
2261
2572
|
)
|
|
2262
2573
|
.get() as { oldest: number | null; bytes: number }
|
|
2263
|
-
const ageDue =
|
|
2574
|
+
const ageDue =
|
|
2575
|
+
olderThanMs !== undefined && oldest !== null && oldest + ACCESS_REFRESH_MS < olderThanMs
|
|
2264
2576
|
const sizeDue = policy.maxBytes !== undefined && bytes > policy.maxBytes
|
|
2265
2577
|
if (ageDue || sizeDue) {
|
|
2266
2578
|
return this.prune({
|
|
@@ -2308,7 +2620,7 @@ export class Cache implements CacheLayer {
|
|
|
2308
2620
|
// them and evicted real entries to make room for them (item 975). None
|
|
2309
2621
|
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
2310
2622
|
// grace window are dropped.
|
|
2311
|
-
const { phantoms, stale, rows } = await this.phantomRows()
|
|
2623
|
+
const { phantoms, stale, rows, inline } = await this.phantomRows()
|
|
2312
2624
|
const picked = pickVictims(
|
|
2313
2625
|
rows.filter((r) => !phantoms.has(r.hash)),
|
|
2314
2626
|
await this.scanOrphans(),
|
|
@@ -2337,10 +2649,14 @@ export class Cache implements CacheLayer {
|
|
|
2337
2649
|
// Artifacts first: a row whose artifact is gone reads as a miss, so the
|
|
2338
2650
|
// order is safe, and on a full disk the delete's own journal needs the
|
|
2339
2651
|
// space the unlinks free. Rows-first failed there before any file went,
|
|
2340
|
-
// and the one verb meant to free a full disk freed nothing (A-14).
|
|
2341
|
-
|
|
2652
|
+
// and the one verb meant to free a full disk freed nothing (A-14). An
|
|
2653
|
+
// inline artifact goes with its rows, in their transaction.
|
|
2654
|
+
await Promise.all(
|
|
2655
|
+
hashes.filter((h) => !inline.has(h)).map((h) => rm(this.tarPath(h), { force: true })),
|
|
2656
|
+
)
|
|
2342
2657
|
const { test, params } = inHashes(rows)
|
|
2343
2658
|
const deleteRows = this.db.transaction(() => {
|
|
2659
|
+
this.db.prepare(`DELETE FROM artifacts WHERE hash ${test}`).run(...params)
|
|
2344
2660
|
this.db.prepare(`DELETE FROM entries WHERE hash ${test}`).run(...params)
|
|
2345
2661
|
this.outputs.forget(rows)
|
|
2346
2662
|
})
|
|
@@ -2366,10 +2682,11 @@ export class Cache implements CacheLayer {
|
|
|
2366
2682
|
/**
|
|
2367
2683
|
* Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
|
|
2368
2684
|
* no `entries` row (another vx version's open dropped it, a `cache.db`
|
|
2369
|
-
* deleted by hand) past the policy,
|
|
2370
|
-
*
|
|
2371
|
-
*
|
|
2372
|
-
*
|
|
2685
|
+
* deleted by hand) past the policy, a `<hash>.tar.zst.tmp-*` a crashed
|
|
2686
|
+
* save never renamed, and a file an inline artifact shadows; and delete
|
|
2687
|
+
* the row-less inline artifacts it took. Nothing else reaps them — a
|
|
2688
|
+
* lookup starts at the row, and a save of the same key replaces the
|
|
2689
|
+
* bytes, so a key that never recurs leaks its bytes forever.
|
|
2373
2690
|
*/
|
|
2374
2691
|
private async reapOrphans(
|
|
2375
2692
|
taken: readonly RowlessFile[],
|
|
@@ -2385,8 +2702,29 @@ export class Cache implements CacheLayer {
|
|
|
2385
2702
|
}
|
|
2386
2703
|
let orphans = 0
|
|
2387
2704
|
let orphanBytes = 0
|
|
2705
|
+
const blobs = taken.filter((o) => o.blob !== undefined)
|
|
2706
|
+
if (blobs.length > 0) {
|
|
2707
|
+
// Only while still row-less: an adopt since the scan keeps it.
|
|
2708
|
+
const drop = this.db.prepare(
|
|
2709
|
+
'DELETE FROM artifacts WHERE hash = ? AND NOT EXISTS (SELECT 1 FROM entries WHERE hash = ?)',
|
|
2710
|
+
)
|
|
2711
|
+
try {
|
|
2712
|
+
this.db.transaction(() => {
|
|
2713
|
+
for (const o of blobs) {
|
|
2714
|
+
if (drop.run(o.blob!, o.blob!).changes === 0) continue
|
|
2715
|
+
orphans += 1
|
|
2716
|
+
orphanBytes += o.size
|
|
2717
|
+
}
|
|
2718
|
+
})()
|
|
2719
|
+
} catch (err) {
|
|
2720
|
+
if (!isIndexFull(err)) throw err
|
|
2721
|
+
orphans = 0
|
|
2722
|
+
orphanBytes = 0
|
|
2723
|
+
}
|
|
2724
|
+
}
|
|
2388
2725
|
await Promise.all(
|
|
2389
2726
|
taken.map(async (o) => {
|
|
2727
|
+
if (o.file === undefined) return
|
|
2390
2728
|
try {
|
|
2391
2729
|
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
2392
2730
|
// concurrent prune took first must not be counted as ours.
|
|
@@ -2414,33 +2752,39 @@ export class Cache implements CacheLayer {
|
|
|
2414
2752
|
phantoms: Map<string, number>
|
|
2415
2753
|
stale: string[]
|
|
2416
2754
|
rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
|
|
2755
|
+
/** The rows whose artifact is inline: present, whatever the directory holds. */
|
|
2756
|
+
inline: Set<string>
|
|
2417
2757
|
}> {
|
|
2418
2758
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
2419
2759
|
const rows = this.db
|
|
2420
|
-
.prepare(
|
|
2760
|
+
.prepare(
|
|
2761
|
+
'SELECT e.hash, e.size_bytes, e.accessed_at, a.hash IS NOT NULL AS inline FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash',
|
|
2762
|
+
)
|
|
2421
2763
|
.all() as Array<{
|
|
2422
2764
|
hash: string
|
|
2423
2765
|
size_bytes: number
|
|
2424
2766
|
accessed_at: number
|
|
2767
|
+
inline: number
|
|
2425
2768
|
}>
|
|
2769
|
+
const inline = new Set(rows.filter((r) => r.inline === 1).map((r) => r.hash))
|
|
2426
2770
|
let names: string[]
|
|
2427
2771
|
try {
|
|
2428
2772
|
names = await readdir(this.artifactDir)
|
|
2429
2773
|
} catch {
|
|
2430
|
-
return { phantoms: new Map(), stale: [], rows }
|
|
2774
|
+
return { phantoms: new Map(), stale: [], rows, inline }
|
|
2431
2775
|
}
|
|
2432
2776
|
const present = new Set(names)
|
|
2433
2777
|
const phantoms = new Map<string, number>()
|
|
2434
2778
|
const stale: string[] = []
|
|
2435
2779
|
for (const r of rows) {
|
|
2436
|
-
if (present.has(`${r.hash}.tar.zst`)) continue
|
|
2780
|
+
if (r.inline === 1 || present.has(`${r.hash}.tar.zst`)) continue
|
|
2437
2781
|
phantoms.set(r.hash, r.size_bytes)
|
|
2438
2782
|
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
2439
2783
|
}
|
|
2440
|
-
return { phantoms, stale, rows }
|
|
2784
|
+
return { phantoms, stale, rows, inline }
|
|
2441
2785
|
}
|
|
2442
2786
|
|
|
2443
|
-
/** Row-less
|
|
2787
|
+
/** Row-less artifacts past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
|
|
2444
2788
|
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2445
2789
|
const found = await this.scanOrphans()
|
|
2446
2790
|
let orphanBytes = 0
|
|
@@ -2448,21 +2792,40 @@ export class Cache implements CacheLayer {
|
|
|
2448
2792
|
return { orphans: found.length, orphanBytes }
|
|
2449
2793
|
}
|
|
2450
2794
|
|
|
2451
|
-
/**
|
|
2795
|
+
/**
|
|
2796
|
+
* Row-less artifacts and temps past the in-flight grace window: one
|
|
2797
|
+
* readdir, one stat per candidate, and the row-less inline artifacts,
|
|
2798
|
+
* their `at` standing for the file time.
|
|
2799
|
+
*/
|
|
2452
2800
|
private async scanOrphans(): Promise<RowlessFile[]> {
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
)
|
|
2461
|
-
|
|
2801
|
+
// Asked once the directory read succeeded, as the rows are (scanOrphanFiles).
|
|
2802
|
+
let rowless: Array<{ hash: string; size: number; at: number }> = []
|
|
2803
|
+
const found = await scanOrphanFiles(this.artifactDir, () => {
|
|
2804
|
+
rowless = this.db
|
|
2805
|
+
.prepare(
|
|
2806
|
+
'SELECT hash, length(bytes) AS size, at FROM artifacts WHERE at <= ? AND hash NOT IN (SELECT hash FROM entries)',
|
|
2807
|
+
)
|
|
2808
|
+
.all(Date.now() - ORPHAN_GRACE_MS) as typeof rowless
|
|
2809
|
+
const rows = this.db
|
|
2810
|
+
.prepare(
|
|
2811
|
+
'SELECT e.hash, a.hash IS NOT NULL AS inline FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash',
|
|
2812
|
+
)
|
|
2813
|
+
.all() as Array<{ hash: string; inline: number }>
|
|
2814
|
+
const blobs = this.db.prepare('SELECT hash FROM artifacts').all() as Array<{ hash: string }>
|
|
2815
|
+
return {
|
|
2816
|
+
files: new Set(rows.filter((r) => r.inline === 0).map((r) => r.hash)),
|
|
2817
|
+
inline: new Set(blobs.map((r) => r.hash)),
|
|
2818
|
+
}
|
|
2819
|
+
})
|
|
2820
|
+
for (const b of rowless) found.push({ blob: b.hash, size: b.size, mtimeMs: b.at, temp: false })
|
|
2462
2821
|
// With a shared store, no artifact in `cacheDir` is indexed: they are
|
|
2463
2822
|
// what this workspace saved before its entries moved out.
|
|
2464
2823
|
if (this.artifactDir === this.cacheDir) return found
|
|
2465
|
-
|
|
2824
|
+
const none = new Set<string>()
|
|
2825
|
+
return [
|
|
2826
|
+
...found,
|
|
2827
|
+
...(await scanOrphanFiles(this.cacheDir, () => ({ files: none, inline: none }))),
|
|
2828
|
+
]
|
|
2466
2829
|
}
|
|
2467
2830
|
|
|
2468
2831
|
close(): void {
|
|
@@ -2578,3 +2941,45 @@ function assertArtifactNames(
|
|
|
2578
2941
|
}
|
|
2579
2942
|
}
|
|
2580
2943
|
}
|
|
2944
|
+
|
|
2945
|
+
/** What `ingest` reads a remote body through. */
|
|
2946
|
+
interface ChunkReader {
|
|
2947
|
+
read(): Promise<{ done: true; value?: undefined } | { done: false; value: Uint8Array }>
|
|
2948
|
+
cancel(reason?: unknown): Promise<void>
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2951
|
+
/**
|
|
2952
|
+
* Read `reader` until it ends or holds more than `max` bytes: `done` and
|
|
2953
|
+
* the whole body, or the bytes read so far for `prepended` to put back.
|
|
2954
|
+
*/
|
|
2955
|
+
async function readUpTo(
|
|
2956
|
+
reader: ChunkReader,
|
|
2957
|
+
max: number,
|
|
2958
|
+
): Promise<{ done: boolean; bytes: Uint8Array }> {
|
|
2959
|
+
const chunks: Uint8Array[] = []
|
|
2960
|
+
let n = 0
|
|
2961
|
+
while (n <= max) {
|
|
2962
|
+
const r = await reader.read()
|
|
2963
|
+
if (r.done) return { done: true, bytes: Buffer.concat(chunks, n) }
|
|
2964
|
+
chunks.push(r.value)
|
|
2965
|
+
n += r.value.byteLength
|
|
2966
|
+
}
|
|
2967
|
+
return { done: false, bytes: Buffer.concat(chunks, n) }
|
|
2968
|
+
}
|
|
2969
|
+
|
|
2970
|
+
/** `head`, then the rest of `reader`: a body `readUpTo` started, whole again. */
|
|
2971
|
+
function prepended(head: Uint8Array, reader: ChunkReader): ReadableStream<Uint8Array> {
|
|
2972
|
+
return new ReadableStream<Uint8Array>({
|
|
2973
|
+
start(controller) {
|
|
2974
|
+
controller.enqueue(head)
|
|
2975
|
+
},
|
|
2976
|
+
async pull(controller) {
|
|
2977
|
+
const r = await reader.read()
|
|
2978
|
+
if (r.done) controller.close()
|
|
2979
|
+
else controller.enqueue(r.value)
|
|
2980
|
+
},
|
|
2981
|
+
cancel(reason) {
|
|
2982
|
+
return reader.cancel(reason)
|
|
2983
|
+
},
|
|
2984
|
+
})
|
|
2985
|
+
}
|