@vzn/vx 0.0.632 → 0.0.634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -10
- package/docs/caching.md +2250 -0
- package/docs/cli.md +3283 -0
- package/docs/execution.md +708 -0
- package/docs/features.md +174 -0
- package/docs/patterns.md +219 -0
- package/docs/schema.md +2151 -0
- package/docs/security.md +209 -0
- package/package.json +12 -5
- package/schemas/docs.json +28 -0
- package/schemas/error.json +5 -1
- package/schemas/init.json +57 -0
- package/schemas/lock.json +21 -0
- package/schemas/plan.json +4 -0
- package/skills/vx/SKILL.md +13 -3
- package/src/cache/cache.ts +556 -122
- package/src/cache/chained-cache.ts +9 -0
- package/src/cache/inputs.ts +71 -30
- package/src/cache/layer.ts +8 -4
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/schema.ts +42 -16
- package/src/cli/docs-corpus.ts +21 -0
- package/src/cli/docs-text.d.ts +5 -0
- package/src/cli/docs.ts +173 -0
- package/src/cli/help.ts +14 -3
- package/src/cli/index.ts +2 -0
- package/src/cli/init.ts +44 -8
- package/src/cli/lock.ts +43 -15
- package/src/cli/plan-format.ts +3 -1
- package/src/cli/run.ts +8 -2
- package/src/cli/upgrade.ts +18 -6
- package/src/config.ts +9 -0
- package/src/graph/priorities.ts +72 -6
- package/src/graph/scheduler.ts +17 -4
- package/src/graph/task-graph.ts +56 -18
- package/src/orchestrator/metrics.ts +11 -4
- package/src/orchestrator/miss-reason.ts +3 -5
- package/src/orchestrator/miss-save.ts +6 -5
- package/src/orchestrator/projects.ts +2 -2
- package/src/orchestrator/remote-prefetch.ts +8 -2
- package/src/orchestrator/run.ts +13 -1
- package/src/orchestrator/task-hash.ts +4 -4
- package/src/util/errors.ts +9 -2
- package/src/util/paths.ts +32 -0
- package/src/util/verbs.ts +1 -0
- package/src/workspace/config-schema.ts +26 -8
- package/src/workspace/lockfile.ts +29 -2
- package/src/workspace/migration.ts +30 -3
package/src/cache/cache.ts
CHANGED
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
} from 'node:fs'
|
|
51
51
|
import { readdir, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
52
52
|
import path from 'node:path'
|
|
53
|
-
import { createTables, inHashes, lazyStatement } from './schema.js'
|
|
53
|
+
import { createArtifactTables, createTables, inHashes, lazyStatement } from './schema.js'
|
|
54
54
|
import {
|
|
55
55
|
UserError,
|
|
56
56
|
formatBytes,
|
|
@@ -122,6 +122,13 @@ import { CACHE_VERSION, foldKey } from './key-fold.js'
|
|
|
122
122
|
*/
|
|
123
123
|
const ORPHAN_GRACE_MS = 60 * 60 * 1000
|
|
124
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Row-less lookups answered by a stat each before the artifact directory
|
|
127
|
+
* is listed once instead: a warm run's few misses keep their stat, a
|
|
128
|
+
* cold run's thousands share one listing.
|
|
129
|
+
*/
|
|
130
|
+
const ADOPT_STATS = 64
|
|
131
|
+
|
|
125
132
|
/**
|
|
126
133
|
* A hit renews an entry's `accessed_at` only once it is older than this:
|
|
127
134
|
* renewing every hit rewrote 2,180 rows at a warm 1,090-package run's
|
|
@@ -250,11 +257,61 @@ function outOfFdsAtOpen(dbFile: string, err: unknown): UserError | undefined {
|
|
|
250
257
|
// output_files' ino + ctime_ms moved to the workspace's own
|
|
251
258
|
// output_stamps, output_dirs lost its foreign key, and the value
|
|
252
259
|
// salt moved to store_meta. The cache KEY is unchanged.
|
|
253
|
-
|
|
260
|
+
// v33: an artifact of at most INLINE_MAX compressed bytes lives in the
|
|
261
|
+
// store's `artifacts` table, written in its rows' transaction, and
|
|
262
|
+
// every entry read joins it. The artifact bytes and the cache KEY are
|
|
263
|
+
// unchanged (CACHE_VERSION not bumped).
|
|
264
|
+
// v34: entry_inputs holds one row per entry, its key components as one
|
|
265
|
+
// JSON array, not a row per component. The cache KEY is unchanged.
|
|
266
|
+
export const SCHEMA_VERSION = 'v34'
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* An entry's key components as its one `entry_inputs` row (v34): JSON
|
|
270
|
+
* `[kind, name, hash]` triples in (kind, name) code-unit order, the first
|
|
271
|
+
* of a repeated (kind, name) kept, as the per-component primary key did.
|
|
272
|
+
*/
|
|
273
|
+
function encodeEntryInputs(
|
|
274
|
+
components: readonly Pick<TaskInputRow, 'kind' | 'name' | 'hash'>[],
|
|
275
|
+
): string {
|
|
276
|
+
const byId = new Map<string, Pick<TaskInputRow, 'kind' | 'name' | 'hash'>>()
|
|
277
|
+
for (const c of components) {
|
|
278
|
+
const id = `${c.kind}\0${c.name}`
|
|
279
|
+
if (!byId.has(id)) byId.set(id, c)
|
|
280
|
+
}
|
|
281
|
+
const ids = [...byId.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
|
282
|
+
return JSON.stringify(
|
|
283
|
+
ids.map((id) => {
|
|
284
|
+
const c = byId.get(id)!
|
|
285
|
+
return [c.kind, c.name, c.hash]
|
|
286
|
+
}),
|
|
287
|
+
)
|
|
288
|
+
}
|
|
254
289
|
|
|
255
290
|
/** The tables a store holds: dropped from a workspace index that held them itself. */
|
|
256
291
|
const STORE_TABLES = ['entry_inputs', 'output_files', 'entry_stdout', 'store_meta', 'entries']
|
|
257
292
|
|
|
293
|
+
/**
|
|
294
|
+
* The inline artifacts' tables (schema.ts): dropped from a workspace index
|
|
295
|
+
* whose entries moved to a store, where an unqualified name would find them
|
|
296
|
+
* first, and never by a schema reset.
|
|
297
|
+
*/
|
|
298
|
+
const ARTIFACT_TABLES = ['artifacts', 'artifacts_meta']
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* The `artifacts` table's own layout (`artifacts_meta.layout`): another value
|
|
302
|
+
* drops the table. It is read only where the schema was not this vx's (a
|
|
303
|
+
* reset, a new index or store, a store moved), so a change to it bumps
|
|
304
|
+
* `SCHEMA_VERSION` too: a warm open reads no statement for it.
|
|
305
|
+
*/
|
|
306
|
+
const ARTIFACT_LAYOUT = 'a1'
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* The largest compressed artifact kept in the index rather than in a file
|
|
310
|
+
* of its own: it is written in its rows' transaction, with no temp and no
|
|
311
|
+
* rename (docs/design/cache-save-cpu-2026-10.md).
|
|
312
|
+
*/
|
|
313
|
+
const INLINE_MAX = 32 * 1024
|
|
314
|
+
|
|
258
315
|
/**
|
|
259
316
|
* An index that records a schema other than this vx's: the one an opener
|
|
260
317
|
* that writes resets. Reads the recorded version alone and touches
|
|
@@ -279,13 +336,26 @@ function indexOfAnotherSchema(dbFile: string): boolean {
|
|
|
279
336
|
return found !== undefined && found !== SCHEMA_VERSION
|
|
280
337
|
}
|
|
281
338
|
|
|
282
|
-
/**
|
|
283
|
-
|
|
284
|
-
|
|
339
|
+
/**
|
|
340
|
+
* Each asked key's entry row with its stdout, which lives apart (v29), none
|
|
341
|
+
* stored reading as '', and its inline artifact's last use, null when the
|
|
342
|
+
* artifact is a file. Driven by the keys, so a key with an inline artifact
|
|
343
|
+
* and no row (a reset kept it) answers too, `hash` null: the miss learns
|
|
344
|
+
* there is something to adopt in the statement that found no entry.
|
|
345
|
+
*/
|
|
346
|
+
const selectEntries = (keys: string): string =>
|
|
347
|
+
`SELECT q.value AS key, e.*, COALESCE(s.stdout, '') AS stdout, a.at AS inline_at FROM ${keys} q LEFT JOIN entries e ON e.hash = q.value LEFT JOIN entry_stdout s ON s.hash = e.hash LEFT JOIN artifacts a ON a.hash = q.value WHERE e.hash IS NOT NULL OR a.at IS NOT NULL`
|
|
348
|
+
const SELECT_ENTRY = selectEntries('(SELECT ? AS value)')
|
|
349
|
+
const SELECT_ENTRIES = selectEntries('json_each(?)')
|
|
285
350
|
|
|
286
|
-
/**
|
|
351
|
+
/**
|
|
352
|
+
* What the orphan sweep may judge: a row-less artifact file or inline
|
|
353
|
+
* artifact (`blob`, its hash), or a dead file — a temp, or a file an
|
|
354
|
+
* inline artifact of its key shadows — which goes whatever the policy.
|
|
355
|
+
*/
|
|
287
356
|
interface RowlessFile {
|
|
288
|
-
file
|
|
357
|
+
file?: string
|
|
358
|
+
blob?: string
|
|
289
359
|
size: number
|
|
290
360
|
mtimeMs: number
|
|
291
361
|
temp: boolean
|
|
@@ -349,7 +419,7 @@ function pickVictims(
|
|
|
349
419
|
*/
|
|
350
420
|
async function scanOrphanFiles(
|
|
351
421
|
cacheDir: string,
|
|
352
|
-
indexedHashes: () => ReadonlySet<string
|
|
422
|
+
indexedHashes: () => { files: ReadonlySet<string>; inline: ReadonlySet<string> },
|
|
353
423
|
): Promise<RowlessFile[]> {
|
|
354
424
|
let names: string[]
|
|
355
425
|
try {
|
|
@@ -363,8 +433,10 @@ async function scanOrphanFiles(
|
|
|
363
433
|
for (const name of names) {
|
|
364
434
|
const m = VX_ARTIFACT_NAME.exec(name)
|
|
365
435
|
if (m === null) continue
|
|
366
|
-
|
|
367
|
-
|
|
436
|
+
// A file whose key has an inline artifact is never read: readers take
|
|
437
|
+
// the blob first. It is what an inline save of a re-built key left.
|
|
438
|
+
const temp = m[2] !== undefined || indexed.inline.has(m[1]!)
|
|
439
|
+
if (temp || !indexed.files.has(m[1]!)) candidates.push({ name, temp })
|
|
368
440
|
}
|
|
369
441
|
const found: RowlessFile[] = []
|
|
370
442
|
await Promise.all(
|
|
@@ -415,8 +487,12 @@ export * from './layer.js'
|
|
|
415
487
|
export * from './policy.js'
|
|
416
488
|
export { zstdContentSize } from './zstd.js'
|
|
417
489
|
|
|
490
|
+
type LiveRow = EntryRow & { hash: string }
|
|
491
|
+
|
|
418
492
|
interface EntryRow {
|
|
419
|
-
|
|
493
|
+
key: string
|
|
494
|
+
/** Null for a row-less inline artifact. */
|
|
495
|
+
hash: string | null
|
|
420
496
|
project: string
|
|
421
497
|
task: string
|
|
422
498
|
command: string
|
|
@@ -428,9 +504,10 @@ interface EntryRow {
|
|
|
428
504
|
accessed_at: number
|
|
429
505
|
cpu_ms: number | null
|
|
430
506
|
peak_rss_bytes: number | null
|
|
507
|
+
inline_at: number | null
|
|
431
508
|
}
|
|
432
509
|
|
|
433
|
-
function entryOf(row:
|
|
510
|
+
function entryOf(row: LiveRow, fileRows: OutputFileRow[]): CacheEntry {
|
|
434
511
|
return {
|
|
435
512
|
hash: row.hash,
|
|
436
513
|
taskId: `${row.project}#${row.task}`,
|
|
@@ -527,6 +604,9 @@ function ensurePrivateDir(dir: string): string | null {
|
|
|
527
604
|
if (!st.isDirectory()) return `${dir} is not a directory`
|
|
528
605
|
const uid = process.getuid?.()
|
|
529
606
|
if (uid !== undefined && st.uid !== uid) return `${dir} belongs to another user`
|
|
607
|
+
// One others could write may already hold what they planted (a link, an
|
|
608
|
+
// entry of their own), and closing it now would keep those: not used.
|
|
609
|
+
if ((st.mode & 0o022) !== 0) return `${dir} is writable by other users`
|
|
530
610
|
// The user's own directory open to others (a `~/.vx` made at the
|
|
531
611
|
// default umask, 0755) is vx's to close: refusing it left the store
|
|
532
612
|
// unused on a stock macOS home (owner, 2026-10-06: "you should own it").
|
|
@@ -642,9 +722,25 @@ export class Cache implements CacheLayer {
|
|
|
642
722
|
private readonly upsertStdout: ReturnType<Database['prepare']>
|
|
643
723
|
private readonly deleteStdout: ReturnType<Database['prepare']>
|
|
644
724
|
private readonly entryExists: ReturnType<Database['prepare']>
|
|
725
|
+
private readonly selectBlob: ReturnType<Database['prepare']>
|
|
726
|
+
private readonly upsertBlob: ReturnType<Database['prepare']>
|
|
727
|
+
private readonly deleteBlob: ReturnType<Database['prepare']>
|
|
645
728
|
private readonly touched = new Set<string>()
|
|
646
729
|
/** Hit artifacts whose file time is past the grace: renewed at the flush (`ORPHAN_GRACE_MS`). */
|
|
647
730
|
private readonly staleTimes = new Set<string>()
|
|
731
|
+
/** The same for inline artifacts, whose `at` stands for the file time. */
|
|
732
|
+
private readonly staleInline = new Set<string>()
|
|
733
|
+
/**
|
|
734
|
+
* The `at` of each inline artifact the last probe held rows for: a restore
|
|
735
|
+
* serves those rows only while the blob still carries it.
|
|
736
|
+
*/
|
|
737
|
+
private readonly heldInlineAt = new Map<string, number>()
|
|
738
|
+
/** Keys the last lookups found an inline artifact for and no row: `adopt`'s to read. */
|
|
739
|
+
private readonly rowless = new Set<string>()
|
|
740
|
+
/** Row-less lookups that stat'ed their artifact path (`artifactFileMayExist`). */
|
|
741
|
+
private adoptStats = 0
|
|
742
|
+
/** The artifact directory's names, listed once past `ADOPT_STATS`; this process's writes kept in it. */
|
|
743
|
+
private artifactNames: Set<string> | undefined
|
|
648
744
|
private readonly insertEntryInput: ReturnType<Database['prepare']>
|
|
649
745
|
/** The per-file (mtime, size) → blob-OID memo behind `hashFile`. */
|
|
650
746
|
private readonly files: FileHashStore
|
|
@@ -693,6 +789,8 @@ export class Cache implements CacheLayer {
|
|
|
693
789
|
* indexed again as they hit. Never printed: the cache is vx's to keep.
|
|
694
790
|
*/
|
|
695
791
|
readonly storeReset: SchemaReset | null = null
|
|
792
|
+
/** This open stamped the store's schema: it was new or another vx's. */
|
|
793
|
+
private storeStamped = false
|
|
696
794
|
|
|
697
795
|
/** A reading verb's open: it never resets the index (the constructor's `mode`). */
|
|
698
796
|
static inspect(cacheDir: string): Cache {
|
|
@@ -872,7 +970,8 @@ export class Cache implements CacheLayer {
|
|
|
872
970
|
.prepare("SELECT 1 FROM schema_meta WHERE key = 'store_dir'")
|
|
873
971
|
.get()
|
|
874
972
|
if (storeRoot === undefined ? recordedStore != null : storeRoot !== null) {
|
|
875
|
-
for (const t of STORE_TABLES
|
|
973
|
+
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES])
|
|
974
|
+
this.db.exec(`DROP TABLE main.${t}`)
|
|
876
975
|
}
|
|
877
976
|
this.db
|
|
878
977
|
.prepare(
|
|
@@ -925,17 +1024,22 @@ export class Cache implements CacheLayer {
|
|
|
925
1024
|
if (storeDir !== undefined && mode === 'open' && this.writeBlocked === null) {
|
|
926
1025
|
// An index that held its entries itself would shadow the store's:
|
|
927
1026
|
// SQLite resolves an unqualified name in `main` first. Its history
|
|
928
|
-
// and memos stay; its entries'
|
|
1027
|
+
// and memos stay; its entries' artifact files are orphans `prune`
|
|
1028
|
+
// reaps, and its inline artifacts go with their table.
|
|
929
1029
|
const holds = readable(
|
|
930
1030
|
() =>
|
|
931
1031
|
this.db
|
|
932
|
-
.prepare(
|
|
1032
|
+
.prepare(
|
|
1033
|
+
"SELECT 1 FROM main.sqlite_master WHERE type = 'table' AND name IN ('entries', 'artifacts')",
|
|
1034
|
+
)
|
|
933
1035
|
.get() != null,
|
|
934
1036
|
)
|
|
935
1037
|
if (holds) {
|
|
936
1038
|
this.db
|
|
937
1039
|
.transaction(() => {
|
|
938
|
-
for (const t of STORE_TABLES)
|
|
1040
|
+
for (const t of [...STORE_TABLES, ...ARTIFACT_TABLES]) {
|
|
1041
|
+
this.db.exec(`DROP TABLE IF EXISTS main.${t}`)
|
|
1042
|
+
}
|
|
939
1043
|
})
|
|
940
1044
|
.immediate()
|
|
941
1045
|
}
|
|
@@ -947,6 +1051,11 @@ export class Cache implements CacheLayer {
|
|
|
947
1051
|
this.storeReset = this.matchStoreSchema(mode === 'open' && this.writeBlocked === null)
|
|
948
1052
|
} else {
|
|
949
1053
|
createTables(this.db, 'main')
|
|
1054
|
+
if (absent && existsSync(dbFile)) this.keepArtifacts(dbFile)
|
|
1055
|
+
}
|
|
1056
|
+
const foreign = storeDir === undefined ? current !== SCHEMA_VERSION : this.storeStamped
|
|
1057
|
+
if (mode === 'open' && this.writeBlocked === null && (foreign || storeDir !== recorded)) {
|
|
1058
|
+
readable(() => this.matchArtifactLayout(storeDir === undefined ? 'main' : 'store'))
|
|
950
1059
|
}
|
|
951
1060
|
if (mode === 'open' && this.writeBlocked === null && storeDir !== recorded) {
|
|
952
1061
|
if (storeDir === undefined)
|
|
@@ -982,20 +1091,29 @@ export class Cache implements CacheLayer {
|
|
|
982
1091
|
'INSERT INTO entry_stdout(hash, stdout) VALUES (?, CAST(? AS TEXT)) ON CONFLICT(hash) DO UPDATE SET stdout = excluded.stdout',
|
|
983
1092
|
)
|
|
984
1093
|
this.deleteStdout = lazyStatement(this.db, 'DELETE FROM entry_stdout WHERE hash = ?')
|
|
985
|
-
this.selectEntry = lazyStatement(this.db,
|
|
986
|
-
// `has` asks only whether the row is there
|
|
987
|
-
// column off the index, so the table row is
|
|
988
|
-
// table refuses here as it does on `get`
|
|
989
|
-
// index alone).
|
|
990
|
-
this.entryExists = lazyStatement(
|
|
1094
|
+
this.selectEntry = lazyStatement(this.db, SELECT_ENTRY)
|
|
1095
|
+
// `has` asks only whether the row is there and where its artifact is,
|
|
1096
|
+
// with no stdout joined. A column off the index, so the table row is
|
|
1097
|
+
// still read and a corrupt table refuses here as it does on `get`
|
|
1098
|
+
// (`SELECT 1` answers from the index alone).
|
|
1099
|
+
this.entryExists = lazyStatement(
|
|
1100
|
+
this.db,
|
|
1101
|
+
'SELECT e.exit_code, a.at AS inline_at FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash WHERE e.hash = ?',
|
|
1102
|
+
)
|
|
1103
|
+
this.selectBlob = lazyStatement(this.db, 'SELECT at, bytes FROM artifacts WHERE hash = ?')
|
|
1104
|
+
this.upsertBlob = lazyStatement(
|
|
1105
|
+
this.db,
|
|
1106
|
+
'INSERT INTO artifacts(hash, at, bytes) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET at = excluded.at, bytes = excluded.bytes',
|
|
1107
|
+
)
|
|
1108
|
+
this.deleteBlob = lazyStatement(this.db, 'DELETE FROM artifacts WHERE hash = ?')
|
|
991
1109
|
// INSERT OR IGNORE: re-saving the same hash (idempotent ingest /
|
|
992
1110
|
// overlapping concurrent saves) leaves the existing rows untouched —
|
|
993
1111
|
// identical inputs derive the identical hash, so the rows are too.
|
|
994
1112
|
this.insertEntryInput = lazyStatement(
|
|
995
1113
|
this.db,
|
|
996
1114
|
`
|
|
997
|
-
INSERT OR IGNORE INTO entry_inputs(entry_hash,
|
|
998
|
-
VALUES (?,
|
|
1115
|
+
INSERT OR IGNORE INTO entry_inputs(entry_hash, components)
|
|
1116
|
+
VALUES (?, ?)
|
|
999
1117
|
`,
|
|
1000
1118
|
)
|
|
1001
1119
|
// The slices: each owns its statements over this handle and its table(s);
|
|
@@ -1117,11 +1235,17 @@ export class Cache implements CacheLayer {
|
|
|
1117
1235
|
return null
|
|
1118
1236
|
}
|
|
1119
1237
|
if (!writable) {
|
|
1238
|
+
const file = this.db
|
|
1239
|
+
.prepare("SELECT file FROM pragma_database_list WHERE name = 'store'")
|
|
1240
|
+
.get() as {
|
|
1241
|
+
file: string
|
|
1242
|
+
}
|
|
1120
1243
|
if (seen.has) {
|
|
1121
1244
|
this.db.exec('DETACH DATABASE store')
|
|
1122
1245
|
this.db.prepare('ATTACH DATABASE ? AS store').run(':memory:')
|
|
1123
1246
|
}
|
|
1124
1247
|
createTables(this.db, 'store')
|
|
1248
|
+
if (seen.has && file.file !== '') this.keepArtifacts(file.file)
|
|
1125
1249
|
return null
|
|
1126
1250
|
}
|
|
1127
1251
|
// Re-read, drop, create and stamp under one write lock: as separate
|
|
@@ -1129,6 +1253,7 @@ export class Cache implements CacheLayer {
|
|
|
1129
1253
|
// tables, and this one's `CREATE IF NOT EXISTS` kept them under its stamp.
|
|
1130
1254
|
return this.db
|
|
1131
1255
|
.transaction((): SchemaReset | null => {
|
|
1256
|
+
this.storeStamped = true
|
|
1132
1257
|
const now = read()
|
|
1133
1258
|
let reset: SchemaReset | null = null
|
|
1134
1259
|
if (now.has && now.found !== SCHEMA_VERSION) {
|
|
@@ -1148,6 +1273,64 @@ export class Cache implements CacheLayer {
|
|
|
1148
1273
|
.immediate()
|
|
1149
1274
|
}
|
|
1150
1275
|
|
|
1276
|
+
/**
|
|
1277
|
+
* A reading open over an index or store of another schema reads it as
|
|
1278
|
+
* the reset a writing open makes leaves it: empty, but for the inline
|
|
1279
|
+
* artifacts, which a reset keeps. They are read from the file in place,
|
|
1280
|
+
* through a view that shadows the empty table (a TEMP name is found
|
|
1281
|
+
* first), so a dry prune names the row-less ones the real prune reaps.
|
|
1282
|
+
*/
|
|
1283
|
+
private keepArtifacts(file: string): void {
|
|
1284
|
+
this.db.prepare('ATTACH DATABASE ? AS kept').run(file)
|
|
1285
|
+
const layout = this.db
|
|
1286
|
+
.prepare("SELECT 1 FROM kept.sqlite_master WHERE type = 'table' AND name = 'artifacts_meta'")
|
|
1287
|
+
.get()
|
|
1288
|
+
? (
|
|
1289
|
+
this.db.prepare("SELECT value FROM kept.artifacts_meta WHERE key = 'layout'").get() as {
|
|
1290
|
+
value: string
|
|
1291
|
+
} | null
|
|
1292
|
+
)?.value
|
|
1293
|
+
: undefined
|
|
1294
|
+
if (layout === ARTIFACT_LAYOUT) {
|
|
1295
|
+
this.db.exec('CREATE TEMP VIEW artifacts AS SELECT hash, at, bytes FROM kept.artifacts')
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
/**
|
|
1300
|
+
* `artifacts` holds the layout this vx writes, or is dropped and made
|
|
1301
|
+
* again: its version is its own (`ARTIFACT_LAYOUT`), apart from the
|
|
1302
|
+
* schema's, so a schema reset keeps it. Silent, as a reset is. Asked
|
|
1303
|
+
* only where the schema was not this vx's (`ARTIFACT_LAYOUT`).
|
|
1304
|
+
*/
|
|
1305
|
+
private matchArtifactLayout(schema: 'main' | 'store'): void {
|
|
1306
|
+
const read = (): string | undefined =>
|
|
1307
|
+
(
|
|
1308
|
+
this.db
|
|
1309
|
+
.prepare(`SELECT value FROM ${schema}.artifacts_meta WHERE key = 'layout'`)
|
|
1310
|
+
.get() as {
|
|
1311
|
+
value: string
|
|
1312
|
+
} | null
|
|
1313
|
+
)?.value
|
|
1314
|
+
// Read first: a workspace's first open of a current store asks too,
|
|
1315
|
+
// and must not wait on the store's write lock to learn it holds `a1`.
|
|
1316
|
+
if (read() === ARTIFACT_LAYOUT) return
|
|
1317
|
+
this.db
|
|
1318
|
+
.transaction(() => {
|
|
1319
|
+
const found = read()
|
|
1320
|
+
if (found === ARTIFACT_LAYOUT) return
|
|
1321
|
+
if (found !== undefined) {
|
|
1322
|
+
this.db.exec(`DROP TABLE ${schema}.artifacts`)
|
|
1323
|
+
createArtifactTables(this.db, schema)
|
|
1324
|
+
}
|
|
1325
|
+
this.db
|
|
1326
|
+
.prepare(
|
|
1327
|
+
`INSERT INTO ${schema}.artifacts_meta(key, value) VALUES ('layout', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
1328
|
+
)
|
|
1329
|
+
.run(ARTIFACT_LAYOUT)
|
|
1330
|
+
})
|
|
1331
|
+
.immediate()
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1151
1334
|
// --- config evaluations: `ConfigEvalStore`, delegated to `ConfigEvalTable` ---
|
|
1152
1335
|
getConfigEval(key: string): string | null {
|
|
1153
1336
|
return this.guard(() => this.configEvals.getConfigEval(key))
|
|
@@ -1296,11 +1479,25 @@ export class Cache implements CacheLayer {
|
|
|
1296
1479
|
* `store.db` deleted) loses nothing a task asks for again. The artifact
|
|
1297
1480
|
* is checked as an ingest checks a remote's (its recorded key, its names
|
|
1298
1481
|
* against the declared outputs); one that fails is a miss, and the run
|
|
1299
|
-
* that follows saves over it.
|
|
1482
|
+
* that follows saves over it. An inline artifact is read first, as every
|
|
1483
|
+
* reader reads it, and indexed in one transaction with its bytes.
|
|
1300
1484
|
*/
|
|
1301
1485
|
private async adopt(hash: string, ctx: CacheGetContext): Promise<boolean> {
|
|
1486
|
+
// Built only for an artifact found: its command is masked on read.
|
|
1487
|
+
const meta = (): IngestMeta => ({
|
|
1488
|
+
taskId: ctx.taskId,
|
|
1489
|
+
command: ctx.command,
|
|
1490
|
+
durationMs: 0,
|
|
1491
|
+
...(ctx.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
1492
|
+
})
|
|
1493
|
+
// Only a key whose lookup saw a row-less inline artifact asks for it: a
|
|
1494
|
+
// statement per miss otherwise, on every cold save.
|
|
1495
|
+
const blob = this.rowless.delete(hash)
|
|
1496
|
+
? (this.selectBlob.get(hash) as { bytes: Uint8Array } | null)
|
|
1497
|
+
: null
|
|
1498
|
+
if (blob !== null) return this.adopted(this.writeArtifactAndIndex(hash, blob.bytes, meta()))
|
|
1302
1499
|
const finalPath = this.tarPath(hash)
|
|
1303
|
-
if (!
|
|
1500
|
+
if (!this.artifactFileMayExist(hash)) return false
|
|
1304
1501
|
const tmpPath = this.tempPath(hash)
|
|
1305
1502
|
try {
|
|
1306
1503
|
// A second name for the same bytes: the index step renames it over
|
|
@@ -1315,17 +1512,34 @@ export class Cache implements CacheLayer {
|
|
|
1315
1512
|
} catch {
|
|
1316
1513
|
return false
|
|
1317
1514
|
}
|
|
1515
|
+
return this.adopted(this.writeArtifactAndIndex(hash, { tmpPath }, meta()))
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
/**
|
|
1519
|
+
* Whether `hash` may have a file artifact: a stat for the first
|
|
1520
|
+
* `ADOPT_STATS` asks, then one listing of the directory. A cold
|
|
1521
|
+
* 1,090-package run asked 3,270 times before its first task ran. A
|
|
1522
|
+
* name listed and since removed fails the adopt's link, a miss; a file
|
|
1523
|
+
* another process writes after the listing is missed too, and its own
|
|
1524
|
+
* row serves it.
|
|
1525
|
+
*/
|
|
1526
|
+
private artifactFileMayExist(hash: string): boolean {
|
|
1527
|
+
const name = `${hash}.tar.zst`
|
|
1528
|
+
if (this.artifactNames === undefined) {
|
|
1529
|
+
if (++this.adoptStats <= ADOPT_STATS) return existsSync(path.join(this.artifactDir, name))
|
|
1530
|
+
try {
|
|
1531
|
+
this.artifactNames = new Set(readdirSync(this.artifactDir))
|
|
1532
|
+
} catch {
|
|
1533
|
+
this.artifactNames = new Set()
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
return this.artifactNames.has(name)
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
/** An adoption's verdict: an artifact that fails the checks is a miss. */
|
|
1540
|
+
private async adopted(indexing: Promise<void>): Promise<boolean> {
|
|
1318
1541
|
try {
|
|
1319
|
-
await
|
|
1320
|
-
hash,
|
|
1321
|
-
{ tmpPath },
|
|
1322
|
-
{
|
|
1323
|
-
taskId: ctx.taskId,
|
|
1324
|
-
command: ctx.command,
|
|
1325
|
-
durationMs: 0,
|
|
1326
|
-
...(ctx.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
1327
|
-
},
|
|
1328
|
-
)
|
|
1542
|
+
await indexing
|
|
1329
1543
|
return true
|
|
1330
1544
|
} catch (err) {
|
|
1331
1545
|
if (err instanceof CorruptArtifactError || err instanceof ArchiveSecurityError) return false
|
|
@@ -1347,14 +1561,22 @@ export class Cache implements CacheLayer {
|
|
|
1347
1561
|
}
|
|
1348
1562
|
|
|
1349
1563
|
private async readEntry(hash: string): Promise<CacheEntry | null> {
|
|
1350
|
-
const row = this.selectEntry.get(hash) as EntryRow |
|
|
1351
|
-
if (
|
|
1564
|
+
const row = this.selectEntry.get(hash) as EntryRow | null
|
|
1565
|
+
if (row === null) return null
|
|
1566
|
+
if (row.hash === null) {
|
|
1567
|
+
this.rowless.add(hash)
|
|
1568
|
+
return null
|
|
1569
|
+
}
|
|
1352
1570
|
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1571
|
+
if (row.inline_at !== null) {
|
|
1572
|
+
if (row.inline_at < Date.now() - ORPHAN_GRACE_MS) this.staleInline.add(hash)
|
|
1573
|
+
} else {
|
|
1574
|
+
// Verify the artifact file actually exists. The DB and the
|
|
1575
|
+
// filesystem can drift if someone manually deletes the cache dir.
|
|
1576
|
+
const st = statSync(this.tarPath(hash), { throwIfNoEntry: false })
|
|
1577
|
+
if (st === undefined) return null
|
|
1578
|
+
if (st.mtimeMs < Date.now() - ORPHAN_GRACE_MS) this.staleTimes.add(hash)
|
|
1579
|
+
}
|
|
1358
1580
|
|
|
1359
1581
|
// Deferred: per-hit UPDATEs cost ~60 ms across 2000+ probes on a
|
|
1360
1582
|
// full-cache run. Hashes are collected and flushed as ONE batched
|
|
@@ -1368,11 +1590,32 @@ export class Cache implements CacheLayer {
|
|
|
1368
1590
|
// for four up-to-date hits on ~70 MB binaries). restoreOutputs
|
|
1369
1591
|
// reads the artifact itself, only when extraction actually runs.
|
|
1370
1592
|
const fileRows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
1371
|
-
const entry = entryOf(row, fileRows)
|
|
1593
|
+
const entry = entryOf(row as LiveRow, fileRows)
|
|
1372
1594
|
entry.outputDirRows = this.loadOutputDirsBatch([hash]).get(hash) ?? []
|
|
1373
1595
|
return entry
|
|
1374
1596
|
}
|
|
1375
1597
|
|
|
1598
|
+
/**
|
|
1599
|
+
* Whether no lookup here can hit: no artifact, inline or file. Every
|
|
1600
|
+
* entry has one, and a row-less one is still a hit (`adopt`). The listing
|
|
1601
|
+
* is kept for `adopt`.
|
|
1602
|
+
*/
|
|
1603
|
+
holdsNothing(): boolean {
|
|
1604
|
+
if (!this.read) return true
|
|
1605
|
+
if (this.guard(() => this.db.query('SELECT 1 FROM artifacts LIMIT 1').get()) != null) {
|
|
1606
|
+
return false
|
|
1607
|
+
}
|
|
1608
|
+
let names: string[]
|
|
1609
|
+
try {
|
|
1610
|
+
names = readdirSync(this.artifactDir)
|
|
1611
|
+
} catch (err) {
|
|
1612
|
+
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') return false
|
|
1613
|
+
names = []
|
|
1614
|
+
}
|
|
1615
|
+
this.artifactNames ??= new Set(names)
|
|
1616
|
+
return !names.some((n) => n.endsWith('.tar.zst'))
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1376
1619
|
/**
|
|
1377
1620
|
* `get` for many hashes at once: one `entries` query and one
|
|
1378
1621
|
* `output_files` query per chunk instead of two per hash, with the
|
|
@@ -1401,12 +1644,24 @@ export class Cache implements CacheLayer {
|
|
|
1401
1644
|
private async getManyEntries(hashes: readonly string[]): Promise<Map<string, CacheEntry>> {
|
|
1402
1645
|
const out = new Map<string, CacheEntry>()
|
|
1403
1646
|
if (!this.read || hashes.length === 0) return out
|
|
1404
|
-
const
|
|
1405
|
-
|
|
1647
|
+
const found = (
|
|
1648
|
+
hashes.length === 1
|
|
1649
|
+
? [this.selectEntry.get(hashes[0]!)].filter((r) => r !== null)
|
|
1650
|
+
: this.db.query(SELECT_ENTRIES).all(JSON.stringify(hashes))
|
|
1651
|
+
) as EntryRow[]
|
|
1652
|
+
const rows: LiveRow[] = []
|
|
1653
|
+
for (const r of found) {
|
|
1654
|
+
if (r.hash === null) this.rowless.add(r.key)
|
|
1655
|
+
else rows.push(r as LiveRow)
|
|
1656
|
+
}
|
|
1406
1657
|
if (rows.length === 0) return out
|
|
1407
1658
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
1408
1659
|
const refreshStart = Date.now() - ACCESS_REFRESH_MS
|
|
1409
1660
|
const live = rows.filter((r) => {
|
|
1661
|
+
if (r.inline_at !== null) {
|
|
1662
|
+
if (r.inline_at < graceStart) this.staleInline.add(r.hash)
|
|
1663
|
+
return true
|
|
1664
|
+
}
|
|
1410
1665
|
const st = statSync(this.tarPath(r.hash), { throwIfNoEntry: false })
|
|
1411
1666
|
if (st !== undefined && st.mtimeMs < graceStart) this.staleTimes.add(r.hash)
|
|
1412
1667
|
return st !== undefined
|
|
@@ -1414,6 +1669,8 @@ export class Cache implements CacheLayer {
|
|
|
1414
1669
|
const liveHashes = live.map((r) => r.hash)
|
|
1415
1670
|
const fileRows = this.loadOutputFilesBatch(liveHashes)
|
|
1416
1671
|
this.outputs.hold(fileRows)
|
|
1672
|
+
this.heldInlineAt.clear()
|
|
1673
|
+
for (const r of live) if (r.inline_at !== null) this.heldInlineAt.set(r.hash, r.inline_at)
|
|
1417
1674
|
const dirRows = this.loadOutputDirsBatch(liveHashes)
|
|
1418
1675
|
for (const row of live) {
|
|
1419
1676
|
if (row.accessed_at < refreshStart) this.touched.add(row.hash)
|
|
@@ -1424,16 +1681,18 @@ export class Cache implements CacheLayer {
|
|
|
1424
1681
|
return out
|
|
1425
1682
|
}
|
|
1426
1683
|
|
|
1427
|
-
// Existence probe: SQL row + artifact
|
|
1428
|
-
// and no accessed_at bump (the plan path
|
|
1684
|
+
// Existence probe: SQL row + where its artifact is (an inline one, or a
|
|
1685
|
+
// file on disk), no byte reads and no accessed_at bump (the plan path
|
|
1686
|
+
// must stay read-only).
|
|
1429
1687
|
async has(hash: string): Promise<'local' | 'remote' | null> {
|
|
1430
1688
|
return this.guard(() => this.hasEntry(hash))
|
|
1431
1689
|
}
|
|
1432
1690
|
|
|
1433
1691
|
private async hasEntry(hash: string): Promise<'local' | 'remote' | null> {
|
|
1434
1692
|
if (!this.read) return null
|
|
1435
|
-
|
|
1436
|
-
|
|
1693
|
+
const row = this.entryExists.get(hash) as { inline_at: number | null } | null
|
|
1694
|
+
if (row === null) return null
|
|
1695
|
+
return row.inline_at !== null || existsSync(this.tarPath(hash)) ? 'local' : null
|
|
1437
1696
|
}
|
|
1438
1697
|
|
|
1439
1698
|
// Local cache has no slower layer to warm from — prefetch is a no-op.
|
|
@@ -1457,6 +1716,16 @@ export class Cache implements CacheLayer {
|
|
|
1457
1716
|
outputsPath(hash: string): string {
|
|
1458
1717
|
return this.tarPath(hash)
|
|
1459
1718
|
}
|
|
1719
|
+
artifactSize(hash: string): number | undefined {
|
|
1720
|
+
return this.guard(
|
|
1721
|
+
() =>
|
|
1722
|
+
(
|
|
1723
|
+
this.db.query('SELECT size_bytes FROM entries WHERE hash = ?').get(hash) as {
|
|
1724
|
+
size_bytes: number
|
|
1725
|
+
} | null
|
|
1726
|
+
)?.size_bytes,
|
|
1727
|
+
)
|
|
1728
|
+
}
|
|
1460
1729
|
|
|
1461
1730
|
async recordOutputDirs(
|
|
1462
1731
|
hash: string,
|
|
@@ -1497,6 +1766,7 @@ export class Cache implements CacheLayer {
|
|
|
1497
1766
|
private async dropEntry(hash: string): Promise<void> {
|
|
1498
1767
|
await rm(this.tarPath(hash), { force: true })
|
|
1499
1768
|
const drop = this.db.transaction(() => {
|
|
1769
|
+
this.deleteBlob.run(hash)
|
|
1500
1770
|
this.deleteEntryRow.run(hash)
|
|
1501
1771
|
this.outputs.forget([hash])
|
|
1502
1772
|
})
|
|
@@ -1549,7 +1819,25 @@ export class Cache implements CacheLayer {
|
|
|
1549
1819
|
// same truncated expectation against the same truncated tree and agrees
|
|
1550
1820
|
// forever. Refuse instead of silently under-restoring — checked before
|
|
1551
1821
|
// anything is renamed into place.
|
|
1552
|
-
|
|
1822
|
+
//
|
|
1823
|
+
// An inline artifact and its rows are read in one read transaction, so
|
|
1824
|
+
// one snapshot: a re-save of the key in another process replaces both
|
|
1825
|
+
// in one commit, and two reads could pair its rows with the old bytes.
|
|
1826
|
+
// The probe's held rows (X-162) serve only while the blob carries the
|
|
1827
|
+
// `at` the probe saw: a re-save or a renewal since moves it, and then
|
|
1828
|
+
// the rows are read again in the blob's snapshot.
|
|
1829
|
+
let rows: readonly OutputFileRow[] = []
|
|
1830
|
+
let inline: Uint8Array | undefined
|
|
1831
|
+
this.guard(() =>
|
|
1832
|
+
this.db.transaction(() => {
|
|
1833
|
+
const blob = this.selectBlob.get(hash) as { at: number; bytes: Uint8Array } | null
|
|
1834
|
+
inline = blob?.bytes
|
|
1835
|
+
rows =
|
|
1836
|
+
blob === null || blob.at === this.heldInlineAt.get(hash)
|
|
1837
|
+
? this.outputs.rowsOf(hash)
|
|
1838
|
+
: (this.outputs.loadOutputFilesBatch([hash]).get(hash) ?? [])
|
|
1839
|
+
})(),
|
|
1840
|
+
)
|
|
1553
1841
|
const expected = rows
|
|
1554
1842
|
.filter((r) => workspaceRoot !== undefined || !r.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
1555
1843
|
.map((r) => (r.path.startsWith(WORKSPACE_OUTPUT_PREFIX) ? r.path : `outputs/${r.path}`))
|
|
@@ -1580,7 +1868,7 @@ export class Cache implements CacheLayer {
|
|
|
1580
1868
|
// 0.62 ms sequential, 2026-09-10) and 30% slower in the run, where
|
|
1581
1869
|
// four workers overlap their round trips and a blocking one stalls
|
|
1582
1870
|
// the other three (1,000-project restore row 1.1–1.2 s → 1.5 s).
|
|
1583
|
-
const tar = await decodedTar(Bun.file(src), hash, this.artifactCeiling)
|
|
1871
|
+
const tar = await decodedTar(inline ?? Bun.file(src), hash, this.artifactCeiling)
|
|
1584
1872
|
await extractArtifactStream(tar, projectDir, workspaceRoot, verify)
|
|
1585
1873
|
} catch (err) {
|
|
1586
1874
|
// A UserError here is the extractor naming the tree's fault (an
|
|
@@ -1781,9 +2069,12 @@ export class Cache implements CacheLayer {
|
|
|
1781
2069
|
* opened by path read those mid-upload (a digest of one artifact over
|
|
1782
2070
|
* the bytes of another). A `Bun.file` over an fd is no answer — its
|
|
1783
2071
|
* second read starts where the first ended. Throws when the artifact is
|
|
1784
|
-
* gone; `release` unlinks the name.
|
|
2072
|
+
* gone; `release` unlinks the name. An inline artifact is read once into
|
|
2073
|
+
* the body, which no re-save can change.
|
|
1785
2074
|
*/
|
|
1786
2075
|
pinArtifact(hash: string): { body: Blob; release: () => Promise<void> } {
|
|
2076
|
+
const blob = this.guard(() => this.selectBlob.get(hash) as { bytes: Uint8Array } | null)
|
|
2077
|
+
if (blob !== null) return { body: new Blob([blob.bytes]), release: async () => {} }
|
|
1787
2078
|
const pinned = this.tempPath(hash)
|
|
1788
2079
|
linkSync(this.tarPath(hash), pinned)
|
|
1789
2080
|
return { body: Bun.file(pinned), release: () => unlink(pinned).catch(() => undefined) }
|
|
@@ -1794,7 +2085,8 @@ export class Cache implements CacheLayer {
|
|
|
1794
2085
|
* `Response` and copies a file `Blob` without collecting either, so a
|
|
1795
2086
|
* pull never holds the artifact. A body that fails mid-stream (a dropped
|
|
1796
2087
|
* socket) leaves a partial temp behind it, removed here; validation
|
|
1797
|
-
* removes its own.
|
|
2088
|
+
* removes its own. A body of at most `INLINE_MAX` bytes — by its size,
|
|
2089
|
+
* or counted as it streams — is read to memory and stored inline.
|
|
1798
2090
|
*/
|
|
1799
2091
|
async ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void> {
|
|
1800
2092
|
const tmpPath = this.tempPath(hash)
|
|
@@ -1811,33 +2103,51 @@ export class Cache implements CacheLayer {
|
|
|
1811
2103
|
// Every refusal releases the remote body: a held response pins its
|
|
1812
2104
|
// connection, and a layer that settles on the body's end never hears.
|
|
1813
2105
|
let counted: ReadableStream<Uint8Array> | undefined
|
|
2106
|
+
let reader: ChunkReader | undefined
|
|
2107
|
+
let small: Uint8Array | undefined
|
|
1814
2108
|
try {
|
|
2109
|
+
const length =
|
|
2110
|
+
body instanceof Blob ? body.size : Number(body.headers.get('content-length') ?? 0)
|
|
2111
|
+
if (length > cap) throw past()
|
|
1815
2112
|
if (body instanceof Blob) {
|
|
1816
|
-
if (
|
|
1817
|
-
await Bun.write(tmpPath, body)
|
|
1818
|
-
} else if (Number(body.headers.get('content-length') ?? 0) > cap) {
|
|
1819
|
-
throw past()
|
|
2113
|
+
if (length <= INLINE_MAX) small = await body.bytes()
|
|
2114
|
+
else await Bun.write(tmpPath, body)
|
|
1820
2115
|
} else {
|
|
1821
|
-
let
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
2116
|
+
let source = body.body
|
|
2117
|
+
if (source === null) small = new Uint8Array(0)
|
|
2118
|
+
else if (length <= INLINE_MAX) {
|
|
2119
|
+
// Counted as it streams: a length the server did not send, or
|
|
2120
|
+
// one it sent short, cannot make a large body inline.
|
|
2121
|
+
const r: ChunkReader = source.getReader()
|
|
2122
|
+
reader = r
|
|
2123
|
+
const head = await readUpTo(r, INLINE_MAX)
|
|
2124
|
+
if (head.done) small = head.bytes
|
|
2125
|
+
else source = prepended(head.bytes, r)
|
|
2126
|
+
}
|
|
2127
|
+
if (small === undefined) {
|
|
2128
|
+
let n = 0
|
|
2129
|
+
counted = source?.pipeThrough(
|
|
2130
|
+
new TransformStream<Uint8Array, Uint8Array>({
|
|
2131
|
+
transform(chunk, controller) {
|
|
2132
|
+
n += chunk.byteLength
|
|
2133
|
+
if (n > cap) controller.error(past())
|
|
2134
|
+
else controller.enqueue(chunk)
|
|
2135
|
+
},
|
|
2136
|
+
}),
|
|
2137
|
+
)
|
|
2138
|
+
await Bun.write(tmpPath, new Response(counted ?? null))
|
|
2139
|
+
}
|
|
1832
2140
|
}
|
|
1833
2141
|
} catch (err) {
|
|
1834
2142
|
// A failed write leaves the pipe's end unlocked; cancelling it
|
|
1835
2143
|
// cancels the source. A body refused by its length is cancelled itself.
|
|
1836
|
-
if (!(body instanceof Blob))
|
|
2144
|
+
if (!(body instanceof Blob)) {
|
|
2145
|
+
void (counted ?? reader ?? body.body)?.cancel(err).catch(() => undefined)
|
|
2146
|
+
}
|
|
1837
2147
|
await unlink(tmpPath).catch(() => undefined)
|
|
1838
2148
|
throw err
|
|
1839
2149
|
}
|
|
1840
|
-
await this.guard(() => this.writeArtifactAndIndex(hash, { tmpPath }, meta))
|
|
2150
|
+
await this.guard(() => this.writeArtifactAndIndex(hash, small ?? { tmpPath }, meta))
|
|
1841
2151
|
}
|
|
1842
2152
|
|
|
1843
2153
|
/** Archive name → absolute source path for every declared output. */
|
|
@@ -1954,7 +2264,8 @@ export class Cache implements CacheLayer {
|
|
|
1954
2264
|
}
|
|
1955
2265
|
|
|
1956
2266
|
/**
|
|
1957
|
-
* Atomically write `compressed` to `<hash>.tar.zst
|
|
2267
|
+
* Atomically write `compressed` to `<hash>.tar.zst`, or inline into
|
|
2268
|
+
* `artifacts` when it is at most `INLINE_MAX` bytes, and (re)build the
|
|
1958
2269
|
* entries + output_files SQL rows from the archive itself. Shared by
|
|
1959
2270
|
* `save()` (we just packed the bytes) and `ingest()` (it streamed them
|
|
1960
2271
|
* from the remote layer into a temp) — both index the identical values, because both
|
|
@@ -1979,9 +2290,14 @@ export class Cache implements CacheLayer {
|
|
|
1979
2290
|
// restore, so the size/mode/mtime fingerprint we store matches
|
|
1980
2291
|
// what isOutputsCurrent will compare against post-restore.
|
|
1981
2292
|
const finalPath = this.tarPath(hash)
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
2293
|
+
// An inline artifact is written in the rows' transaction below: no
|
|
2294
|
+
// temp, no rename.
|
|
2295
|
+
const inline =
|
|
2296
|
+
compressed instanceof Uint8Array && compressed.byteLength <= INLINE_MAX ? compressed : null
|
|
2297
|
+
let tmpPath: string | undefined
|
|
2298
|
+
if (inline === null && compressed instanceof Uint8Array) {
|
|
2299
|
+
const temp = this.tempPath(hash)
|
|
2300
|
+
tmpPath = temp
|
|
1985
2301
|
// The temp is written BEFORE validation so a large artifact can be
|
|
1986
2302
|
// scanned from the file as it decodes — a file stream reads in
|
|
1987
2303
|
// bounded pieces; the bytes in memory would not (see `decodedTar`).
|
|
@@ -1993,12 +2309,12 @@ export class Cache implements CacheLayer {
|
|
|
1993
2309
|
// and a fresh one is an orphan only an hour on (A-14).
|
|
1994
2310
|
// Off this thread even when small: a file's creation on the main
|
|
1995
2311
|
// thread cost more CPU and wall than the round trip (A/B, 2026-10-04).
|
|
1996
|
-
await writeFile(
|
|
1997
|
-
await unlink(
|
|
2312
|
+
await writeFile(temp, compressed).catch(async (err: unknown) => {
|
|
2313
|
+
await unlink(temp).catch(() => undefined)
|
|
1998
2314
|
throw err
|
|
1999
2315
|
})
|
|
2000
2316
|
endWrite()
|
|
2001
|
-
} else {
|
|
2317
|
+
} else if (!(compressed instanceof Uint8Array)) {
|
|
2002
2318
|
// `save` or `ingest` already streamed the artifact into its temp.
|
|
2003
2319
|
tmpPath = compressed.tmpPath
|
|
2004
2320
|
}
|
|
@@ -2015,7 +2331,7 @@ export class Cache implements CacheLayer {
|
|
|
2015
2331
|
const source =
|
|
2016
2332
|
compressed instanceof Uint8Array && compressed.byteLength <= STREAM_DECODE_FROM
|
|
2017
2333
|
? compressed
|
|
2018
|
-
: Bun.file(tmpPath)
|
|
2334
|
+
: Bun.file(tmpPath!)
|
|
2019
2335
|
scanned = await scanArtifact(await decodedTar(source, hash, this.artifactCeiling))
|
|
2020
2336
|
}
|
|
2021
2337
|
endScan()
|
|
@@ -2034,13 +2350,13 @@ export class Cache implements CacheLayer {
|
|
|
2034
2350
|
}
|
|
2035
2351
|
assertArtifactNames(hash, scanned.entries, meta.outputs)
|
|
2036
2352
|
} catch (err) {
|
|
2037
|
-
await unlink(tmpPath).catch(() => undefined)
|
|
2353
|
+
if (tmpPath !== undefined) await unlink(tmpPath).catch(() => undefined)
|
|
2038
2354
|
if (err instanceof ArchiveSecurityError || err instanceof CorruptArtifactError) throw err
|
|
2039
2355
|
throw new CorruptArtifactError(hash, 'artifact is not a readable archive', err)
|
|
2040
2356
|
}
|
|
2041
2357
|
const { entries } = scanned
|
|
2042
2358
|
const totalBytes =
|
|
2043
|
-
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath).size
|
|
2359
|
+
compressed instanceof Uint8Array ? compressed.byteLength : Bun.file(tmpPath!).size
|
|
2044
2360
|
const outputFileRows: Array<[string, number, number, number]> = []
|
|
2045
2361
|
// Per-output-file fingerprint rows feed the skip-restore check.
|
|
2046
2362
|
// Row paths: project entries store the bare rel (`outputs/`
|
|
@@ -2102,18 +2418,37 @@ export class Cache implements CacheLayer {
|
|
|
2102
2418
|
// renames finds no artifact, a miss (`ArtifactVanishedError`); one
|
|
2103
2419
|
// holding the old file still reads it whole. The aside name is a temp
|
|
2104
2420
|
// name, so a crash before its unlink leaves an orphan the sweep takes.
|
|
2105
|
-
|
|
2106
|
-
|
|
2421
|
+
//
|
|
2422
|
+
// An inline artifact is upserted in the same transaction, and a file
|
|
2423
|
+
// save deletes the key's inline one: as of every commit a key's live
|
|
2424
|
+
// bytes are in one place. An inline save moves aside whatever file the
|
|
2425
|
+
// key has, as a file save does: one rename that finds nothing on a
|
|
2426
|
+
// first save, where asking the index first was a statement per save.
|
|
2427
|
+
let aside: string | undefined
|
|
2107
2428
|
let renamed = false
|
|
2108
|
-
const
|
|
2429
|
+
const upsertBlob = this.upsertBlob
|
|
2430
|
+
const deleteBlob = this.deleteBlob
|
|
2431
|
+
const moveAside = (): string | undefined => {
|
|
2432
|
+
const to = this.tempPath(hash)
|
|
2109
2433
|
try {
|
|
2110
|
-
renameSync(finalPath,
|
|
2111
|
-
|
|
2434
|
+
renameSync(finalPath, to)
|
|
2435
|
+
return to
|
|
2112
2436
|
} catch (err) {
|
|
2113
2437
|
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
|
|
2438
|
+
return undefined
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
const tx = this.db.transaction(() => {
|
|
2442
|
+
if (inline !== null) {
|
|
2443
|
+
aside = moveAside()
|
|
2444
|
+
upsertBlob.run(hash, now, inline)
|
|
2445
|
+
} else {
|
|
2446
|
+
aside = moveAside()
|
|
2447
|
+
renameSync(tmpPath!, finalPath)
|
|
2448
|
+
this.artifactNames?.add(path.basename(finalPath))
|
|
2449
|
+
renamed = true
|
|
2450
|
+
deleteBlob.run(hash)
|
|
2114
2451
|
}
|
|
2115
|
-
renameSync(tmpPath, finalPath)
|
|
2116
|
-
renamed = true
|
|
2117
2452
|
insertEntry.run(
|
|
2118
2453
|
hash,
|
|
2119
2454
|
project,
|
|
@@ -2144,11 +2479,9 @@ export class Cache implements CacheLayer {
|
|
|
2144
2479
|
)
|
|
2145
2480
|
outputs.replaceFileRows(hash, outputFileRows)
|
|
2146
2481
|
// INSERT OR IGNORE: identical inputs derive this same hash, so a
|
|
2147
|
-
// re-save's
|
|
2148
|
-
if (inputComponents !== undefined) {
|
|
2149
|
-
|
|
2150
|
-
insertEntryInput.run(hash, c.kind, c.name, c.hash)
|
|
2151
|
-
}
|
|
2482
|
+
// re-save's row is identical — keep the first, skip the rest.
|
|
2483
|
+
if (inputComponents !== undefined && inputComponents.length > 0) {
|
|
2484
|
+
insertEntryInput.run(hash, encodeEntryInputs(inputComponents))
|
|
2152
2485
|
}
|
|
2153
2486
|
})
|
|
2154
2487
|
const endTx = span('save: index tx')
|
|
@@ -2156,18 +2489,21 @@ export class Cache implements CacheLayer {
|
|
|
2156
2489
|
tx.immediate()
|
|
2157
2490
|
} catch (err) {
|
|
2158
2491
|
endTx()
|
|
2159
|
-
|
|
2160
|
-
if (
|
|
2492
|
+
const left = renamed ? finalPath : tmpPath
|
|
2493
|
+
if (left !== undefined) await unlink(left).catch(() => undefined)
|
|
2494
|
+
if (aside !== undefined) await unlink(aside).catch(() => undefined)
|
|
2161
2495
|
throw err
|
|
2162
2496
|
}
|
|
2163
2497
|
endTx()
|
|
2164
|
-
if (
|
|
2498
|
+
if (aside !== undefined) await unlink(aside).catch(() => undefined)
|
|
2165
2499
|
}
|
|
2166
2500
|
|
|
2167
2501
|
/** Apply the deferred accessed_at bumps in one statement. */
|
|
2168
2502
|
private flushAccessed(): void {
|
|
2169
2503
|
// Apart: a hit whose row is fresh still renews an old file time.
|
|
2170
|
-
if (this.touched.size === 0 && this.staleTimes.size === 0)
|
|
2504
|
+
if (this.touched.size === 0 && this.staleTimes.size === 0 && this.staleInline.size === 0) {
|
|
2505
|
+
return
|
|
2506
|
+
}
|
|
2171
2507
|
const hashes = [...this.touched]
|
|
2172
2508
|
this.touched.clear()
|
|
2173
2509
|
const now = Date.now()
|
|
@@ -2182,11 +2518,16 @@ export class Cache implements CacheLayer {
|
|
|
2182
2518
|
}
|
|
2183
2519
|
}
|
|
2184
2520
|
this.staleTimes.clear()
|
|
2185
|
-
|
|
2521
|
+
const inline = [...this.staleInline]
|
|
2522
|
+
this.staleInline.clear()
|
|
2186
2523
|
// LRU bookkeeping: on a full disk the bumps are dropped, never the prune
|
|
2187
2524
|
// or the stats that asked for them (A-14).
|
|
2188
2525
|
try {
|
|
2189
|
-
|
|
2526
|
+
if (inline.length > 0) {
|
|
2527
|
+
const { test, params } = inHashes(inline)
|
|
2528
|
+
this.db.prepare(`UPDATE artifacts SET at = ? WHERE hash ${test}`).run(now, ...params)
|
|
2529
|
+
}
|
|
2530
|
+
if (hashes.length > 0) this.writeAccessed(hashes, now)
|
|
2190
2531
|
} catch (err) {
|
|
2191
2532
|
if (!isIndexFull(err)) throw err
|
|
2192
2533
|
}
|
|
@@ -2334,7 +2675,7 @@ export class Cache implements CacheLayer {
|
|
|
2334
2675
|
// them and evicted real entries to make room for them (item 975). None
|
|
2335
2676
|
// is evicted or freed, whatever its age (item 1081); the ones past the
|
|
2336
2677
|
// grace window are dropped.
|
|
2337
|
-
const { phantoms, stale, rows } = await this.phantomRows()
|
|
2678
|
+
const { phantoms, stale, rows, inline } = await this.phantomRows()
|
|
2338
2679
|
const picked = pickVictims(
|
|
2339
2680
|
rows.filter((r) => !phantoms.has(r.hash)),
|
|
2340
2681
|
await this.scanOrphans(),
|
|
@@ -2363,10 +2704,14 @@ export class Cache implements CacheLayer {
|
|
|
2363
2704
|
// Artifacts first: a row whose artifact is gone reads as a miss, so the
|
|
2364
2705
|
// order is safe, and on a full disk the delete's own journal needs the
|
|
2365
2706
|
// space the unlinks free. Rows-first failed there before any file went,
|
|
2366
|
-
// and the one verb meant to free a full disk freed nothing (A-14).
|
|
2367
|
-
|
|
2707
|
+
// and the one verb meant to free a full disk freed nothing (A-14). An
|
|
2708
|
+
// inline artifact goes with its rows, in their transaction.
|
|
2709
|
+
await Promise.all(
|
|
2710
|
+
hashes.filter((h) => !inline.has(h)).map((h) => rm(this.tarPath(h), { force: true })),
|
|
2711
|
+
)
|
|
2368
2712
|
const { test, params } = inHashes(rows)
|
|
2369
2713
|
const deleteRows = this.db.transaction(() => {
|
|
2714
|
+
this.db.prepare(`DELETE FROM artifacts WHERE hash ${test}`).run(...params)
|
|
2370
2715
|
this.db.prepare(`DELETE FROM entries WHERE hash ${test}`).run(...params)
|
|
2371
2716
|
this.outputs.forget(rows)
|
|
2372
2717
|
})
|
|
@@ -2392,10 +2737,11 @@ export class Cache implements CacheLayer {
|
|
|
2392
2737
|
/**
|
|
2393
2738
|
* Unlink the row-less files `pickVictims` took: a `<hash>.tar.zst` with
|
|
2394
2739
|
* no `entries` row (another vx version's open dropped it, a `cache.db`
|
|
2395
|
-
* deleted by hand) past the policy,
|
|
2396
|
-
*
|
|
2397
|
-
*
|
|
2398
|
-
*
|
|
2740
|
+
* deleted by hand) past the policy, a `<hash>.tar.zst.tmp-*` a crashed
|
|
2741
|
+
* save never renamed, and a file an inline artifact shadows; and delete
|
|
2742
|
+
* the row-less inline artifacts it took. Nothing else reaps them — a
|
|
2743
|
+
* lookup starts at the row, and a save of the same key replaces the
|
|
2744
|
+
* bytes, so a key that never recurs leaks its bytes forever.
|
|
2399
2745
|
*/
|
|
2400
2746
|
private async reapOrphans(
|
|
2401
2747
|
taken: readonly RowlessFile[],
|
|
@@ -2411,8 +2757,29 @@ export class Cache implements CacheLayer {
|
|
|
2411
2757
|
}
|
|
2412
2758
|
let orphans = 0
|
|
2413
2759
|
let orphanBytes = 0
|
|
2760
|
+
const blobs = taken.filter((o) => o.blob !== undefined)
|
|
2761
|
+
if (blobs.length > 0) {
|
|
2762
|
+
// Only while still row-less: an adopt since the scan keeps it.
|
|
2763
|
+
const drop = this.db.prepare(
|
|
2764
|
+
'DELETE FROM artifacts WHERE hash = ? AND NOT EXISTS (SELECT 1 FROM entries WHERE hash = ?)',
|
|
2765
|
+
)
|
|
2766
|
+
try {
|
|
2767
|
+
this.db.transaction(() => {
|
|
2768
|
+
for (const o of blobs) {
|
|
2769
|
+
if (drop.run(o.blob!, o.blob!).changes === 0) continue
|
|
2770
|
+
orphans += 1
|
|
2771
|
+
orphanBytes += o.size
|
|
2772
|
+
}
|
|
2773
|
+
})()
|
|
2774
|
+
} catch (err) {
|
|
2775
|
+
if (!isIndexFull(err)) throw err
|
|
2776
|
+
orphans = 0
|
|
2777
|
+
orphanBytes = 0
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2414
2780
|
await Promise.all(
|
|
2415
2781
|
taken.map(async (o) => {
|
|
2782
|
+
if (o.file === undefined) return
|
|
2416
2783
|
try {
|
|
2417
2784
|
// unlink, not `rm({ force })`: force swallows ENOENT, and a file a
|
|
2418
2785
|
// concurrent prune took first must not be counted as ours.
|
|
@@ -2440,33 +2807,39 @@ export class Cache implements CacheLayer {
|
|
|
2440
2807
|
phantoms: Map<string, number>
|
|
2441
2808
|
stale: string[]
|
|
2442
2809
|
rows: Array<{ hash: string; size_bytes: number; accessed_at: number }>
|
|
2810
|
+
/** The rows whose artifact is inline: present, whatever the directory holds. */
|
|
2811
|
+
inline: Set<string>
|
|
2443
2812
|
}> {
|
|
2444
2813
|
const graceStart = Date.now() - ORPHAN_GRACE_MS
|
|
2445
2814
|
const rows = this.db
|
|
2446
|
-
.prepare(
|
|
2815
|
+
.prepare(
|
|
2816
|
+
'SELECT e.hash, e.size_bytes, e.accessed_at, a.hash IS NOT NULL AS inline FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash',
|
|
2817
|
+
)
|
|
2447
2818
|
.all() as Array<{
|
|
2448
2819
|
hash: string
|
|
2449
2820
|
size_bytes: number
|
|
2450
2821
|
accessed_at: number
|
|
2822
|
+
inline: number
|
|
2451
2823
|
}>
|
|
2824
|
+
const inline = new Set(rows.filter((r) => r.inline === 1).map((r) => r.hash))
|
|
2452
2825
|
let names: string[]
|
|
2453
2826
|
try {
|
|
2454
2827
|
names = await readdir(this.artifactDir)
|
|
2455
2828
|
} catch {
|
|
2456
|
-
return { phantoms: new Map(), stale: [], rows }
|
|
2829
|
+
return { phantoms: new Map(), stale: [], rows, inline }
|
|
2457
2830
|
}
|
|
2458
2831
|
const present = new Set(names)
|
|
2459
2832
|
const phantoms = new Map<string, number>()
|
|
2460
2833
|
const stale: string[] = []
|
|
2461
2834
|
for (const r of rows) {
|
|
2462
|
-
if (present.has(`${r.hash}.tar.zst`)) continue
|
|
2835
|
+
if (r.inline === 1 || present.has(`${r.hash}.tar.zst`)) continue
|
|
2463
2836
|
phantoms.set(r.hash, r.size_bytes)
|
|
2464
2837
|
if (r.accessed_at < graceStart) stale.push(r.hash)
|
|
2465
2838
|
}
|
|
2466
|
-
return { phantoms, stale, rows }
|
|
2839
|
+
return { phantoms, stale, rows, inline }
|
|
2467
2840
|
}
|
|
2468
2841
|
|
|
2469
|
-
/** Row-less
|
|
2842
|
+
/** Row-less artifacts past the in-flight grace, for `vx info`: a prune reaps them by its policy. */
|
|
2470
2843
|
async orphanStats(): Promise<{ orphans: number; orphanBytes: number }> {
|
|
2471
2844
|
const found = await this.scanOrphans()
|
|
2472
2845
|
let orphanBytes = 0
|
|
@@ -2474,21 +2847,40 @@ export class Cache implements CacheLayer {
|
|
|
2474
2847
|
return { orphans: found.length, orphanBytes }
|
|
2475
2848
|
}
|
|
2476
2849
|
|
|
2477
|
-
/**
|
|
2850
|
+
/**
|
|
2851
|
+
* Row-less artifacts and temps past the in-flight grace window: one
|
|
2852
|
+
* readdir, one stat per candidate, and the row-less inline artifacts,
|
|
2853
|
+
* their `at` standing for the file time.
|
|
2854
|
+
*/
|
|
2478
2855
|
private async scanOrphans(): Promise<RowlessFile[]> {
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
)
|
|
2487
|
-
|
|
2856
|
+
// Asked once the directory read succeeded, as the rows are (scanOrphanFiles).
|
|
2857
|
+
let rowless: Array<{ hash: string; size: number; at: number }> = []
|
|
2858
|
+
const found = await scanOrphanFiles(this.artifactDir, () => {
|
|
2859
|
+
rowless = this.db
|
|
2860
|
+
.prepare(
|
|
2861
|
+
'SELECT hash, length(bytes) AS size, at FROM artifacts WHERE at <= ? AND hash NOT IN (SELECT hash FROM entries)',
|
|
2862
|
+
)
|
|
2863
|
+
.all(Date.now() - ORPHAN_GRACE_MS) as typeof rowless
|
|
2864
|
+
const rows = this.db
|
|
2865
|
+
.prepare(
|
|
2866
|
+
'SELECT e.hash, a.hash IS NOT NULL AS inline FROM entries e LEFT JOIN artifacts a ON a.hash = e.hash',
|
|
2867
|
+
)
|
|
2868
|
+
.all() as Array<{ hash: string; inline: number }>
|
|
2869
|
+
const blobs = this.db.prepare('SELECT hash FROM artifacts').all() as Array<{ hash: string }>
|
|
2870
|
+
return {
|
|
2871
|
+
files: new Set(rows.filter((r) => r.inline === 0).map((r) => r.hash)),
|
|
2872
|
+
inline: new Set(blobs.map((r) => r.hash)),
|
|
2873
|
+
}
|
|
2874
|
+
})
|
|
2875
|
+
for (const b of rowless) found.push({ blob: b.hash, size: b.size, mtimeMs: b.at, temp: false })
|
|
2488
2876
|
// With a shared store, no artifact in `cacheDir` is indexed: they are
|
|
2489
2877
|
// what this workspace saved before its entries moved out.
|
|
2490
2878
|
if (this.artifactDir === this.cacheDir) return found
|
|
2491
|
-
|
|
2879
|
+
const none = new Set<string>()
|
|
2880
|
+
return [
|
|
2881
|
+
...found,
|
|
2882
|
+
...(await scanOrphanFiles(this.cacheDir, () => ({ files: none, inline: none }))),
|
|
2883
|
+
]
|
|
2492
2884
|
}
|
|
2493
2885
|
|
|
2494
2886
|
close(): void {
|
|
@@ -2604,3 +2996,45 @@ function assertArtifactNames(
|
|
|
2604
2996
|
}
|
|
2605
2997
|
}
|
|
2606
2998
|
}
|
|
2999
|
+
|
|
3000
|
+
/** What `ingest` reads a remote body through. */
|
|
3001
|
+
interface ChunkReader {
|
|
3002
|
+
read(): Promise<{ done: true; value?: undefined } | { done: false; value: Uint8Array }>
|
|
3003
|
+
cancel(reason?: unknown): Promise<void>
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
/**
|
|
3007
|
+
* Read `reader` until it ends or holds more than `max` bytes: `done` and
|
|
3008
|
+
* the whole body, or the bytes read so far for `prepended` to put back.
|
|
3009
|
+
*/
|
|
3010
|
+
async function readUpTo(
|
|
3011
|
+
reader: ChunkReader,
|
|
3012
|
+
max: number,
|
|
3013
|
+
): Promise<{ done: boolean; bytes: Uint8Array }> {
|
|
3014
|
+
const chunks: Uint8Array[] = []
|
|
3015
|
+
let n = 0
|
|
3016
|
+
while (n <= max) {
|
|
3017
|
+
const r = await reader.read()
|
|
3018
|
+
if (r.done) return { done: true, bytes: Buffer.concat(chunks, n) }
|
|
3019
|
+
chunks.push(r.value)
|
|
3020
|
+
n += r.value.byteLength
|
|
3021
|
+
}
|
|
3022
|
+
return { done: false, bytes: Buffer.concat(chunks, n) }
|
|
3023
|
+
}
|
|
3024
|
+
|
|
3025
|
+
/** `head`, then the rest of `reader`: a body `readUpTo` started, whole again. */
|
|
3026
|
+
function prepended(head: Uint8Array, reader: ChunkReader): ReadableStream<Uint8Array> {
|
|
3027
|
+
return new ReadableStream<Uint8Array>({
|
|
3028
|
+
start(controller) {
|
|
3029
|
+
controller.enqueue(head)
|
|
3030
|
+
},
|
|
3031
|
+
async pull(controller) {
|
|
3032
|
+
const r = await reader.read()
|
|
3033
|
+
if (r.done) controller.close()
|
|
3034
|
+
else controller.enqueue(r.value)
|
|
3035
|
+
},
|
|
3036
|
+
cancel(reason) {
|
|
3037
|
+
return reader.cancel(reason)
|
|
3038
|
+
},
|
|
3039
|
+
})
|
|
3040
|
+
}
|