@vzn/vx 0.0.15 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -130
- package/index.ts +10 -0
- package/launcher.mjs +10 -11
- package/package.json +19 -5
- package/plugins/local-cache/index.ts +2 -0
- package/plugins/local-executor/index.ts +2 -0
- package/plugins/schedule-history/index.ts +2 -0
- package/src/bin.ts +2 -2
- package/src/cache/archive.ts +557 -0
- package/src/cache/cache.ts +929 -250
- package/src/cache/cas-backend.ts +20 -2
- package/src/cache/chained-cache.ts +202 -0
- package/src/cache/digest.ts +0 -15
- package/src/cache/index.ts +13 -3
- package/src/cache/inputs.ts +445 -41
- package/src/cache/layered-cache.ts +129 -30
- package/src/cache/tar-stream.ts +358 -0
- package/src/cli/cache.ts +36 -10
- package/src/cli/help.ts +167 -128
- package/src/cli/index.ts +74 -43
- package/src/cli/info.ts +30 -0
- package/src/cli/last.ts +140 -0
- package/src/cli/migrate-emit.ts +21 -0
- package/src/cli/migrate-nx.ts +23 -0
- package/src/cli/migrate-scripts.ts +127 -0
- package/src/cli/migrate-turbo.ts +43 -6
- package/src/cli/migrate.ts +119 -33
- package/src/cli/plan-format.ts +58 -3
- package/src/cli/plugin-commands.ts +96 -0
- package/src/cli/prune.ts +291 -0
- package/src/cli/run.ts +272 -80
- package/src/cli/upgrade.ts +7 -6
- package/src/cli/watch.ts +259 -61
- package/src/cli/why.ts +193 -0
- package/src/config.ts +40 -23
- package/src/exec/env.ts +7 -1
- package/src/exec/executor.ts +239 -0
- package/src/exec/index.ts +19 -1
- package/src/exec/runner.ts +116 -29
- package/src/exec/sandbox-runtime.ts +194 -24
- package/src/graph/index.ts +3 -1
- package/src/graph/scheduler.ts +92 -18
- package/src/graph/task-graph.ts +220 -6
- package/src/index.ts +82 -84
- package/src/orchestrator/deferred-outputs.ts +144 -0
- package/src/orchestrator/download-policy.ts +171 -0
- package/src/orchestrator/events.ts +30 -0
- package/src/orchestrator/execute-task.ts +328 -94
- package/src/orchestrator/failure-mode.ts +61 -0
- package/src/orchestrator/framed-output.ts +87 -35
- package/src/orchestrator/history.ts +78 -31
- package/src/orchestrator/index.ts +43 -90
- package/src/orchestrator/local-shortcircuit.ts +22 -0
- package/src/orchestrator/logger.ts +196 -14
- package/src/orchestrator/metrics.ts +57 -1376
- package/src/orchestrator/missing-plugin.ts +9 -0
- package/src/orchestrator/options.ts +22 -4
- package/src/orchestrator/plan.ts +144 -1
- package/src/orchestrator/plugin-host.ts +199 -119
- package/src/orchestrator/plugin.ts +144 -57
- package/src/orchestrator/prepare.ts +220 -69
- package/src/orchestrator/remote-prefetch.ts +27 -8
- package/src/orchestrator/resources.ts +20 -34
- package/src/orchestrator/run-artifacts.ts +41 -16
- package/src/orchestrator/run-context.ts +167 -29
- package/src/orchestrator/run-report.ts +73 -46
- package/src/orchestrator/run.ts +590 -122
- package/src/orchestrator/stable-keys.ts +74 -33
- package/src/orchestrator/status-line.ts +47 -8
- package/src/orchestrator/summary.ts +22 -1
- package/src/orchestrator/tally.ts +78 -35
- package/src/orchestrator/task-hash.ts +105 -11
- package/src/orchestrator/task-log-buffer.ts +291 -0
- package/src/orchestrator/telemetry-host.ts +24 -6
- package/src/orchestrator/telemetry.ts +181 -21
- package/src/orchestrator/upstream.ts +72 -31
- package/src/orchestrator/verify.ts +12 -2
- package/src/plugins/local-cache/index.ts +11 -0
- package/src/plugins/local-executor/index.ts +45 -0
- package/src/plugins/schedule-history/index.ts +109 -0
- package/src/util/edit-distance.ts +21 -0
- package/src/util/errors.ts +15 -1
- package/src/util/index.ts +7 -2
- package/src/util/num.ts +37 -0
- package/src/util/paths.ts +43 -0
- package/src/util/settle.ts +48 -0
- package/src/util/size.ts +5 -1
- package/src/util/tail.ts +66 -0
- package/src/util/timing.ts +63 -0
- package/src/workspace/affected.ts +142 -15
- package/src/workspace/config-cache.ts +304 -0
- package/src/workspace/config-eval.ts +210 -0
- package/src/workspace/config-imports.ts +161 -0
- package/src/workspace/filter.ts +22 -4
- package/src/workspace/fingerprint.ts +16 -1
- package/src/workspace/index.ts +15 -3
- package/src/workspace/lockfile.ts +13 -2
- package/src/workspace/nested-dirs.ts +21 -7
- package/src/workspace/package-graph.ts +23 -20
- package/src/workspace/project-loader.ts +503 -59
- package/src/workspace/workspace.ts +213 -72
- package/src/cache/tar.ts +0 -343
- package/src/cli/backend.ts +0 -23
- package/src/cli/mcp-rpc.ts +0 -292
- package/src/cli/mcp.ts +0 -92
- package/src/orchestrator/devframe-surface.ts +0 -74
- package/src/orchestrator/predict.ts +0 -106
- package/src/orchestrator/protocol.ts +0 -156
- package/src/orchestrator/run-state.ts +0 -145
- package/src/orchestrator/wire-render.ts +0 -92
- package/src/orchestrator/wire.ts +0 -212
|
@@ -23,6 +23,7 @@ import type {
|
|
|
23
23
|
CacheLayer,
|
|
24
24
|
CachePolicy,
|
|
25
25
|
CacheStats,
|
|
26
|
+
CacheStatsOptions,
|
|
26
27
|
IngestMeta,
|
|
27
28
|
InvocationRecord,
|
|
28
29
|
OutputFileRow,
|
|
@@ -31,7 +32,7 @@ import type {
|
|
|
31
32
|
RunRecord,
|
|
32
33
|
SaveArgs,
|
|
33
34
|
} from './cache.js'
|
|
34
|
-
import { FULL_CACHE_POLICY } from './cache.js'
|
|
35
|
+
import { FULL_CACHE_POLICY, type OutputDirRow } from './cache.js'
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
38
|
* What a remote cache layer must provide — THE plugin seam for remote
|
|
@@ -47,6 +48,15 @@ import { FULL_CACHE_POLICY } from './cache.js'
|
|
|
47
48
|
export interface RemoteCacheLayer {
|
|
48
49
|
/** Existence probe (drives the plan path's `--dry` remote prediction). */
|
|
49
50
|
has(hash: string): Promise<boolean>
|
|
51
|
+
/**
|
|
52
|
+
* Optional batch existence probe: given N hashes, return the subset stored
|
|
53
|
+
* remotely in ONE round-trip. Lets the prefetch pass collapse N per-hash
|
|
54
|
+
* network probes into one, then fetch only the hits. A remote that can't
|
|
55
|
+
* batch omits this method (or returns `null`) and the layer falls back to
|
|
56
|
+
* the per-hash path. Never throws for control flow — `null` means "no batch
|
|
57
|
+
* info; use per-hash".
|
|
58
|
+
*/
|
|
59
|
+
hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
|
|
50
60
|
/** Fetch an artifact's bytes; `null` = miss. `durationMs` is the
|
|
51
61
|
* producing task's duration when the wire carries it. */
|
|
52
62
|
get(hash: string): Promise<{ body: ArrayBuffer; durationMs: number | undefined } | null>
|
|
@@ -73,6 +83,9 @@ export interface LayeredCacheOptions {
|
|
|
73
83
|
}
|
|
74
84
|
|
|
75
85
|
export class LayeredCache implements CacheLayer {
|
|
86
|
+
/** A remote layer is composed in by construction — see `CacheLayer.hasRemote`. */
|
|
87
|
+
readonly hasRemote = true
|
|
88
|
+
|
|
76
89
|
/**
|
|
77
90
|
* In-flight remote pulls keyed by hash. `prefetch` and `get` both go
|
|
78
91
|
* through here, so a key probed concurrently by both resolves a
|
|
@@ -109,7 +122,7 @@ export class LayeredCache implements CacheLayer {
|
|
|
109
122
|
private readonly policy: CachePolicy
|
|
110
123
|
|
|
111
124
|
constructor(
|
|
112
|
-
|
|
125
|
+
readonly local: Cache,
|
|
113
126
|
private readonly remote: RemoteCacheLayer,
|
|
114
127
|
private readonly options: LayeredCacheOptions = {},
|
|
115
128
|
) {
|
|
@@ -123,9 +136,45 @@ export class LayeredCache implements CacheLayer {
|
|
|
123
136
|
async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
124
137
|
// No-op when remote reads are off — there's nothing to warm from.
|
|
125
138
|
if (!this.policy.remoteRead) return false
|
|
139
|
+
// The local-first skip lives in doPullFromRemote (the shared choke
|
|
140
|
+
// point): pullFromRemote registers the `inflight` entry SYNCHRONOUSLY,
|
|
141
|
+
// so a concurrent markRemoteAbsent can't clobber a pending pull — a
|
|
142
|
+
// guard done here (behind an async local.has) would reopen that race.
|
|
126
143
|
return await this.pullFromRemote(hash, ctx)
|
|
127
144
|
}
|
|
128
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Batch existence probe over the remote layer — the subset of `hashes`
|
|
148
|
+
* present remotely, in one round-trip, or `null` when the remote can't
|
|
149
|
+
* batch (no `hasMany`, reads disabled, or an error). The prefetch pass uses
|
|
150
|
+
* this to fetch only the hits and to pre-mark the misses (`markRemoteAbsent`)
|
|
151
|
+
* so their lazy `get` skips the network. Never throws — a batch failure
|
|
152
|
+
* degrades to "no batch info" and the caller falls back to per-hash.
|
|
153
|
+
*/
|
|
154
|
+
async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
|
|
155
|
+
if (!this.policy.remoteRead || this.remote.hasMany === undefined) return null
|
|
156
|
+
try {
|
|
157
|
+
return await this.remote.hasMany(hashes)
|
|
158
|
+
} catch (err) {
|
|
159
|
+
this.reportRemoteError(err)
|
|
160
|
+
return null
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Record that the remote layer has NO artifact for each of `hashes` (from a
|
|
166
|
+
* batch probe), so a later `get`/`prefetch` short-circuits to a miss WITHOUT
|
|
167
|
+
* a network round-trip. Byte-for-byte equivalent to a background prefetch GET
|
|
168
|
+
* having resolved `false` into `inflight` — same at-most-once semantics, same
|
|
169
|
+
* point-in-time staleness window — but without spending the GET. Never
|
|
170
|
+
* overwrites an entry already in flight.
|
|
171
|
+
*/
|
|
172
|
+
markRemoteAbsent(hashes: Iterable<string>): void {
|
|
173
|
+
for (const hash of hashes) {
|
|
174
|
+
if (!this.inflight.has(hash)) this.inflight.set(hash, Promise.resolve(false))
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
129
178
|
async get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
|
|
130
179
|
const localHit = await this.local.get(hash, ctx)
|
|
131
180
|
if (localHit) {
|
|
@@ -147,11 +196,23 @@ export class LayeredCache implements CacheLayer {
|
|
|
147
196
|
const ingested = await this.pullFromRemote(hash, ctx)
|
|
148
197
|
if (!ingested) return null
|
|
149
198
|
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
|
|
154
|
-
|
|
199
|
+
// Read past the local READ gate: `ingest` is deliberately ungated, so
|
|
200
|
+
// the artifact + index row this pull just wrote exist regardless of
|
|
201
|
+
// policy, and the gate ("don't serve hits from the pre-existing local
|
|
202
|
+
// cache") must not discard them. Going through the gated `get` here
|
|
203
|
+
// made `--cache=local:,remote:rw` download the artifact, throw the hit
|
|
204
|
+
// away, re-execute and re-upload — on every run, forever.
|
|
205
|
+
const materialized = await this.local.getIngested(hash)
|
|
206
|
+
if (!materialized) return null
|
|
207
|
+
|
|
208
|
+
// `remoteSourced` — not merely "the pull returned true" — is what says
|
|
209
|
+
// the remote cache saved this work. `doPullFromRemote` also returns true
|
|
210
|
+
// for its local-first skip, which issues NO remote GET: when a run
|
|
211
|
+
// sharing this cache dir ingests the artifact between the local read
|
|
212
|
+
// above and that skip's `local.has`, stamping 'remote' here reported a
|
|
213
|
+
// purely-local hit as `cache-hit-remote` and inflated the remote
|
|
214
|
+
// hit-rate. Whatever provenance the entry has is the truth.
|
|
215
|
+
return this.remoteSourced.has(hash) ? { ...materialized, source: 'remote' } : materialized
|
|
155
216
|
}
|
|
156
217
|
|
|
157
218
|
// Existence probe: local first, then a remote HEAD — no body
|
|
@@ -185,6 +246,16 @@ export class LayeredCache implements CacheLayer {
|
|
|
185
246
|
}
|
|
186
247
|
|
|
187
248
|
private async doPullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
249
|
+
// Local-first, mirroring get()/has(): if local ALREADY holds the artifact
|
|
250
|
+
// there is nothing to pull — skip the remote GET (a warm-local run would
|
|
251
|
+
// otherwise re-download every artifact it already has) and DON'T mark it
|
|
252
|
+
// `remoteSourced` (that would mislabel a purely-local warm hit as
|
|
253
|
+
// cache-hit-remote and inflate the remote hit-rate). Returning `true` is
|
|
254
|
+
// correct for a get() read-through too: "the artifact is in local" — the
|
|
255
|
+
// caller re-reads it, keeping whatever provenance it already had (local,
|
|
256
|
+
// or remote if a concurrent prefetch set it).
|
|
257
|
+
if ((await this.local.has(hash)) === 'local') return true
|
|
258
|
+
|
|
188
259
|
let remoteResult
|
|
189
260
|
try {
|
|
190
261
|
remoteResult = await this.remote.get(hash)
|
|
@@ -233,32 +304,48 @@ export class LayeredCache implements CacheLayer {
|
|
|
233
304
|
}
|
|
234
305
|
|
|
235
306
|
async save(args: SaveArgs): Promise<void> {
|
|
236
|
-
// Local write honors its own
|
|
237
|
-
//
|
|
307
|
+
// Local write honors its own gates inside Cache.save (no-op when local
|
|
308
|
+
// writes are disabled, or when ChainedCache marked this a duplicate save
|
|
309
|
+
// to a shared local handle — see `skipLocalWrite` on the contract).
|
|
238
310
|
await this.local.save(args)
|
|
239
311
|
if (!this.policy.remoteWrite) return
|
|
240
|
-
// Write-through upload, OFF the task's critical path
|
|
241
|
-
//
|
|
242
|
-
//
|
|
243
|
-
// are
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
312
|
+
// Write-through upload, OFF the task's critical path: the PUT runs in
|
|
313
|
+
// the bounded background pool so the task's worker slot is released
|
|
314
|
+
// immediately, and `run()` awaits `drainUploads()` before closing the
|
|
315
|
+
// cache. Errors are logged, not propagated: the task already
|
|
316
|
+
// succeeded; we don't fail it on cache-server issues.
|
|
317
|
+
//
|
|
318
|
+
// The artifact bytes are read INSIDE the job, not here.
|
|
319
|
+
// UPLOAD_CONCURRENCY bounds sockets, not memory: a queued closure
|
|
320
|
+
// holding its own artifact keeps the WHOLE backlog resident, so peak
|
|
321
|
+
// RSS scaled with a run's total miss artifact bytes rather than with
|
|
322
|
+
// the pool — any cold monorepo run whose remote uploads slower than the
|
|
323
|
+
// build produces artifacts held every pending one at once. Reading in
|
|
324
|
+
// the job caps resident artifact bytes at UPLOAD_CONCURRENCY. The
|
|
325
|
+
// artifact is content-addressed and immutable, so a deferred read sees
|
|
326
|
+
// the same bytes; if a concurrent `vx cache prune` removed it first the
|
|
327
|
+
// read throws and this upload is skipped — the never-fail contract.
|
|
328
|
+
const hash = args.hash
|
|
258
329
|
const durationMs = args.entry.durationMs
|
|
330
|
+
// Local writes disabled (`--cache=local:,remote:rw`): there is no
|
|
331
|
+
// on-disk artifact to read later, so the bytes must be packed NOW,
|
|
332
|
+
// while this task's output files are still on disk. Deferring THIS
|
|
333
|
+
// read would pack whatever the tree happens to hold when the job
|
|
334
|
+
// runs. Such a run keeps the old memory profile by necessity — the
|
|
335
|
+
// bytes exist nowhere else.
|
|
336
|
+
let packed: Uint8Array | undefined
|
|
337
|
+
if (!this.local.localWritesEnabled) {
|
|
338
|
+
try {
|
|
339
|
+
packed = await this.local.packArtifactBytes(args)
|
|
340
|
+
} catch (err) {
|
|
341
|
+
this.reportRemoteError(err)
|
|
342
|
+
return
|
|
343
|
+
}
|
|
344
|
+
}
|
|
259
345
|
this.enqueueUpload(async () => {
|
|
260
346
|
try {
|
|
261
|
-
await this.
|
|
347
|
+
const bytes = packed ?? (await Bun.file(this.local.outputsPath(hash)).bytes())
|
|
348
|
+
await this.remote.put(hash, bytes, { durationMs })
|
|
262
349
|
} catch (err) {
|
|
263
350
|
this.reportRemoteError(err)
|
|
264
351
|
}
|
|
@@ -311,6 +398,18 @@ export class LayeredCache implements CacheLayer {
|
|
|
311
398
|
return this.local.isOutputsCurrent(projectDir, expected)
|
|
312
399
|
}
|
|
313
400
|
|
|
401
|
+
recordOutputDirs(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void> {
|
|
402
|
+
return this.local.recordOutputDirs(hash, projectDir, prefixes)
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
|
|
406
|
+
return this.local.loadOutputDirsBatch(hashes)
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
|
|
410
|
+
return this.local.outputDirsCurrent(projectDir, rows)
|
|
411
|
+
}
|
|
412
|
+
|
|
314
413
|
recordRun(run: RunRecord): void {
|
|
315
414
|
this.local.recordRun(run)
|
|
316
415
|
}
|
|
@@ -323,8 +422,8 @@ export class LayeredCache implements CacheLayer {
|
|
|
323
422
|
this.local.recordRunBundle(bundle)
|
|
324
423
|
}
|
|
325
424
|
|
|
326
|
-
stats(): CacheStats {
|
|
327
|
-
return this.local.stats()
|
|
425
|
+
stats(opts?: CacheStatsOptions): CacheStats {
|
|
426
|
+
return this.local.stats(opts)
|
|
328
427
|
}
|
|
329
428
|
|
|
330
429
|
async prune(options: PruneOptions): Promise<PruneResult> {
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
// A streaming tar reader for the restore path. `Bun.Archive` needs the whole
|
|
2
|
+
// tar in memory and `.files()` copies every entry, so restoring a 150 MiB
|
|
3
|
+
// artifact peaked at 3.2× its size (measured 2026-09-03). This reads the
|
|
4
|
+
// tar as it streams out of the zstd decoder and hands each entry's body to
|
|
5
|
+
// the caller as chunks, so memory is bounded by one chunk, not the archive.
|
|
6
|
+
//
|
|
7
|
+
// Dialect: what libarchive (Bun.Archive) writes — ustar with the name/prefix
|
|
8
|
+
// split, pax extended headers (`x`: path, size) past ustar's limits — plus
|
|
9
|
+
// GNU long names (`L`) for foreign artifacts. Every header's checksum is
|
|
10
|
+
// verified; an archive that ends before its data does is an error, never a
|
|
11
|
+
// short entry. Non-regular entries are reported with their type and their
|
|
12
|
+
// bodies skipped; the caller decides what to materialise (nothing but `0`).
|
|
13
|
+
|
|
14
|
+
export interface TarEntry {
|
|
15
|
+
name: string
|
|
16
|
+
size: number
|
|
17
|
+
/** POSIX typeflag: '0' regular, '5' directory, '2' symlink, … */
|
|
18
|
+
type: string
|
|
19
|
+
/** Header mtime in ms (second precision — the sidecar carries the real one). */
|
|
20
|
+
mtimeMs: number
|
|
21
|
+
/** The entry's bytes, in stream order. Must be drained before the next entry. */
|
|
22
|
+
body: AsyncIterable<Uint8Array>
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export class TarFormatError extends Error {
|
|
26
|
+
constructor(message: string) {
|
|
27
|
+
super(message)
|
|
28
|
+
this.name = 'TarFormatError'
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const BLOCK = 512
|
|
33
|
+
const decoder = new TextDecoder()
|
|
34
|
+
|
|
35
|
+
function field(h: Uint8Array, off: number, len: number): string {
|
|
36
|
+
let end = off
|
|
37
|
+
while (end < off + len && h[end] !== 0) end++
|
|
38
|
+
return decoder.decode(h.subarray(off, end))
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function octal(h: Uint8Array, off: number, len: number): number {
|
|
42
|
+
// GNU base-256 (high bit set) for sizes ≥ 8 GiB.
|
|
43
|
+
if (h[off]! & 0x80) {
|
|
44
|
+
let n = 0
|
|
45
|
+
for (let i = off + 1; i < off + len; i++) n = n * 256 + h[i]!
|
|
46
|
+
return n
|
|
47
|
+
}
|
|
48
|
+
const s = field(h, off, len).trim()
|
|
49
|
+
if (s === '') return 0
|
|
50
|
+
// Every digit, not the longest parseable prefix: `parseInt` would read
|
|
51
|
+
// a corrupt `5zz` as 5 and let the damage surface later, if at all.
|
|
52
|
+
if (!/^[0-7]+$/.test(s)) throw new TarFormatError(`bad octal field: ${JSON.stringify(s)}`)
|
|
53
|
+
return parseInt(s, 8)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function checksumOk(h: Uint8Array): boolean {
|
|
57
|
+
const stored = octal(h, 148, 8)
|
|
58
|
+
let sum = 0
|
|
59
|
+
for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 32 : h[i]!
|
|
60
|
+
return sum === stored
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** pax `x` body: `<len> <key>=<value>\n` records. */
|
|
64
|
+
function parsePax(body: Uint8Array): Map<string, string> {
|
|
65
|
+
const out = new Map<string, string>()
|
|
66
|
+
let i = 0
|
|
67
|
+
while (i < body.byteLength) {
|
|
68
|
+
let sp = i
|
|
69
|
+
while (sp < body.byteLength && body[sp] !== 32) sp++
|
|
70
|
+
const len = Number(decoder.decode(body.subarray(i, sp)))
|
|
71
|
+
if (!Number.isInteger(len) || len <= 0 || i + len > body.byteLength) {
|
|
72
|
+
throw new TarFormatError('malformed pax record')
|
|
73
|
+
}
|
|
74
|
+
const rec = decoder.decode(body.subarray(sp + 1, i + len - 1))
|
|
75
|
+
const eq = rec.indexOf('=')
|
|
76
|
+
if (eq > 0) out.set(rec.slice(0, eq), rec.slice(eq + 1))
|
|
77
|
+
i += len
|
|
78
|
+
}
|
|
79
|
+
return out
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** A byte source with exact-size reads over a stream of arbitrary chunks. */
|
|
83
|
+
class Source {
|
|
84
|
+
private chunks: Uint8Array[] = []
|
|
85
|
+
private head = 0
|
|
86
|
+
private done = false
|
|
87
|
+
constructor(private readonly reader: AsyncIterator<Uint8Array>) {}
|
|
88
|
+
|
|
89
|
+
private async fill(): Promise<boolean> {
|
|
90
|
+
if (this.done) return false
|
|
91
|
+
const { done, value } = await this.reader.next()
|
|
92
|
+
if (done === true || value === undefined) {
|
|
93
|
+
this.done = true
|
|
94
|
+
return false
|
|
95
|
+
}
|
|
96
|
+
if (value.byteLength > 0) this.chunks.push(value)
|
|
97
|
+
return true
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Exactly `n` bytes, or null at a clean end (nothing buffered), or a TarFormatError mid-entry. */
|
|
101
|
+
async exact(n: number, what: string): Promise<Uint8Array | null> {
|
|
102
|
+
const out = new Uint8Array(n)
|
|
103
|
+
let got = 0
|
|
104
|
+
while (got < n) {
|
|
105
|
+
if (this.chunks.length === 0 && !(await this.fill())) {
|
|
106
|
+
if (got === 0) return null
|
|
107
|
+
throw new TarFormatError(`archive ends inside ${what} (${got} of ${n} bytes)`)
|
|
108
|
+
}
|
|
109
|
+
const c = this.chunks[0]!
|
|
110
|
+
const take = Math.min(n - got, c.byteLength - this.head)
|
|
111
|
+
out.set(c.subarray(this.head, this.head + take), got)
|
|
112
|
+
got += take
|
|
113
|
+
this.head += take
|
|
114
|
+
if (this.head === c.byteLength) {
|
|
115
|
+
this.chunks.shift()
|
|
116
|
+
this.head = 0
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return out
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Up to `n` bytes as they arrive (never more), for streaming a body. */
|
|
123
|
+
async *take(n: number, what: string): AsyncIterable<Uint8Array> {
|
|
124
|
+
let left = n
|
|
125
|
+
while (left > 0) {
|
|
126
|
+
if (this.chunks.length === 0 && !(await this.fill())) {
|
|
127
|
+
throw new TarFormatError(`archive ends inside ${what} (${n - left} of ${n} bytes)`)
|
|
128
|
+
}
|
|
129
|
+
const c = this.chunks[0]!
|
|
130
|
+
const take = Math.min(left, c.byteLength - this.head)
|
|
131
|
+
const piece = c.subarray(this.head, this.head + take)
|
|
132
|
+
left -= take
|
|
133
|
+
this.head += take
|
|
134
|
+
if (this.head === c.byteLength) {
|
|
135
|
+
this.chunks.shift()
|
|
136
|
+
this.head = 0
|
|
137
|
+
}
|
|
138
|
+
yield piece
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Iterate a tar stream entry by entry. Each entry's `body` MUST be fully
|
|
145
|
+
* consumed (or the iterator drains it) before the next entry is read.
|
|
146
|
+
*/
|
|
147
|
+
export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGenerator<TarEntry> {
|
|
148
|
+
const src = new Source(stream[Symbol.asyncIterator]())
|
|
149
|
+
let pendingPath: string | undefined
|
|
150
|
+
let pendingSize: number | undefined
|
|
151
|
+
let zeroBlocks = 0
|
|
152
|
+
for (;;) {
|
|
153
|
+
const h = await src.exact(BLOCK, 'a header')
|
|
154
|
+
if (h === null) {
|
|
155
|
+
if (zeroBlocks === 0) throw new TarFormatError('archive has no end-of-archive marker')
|
|
156
|
+
return
|
|
157
|
+
}
|
|
158
|
+
if (h.every((b) => b === 0)) {
|
|
159
|
+
zeroBlocks++
|
|
160
|
+
if (zeroBlocks >= 2) return
|
|
161
|
+
continue
|
|
162
|
+
}
|
|
163
|
+
zeroBlocks = 0
|
|
164
|
+
if (!checksumOk(h)) throw new TarFormatError('header checksum mismatch')
|
|
165
|
+
const type = h[156] === 0 ? '0' : String.fromCharCode(h[156]!)
|
|
166
|
+
const size = pendingSize ?? octal(h, 124, 12)
|
|
167
|
+
const mtimeMs = octal(h, 136, 12) * 1000
|
|
168
|
+
const padded = Math.ceil(size / BLOCK) * BLOCK
|
|
169
|
+
const magic = field(h, 257, 6)
|
|
170
|
+
const prefix = magic.startsWith('ustar') ? field(h, 345, 155) : ''
|
|
171
|
+
const rawName = field(h, 0, 100)
|
|
172
|
+
let name = pendingPath ?? (prefix ? `${prefix}/${rawName}` : rawName)
|
|
173
|
+
pendingPath = undefined
|
|
174
|
+
pendingSize = undefined
|
|
175
|
+
|
|
176
|
+
if (type === 'x' || type === 'L') {
|
|
177
|
+
// Extended header: applies to the NEXT entry only.
|
|
178
|
+
const body = await src.exact(padded, 'an extended header')
|
|
179
|
+
if (body === null) throw new TarFormatError('archive ends inside an extended header')
|
|
180
|
+
if (type === 'x') {
|
|
181
|
+
const pax = parsePax(body.subarray(0, size))
|
|
182
|
+
const p = pax.get('path')
|
|
183
|
+
if (p !== undefined) pendingPath = p
|
|
184
|
+
const s = pax.get('size')
|
|
185
|
+
if (s !== undefined) pendingSize = Number(s)
|
|
186
|
+
} else {
|
|
187
|
+
pendingPath = field(body, 0, size)
|
|
188
|
+
}
|
|
189
|
+
continue
|
|
190
|
+
}
|
|
191
|
+
if (type === 'g') {
|
|
192
|
+
// pax global header: not used by Bun.Archive; skip its body.
|
|
193
|
+
await src.exact(padded, 'a global header')
|
|
194
|
+
continue
|
|
195
|
+
}
|
|
196
|
+
name = name.replace(/\/+$/, type === '5' ? '/' : '')
|
|
197
|
+
|
|
198
|
+
let drained = false
|
|
199
|
+
const body = (async function* (): AsyncIterable<Uint8Array> {
|
|
200
|
+
if (size > 0) for await (const c of src.take(size, `entry ${name}`)) yield c
|
|
201
|
+
const pad = padded - size
|
|
202
|
+
if (pad > 0 && (await src.exact(pad, `padding after ${name}`)) === null) {
|
|
203
|
+
throw new TarFormatError(`archive ends inside padding after ${name}`)
|
|
204
|
+
}
|
|
205
|
+
drained = true
|
|
206
|
+
})()
|
|
207
|
+
yield { name, size, type, mtimeMs, body }
|
|
208
|
+
if (!drained) {
|
|
209
|
+
// The caller skipped this body: drain it so the next header lines up.
|
|
210
|
+
for await (const _ of body) {
|
|
211
|
+
// discard
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ─── Writer ───────────────────────────────────────────────────────────
|
|
218
|
+
|
|
219
|
+
/** One regular file to pack. `body` is read as it is written. */
|
|
220
|
+
export interface TarInput {
|
|
221
|
+
name: string
|
|
222
|
+
size: number
|
|
223
|
+
/** Permission bits; the sidecar carries the exact value, this is for foreign readers. */
|
|
224
|
+
mode?: number
|
|
225
|
+
/** Seconds since epoch; the sidecar carries the millisecond value. */
|
|
226
|
+
mtime?: number
|
|
227
|
+
body: Blob | Uint8Array | string
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const encoder = new TextEncoder()
|
|
231
|
+
|
|
232
|
+
function writeOctal(h: Uint8Array, off: number, len: number, n: number): void {
|
|
233
|
+
const digits = n.toString(8).padStart(len - 1, '0')
|
|
234
|
+
if (digits.length > len - 1)
|
|
235
|
+
throw new TarFormatError(`value ${n} does not fit a ${len}-byte field`)
|
|
236
|
+
h.set(encoder.encode(digits), off)
|
|
237
|
+
h[off + len - 1] = 0
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function header(
|
|
241
|
+
name: string | Uint8Array,
|
|
242
|
+
size: number,
|
|
243
|
+
type: string,
|
|
244
|
+
mode: number,
|
|
245
|
+
mtime: number,
|
|
246
|
+
): Uint8Array {
|
|
247
|
+
const h = new Uint8Array(BLOCK)
|
|
248
|
+
const nameBytes = typeof name === 'string' ? encoder.encode(name) : name
|
|
249
|
+
if (nameBytes.byteLength <= 100) {
|
|
250
|
+
h.set(nameBytes, 0)
|
|
251
|
+
} else {
|
|
252
|
+
// ustar prefix split: the longest tail that fits 100 bytes, split at
|
|
253
|
+
// a `/`, with the head fitting 155. Anything else needs pax.
|
|
254
|
+
const cut = splitForUstar(nameBytes)
|
|
255
|
+
if (cut === null)
|
|
256
|
+
throw new TarFormatError(`name too long for ustar: ${decoder.decode(nameBytes)}`)
|
|
257
|
+
h.set(nameBytes.subarray(cut + 1), 0)
|
|
258
|
+
h.set(nameBytes.subarray(0, cut), 345)
|
|
259
|
+
}
|
|
260
|
+
writeOctal(h, 100, 8, mode & 0o7777)
|
|
261
|
+
writeOctal(h, 108, 8, 0)
|
|
262
|
+
writeOctal(h, 116, 8, 0)
|
|
263
|
+
writeOctal(h, 124, 12, size)
|
|
264
|
+
writeOctal(h, 136, 12, mtime)
|
|
265
|
+
h[156] = type.charCodeAt(0)
|
|
266
|
+
h.set(encoder.encode('ustar\0'), 257)
|
|
267
|
+
h.set(encoder.encode('00'), 263)
|
|
268
|
+
h.fill(32, 148, 156)
|
|
269
|
+
let sum = 0
|
|
270
|
+
for (let i = 0; i < BLOCK; i++) sum += h[i]!
|
|
271
|
+
h.set(encoder.encode(sum.toString(8).padStart(6, '0')), 148)
|
|
272
|
+
h[154] = 0
|
|
273
|
+
h[155] = 32
|
|
274
|
+
return h
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Byte index of the `/` to split at, or null when no split satisfies both fields. */
|
|
278
|
+
function splitForUstar(name: Uint8Array): number | null {
|
|
279
|
+
for (let i = name.byteLength - 1; i > 0; i--) {
|
|
280
|
+
if (name[i] !== 47) continue
|
|
281
|
+
const tail = name.byteLength - i - 1
|
|
282
|
+
if (tail > 100) return null
|
|
283
|
+
if (tail >= 1 && i <= 155) return i
|
|
284
|
+
}
|
|
285
|
+
return null
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function paxRecord(key: string, value: string): Uint8Array {
|
|
289
|
+
const body = encoder.encode(` ${key}=${value}\n`)
|
|
290
|
+
let len = body.byteLength + 1
|
|
291
|
+
while (String(len).length + body.byteLength !== len) len = String(len).length + body.byteLength
|
|
292
|
+
const out = new Uint8Array(len)
|
|
293
|
+
out.set(encoder.encode(String(len)), 0)
|
|
294
|
+
out.set(body, String(len).length)
|
|
295
|
+
return out
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const padding = (size: number): Uint8Array => new Uint8Array((BLOCK - (size % BLOCK)) % BLOCK)
|
|
299
|
+
const padded = (size: number): number => size + ((BLOCK - (size % BLOCK)) % BLOCK)
|
|
300
|
+
|
|
301
|
+
/** Whether `name` needs a pax `path` record (fits neither ustar field). */
|
|
302
|
+
function needsPax(name: string): boolean {
|
|
303
|
+
const bytes = encoder.encode(name)
|
|
304
|
+
return bytes.byteLength > 100 && splitForUstar(bytes) === null
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** The exact number of bytes `tarPack` writes for these inputs. */
|
|
308
|
+
export function tarSize(inputs: readonly TarInput[]): number {
|
|
309
|
+
let size = BLOCK * 2
|
|
310
|
+
for (const i of inputs) {
|
|
311
|
+
if (needsPax(i.name)) size += BLOCK + padded(paxRecord('path', i.name).byteLength)
|
|
312
|
+
size += BLOCK + padded(i.size)
|
|
313
|
+
}
|
|
314
|
+
return size
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Pack regular files as a tar stream: ustar with the name/prefix split,
|
|
319
|
+
* a pax `path` record when a name fits neither field — exactly the
|
|
320
|
+
* dialect `tarEntries` reads — and the two-block end marker. Bodies are
|
|
321
|
+
* streamed, so memory is bounded by a chunk, never by an entry.
|
|
322
|
+
*/
|
|
323
|
+
export async function* tarPack(
|
|
324
|
+
inputs: AsyncIterable<TarInput> | Iterable<TarInput>,
|
|
325
|
+
): AsyncGenerator<Uint8Array> {
|
|
326
|
+
for await (const input of inputs) {
|
|
327
|
+
const mode = input.mode ?? 0o644
|
|
328
|
+
const mtime = input.mtime ?? 0
|
|
329
|
+
// Under a pax record the ustar name is a courtesy for readers that
|
|
330
|
+
// ignore pax: its first 100 BYTES, never characters — a multibyte
|
|
331
|
+
// name sliced by characters can be over 100 bytes again.
|
|
332
|
+
let headerName: string | Uint8Array = input.name
|
|
333
|
+
if (needsPax(input.name)) {
|
|
334
|
+
const pax = paxRecord('path', input.name)
|
|
335
|
+
yield header('PaxHeaders/entry', pax.byteLength, 'x', 0o644, mtime)
|
|
336
|
+
yield pax
|
|
337
|
+
yield padding(pax.byteLength)
|
|
338
|
+
headerName = encoder.encode(input.name).subarray(0, 100)
|
|
339
|
+
}
|
|
340
|
+
yield header(headerName, input.size, '0', mode, mtime)
|
|
341
|
+
if (input.body instanceof Blob) {
|
|
342
|
+
let n = 0
|
|
343
|
+
for await (const chunk of input.body.stream()) {
|
|
344
|
+
n += chunk.byteLength
|
|
345
|
+
yield chunk
|
|
346
|
+
}
|
|
347
|
+
if (n !== input.size)
|
|
348
|
+
throw new TarFormatError(`${input.name}: ${n} bytes read, ${input.size} declared`)
|
|
349
|
+
} else {
|
|
350
|
+
const bytes = typeof input.body === 'string' ? encoder.encode(input.body) : input.body
|
|
351
|
+
if (bytes.byteLength !== input.size)
|
|
352
|
+
throw new TarFormatError(`${input.name}: ${bytes.byteLength} bytes, ${input.size} declared`)
|
|
353
|
+
yield bytes
|
|
354
|
+
}
|
|
355
|
+
yield padding(input.size)
|
|
356
|
+
}
|
|
357
|
+
yield new Uint8Array(BLOCK * 2)
|
|
358
|
+
}
|
package/src/cli/cache.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Cache } from '../cache/index.js'
|
|
2
|
-
import { parseSize } from '../util/index.js'
|
|
2
|
+
import { parseDecimalInt, parseSize } from '../util/index.js'
|
|
3
3
|
import { findWorkspaceRoot, loadWorkspaceConfig, resolveCacheDir } from '../workspace/index.js'
|
|
4
4
|
import { formatBytes } from './format.js'
|
|
5
5
|
|
|
@@ -31,17 +31,35 @@ export function parsePruneArgs(args: readonly string[]): PruneArgs {
|
|
|
31
31
|
const out: PruneArgs = {}
|
|
32
32
|
for (let i = 0; i < args.length; i++) {
|
|
33
33
|
const a = args[i]
|
|
34
|
-
if (a === '--older-than') {
|
|
35
|
-
const v = args[++i]
|
|
36
|
-
if (v === undefined
|
|
34
|
+
if (a === '--older-than' || a?.startsWith('--older-than=')) {
|
|
35
|
+
const v = a === '--older-than' ? args[++i] : a.slice('--older-than='.length)
|
|
36
|
+
if (v === undefined || v === '') {
|
|
37
|
+
return { error: `--older-than requires a value (e.g. 30d, 24h, 60m)` }
|
|
38
|
+
}
|
|
37
39
|
const ms = parseDuration(v)
|
|
38
40
|
if (ms === null) return { error: `invalid duration: ${v}` }
|
|
41
|
+
// `0d` evicts everything ever cached. That is far more often a
|
|
42
|
+
// computed-to-zero retention than an intent, and no other flag
|
|
43
|
+
// combination expresses "wipe the cache" — so refuse it and name
|
|
44
|
+
// the deliberate way to do it.
|
|
45
|
+
if (ms === 0) {
|
|
46
|
+
return {
|
|
47
|
+
error: `--older-than 0 would evict every entry — delete the cache directory instead`,
|
|
48
|
+
}
|
|
49
|
+
}
|
|
39
50
|
out.olderThanMs = Date.now() - ms
|
|
40
|
-
} else if (a === '--max-size') {
|
|
41
|
-
const v = args[++i]
|
|
42
|
-
if (v === undefined
|
|
51
|
+
} else if (a === '--max-size' || a?.startsWith('--max-size=')) {
|
|
52
|
+
const v = a === '--max-size' ? args[++i] : a.slice('--max-size='.length)
|
|
53
|
+
if (v === undefined || v === '') {
|
|
54
|
+
return { error: `--max-size requires a value (e.g. 500M, 1G)` }
|
|
55
|
+
}
|
|
43
56
|
const bytes = parseSize(v)
|
|
44
57
|
if (bytes === null) return { error: `invalid size: ${v}` }
|
|
58
|
+
if (bytes === 0) {
|
|
59
|
+
return {
|
|
60
|
+
error: `--max-size 0 would evict every entry — delete the cache directory instead`,
|
|
61
|
+
}
|
|
62
|
+
}
|
|
45
63
|
out.maxBytes = bytes
|
|
46
64
|
} else {
|
|
47
65
|
return { error: `unknown argument: ${a}` }
|
|
@@ -86,11 +104,19 @@ async function pruneCmd(args: readonly string[]): Promise<number> {
|
|
|
86
104
|
return 0
|
|
87
105
|
}
|
|
88
106
|
|
|
107
|
+
/**
|
|
108
|
+
* Parse `<n><unit>` where unit is s/m/h/d, case-insensitively (`parseSize`
|
|
109
|
+
* has always been case-insensitive; this matched only lowercase, so `30D`
|
|
110
|
+
* was rejected while `1GB` and `1gb` both worked).
|
|
111
|
+
*/
|
|
89
112
|
export function parseDuration(input: string): number | null {
|
|
90
|
-
const m = input.match(/^(\d+)([smhd])$/)
|
|
113
|
+
const m = input.match(/^(\d+)([smhd])$/i)
|
|
91
114
|
if (!m) return null
|
|
92
|
-
const n =
|
|
93
|
-
|
|
115
|
+
const n = parseDecimalInt(m[1]!)
|
|
116
|
+
if (n === null) return null
|
|
117
|
+
// Lowercase before the switch: with the /i flag an uppercase `M` would
|
|
118
|
+
// otherwise fall through to the days branch.
|
|
119
|
+
const unit = m[2]!.toLowerCase()
|
|
94
120
|
const mult = unit === 's' ? 1000 : unit === 'm' ? 60_000 : unit === 'h' ? 3_600_000 : 86_400_000
|
|
95
121
|
return n * mult
|
|
96
122
|
}
|