@vzn/vx 0.0.15 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +91 -130
  2. package/index.ts +10 -0
  3. package/launcher.mjs +10 -11
  4. package/package.json +19 -5
  5. package/plugins/local-cache/index.ts +2 -0
  6. package/plugins/local-executor/index.ts +2 -0
  7. package/plugins/schedule-history/index.ts +2 -0
  8. package/src/bin.ts +2 -2
  9. package/src/cache/archive.ts +557 -0
  10. package/src/cache/cache.ts +929 -250
  11. package/src/cache/cas-backend.ts +20 -2
  12. package/src/cache/chained-cache.ts +202 -0
  13. package/src/cache/digest.ts +0 -15
  14. package/src/cache/index.ts +13 -3
  15. package/src/cache/inputs.ts +445 -41
  16. package/src/cache/layered-cache.ts +129 -30
  17. package/src/cache/tar-stream.ts +358 -0
  18. package/src/cli/cache.ts +36 -10
  19. package/src/cli/help.ts +167 -128
  20. package/src/cli/index.ts +74 -43
  21. package/src/cli/info.ts +30 -0
  22. package/src/cli/last.ts +140 -0
  23. package/src/cli/migrate-emit.ts +21 -0
  24. package/src/cli/migrate-nx.ts +23 -0
  25. package/src/cli/migrate-scripts.ts +127 -0
  26. package/src/cli/migrate-turbo.ts +43 -6
  27. package/src/cli/migrate.ts +119 -33
  28. package/src/cli/plan-format.ts +58 -3
  29. package/src/cli/plugin-commands.ts +96 -0
  30. package/src/cli/prune.ts +291 -0
  31. package/src/cli/run.ts +272 -80
  32. package/src/cli/upgrade.ts +7 -6
  33. package/src/cli/watch.ts +259 -61
  34. package/src/cli/why.ts +193 -0
  35. package/src/config.ts +40 -23
  36. package/src/exec/env.ts +7 -1
  37. package/src/exec/executor.ts +239 -0
  38. package/src/exec/index.ts +19 -1
  39. package/src/exec/runner.ts +116 -29
  40. package/src/exec/sandbox-runtime.ts +194 -24
  41. package/src/graph/index.ts +3 -1
  42. package/src/graph/scheduler.ts +92 -18
  43. package/src/graph/task-graph.ts +220 -6
  44. package/src/index.ts +82 -84
  45. package/src/orchestrator/deferred-outputs.ts +144 -0
  46. package/src/orchestrator/download-policy.ts +171 -0
  47. package/src/orchestrator/events.ts +30 -0
  48. package/src/orchestrator/execute-task.ts +328 -94
  49. package/src/orchestrator/failure-mode.ts +61 -0
  50. package/src/orchestrator/framed-output.ts +87 -35
  51. package/src/orchestrator/history.ts +78 -31
  52. package/src/orchestrator/index.ts +43 -90
  53. package/src/orchestrator/local-shortcircuit.ts +22 -0
  54. package/src/orchestrator/logger.ts +196 -14
  55. package/src/orchestrator/metrics.ts +57 -1376
  56. package/src/orchestrator/missing-plugin.ts +9 -0
  57. package/src/orchestrator/options.ts +22 -4
  58. package/src/orchestrator/plan.ts +144 -1
  59. package/src/orchestrator/plugin-host.ts +199 -119
  60. package/src/orchestrator/plugin.ts +144 -57
  61. package/src/orchestrator/prepare.ts +220 -69
  62. package/src/orchestrator/remote-prefetch.ts +27 -8
  63. package/src/orchestrator/resources.ts +20 -34
  64. package/src/orchestrator/run-artifacts.ts +41 -16
  65. package/src/orchestrator/run-context.ts +167 -29
  66. package/src/orchestrator/run-report.ts +73 -46
  67. package/src/orchestrator/run.ts +590 -122
  68. package/src/orchestrator/stable-keys.ts +74 -33
  69. package/src/orchestrator/status-line.ts +47 -8
  70. package/src/orchestrator/summary.ts +22 -1
  71. package/src/orchestrator/tally.ts +78 -35
  72. package/src/orchestrator/task-hash.ts +105 -11
  73. package/src/orchestrator/task-log-buffer.ts +291 -0
  74. package/src/orchestrator/telemetry-host.ts +24 -6
  75. package/src/orchestrator/telemetry.ts +181 -21
  76. package/src/orchestrator/upstream.ts +72 -31
  77. package/src/orchestrator/verify.ts +12 -2
  78. package/src/plugins/local-cache/index.ts +11 -0
  79. package/src/plugins/local-executor/index.ts +45 -0
  80. package/src/plugins/schedule-history/index.ts +109 -0
  81. package/src/util/edit-distance.ts +21 -0
  82. package/src/util/errors.ts +15 -1
  83. package/src/util/index.ts +7 -2
  84. package/src/util/num.ts +37 -0
  85. package/src/util/paths.ts +43 -0
  86. package/src/util/settle.ts +48 -0
  87. package/src/util/size.ts +5 -1
  88. package/src/util/tail.ts +66 -0
  89. package/src/util/timing.ts +63 -0
  90. package/src/workspace/affected.ts +142 -15
  91. package/src/workspace/config-cache.ts +304 -0
  92. package/src/workspace/config-eval.ts +210 -0
  93. package/src/workspace/config-imports.ts +161 -0
  94. package/src/workspace/filter.ts +22 -4
  95. package/src/workspace/fingerprint.ts +16 -1
  96. package/src/workspace/index.ts +15 -3
  97. package/src/workspace/lockfile.ts +13 -2
  98. package/src/workspace/nested-dirs.ts +21 -7
  99. package/src/workspace/package-graph.ts +23 -20
  100. package/src/workspace/project-loader.ts +503 -59
  101. package/src/workspace/workspace.ts +213 -72
  102. package/src/cache/tar.ts +0 -343
  103. package/src/cli/backend.ts +0 -23
  104. package/src/cli/mcp-rpc.ts +0 -292
  105. package/src/cli/mcp.ts +0 -92
  106. package/src/orchestrator/devframe-surface.ts +0 -74
  107. package/src/orchestrator/predict.ts +0 -106
  108. package/src/orchestrator/protocol.ts +0 -156
  109. package/src/orchestrator/run-state.ts +0 -145
  110. package/src/orchestrator/wire-render.ts +0 -92
  111. package/src/orchestrator/wire.ts +0 -212
@@ -23,6 +23,7 @@ import type {
23
23
  CacheLayer,
24
24
  CachePolicy,
25
25
  CacheStats,
26
+ CacheStatsOptions,
26
27
  IngestMeta,
27
28
  InvocationRecord,
28
29
  OutputFileRow,
@@ -31,7 +32,7 @@ import type {
31
32
  RunRecord,
32
33
  SaveArgs,
33
34
  } from './cache.js'
34
- import { FULL_CACHE_POLICY } from './cache.js'
35
+ import { FULL_CACHE_POLICY, type OutputDirRow } from './cache.js'
35
36
 
36
37
  /**
37
38
  * What a remote cache layer must provide — THE plugin seam for remote
@@ -47,6 +48,15 @@ import { FULL_CACHE_POLICY } from './cache.js'
47
48
  export interface RemoteCacheLayer {
48
49
  /** Existence probe (drives the plan path's `--dry` remote prediction). */
49
50
  has(hash: string): Promise<boolean>
51
+ /**
52
+ * Optional batch existence probe: given N hashes, return the subset stored
53
+ * remotely in ONE round-trip. Lets the prefetch pass collapse N per-hash
54
+ * network probes into one, then fetch only the hits. A remote that can't
55
+ * batch omits this method (or returns `null`) and the layer falls back to
56
+ * the per-hash path. Never throws for control flow — `null` means "no batch
57
+ * info; use per-hash".
58
+ */
59
+ hasMany?(hashes: readonly string[]): Promise<Set<string> | null>
50
60
  /** Fetch an artifact's bytes; `null` = miss. `durationMs` is the
51
61
  * producing task's duration when the wire carries it. */
52
62
  get(hash: string): Promise<{ body: ArrayBuffer; durationMs: number | undefined } | null>
@@ -73,6 +83,9 @@ export interface LayeredCacheOptions {
73
83
  }
74
84
 
75
85
  export class LayeredCache implements CacheLayer {
86
+ /** A remote layer is composed in by construction — see `CacheLayer.hasRemote`. */
87
+ readonly hasRemote = true
88
+
76
89
  /**
77
90
  * In-flight remote pulls keyed by hash. `prefetch` and `get` both go
78
91
  * through here, so a key probed concurrently by both resolves a
@@ -109,7 +122,7 @@ export class LayeredCache implements CacheLayer {
109
122
  private readonly policy: CachePolicy
110
123
 
111
124
  constructor(
112
- private readonly local: Cache,
125
+ readonly local: Cache,
113
126
  private readonly remote: RemoteCacheLayer,
114
127
  private readonly options: LayeredCacheOptions = {},
115
128
  ) {
@@ -123,9 +136,45 @@ export class LayeredCache implements CacheLayer {
123
136
  async prefetch(hash: string, ctx?: CacheGetContext): Promise<boolean> {
124
137
  // No-op when remote reads are off — there's nothing to warm from.
125
138
  if (!this.policy.remoteRead) return false
139
+ // The local-first skip lives in doPullFromRemote (the shared choke
140
+ // point): pullFromRemote registers the `inflight` entry SYNCHRONOUSLY,
141
+ // so a concurrent markRemoteAbsent can't clobber a pending pull — a
142
+ // guard done here (behind an async local.has) would reopen that race.
126
143
  return await this.pullFromRemote(hash, ctx)
127
144
  }
128
145
 
146
+ /**
147
+ * Batch existence probe over the remote layer — the subset of `hashes`
148
+ * present remotely, in one round-trip, or `null` when the remote can't
149
+ * batch (no `hasMany`, reads disabled, or an error). The prefetch pass uses
150
+ * this to fetch only the hits and to pre-mark the misses (`markRemoteAbsent`)
151
+ * so their lazy `get` skips the network. Never throws — a batch failure
152
+ * degrades to "no batch info" and the caller falls back to per-hash.
153
+ */
154
+ async remoteHasMany(hashes: readonly string[]): Promise<Set<string> | null> {
155
+ if (!this.policy.remoteRead || this.remote.hasMany === undefined) return null
156
+ try {
157
+ return await this.remote.hasMany(hashes)
158
+ } catch (err) {
159
+ this.reportRemoteError(err)
160
+ return null
161
+ }
162
+ }
163
+
164
+ /**
165
+ * Record that the remote layer has NO artifact for each of `hashes` (from a
166
+ * batch probe), so a later `get`/`prefetch` short-circuits to a miss WITHOUT
167
+ * a network round-trip. Byte-for-byte equivalent to a background prefetch GET
168
+ * having resolved `false` into `inflight` — same at-most-once semantics, same
169
+ * point-in-time staleness window — but without spending the GET. Never
170
+ * overwrites an entry already in flight.
171
+ */
172
+ markRemoteAbsent(hashes: Iterable<string>): void {
173
+ for (const hash of hashes) {
174
+ if (!this.inflight.has(hash)) this.inflight.set(hash, Promise.resolve(false))
175
+ }
176
+ }
177
+
129
178
  async get(hash: string, ctx?: CacheGetContext): Promise<CacheEntry | null> {
130
179
  const localHit = await this.local.get(hash, ctx)
131
180
  if (localHit) {
@@ -147,11 +196,23 @@ export class LayeredCache implements CacheLayer {
147
196
  const ingested = await this.pullFromRemote(hash, ctx)
148
197
  if (!ingested) return null
149
198
 
150
- // The artifact is now in local, but this *lookup* was a remote
151
- // hit — flip the source so callers can distinguish "saved work
152
- // via the remote cache" from "saved work via a prior local run".
153
- const materialized = await this.local.get(hash, ctx)
154
- return materialized ? { ...materialized, source: 'remote' } : null
199
+ // Read past the local READ gate: `ingest` is deliberately ungated, so
200
+ // the artifact + index row this pull just wrote exist regardless of
201
+ // policy, and the gate ("don't serve hits from the pre-existing local
202
+ // cache") must not discard them. Going through the gated `get` here
203
+ // made `--cache=local:,remote:rw` download the artifact, throw the hit
204
+ // away, re-execute and re-upload — on every run, forever.
205
+ const materialized = await this.local.getIngested(hash)
206
+ if (!materialized) return null
207
+
208
+ // `remoteSourced` — not merely "the pull returned true" — is what says
209
+ // the remote cache saved this work. `doPullFromRemote` also returns true
210
+ // for its local-first skip, which issues NO remote GET: when a run
211
+ // sharing this cache dir ingests the artifact between the local read
212
+ // above and that skip's `local.has`, stamping 'remote' here reported a
213
+ // purely-local hit as `cache-hit-remote` and inflated the remote
214
+ // hit-rate. Whatever provenance the entry has is the truth.
215
+ return this.remoteSourced.has(hash) ? { ...materialized, source: 'remote' } : materialized
155
216
  }
156
217
 
157
218
  // Existence probe: local first, then a remote HEAD — no body
@@ -185,6 +246,16 @@ export class LayeredCache implements CacheLayer {
185
246
  }
186
247
 
187
248
  private async doPullFromRemote(hash: string, ctx?: CacheGetContext): Promise<boolean> {
249
+ // Local-first, mirroring get()/has(): if local ALREADY holds the artifact
250
+ // there is nothing to pull — skip the remote GET (a warm-local run would
251
+ // otherwise re-download every artifact it already has) and DON'T mark it
252
+ // `remoteSourced` (that would mislabel a purely-local warm hit as
253
+ // cache-hit-remote and inflate the remote hit-rate). Returning `true` is
254
+ // correct for a get() read-through too: "the artifact is in local" — the
255
+ // caller re-reads it, keeping whatever provenance it already had (local,
256
+ // or remote if a concurrent prefetch set it).
257
+ if ((await this.local.has(hash)) === 'local') return true
258
+
188
259
  let remoteResult
189
260
  try {
190
261
  remoteResult = await this.remote.get(hash)
@@ -233,32 +304,48 @@ export class LayeredCache implements CacheLayer {
233
304
  }
234
305
 
235
306
  async save(args: SaveArgs): Promise<void> {
236
- // Local write honors its own gate inside Cache.save (no-op when
237
- // local writes are disabled).
307
+ // Local write honors its own gates inside Cache.save (no-op when local
308
+ // writes are disabled, or when ChainedCache marked this a duplicate save
309
+ // to a shared local handle — see `skipLocalWrite` on the contract).
238
310
  await this.local.save(args)
239
311
  if (!this.policy.remoteWrite) return
240
- // Write-through upload, OFF the task's critical path. The bytes are
241
- // captured NOW — read from the artifact the local layer just wrote
242
- // (same format on both sides, no repacking), or, when local writes
243
- // are disabled (`--cache=local:,remote:rw`), packed in memory while
244
- // the output files are guaranteed still on disk. The PUT itself then
245
- // runs in the bounded background pool so the task's worker slot is
246
- // released immediately; `run()` awaits `drainUploads()` before
247
- // closing the cache. Errors are logged, not propagated: the task
248
- // already succeeded; we don't fail it on cache-server issues.
249
- let bytes: Uint8Array
250
- try {
251
- bytes = this.local.localWritesEnabled
252
- ? await Bun.file(this.local.outputsPath(args.hash)).bytes()
253
- : await this.local.packArtifactBytes(args)
254
- } catch (err) {
255
- this.reportRemoteError(err)
256
- return
257
- }
312
+ // Write-through upload, OFF the task's critical path: the PUT runs in
313
+ // the bounded background pool so the task's worker slot is released
314
+ // immediately, and `run()` awaits `drainUploads()` before closing the
315
+ // cache. Errors are logged, not propagated: the task already
316
+ // succeeded; we don't fail it on cache-server issues.
317
+ //
318
+ // The artifact bytes are read INSIDE the job, not here.
319
+ // UPLOAD_CONCURRENCY bounds sockets, not memory: a queued closure
320
+ // holding its own artifact keeps the WHOLE backlog resident, so peak
321
+ // RSS scaled with a run's total miss artifact bytes rather than with
322
+ // the pool — any cold monorepo run whose remote uploads slower than the
323
+ // build produces artifacts held every pending one at once. Reading in
324
+ // the job caps resident artifact bytes at UPLOAD_CONCURRENCY. The
325
+ // artifact is content-addressed and immutable, so a deferred read sees
326
+ // the same bytes; if a concurrent `vx cache prune` removed it first the
327
+ // read throws and this upload is skipped — the never-fail contract.
328
+ const hash = args.hash
258
329
  const durationMs = args.entry.durationMs
330
+ // Local writes disabled (`--cache=local:,remote:rw`): there is no
331
+ // on-disk artifact to read later, so the bytes must be packed NOW,
332
+ // while this task's output files are still on disk. Deferring THIS
333
+ // read would pack whatever the tree happens to hold when the job
334
+ // runs. Such a run keeps the old memory profile by necessity — the
335
+ // bytes exist nowhere else.
336
+ let packed: Uint8Array | undefined
337
+ if (!this.local.localWritesEnabled) {
338
+ try {
339
+ packed = await this.local.packArtifactBytes(args)
340
+ } catch (err) {
341
+ this.reportRemoteError(err)
342
+ return
343
+ }
344
+ }
259
345
  this.enqueueUpload(async () => {
260
346
  try {
261
- await this.remote.put(args.hash, bytes, { durationMs })
347
+ const bytes = packed ?? (await Bun.file(this.local.outputsPath(hash)).bytes())
348
+ await this.remote.put(hash, bytes, { durationMs })
262
349
  } catch (err) {
263
350
  this.reportRemoteError(err)
264
351
  }
@@ -311,6 +398,18 @@ export class LayeredCache implements CacheLayer {
311
398
  return this.local.isOutputsCurrent(projectDir, expected)
312
399
  }
313
400
 
401
+ recordOutputDirs(hash: string, projectDir: string, prefixes: readonly string[]): Promise<void> {
402
+ return this.local.recordOutputDirs(hash, projectDir, prefixes)
403
+ }
404
+
405
+ loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
406
+ return this.local.loadOutputDirsBatch(hashes)
407
+ }
408
+
409
+ outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
410
+ return this.local.outputDirsCurrent(projectDir, rows)
411
+ }
412
+
314
413
  recordRun(run: RunRecord): void {
315
414
  this.local.recordRun(run)
316
415
  }
@@ -323,8 +422,8 @@ export class LayeredCache implements CacheLayer {
323
422
  this.local.recordRunBundle(bundle)
324
423
  }
325
424
 
326
- stats(): CacheStats {
327
- return this.local.stats()
425
+ stats(opts?: CacheStatsOptions): CacheStats {
426
+ return this.local.stats(opts)
328
427
  }
329
428
 
330
429
  async prune(options: PruneOptions): Promise<PruneResult> {
@@ -0,0 +1,358 @@
1
+ // A streaming tar reader for the restore path. `Bun.Archive` needs the whole
2
+ // tar in memory and `.files()` copies every entry, so restoring a 150 MiB
3
+ // artifact peaked at 3.2× its size (measured 2026-09-03). This reads the
4
+ // tar as it streams out of the zstd decoder and hands each entry's body to
5
+ // the caller as chunks, so memory is bounded by one chunk, not the archive.
6
+ //
7
+ // Dialect: what libarchive (Bun.Archive) writes — ustar with the name/prefix
8
+ // split, pax extended headers (`x`: path, size) past ustar's limits — plus
9
+ // GNU long names (`L`) for foreign artifacts. Every header's checksum is
10
+ // verified; an archive that ends before its data does is an error, never a
11
+ // short entry. Non-regular entries are reported with their type and their
12
+ // bodies skipped; the caller decides what to materialise (nothing but `0`).
13
+
14
+ export interface TarEntry {
15
+ name: string
16
+ size: number
17
+ /** POSIX typeflag: '0' regular, '5' directory, '2' symlink, … */
18
+ type: string
19
+ /** Header mtime in ms (second precision — the sidecar carries the real one). */
20
+ mtimeMs: number
21
+ /** The entry's bytes, in stream order. Must be drained before the next entry. */
22
+ body: AsyncIterable<Uint8Array>
23
+ }
24
+
25
+ export class TarFormatError extends Error {
26
+ constructor(message: string) {
27
+ super(message)
28
+ this.name = 'TarFormatError'
29
+ }
30
+ }
31
+
32
+ const BLOCK = 512
33
+ const decoder = new TextDecoder()
34
+
35
+ function field(h: Uint8Array, off: number, len: number): string {
36
+ let end = off
37
+ while (end < off + len && h[end] !== 0) end++
38
+ return decoder.decode(h.subarray(off, end))
39
+ }
40
+
41
+ function octal(h: Uint8Array, off: number, len: number): number {
42
+ // GNU base-256 (high bit set) for sizes ≥ 8 GiB.
43
+ if (h[off]! & 0x80) {
44
+ let n = 0
45
+ for (let i = off + 1; i < off + len; i++) n = n * 256 + h[i]!
46
+ return n
47
+ }
48
+ const s = field(h, off, len).trim()
49
+ if (s === '') return 0
50
+ // Every digit, not the longest parseable prefix: `parseInt` would read
51
+ // a corrupt `5zz` as 5 and let the damage surface later, if at all.
52
+ if (!/^[0-7]+$/.test(s)) throw new TarFormatError(`bad octal field: ${JSON.stringify(s)}`)
53
+ return parseInt(s, 8)
54
+ }
55
+
56
+ function checksumOk(h: Uint8Array): boolean {
57
+ const stored = octal(h, 148, 8)
58
+ let sum = 0
59
+ for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 32 : h[i]!
60
+ return sum === stored
61
+ }
62
+
63
+ /** pax `x` body: `<len> <key>=<value>\n` records. */
64
+ function parsePax(body: Uint8Array): Map<string, string> {
65
+ const out = new Map<string, string>()
66
+ let i = 0
67
+ while (i < body.byteLength) {
68
+ let sp = i
69
+ while (sp < body.byteLength && body[sp] !== 32) sp++
70
+ const len = Number(decoder.decode(body.subarray(i, sp)))
71
+ if (!Number.isInteger(len) || len <= 0 || i + len > body.byteLength) {
72
+ throw new TarFormatError('malformed pax record')
73
+ }
74
+ const rec = decoder.decode(body.subarray(sp + 1, i + len - 1))
75
+ const eq = rec.indexOf('=')
76
+ if (eq > 0) out.set(rec.slice(0, eq), rec.slice(eq + 1))
77
+ i += len
78
+ }
79
+ return out
80
+ }
81
+
82
+ /** A byte source with exact-size reads over a stream of arbitrary chunks. */
83
+ class Source {
84
+ private chunks: Uint8Array[] = []
85
+ private head = 0
86
+ private done = false
87
+ constructor(private readonly reader: AsyncIterator<Uint8Array>) {}
88
+
89
+ private async fill(): Promise<boolean> {
90
+ if (this.done) return false
91
+ const { done, value } = await this.reader.next()
92
+ if (done === true || value === undefined) {
93
+ this.done = true
94
+ return false
95
+ }
96
+ if (value.byteLength > 0) this.chunks.push(value)
97
+ return true
98
+ }
99
+
100
+ /** Exactly `n` bytes, or null at a clean end (nothing buffered), or a TarFormatError mid-entry. */
101
+ async exact(n: number, what: string): Promise<Uint8Array | null> {
102
+ const out = new Uint8Array(n)
103
+ let got = 0
104
+ while (got < n) {
105
+ if (this.chunks.length === 0 && !(await this.fill())) {
106
+ if (got === 0) return null
107
+ throw new TarFormatError(`archive ends inside ${what} (${got} of ${n} bytes)`)
108
+ }
109
+ const c = this.chunks[0]!
110
+ const take = Math.min(n - got, c.byteLength - this.head)
111
+ out.set(c.subarray(this.head, this.head + take), got)
112
+ got += take
113
+ this.head += take
114
+ if (this.head === c.byteLength) {
115
+ this.chunks.shift()
116
+ this.head = 0
117
+ }
118
+ }
119
+ return out
120
+ }
121
+
122
+ /** Up to `n` bytes as they arrive (never more), for streaming a body. */
123
+ async *take(n: number, what: string): AsyncIterable<Uint8Array> {
124
+ let left = n
125
+ while (left > 0) {
126
+ if (this.chunks.length === 0 && !(await this.fill())) {
127
+ throw new TarFormatError(`archive ends inside ${what} (${n - left} of ${n} bytes)`)
128
+ }
129
+ const c = this.chunks[0]!
130
+ const take = Math.min(left, c.byteLength - this.head)
131
+ const piece = c.subarray(this.head, this.head + take)
132
+ left -= take
133
+ this.head += take
134
+ if (this.head === c.byteLength) {
135
+ this.chunks.shift()
136
+ this.head = 0
137
+ }
138
+ yield piece
139
+ }
140
+ }
141
+ }
142
+
143
+ /**
144
+ * Iterate a tar stream entry by entry. Each entry's `body` MUST be fully
145
+ * consumed (or the iterator drains it) before the next entry is read.
146
+ */
147
+ export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGenerator<TarEntry> {
148
+ const src = new Source(stream[Symbol.asyncIterator]())
149
+ let pendingPath: string | undefined
150
+ let pendingSize: number | undefined
151
+ let zeroBlocks = 0
152
+ for (;;) {
153
+ const h = await src.exact(BLOCK, 'a header')
154
+ if (h === null) {
155
+ if (zeroBlocks === 0) throw new TarFormatError('archive has no end-of-archive marker')
156
+ return
157
+ }
158
+ if (h.every((b) => b === 0)) {
159
+ zeroBlocks++
160
+ if (zeroBlocks >= 2) return
161
+ continue
162
+ }
163
+ zeroBlocks = 0
164
+ if (!checksumOk(h)) throw new TarFormatError('header checksum mismatch')
165
+ const type = h[156] === 0 ? '0' : String.fromCharCode(h[156]!)
166
+ const size = pendingSize ?? octal(h, 124, 12)
167
+ const mtimeMs = octal(h, 136, 12) * 1000
168
+ const padded = Math.ceil(size / BLOCK) * BLOCK
169
+ const magic = field(h, 257, 6)
170
+ const prefix = magic.startsWith('ustar') ? field(h, 345, 155) : ''
171
+ const rawName = field(h, 0, 100)
172
+ let name = pendingPath ?? (prefix ? `${prefix}/${rawName}` : rawName)
173
+ pendingPath = undefined
174
+ pendingSize = undefined
175
+
176
+ if (type === 'x' || type === 'L') {
177
+ // Extended header: applies to the NEXT entry only.
178
+ const body = await src.exact(padded, 'an extended header')
179
+ if (body === null) throw new TarFormatError('archive ends inside an extended header')
180
+ if (type === 'x') {
181
+ const pax = parsePax(body.subarray(0, size))
182
+ const p = pax.get('path')
183
+ if (p !== undefined) pendingPath = p
184
+ const s = pax.get('size')
185
+ if (s !== undefined) pendingSize = Number(s)
186
+ } else {
187
+ pendingPath = field(body, 0, size)
188
+ }
189
+ continue
190
+ }
191
+ if (type === 'g') {
192
+ // pax global header: not used by Bun.Archive; skip its body.
193
+ await src.exact(padded, 'a global header')
194
+ continue
195
+ }
196
+ name = name.replace(/\/+$/, type === '5' ? '/' : '')
197
+
198
+ let drained = false
199
+ const body = (async function* (): AsyncIterable<Uint8Array> {
200
+ if (size > 0) for await (const c of src.take(size, `entry ${name}`)) yield c
201
+ const pad = padded - size
202
+ if (pad > 0 && (await src.exact(pad, `padding after ${name}`)) === null) {
203
+ throw new TarFormatError(`archive ends inside padding after ${name}`)
204
+ }
205
+ drained = true
206
+ })()
207
+ yield { name, size, type, mtimeMs, body }
208
+ if (!drained) {
209
+ // The caller skipped this body: drain it so the next header lines up.
210
+ for await (const _ of body) {
211
+ // discard
212
+ }
213
+ }
214
+ }
215
+ }
216
+
217
+ // ─── Writer ───────────────────────────────────────────────────────────
218
+
219
+ /** One regular file to pack. `body` is read as it is written. */
220
+ export interface TarInput {
221
+ name: string
222
+ size: number
223
+ /** Permission bits; the sidecar carries the exact value, this is for foreign readers. */
224
+ mode?: number
225
+ /** Seconds since epoch; the sidecar carries the millisecond value. */
226
+ mtime?: number
227
+ body: Blob | Uint8Array | string
228
+ }
229
+
230
+ const encoder = new TextEncoder()
231
+
232
+ function writeOctal(h: Uint8Array, off: number, len: number, n: number): void {
233
+ const digits = n.toString(8).padStart(len - 1, '0')
234
+ if (digits.length > len - 1)
235
+ throw new TarFormatError(`value ${n} does not fit a ${len}-byte field`)
236
+ h.set(encoder.encode(digits), off)
237
+ h[off + len - 1] = 0
238
+ }
239
+
240
+ function header(
241
+ name: string | Uint8Array,
242
+ size: number,
243
+ type: string,
244
+ mode: number,
245
+ mtime: number,
246
+ ): Uint8Array {
247
+ const h = new Uint8Array(BLOCK)
248
+ const nameBytes = typeof name === 'string' ? encoder.encode(name) : name
249
+ if (nameBytes.byteLength <= 100) {
250
+ h.set(nameBytes, 0)
251
+ } else {
252
+ // ustar prefix split: the longest tail that fits 100 bytes, split at
253
+ // a `/`, with the head fitting 155. Anything else needs pax.
254
+ const cut = splitForUstar(nameBytes)
255
+ if (cut === null)
256
+ throw new TarFormatError(`name too long for ustar: ${decoder.decode(nameBytes)}`)
257
+ h.set(nameBytes.subarray(cut + 1), 0)
258
+ h.set(nameBytes.subarray(0, cut), 345)
259
+ }
260
+ writeOctal(h, 100, 8, mode & 0o7777)
261
+ writeOctal(h, 108, 8, 0)
262
+ writeOctal(h, 116, 8, 0)
263
+ writeOctal(h, 124, 12, size)
264
+ writeOctal(h, 136, 12, mtime)
265
+ h[156] = type.charCodeAt(0)
266
+ h.set(encoder.encode('ustar\0'), 257)
267
+ h.set(encoder.encode('00'), 263)
268
+ h.fill(32, 148, 156)
269
+ let sum = 0
270
+ for (let i = 0; i < BLOCK; i++) sum += h[i]!
271
+ h.set(encoder.encode(sum.toString(8).padStart(6, '0')), 148)
272
+ h[154] = 0
273
+ h[155] = 32
274
+ return h
275
+ }
276
+
277
+ /** Byte index of the `/` to split at, or null when no split satisfies both fields. */
278
+ function splitForUstar(name: Uint8Array): number | null {
279
+ for (let i = name.byteLength - 1; i > 0; i--) {
280
+ if (name[i] !== 47) continue
281
+ const tail = name.byteLength - i - 1
282
+ if (tail > 100) return null
283
+ if (tail >= 1 && i <= 155) return i
284
+ }
285
+ return null
286
+ }
287
+
288
+ function paxRecord(key: string, value: string): Uint8Array {
289
+ const body = encoder.encode(` ${key}=${value}\n`)
290
+ let len = body.byteLength + 1
291
+ while (String(len).length + body.byteLength !== len) len = String(len).length + body.byteLength
292
+ const out = new Uint8Array(len)
293
+ out.set(encoder.encode(String(len)), 0)
294
+ out.set(body, String(len).length)
295
+ return out
296
+ }
297
+
298
+ const padding = (size: number): Uint8Array => new Uint8Array((BLOCK - (size % BLOCK)) % BLOCK)
299
+ const padded = (size: number): number => size + ((BLOCK - (size % BLOCK)) % BLOCK)
300
+
301
+ /** Whether `name` needs a pax `path` record (fits neither ustar field). */
302
+ function needsPax(name: string): boolean {
303
+ const bytes = encoder.encode(name)
304
+ return bytes.byteLength > 100 && splitForUstar(bytes) === null
305
+ }
306
+
307
+ /** The exact number of bytes `tarPack` writes for these inputs. */
308
+ export function tarSize(inputs: readonly TarInput[]): number {
309
+ let size = BLOCK * 2
310
+ for (const i of inputs) {
311
+ if (needsPax(i.name)) size += BLOCK + padded(paxRecord('path', i.name).byteLength)
312
+ size += BLOCK + padded(i.size)
313
+ }
314
+ return size
315
+ }
316
+
317
+ /**
318
+ * Pack regular files as a tar stream: ustar with the name/prefix split,
319
+ * a pax `path` record when a name fits neither field — exactly the
320
+ * dialect `tarEntries` reads — and the two-block end marker. Bodies are
321
+ * streamed, so memory is bounded by a chunk, never by an entry.
322
+ */
323
+ export async function* tarPack(
324
+ inputs: AsyncIterable<TarInput> | Iterable<TarInput>,
325
+ ): AsyncGenerator<Uint8Array> {
326
+ for await (const input of inputs) {
327
+ const mode = input.mode ?? 0o644
328
+ const mtime = input.mtime ?? 0
329
+ // Under a pax record the ustar name is a courtesy for readers that
330
+ // ignore pax: its first 100 BYTES, never characters — a multibyte
331
+ // name sliced by characters can be over 100 bytes again.
332
+ let headerName: string | Uint8Array = input.name
333
+ if (needsPax(input.name)) {
334
+ const pax = paxRecord('path', input.name)
335
+ yield header('PaxHeaders/entry', pax.byteLength, 'x', 0o644, mtime)
336
+ yield pax
337
+ yield padding(pax.byteLength)
338
+ headerName = encoder.encode(input.name).subarray(0, 100)
339
+ }
340
+ yield header(headerName, input.size, '0', mode, mtime)
341
+ if (input.body instanceof Blob) {
342
+ let n = 0
343
+ for await (const chunk of input.body.stream()) {
344
+ n += chunk.byteLength
345
+ yield chunk
346
+ }
347
+ if (n !== input.size)
348
+ throw new TarFormatError(`${input.name}: ${n} bytes read, ${input.size} declared`)
349
+ } else {
350
+ const bytes = typeof input.body === 'string' ? encoder.encode(input.body) : input.body
351
+ if (bytes.byteLength !== input.size)
352
+ throw new TarFormatError(`${input.name}: ${bytes.byteLength} bytes, ${input.size} declared`)
353
+ yield bytes
354
+ }
355
+ yield padding(input.size)
356
+ }
357
+ yield new Uint8Array(BLOCK * 2)
358
+ }
package/src/cli/cache.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { Cache } from '../cache/index.js'
2
- import { parseSize } from '../util/index.js'
2
+ import { parseDecimalInt, parseSize } from '../util/index.js'
3
3
  import { findWorkspaceRoot, loadWorkspaceConfig, resolveCacheDir } from '../workspace/index.js'
4
4
  import { formatBytes } from './format.js'
5
5
 
@@ -31,17 +31,35 @@ export function parsePruneArgs(args: readonly string[]): PruneArgs {
31
31
  const out: PruneArgs = {}
32
32
  for (let i = 0; i < args.length; i++) {
33
33
  const a = args[i]
34
- if (a === '--older-than') {
35
- const v = args[++i]
36
- if (v === undefined) return { error: `${a} requires a value (e.g. 30d, 24h, 60m)` }
34
+ if (a === '--older-than' || a?.startsWith('--older-than=')) {
35
+ const v = a === '--older-than' ? args[++i] : a.slice('--older-than='.length)
36
+ if (v === undefined || v === '') {
37
+ return { error: `--older-than requires a value (e.g. 30d, 24h, 60m)` }
38
+ }
37
39
  const ms = parseDuration(v)
38
40
  if (ms === null) return { error: `invalid duration: ${v}` }
41
+ // `0d` evicts everything ever cached. That is far more often a
42
+ // computed-to-zero retention than an intent, and no other flag
43
+ // combination expresses "wipe the cache" — so refuse it and name
44
+ // the deliberate way to do it.
45
+ if (ms === 0) {
46
+ return {
47
+ error: `--older-than 0 would evict every entry — delete the cache directory instead`,
48
+ }
49
+ }
39
50
  out.olderThanMs = Date.now() - ms
40
- } else if (a === '--max-size') {
41
- const v = args[++i]
42
- if (v === undefined) return { error: `${a} requires a value (e.g. 500M, 1G)` }
51
+ } else if (a === '--max-size' || a?.startsWith('--max-size=')) {
52
+ const v = a === '--max-size' ? args[++i] : a.slice('--max-size='.length)
53
+ if (v === undefined || v === '') {
54
+ return { error: `--max-size requires a value (e.g. 500M, 1G)` }
55
+ }
43
56
  const bytes = parseSize(v)
44
57
  if (bytes === null) return { error: `invalid size: ${v}` }
58
+ if (bytes === 0) {
59
+ return {
60
+ error: `--max-size 0 would evict every entry — delete the cache directory instead`,
61
+ }
62
+ }
45
63
  out.maxBytes = bytes
46
64
  } else {
47
65
  return { error: `unknown argument: ${a}` }
@@ -86,11 +104,19 @@ async function pruneCmd(args: readonly string[]): Promise<number> {
86
104
  return 0
87
105
  }
88
106
 
107
+ /**
108
+ * Parse `<n><unit>` where unit is s/m/h/d, case-insensitively (`parseSize`
109
+ * has always been case-insensitive; this matched only lowercase, so `30D`
110
+ * was rejected while `1GB` and `1gb` both worked).
111
+ */
89
112
  export function parseDuration(input: string): number | null {
90
- const m = input.match(/^(\d+)([smhd])$/)
113
+ const m = input.match(/^(\d+)([smhd])$/i)
91
114
  if (!m) return null
92
- const n = Number(m[1])
93
- const unit = m[2]
115
+ const n = parseDecimalInt(m[1]!)
116
+ if (n === null) return null
117
+ // Lowercase before the switch: with the /i flag an uppercase `M` would
118
+ // otherwise fall through to the days branch.
119
+ const unit = m[2]!.toLowerCase()
94
120
  const mult = unit === 's' ? 1000 : unit === 'm' ? 60_000 : unit === 'h' ? 3_600_000 : 86_400_000
95
121
  return n * mult
96
122
  }