@vzn/vx 0.0.631 → 0.0.633
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -18
- package/docs/caching.md +2247 -0
- package/docs/cli.md +3214 -0
- package/docs/execution.md +708 -0
- package/docs/features.md +172 -0
- package/docs/patterns.md +219 -0
- package/docs/schema.md +2137 -0
- package/docs/security.md +209 -0
- package/package.json +14 -5
- package/schemas/cache.json +24 -0
- package/schemas/docs.json +28 -0
- package/schemas/error.json +23 -0
- package/schemas/info.json +147 -0
- package/schemas/init.json +57 -0
- package/schemas/last.json +188 -0
- package/schemas/lock.json +21 -0
- package/schemas/plan.json +88 -0
- package/schemas/show.json +124 -0
- package/schemas/summary.json +133 -0
- package/schemas/why.json +168 -0
- package/skills/vx/SKILL.md +89 -0
- package/src/bin.ts +9 -0
- package/src/cache/archive.ts +153 -51
- package/src/cache/cache.ts +542 -137
- package/src/cache/chained-cache.ts +9 -0
- package/src/cache/inputs.ts +5 -0
- package/src/cache/layer.ts +10 -5
- package/src/cache/layered-cache.ts +4 -0
- package/src/cache/schema.ts +44 -17
- package/src/cache/tar-stream.ts +204 -66
- package/src/cache/zstd.ts +1 -1
- package/src/cli/cache.ts +14 -14
- package/src/cli/docs-corpus.ts +21 -0
- package/src/cli/docs-text.d.ts +5 -0
- package/src/cli/docs.ts +167 -0
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/help.ts +42 -7
- package/src/cli/index.ts +25 -11
- package/src/cli/info.ts +38 -9
- package/src/cli/init.ts +131 -24
- package/src/cli/last.ts +54 -3
- package/src/cli/lock.ts +43 -15
- package/src/cli/plan-format.ts +23 -1
- package/src/cli/plugin-commands.ts +3 -0
- package/src/cli/plugin-templates.ts +1 -1
- package/src/cli/run.ts +40 -21
- package/src/cli/show.ts +10 -7
- package/src/cli/upgrade.ts +19 -7
- package/src/cli/watch.ts +7 -2
- package/src/cli/why.ts +14 -40
- package/src/exec/runner.ts +27 -10
- package/src/graph/scheduler.ts +36 -0
- package/src/graph/task-graph.ts +4 -4
- package/src/index.ts +22 -5
- package/src/orchestrator/admission.ts +29 -1
- package/src/orchestrator/affected-tasks.ts +87 -18
- package/src/orchestrator/doctor.ts +14 -1
- package/src/orchestrator/execute-task.ts +149 -64
- package/src/orchestrator/failure-mode.ts +5 -3
- package/src/orchestrator/framed-output.ts +17 -3
- package/src/orchestrator/index.ts +11 -0
- package/src/orchestrator/logger.ts +57 -6
- package/src/orchestrator/metrics.ts +76 -4
- package/src/orchestrator/miss-reason.ts +3 -5
- package/src/orchestrator/miss-save.ts +6 -5
- package/src/orchestrator/options.ts +12 -0
- package/src/orchestrator/output-log.ts +14 -1
- package/src/orchestrator/path-links.ts +79 -0
- package/src/orchestrator/plan.ts +20 -3
- package/src/orchestrator/plugin.ts +7 -0
- package/src/orchestrator/prepare.ts +14 -4
- package/src/orchestrator/projects.ts +2 -2
- package/src/orchestrator/run-artifacts.ts +18 -2
- package/src/orchestrator/run-failures.ts +192 -0
- package/src/orchestrator/run-records.ts +10 -8
- package/src/orchestrator/run.ts +87 -22
- package/src/orchestrator/task-hash.ts +4 -4
- package/src/orchestrator/terminal-signals.ts +16 -0
- package/src/orchestrator/upstream.ts +65 -10
- package/src/util/errors.ts +55 -1
- package/src/util/index.ts +3 -0
- package/src/util/secret-mask.ts +13 -1
- package/src/util/verbs.ts +1 -0
- package/src/workspace/config-schema.ts +10 -8
- package/src/workspace/migration.ts +30 -3
- package/src/workspace/workspace.ts +1 -0
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// (item 1020). A save that fails in EVERY layer still throws, so the
|
|
13
13
|
// caller's own "cache save failed" line stays the one word on it.
|
|
14
14
|
|
|
15
|
+
import { statSync } from 'node:fs'
|
|
15
16
|
import type {
|
|
16
17
|
Cache,
|
|
17
18
|
CacheEntry,
|
|
@@ -248,6 +249,14 @@ export class ChainedCache implements CacheLayer {
|
|
|
248
249
|
return this.owner(hash).outputsPath(hash)
|
|
249
250
|
}
|
|
250
251
|
|
|
252
|
+
/** A layer that cannot say keeps a file at `outputsPath`, or nothing. */
|
|
253
|
+
artifactSize(hash: string): number | undefined {
|
|
254
|
+
const owner = this.owner(hash)
|
|
255
|
+
return owner.artifactSize !== undefined
|
|
256
|
+
? owner.artifactSize(hash)
|
|
257
|
+
: statSync(owner.outputsPath(hash), { throwIfNoEntry: false })?.size
|
|
258
|
+
}
|
|
259
|
+
|
|
251
260
|
prune(options: PruneOptions): Promise<PruneResult> {
|
|
252
261
|
return this.layers[0]!.prune(options)
|
|
253
262
|
}
|
package/src/cache/inputs.ts
CHANGED
|
@@ -92,6 +92,11 @@ const ALWAYS_IGNORE = [
|
|
|
92
92
|
// extracts the downloaded runtime into, `<cwd>/.<16 hex>-<8 hex>.tmp/`,
|
|
93
93
|
// 60 MB mid-write under the same race (2026-09-29).
|
|
94
94
|
'**/.????????????????-????????.tmp/**',
|
|
95
|
+
// vx's own restore temp (archive.ts), live only while an artifact lands
|
|
96
|
+
// beside its target. A key taken meanwhile listed it, the rename removed
|
|
97
|
+
// it, and every task reading that directory was withheld its save
|
|
98
|
+
// (withastro/astro: 25 of 33 builds re-ran on a restore, X-209).
|
|
99
|
+
'**/.vx-tmp-*',
|
|
95
100
|
]
|
|
96
101
|
|
|
97
102
|
const DEFAULT_FILE_GLOBS: readonly string[] = ['**/*']
|
package/src/cache/layer.ts
CHANGED
|
@@ -215,7 +215,8 @@ export interface RunRecord {
|
|
|
215
215
|
/**
|
|
216
216
|
* Why the task failed or was skipped, as the run's own footer said it
|
|
217
217
|
* (v27 columns; NULL on older rows and where the reason does not apply):
|
|
218
|
-
* a skip's root blocker
|
|
218
|
+
* a skip's root blocker (or the one a task that ran behind a failure ran
|
|
219
|
+
* behind), vx's own timeout, the sandbox's violation count,
|
|
219
220
|
* and why a persistent task never became ready.
|
|
220
221
|
*/
|
|
221
222
|
blockedBy?: string
|
|
@@ -266,7 +267,7 @@ export interface InvocationRecord {
|
|
|
266
267
|
}
|
|
267
268
|
|
|
268
269
|
/**
|
|
269
|
-
* One cache-key component
|
|
270
|
+
* One cache-key component of an `entry_inputs` row (v34: one row per entry) — keyed by
|
|
270
271
|
* the cache-entry HASH it belongs to, not a run. Written inside the
|
|
271
272
|
* entry-save transaction (`writeArtifactAndIndex`) on a miss/save; a
|
|
272
273
|
* cache HIT does not save, so it persists nothing (warm runs are free).
|
|
@@ -729,12 +730,16 @@ export interface CacheLayer {
|
|
|
729
730
|
*/
|
|
730
731
|
hashFile(filePath: string): Promise<string>
|
|
731
732
|
/**
|
|
732
|
-
* Absolute path
|
|
733
|
+
* Absolute path a file-backed artifact for a hash has —
|
|
733
734
|
* `<cacheDir>/<hash>.tar.zst`. Returns the path whether or
|
|
734
|
-
* not the artifact exists
|
|
735
|
-
* `
|
|
735
|
+
* not the artifact exists: a small artifact lives in the index, not
|
|
736
|
+
* in a file (`artifactSize` is its size). Exposed for telemetry /
|
|
737
|
+
* dashboards; `restoreOutputs` is the canonical way to materialize
|
|
738
|
+
* the bytes.
|
|
736
739
|
*/
|
|
737
740
|
outputsPath(hash: string): string
|
|
741
|
+
/** The stored artifact's compressed size, wherever it lives; undefined when none is stored. */
|
|
742
|
+
artifactSize?(hash: string): number | undefined
|
|
738
743
|
prune(options: PruneOptions): Promise<PruneResult>
|
|
739
744
|
close(): void
|
|
740
745
|
}
|
|
@@ -529,6 +529,10 @@ export class LayeredCache implements CacheLayer {
|
|
|
529
529
|
return this.local.outputsPath(hash)
|
|
530
530
|
}
|
|
531
531
|
|
|
532
|
+
artifactSize(hash: string): number | undefined {
|
|
533
|
+
return this.local.artifactSize(hash)
|
|
534
|
+
}
|
|
535
|
+
|
|
532
536
|
hashFile(filePath: string): Promise<string> {
|
|
533
537
|
return this.local.hashFile(filePath)
|
|
534
538
|
}
|
package/src/cache/schema.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// The index's tables, one concern: what a cache.db holds. `Cache` opens the
|
|
2
2
|
// file, checks `SCHEMA_VERSION` and creates these; every statement against
|
|
3
3
|
// them lives in the store that owns the table (file-hashes, config-evals,
|
|
4
|
-
// output-index, history) or in cache.ts for `entries` and
|
|
4
|
+
// output-index, history) or in cache.ts for `entries`, `entry_inputs` and
|
|
5
|
+
// `artifacts`.
|
|
5
6
|
|
|
6
7
|
import type { Database, SQLQueryBindings } from 'bun:sqlite'
|
|
7
8
|
|
|
@@ -95,7 +96,8 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
95
96
|
-- time by design; NULL on rows older than the column.
|
|
96
97
|
cached INTEGER,
|
|
97
98
|
-- v27: why a task failed or was skipped, as the run's own footer
|
|
98
|
-
-- said it (items 267–270): a skip's root blocker (a task id),
|
|
99
|
+
-- said it (items 267–270): a skip's root blocker (a task id), or
|
|
100
|
+
-- the one a task that ran behind a failure ran behind (X-183), vx's
|
|
99
101
|
-- own timeout (1), the sandbox's violation count, and why a
|
|
100
102
|
-- persistent task never became ready ('timeout' | 'exited' |
|
|
101
103
|
-- 'spawn'). NULL where the reason does not apply.
|
|
@@ -254,22 +256,18 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
254
256
|
CREATE INDEX IF NOT EXISTS main.invocations_started ON invocations(started_at);
|
|
255
257
|
CREATE INDEX IF NOT EXISTS main.invocations_branch ON invocations(branch);
|
|
256
258
|
CREATE INDEX IF NOT EXISTS main.invocations_ci ON invocations(ci);
|
|
257
|
-
-- v22 (Tier 3): the input-fingerprint moat
|
|
258
|
-
--
|
|
259
|
-
--
|
|
260
|
-
--
|
|
261
|
-
--
|
|
262
|
-
--
|
|
263
|
-
--
|
|
264
|
-
--
|
|
265
|
-
--
|
|
266
|
-
-- rows in sync with entries (a prune sweeps them automatically).
|
|
259
|
+
-- v22 (Tier 3): the input-fingerprint moat: each cache ENTRY's key
|
|
260
|
+
-- components (NOT a run's). Written inside the entry-save transaction,
|
|
261
|
+
-- only on a miss/save, so a warm all-cache-hit run writes nothing
|
|
262
|
+
-- here. v34: one row per entry, the components a JSON array of
|
|
263
|
+
-- [kind, name, hash] in (kind, name) code-unit order. A row per
|
|
264
|
+
-- component cost a b-tree insert each, and a task with 50 input
|
|
265
|
+
-- files spent ~0.5 ms a save on them; every reader loads the whole
|
|
266
|
+
-- set by entry hash and diffs it app-side anyway. ON DELETE CASCADE
|
|
267
|
+
-- keeps it in sync with entries (a prune sweeps it automatically).
|
|
267
268
|
CREATE TABLE IF NOT EXISTS ${store}.entry_inputs (
|
|
268
|
-
entry_hash TEXT
|
|
269
|
-
|
|
270
|
-
name TEXT NOT NULL,
|
|
271
|
-
hash TEXT NOT NULL,
|
|
272
|
-
PRIMARY KEY (entry_hash, kind, name),
|
|
269
|
+
entry_hash TEXT PRIMARY KEY,
|
|
270
|
+
components TEXT NOT NULL,
|
|
273
271
|
FOREIGN KEY (entry_hash) REFERENCES entries(hash) ON DELETE CASCADE
|
|
274
272
|
);
|
|
275
273
|
-- v32: what belongs to the entries rather than to one workspace: the
|
|
@@ -279,9 +277,38 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
279
277
|
key TEXT PRIMARY KEY,
|
|
280
278
|
value TEXT NOT NULL
|
|
281
279
|
);
|
|
280
|
+
${artifactTables(store)}
|
|
282
281
|
`)
|
|
283
282
|
}
|
|
284
283
|
|
|
284
|
+
/**
|
|
285
|
+
* v33: an artifact of at most `INLINE_MAX` compressed bytes lives here, its
|
|
286
|
+
* exact `<hash>.tar.zst` bytes, written in the transaction that writes its
|
|
287
|
+
* rows. No foreign key and outside every drop a `SCHEMA_VERSION` reset makes:
|
|
288
|
+
* the artifact is the record and the entries only its inventory, so a reset
|
|
289
|
+
* drops the rows and keeps these, as it keeps the files. `artifacts_meta`
|
|
290
|
+
* holds the table's own layout version (`'layout'`), the one thing that may
|
|
291
|
+
* drop it. `at` is the last use a row-less sweep judges, as a file's mtime;
|
|
292
|
+
* it sits before `bytes` so reading it never walks a blob's overflow pages.
|
|
293
|
+
*/
|
|
294
|
+
export function createArtifactTables(db: Database, store: 'main' | 'store'): void {
|
|
295
|
+
db.exec(artifactTables(store))
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// In the entry tables' exec, so an open runs no statement more for them.
|
|
299
|
+
function artifactTables(store: 'main' | 'store'): string {
|
|
300
|
+
return `
|
|
301
|
+
CREATE TABLE IF NOT EXISTS ${store}.artifacts (
|
|
302
|
+
hash TEXT PRIMARY KEY,
|
|
303
|
+
at INTEGER NOT NULL,
|
|
304
|
+
bytes BLOB NOT NULL
|
|
305
|
+
) WITHOUT ROWID;
|
|
306
|
+
CREATE TABLE IF NOT EXISTS ${store}.artifacts_meta (
|
|
307
|
+
key TEXT PRIMARY KEY,
|
|
308
|
+
value TEXT NOT NULL
|
|
309
|
+
);`
|
|
310
|
+
}
|
|
311
|
+
|
|
285
312
|
/**
|
|
286
313
|
* A `WHERE <column> …` test against many hashes, as one statement whatever
|
|
287
314
|
* their number. A list of `?` is a new statement per length, compiled each
|
package/src/cache/tar-stream.ts
CHANGED
|
@@ -187,30 +187,45 @@ class Source {
|
|
|
187
187
|
}
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
+
/** What one header block means, decided by `TarDecoder.header`. */
|
|
191
|
+
type HeaderStep =
|
|
192
|
+
/** A zero block: the first half of the end marker. */
|
|
193
|
+
| { kind: 'zero' }
|
|
194
|
+
/** The second zero block in a row: the archive is over. */
|
|
195
|
+
| { kind: 'end' }
|
|
196
|
+
/** An extended header (`x`, `L`) or a global one (`g`); its padded body follows. */
|
|
197
|
+
| { kind: 'extended'; type: 'x' | 'L' | 'g'; size: number; padded: number }
|
|
198
|
+
/** An entry; its padded body follows. */
|
|
199
|
+
| { kind: 'entry'; name: string; size: number; type: string; mtimeMs: number; padded: number }
|
|
200
|
+
|
|
201
|
+
const ZERO: HeaderStep = { kind: 'zero' }
|
|
202
|
+
const END: HeaderStep = { kind: 'end' }
|
|
203
|
+
|
|
190
204
|
/**
|
|
191
|
-
*
|
|
192
|
-
*
|
|
205
|
+
* The dialect, decoded off whole blocks with no I/O: the one copy of the
|
|
206
|
+
* header rules both readers run — `tarEntries` over a stream (restore,
|
|
207
|
+
* ingest) and `tarEntriesSync` over a tar in memory (a save's own) — so
|
|
208
|
+
* a name a save indexes is the name a restore reads.
|
|
193
209
|
*/
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if (
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
210
|
+
class TarDecoder {
|
|
211
|
+
private pendingPath: string | undefined
|
|
212
|
+
private pendingSize: number | undefined
|
|
213
|
+
private zeroBlocks = 0
|
|
214
|
+
|
|
215
|
+
/** The bytes ended at a header boundary: clean only after a zero block. */
|
|
216
|
+
eof(): void {
|
|
217
|
+
if (this.zeroBlocks === 0) throw new TarFormatError('archive has no end-of-archive marker')
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
header(h: Uint8Array): HeaderStep {
|
|
205
221
|
if (isZeroBlock(h)) {
|
|
206
|
-
zeroBlocks++
|
|
207
|
-
|
|
208
|
-
continue
|
|
222
|
+
this.zeroBlocks++
|
|
223
|
+
return this.zeroBlocks >= 2 ? END : ZERO
|
|
209
224
|
}
|
|
210
|
-
zeroBlocks = 0
|
|
225
|
+
this.zeroBlocks = 0
|
|
211
226
|
if (!checksumOk(h)) throw new TarFormatError('header checksum mismatch')
|
|
212
227
|
const type = h[156] === 0 ? '0' : String.fromCharCode(h[156]!)
|
|
213
|
-
const size = pendingSize ?? octal(h, 124, 12)
|
|
228
|
+
const size = this.pendingSize ?? octal(h, 124, 12)
|
|
214
229
|
const mtimeMs = octal(h, 136, 12) * 1000
|
|
215
230
|
const padded = Math.ceil(size / BLOCK) * BLOCK
|
|
216
231
|
// POSIX `ustar\0` only: old GNU's `ustar ` keeps atime and ctime at
|
|
@@ -218,44 +233,67 @@ export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGene
|
|
|
218
233
|
// `<atime>…/<name>`.
|
|
219
234
|
const prefix = field(h, 257, 6) === 'ustar' ? field(h, 345, 155) : ''
|
|
220
235
|
const rawName = field(h, 0, 100)
|
|
221
|
-
|
|
222
|
-
pendingPath = undefined
|
|
223
|
-
pendingSize = undefined
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
236
|
+
const name = this.pendingPath ?? (prefix ? `${prefix}/${rawName}` : rawName)
|
|
237
|
+
this.pendingPath = undefined
|
|
238
|
+
this.pendingSize = undefined
|
|
239
|
+
if (type === 'x' || type === 'L' || type === 'g') {
|
|
240
|
+
if (size > MAX_EXTENDED_HEADER) {
|
|
241
|
+
throw new TarFormatError(
|
|
242
|
+
`extended header of ${size} bytes, past ${MAX_EXTENDED_HEADER} (a hostile archive?)`,
|
|
243
|
+
)
|
|
244
|
+
}
|
|
245
|
+
return { kind: 'extended', type, size, padded }
|
|
229
246
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
247
|
+
const trimmed = name.replace(/\/+$/, type === '5' ? '/' : '')
|
|
248
|
+
return { kind: 'entry', name: trimmed, size, type, mtimeMs, padded }
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** An `x` or `L` header's padded body: it applies to the NEXT entry only. */
|
|
252
|
+
extended(type: 'x' | 'L', body: Uint8Array, size: number): void {
|
|
253
|
+
if (type === 'L') {
|
|
254
|
+
this.pendingPath = field(body, 0, size)
|
|
255
|
+
return
|
|
256
|
+
}
|
|
257
|
+
const pax = parsePax(body.subarray(0, size))
|
|
258
|
+
const p = pax.get('path')
|
|
259
|
+
if (p !== undefined) this.pendingPath = p
|
|
260
|
+
const s = pax.get('size')
|
|
261
|
+
if (s !== undefined) {
|
|
262
|
+
// Digits only: `Number` reads `0.5` and `-1`, and a fractional
|
|
263
|
+
// size moved the reader to a fractional offset.
|
|
264
|
+
if (!/^[0-9]+$/.test(s) || !Number.isSafeInteger(Number(s))) {
|
|
265
|
+
throw new TarFormatError(`bad pax size: ${JSON.stringify(s)}`)
|
|
249
266
|
}
|
|
250
|
-
|
|
267
|
+
this.pendingSize = Number(s)
|
|
251
268
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const extendedWhat = (type: 'x' | 'L' | 'g'): string =>
|
|
273
|
+
type === 'g' ? 'a global header' : 'an extended header'
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Iterate a tar stream entry by entry. Each entry's `body` MUST be fully
|
|
277
|
+
* consumed (or the iterator drains it) before the next entry is read.
|
|
278
|
+
*/
|
|
279
|
+
export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGenerator<TarEntry> {
|
|
280
|
+
const src = new Source(stream[Symbol.asyncIterator]())
|
|
281
|
+
const tar = new TarDecoder()
|
|
282
|
+
for (;;) {
|
|
283
|
+
const h = await src.exact(BLOCK, 'a header')
|
|
284
|
+
if (h === null) return tar.eof()
|
|
285
|
+
const step = tar.header(h)
|
|
286
|
+
if (step.kind === 'end') return
|
|
287
|
+
if (step.kind === 'zero') continue
|
|
288
|
+
if (step.kind === 'extended') {
|
|
289
|
+
const body = await src.exact(step.padded, extendedWhat(step.type))
|
|
290
|
+
// pax global header: not used by Bun.Archive; its body is skipped.
|
|
291
|
+
if (step.type === 'g') continue
|
|
292
|
+
if (body === null) throw new TarFormatError('archive ends inside an extended header')
|
|
293
|
+
tar.extended(step.type, body, step.size)
|
|
255
294
|
continue
|
|
256
295
|
}
|
|
257
|
-
|
|
258
|
-
|
|
296
|
+
const { name, size, padded } = step
|
|
259
297
|
let drained = false
|
|
260
298
|
const body = (async function* (): AsyncIterable<Uint8Array> {
|
|
261
299
|
if (size > 0) for await (const c of src.take(size, `entry ${name}`)) yield c
|
|
@@ -265,7 +303,7 @@ export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGene
|
|
|
265
303
|
}
|
|
266
304
|
drained = true
|
|
267
305
|
})()
|
|
268
|
-
yield { name, size, type, mtimeMs, body }
|
|
306
|
+
yield { name, size, type: step.type, mtimeMs: step.mtimeMs, body }
|
|
269
307
|
if (!drained) {
|
|
270
308
|
// The caller skipped this body: drain it so the next header lines up.
|
|
271
309
|
for await (const _ of body) {
|
|
@@ -275,6 +313,77 @@ export async function* tarEntries(stream: ReadableStream<Uint8Array>): AsyncGene
|
|
|
275
313
|
}
|
|
276
314
|
}
|
|
277
315
|
|
|
316
|
+
/** One entry of a tar in memory. `body()` is a view, checked against the tar's end when asked for. */
|
|
317
|
+
interface TarBytesEntry {
|
|
318
|
+
name: string
|
|
319
|
+
size: number
|
|
320
|
+
type: string
|
|
321
|
+
mtimeMs: number
|
|
322
|
+
body(): Uint8Array
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* `tarEntries` over a tar already in memory, synchronously, through the
|
|
327
|
+
* same decoder. A body is checked against the tar's end when it is read,
|
|
328
|
+
* as the stream reader checks it when it is drained, so the two refuse
|
|
329
|
+
* a damaged archive with the same class at the same point.
|
|
330
|
+
*/
|
|
331
|
+
export function* tarEntriesSync(tar: Uint8Array): Generator<TarBytesEntry> {
|
|
332
|
+
const dec = new TarDecoder()
|
|
333
|
+
const len = tar.byteLength
|
|
334
|
+
let off = 0
|
|
335
|
+
for (;;) {
|
|
336
|
+
if (off === len) return dec.eof()
|
|
337
|
+
if (len - off < BLOCK) {
|
|
338
|
+
throw new TarFormatError(`archive ends inside a header (${len - off} of ${BLOCK} bytes)`)
|
|
339
|
+
}
|
|
340
|
+
const step = dec.header(tar.subarray(off, off + BLOCK))
|
|
341
|
+
off += BLOCK
|
|
342
|
+
if (step.kind === 'end') return
|
|
343
|
+
if (step.kind === 'zero') continue
|
|
344
|
+
if (step.kind === 'extended') {
|
|
345
|
+
const left = len - off
|
|
346
|
+
if (left < step.padded) {
|
|
347
|
+
if (left > 0) {
|
|
348
|
+
throw new TarFormatError(
|
|
349
|
+
`archive ends inside ${extendedWhat(step.type)} (${left} of ${step.padded} bytes)`,
|
|
350
|
+
)
|
|
351
|
+
}
|
|
352
|
+
// At the very end a global header's body is skipped as absent, and
|
|
353
|
+
// the end marker it lacks is what refuses the archive.
|
|
354
|
+
if (step.type === 'g') continue
|
|
355
|
+
throw new TarFormatError('archive ends inside an extended header')
|
|
356
|
+
}
|
|
357
|
+
if (step.type !== 'g')
|
|
358
|
+
dec.extended(step.type, tar.subarray(off, off + step.padded), step.size)
|
|
359
|
+
off += step.padded
|
|
360
|
+
continue
|
|
361
|
+
}
|
|
362
|
+
const { name, size, padded } = step
|
|
363
|
+
const at = off
|
|
364
|
+
// The stream reader's messages, word for word: the parity rows compare them.
|
|
365
|
+
const body = (): Uint8Array => {
|
|
366
|
+
const left = len - at
|
|
367
|
+
if (left < size) {
|
|
368
|
+
throw new TarFormatError(`archive ends inside entry ${name} (${left} of ${size} bytes)`)
|
|
369
|
+
}
|
|
370
|
+
if (left < padded) {
|
|
371
|
+
const pad = left - size
|
|
372
|
+
throw new TarFormatError(
|
|
373
|
+
pad === 0
|
|
374
|
+
? `archive ends inside padding after ${name}`
|
|
375
|
+
: `archive ends inside padding after ${name} (${pad} of ${padded - size} bytes)`,
|
|
376
|
+
)
|
|
377
|
+
}
|
|
378
|
+
return tar.subarray(at, at + size)
|
|
379
|
+
}
|
|
380
|
+
yield { name, size, type: step.type, mtimeMs: step.mtimeMs, body }
|
|
381
|
+
// A body the caller skipped is still checked, as the stream drains it.
|
|
382
|
+
body()
|
|
383
|
+
off += padded
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
278
387
|
// ─── Writer ───────────────────────────────────────────────────────────
|
|
279
388
|
|
|
280
389
|
/** One regular file to pack. `body` is read as it is written. */
|
|
@@ -379,6 +488,48 @@ export function tarSize(inputs: readonly TarInput[]): number {
|
|
|
379
488
|
return size
|
|
380
489
|
}
|
|
381
490
|
|
|
491
|
+
/** An entry's blocks before its body: a pax `path` record when needed, then the ustar header. */
|
|
492
|
+
function entryHead(input: TarInput): Uint8Array[] {
|
|
493
|
+
const mode = input.mode ?? 0o644
|
|
494
|
+
const mtime = input.mtime ?? 0
|
|
495
|
+
if (!needsPax(input.name)) return [header(input.name, input.size, '0', mode, mtime)]
|
|
496
|
+
// Under a pax record the ustar name is a courtesy for readers that
|
|
497
|
+
// ignore pax: its first 100 BYTES, never characters — a multibyte
|
|
498
|
+
// name sliced by characters can be over 100 bytes again.
|
|
499
|
+
const pax = paxRecord('path', input.name)
|
|
500
|
+
return [
|
|
501
|
+
header('PaxHeaders/entry', pax.byteLength, 'x', 0o644, mtime),
|
|
502
|
+
pax,
|
|
503
|
+
padding(pax.byteLength),
|
|
504
|
+
header(encoder.encode(input.name).subarray(0, 100), input.size, '0', mode, mtime),
|
|
505
|
+
]
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* `tarPack` of in-memory bodies, written into one buffer on this thread:
|
|
510
|
+
* the same bytes, without a generator hop per block (the small-artifact
|
|
511
|
+
* save, `packArtifactBytes`). Returns the bytes written.
|
|
512
|
+
*/
|
|
513
|
+
export function tarPackInto(
|
|
514
|
+
inputs: readonly (TarInput & { body: Uint8Array | string })[],
|
|
515
|
+
out: Uint8Array,
|
|
516
|
+
): number {
|
|
517
|
+
let off = 0
|
|
518
|
+
for (const input of inputs) {
|
|
519
|
+
for (const block of entryHead(input)) {
|
|
520
|
+
out.set(block, off)
|
|
521
|
+
off += block.byteLength
|
|
522
|
+
}
|
|
523
|
+
const bytes = typeof input.body === 'string' ? encoder.encode(input.body) : input.body
|
|
524
|
+
if (bytes.byteLength !== input.size)
|
|
525
|
+
throw new TarFormatError(`${input.name}: ${bytes.byteLength} bytes, ${input.size} declared`)
|
|
526
|
+
out.set(bytes, off)
|
|
527
|
+
// `out` is zeroed: padding and the end marker are a skip.
|
|
528
|
+
off += padded(input.size)
|
|
529
|
+
}
|
|
530
|
+
return off + BLOCK * 2
|
|
531
|
+
}
|
|
532
|
+
|
|
382
533
|
/**
|
|
383
534
|
* Pack regular files as a tar stream: ustar with the name/prefix split,
|
|
384
535
|
* a pax `path` record when a name fits neither field — exactly the
|
|
@@ -389,20 +540,7 @@ export async function* tarPack(
|
|
|
389
540
|
inputs: AsyncIterable<TarInput> | Iterable<TarInput>,
|
|
390
541
|
): AsyncGenerator<Uint8Array> {
|
|
391
542
|
for await (const input of inputs) {
|
|
392
|
-
|
|
393
|
-
const mtime = input.mtime ?? 0
|
|
394
|
-
// Under a pax record the ustar name is a courtesy for readers that
|
|
395
|
-
// ignore pax: its first 100 BYTES, never characters — a multibyte
|
|
396
|
-
// name sliced by characters can be over 100 bytes again.
|
|
397
|
-
let headerName: string | Uint8Array = input.name
|
|
398
|
-
if (needsPax(input.name)) {
|
|
399
|
-
const pax = paxRecord('path', input.name)
|
|
400
|
-
yield header('PaxHeaders/entry', pax.byteLength, 'x', 0o644, mtime)
|
|
401
|
-
yield pax
|
|
402
|
-
yield padding(pax.byteLength)
|
|
403
|
-
headerName = encoder.encode(input.name).subarray(0, 100)
|
|
404
|
-
}
|
|
405
|
-
yield header(headerName, input.size, '0', mode, mtime)
|
|
543
|
+
yield* entryHead(input)
|
|
406
544
|
if (input.body instanceof Blob || isChunks(input.body)) {
|
|
407
545
|
let n = 0
|
|
408
546
|
const chunks = input.body instanceof Blob ? input.body.stream() : input.body
|
package/src/cache/zstd.ts
CHANGED
|
@@ -157,7 +157,7 @@ export const zstdEncoder = (): TransformStream<Uint8Array, Uint8Array> =>
|
|
|
157
157
|
new CompressionStream('zstd') as unknown as TransformStream<Uint8Array, Uint8Array>
|
|
158
158
|
|
|
159
159
|
/** Bytes in memory as a one-chunk stream, for a reader that takes a stream. */
|
|
160
|
-
|
|
160
|
+
const oneChunk = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
|
161
161
|
new ReadableStream({
|
|
162
162
|
start(c) {
|
|
163
163
|
c.enqueue(bytes)
|
package/src/cli/cache.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs'
|
|
2
2
|
import { Cache } from '../cache/index.js'
|
|
3
|
-
import { flagHint, formatValue, refusedWord, seeHelp } from './help.js'
|
|
4
|
-
import { nearest, parseDuration, parseSize } from '../util/index.js'
|
|
3
|
+
import { flagHint, formatValue, refuse, refusedWord, seeHelp } from './help.js'
|
|
4
|
+
import { errorCode, nearest, parseDuration, parseSize } from '../util/index.js'
|
|
5
5
|
import { acquireRunLock } from '../orchestrator/index.js'
|
|
6
6
|
import { findWorkspaceRoot } from '../workspace/index.js'
|
|
7
7
|
import {
|
|
@@ -56,15 +56,19 @@ export async function cacheCmd(args: readonly string[]): Promise<number> {
|
|
|
56
56
|
return await pruneCmd(rest)
|
|
57
57
|
case undefined:
|
|
58
58
|
// A bare `vx cache prune` refuses too: the hint names a whole command.
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
return refuse(
|
|
60
|
+
'cache',
|
|
61
|
+
args,
|
|
62
|
+
`missing subcommand. Try \`vx cache prune --older-than 30d\` or \`vx cache prune --max-size 1G\`${seeHelp('cache')}`,
|
|
63
|
+
'VX_E_USAGE',
|
|
61
64
|
)
|
|
62
|
-
return 1
|
|
63
65
|
default:
|
|
64
|
-
|
|
65
|
-
|
|
66
|
+
return refuse(
|
|
67
|
+
'cache',
|
|
68
|
+
args,
|
|
69
|
+
`unknown subcommand: ${sub}${cacheSubHint(sub)}${seeHelp('cache')}`,
|
|
70
|
+
'VX_E_USAGE',
|
|
66
71
|
)
|
|
67
|
-
return 1
|
|
68
72
|
}
|
|
69
73
|
}
|
|
70
74
|
|
|
@@ -153,17 +157,13 @@ export function parsePruneArgs(args: readonly string[]): PruneArgs {
|
|
|
153
157
|
|
|
154
158
|
async function pruneCmd(args: readonly string[]): Promise<number> {
|
|
155
159
|
const parsed = parsePruneArgs(args)
|
|
156
|
-
if (parsed.error)
|
|
157
|
-
process.stderr.write(`vx cache prune: ${parsed.error}\n`)
|
|
158
|
-
return 1
|
|
159
|
-
}
|
|
160
|
+
if (parsed.error) return refuse('cache prune', args, parsed.error, 'VX_E_USAGE')
|
|
160
161
|
const cwd = process.cwd()
|
|
161
162
|
let root: string
|
|
162
163
|
try {
|
|
163
164
|
root = await findWorkspaceRoot(cwd)
|
|
164
165
|
} catch (err) {
|
|
165
|
-
|
|
166
|
-
return 1
|
|
166
|
+
return refuse('cache prune', args, (err as Error).message, errorCode(err))
|
|
167
167
|
}
|
|
168
168
|
// Honor `--cache-dir`, `defineWorkspace({ cacheDir: '...' })` and a
|
|
169
169
|
// `config` plugin's edit of it — `vx run` and `vx cache prune` must
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// The pages `vx docs` searches: core's reference, imported as text so the
|
|
2
|
+
// compiled binary carries them and an npm install reads them from the
|
|
3
|
+
// package (package.json `files`). Loaded only by `vx docs`.
|
|
4
|
+
import caching from '../../docs/caching.md' with { type: 'text' }
|
|
5
|
+
import cli from '../../docs/cli.md' with { type: 'text' }
|
|
6
|
+
import execution from '../../docs/execution.md' with { type: 'text' }
|
|
7
|
+
import features from '../../docs/features.md' with { type: 'text' }
|
|
8
|
+
import patterns from '../../docs/patterns.md' with { type: 'text' }
|
|
9
|
+
import schema from '../../docs/schema.md' with { type: 'text' }
|
|
10
|
+
import security from '../../docs/security.md' with { type: 'text' }
|
|
11
|
+
|
|
12
|
+
/** Page name (its file, no `.md`) → markdown. */
|
|
13
|
+
export const PAGES: Readonly<Record<string, string>> = {
|
|
14
|
+
cli,
|
|
15
|
+
schema,
|
|
16
|
+
caching,
|
|
17
|
+
execution,
|
|
18
|
+
patterns,
|
|
19
|
+
security,
|
|
20
|
+
features,
|
|
21
|
+
}
|