@vzn/vx 0.0.349 → 0.0.352
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/cache/cache.ts +31 -1
- package/src/cache/file-hashes.ts +30 -0
- package/src/cache/git-inputs.ts +164 -10
- package/src/cache/index.ts +2 -0
- package/src/cache/inputs.ts +84 -22
- package/src/cache/key-fold.ts +5 -1
- package/src/cache/schema.ts +9 -0
- package/src/cli/show.ts +7 -1
- package/src/cli/workspace-config.ts +1 -1
- package/src/exec/index.ts +7 -1
- package/src/exec/sandbox-paths.ts +22 -0
- package/src/exec/sandbox-runtime.ts +71 -15
- package/src/exec/sandbox-violations.ts +41 -6
- package/src/graph/scheduler.ts +9 -1
- package/src/orchestrator/execute-task.ts +2 -0
- package/src/orchestrator/framed-output.ts +13 -0
- package/src/orchestrator/logger.ts +41 -2
- package/src/orchestrator/options.ts +6 -4
- package/src/orchestrator/plugin-host.ts +7 -6
- package/src/orchestrator/prepare.ts +36 -9
- package/src/orchestrator/projects.ts +9 -3
- package/src/orchestrator/run.ts +10 -2
- package/src/orchestrator/sandbox-request.ts +6 -5
- package/src/orchestrator/shell-verdict.ts +10 -1
- package/src/workspace/affected.ts +34 -1
- package/src/workspace/config-schema.ts +28 -0
- package/src/workspace/migrate-scripts.ts +49 -6
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.352",
|
|
4
4
|
"description": "A fast, correct task runner and build cache for JavaScript monorepos.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.352",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.352",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.352",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.352"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/src/cache/cache.ts
CHANGED
|
@@ -605,6 +605,14 @@ export class Cache implements CacheLayer {
|
|
|
605
605
|
// every open there failed `attempt to write a readonly database`
|
|
606
606
|
// before the not-writable refusal could name the directory (O-10).
|
|
607
607
|
if (!absent) this.db.fileControl(SQLITE_FCNTL_PERSIST_WAL, 1)
|
|
608
|
+
// Any non-negative limit makes the last clean close truncate a
|
|
609
|
+
// persistent `-wal` to zero bytes once it is checkpointed (SQLite's
|
|
610
|
+
// walClose). At the default (-1) it kept every frame, so each later
|
|
611
|
+
// connection recovered them at open and copied them into the
|
|
612
|
+
// database again at its close: 738 frames, 4.5 MB, on a 1,000-project
|
|
613
|
+
// cache. The file stays (O-10 above); 64 MiB is far above what a run
|
|
614
|
+
// writes, so the limit never trims a WAL mid-run.
|
|
615
|
+
this.db.exec('PRAGMA journal_size_limit = 67108864')
|
|
608
616
|
this.db.exec('PRAGMA synchronous = NORMAL')
|
|
609
617
|
this.db.exec('PRAGMA foreign_keys = ON')
|
|
610
618
|
this.db.exec(`
|
|
@@ -666,7 +674,7 @@ export class Cache implements CacheLayer {
|
|
|
666
674
|
}
|
|
667
675
|
if (found === SCHEMA_VERSION) return null
|
|
668
676
|
this.db.exec(
|
|
669
|
-
'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals; DROP TABLE IF EXISTS config_closures; DROP TABLE IF EXISTS output_dirs;',
|
|
677
|
+
'DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS runs; DROP TABLE IF EXISTS file_hashes; DROP TABLE IF EXISTS blob_sizes; DROP TABLE IF EXISTS output_files; DROP TABLE IF EXISTS invocations; DROP TABLE IF EXISTS run_task_inputs; DROP TABLE IF EXISTS entry_inputs; DROP TABLE IF EXISTS config_evals; DROP TABLE IF EXISTS config_closures; DROP TABLE IF EXISTS output_dirs;',
|
|
670
678
|
)
|
|
671
679
|
this.db
|
|
672
680
|
.prepare("UPDATE schema_meta SET value = ? WHERE key = 'version'")
|
|
@@ -790,6 +798,14 @@ export class Cache implements CacheLayer {
|
|
|
790
798
|
hashFiles(paths: readonly string[]): Promise<Map<string, string>> {
|
|
791
799
|
return this.guard(() => this.files.hashFiles(paths))
|
|
792
800
|
}
|
|
801
|
+
/** `BlobSizeMemo`: the sizes of these blobs this cache has learned. */
|
|
802
|
+
knownBlobSizes(oids: readonly string[]): Map<string, number> {
|
|
803
|
+
return this.guard(() => this.files.knownBlobSizes(oids))
|
|
804
|
+
}
|
|
805
|
+
/** `BlobSizeMemo`: remember blob sizes, honouring the local WRITE axis. */
|
|
806
|
+
rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
|
|
807
|
+
this.guard(() => this.files.rememberBlobSizes(sizes))
|
|
808
|
+
}
|
|
793
809
|
/**
|
|
794
810
|
* `relPosix` against the run's workspace root, memoized: the same three
|
|
795
811
|
* thousand files are re-relativized for every task of the project, which
|
|
@@ -799,11 +815,25 @@ export class Cache implements CacheLayer {
|
|
|
799
815
|
*/
|
|
800
816
|
private relMemo = new Map<string, string>()
|
|
801
817
|
private relMemoRoot: string | undefined
|
|
818
|
+
/** `root` and a slash, when a file under it is relative by slicing; else undefined. */
|
|
819
|
+
private relPrefix: string | undefined
|
|
802
820
|
private relFor(root: string, file: string): string {
|
|
803
821
|
if (root !== this.relMemoRoot) {
|
|
804
822
|
this.relMemoRoot = root
|
|
805
823
|
this.relMemo.clear()
|
|
824
|
+
this.relPrefix =
|
|
825
|
+
path.sep === '/' && path.isAbsolute(root) && path.normalize(root) === root
|
|
826
|
+
? root.endsWith('/')
|
|
827
|
+
? root
|
|
828
|
+
: `${root}/`
|
|
829
|
+
: undefined
|
|
806
830
|
}
|
|
831
|
+
// An input file is a normalized absolute path (`resolveInputs` makes
|
|
832
|
+
// every one), and under a normalized root that is the root, a slash and
|
|
833
|
+
// its relative path: a slice, where the memo hashed the whole path per
|
|
834
|
+
// file per task (~15 ms over 36,000 files, I-31).
|
|
835
|
+
const prefix = this.relPrefix
|
|
836
|
+
if (prefix !== undefined && file.startsWith(prefix)) return file.slice(prefix.length)
|
|
807
837
|
let rel = this.relMemo.get(file)
|
|
808
838
|
if (rel === undefined) this.relMemo.set(file, (rel = relPosix(root, file)))
|
|
809
839
|
return rel
|
package/src/cache/file-hashes.ts
CHANGED
|
@@ -68,12 +68,42 @@ export class FileHashStore {
|
|
|
68
68
|
if (last !== null && now - Number(last.value) < DAY_MS) return
|
|
69
69
|
this.db.transaction(() => {
|
|
70
70
|
this.db.prepare('DELETE FROM file_hashes WHERE seen_at < ?').run(cutoff)
|
|
71
|
+
this.db.prepare('DELETE FROM blob_sizes WHERE seen_at < ?').run(cutoff)
|
|
71
72
|
this.db
|
|
72
73
|
.prepare('INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)')
|
|
73
74
|
.run(FILE_HASHES_SWEPT_AT, String(now))
|
|
74
75
|
})()
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
/** Each of `oids` whose size a `rememberBlobSizes` stored: one `IN` query per 900. */
|
|
79
|
+
knownBlobSizes(oids: readonly string[]): Map<string, number> {
|
|
80
|
+
const out = new Map<string, number>()
|
|
81
|
+
for (let i = 0; i < oids.length; i += 900) {
|
|
82
|
+
const chunk = oids.slice(i, i + 900)
|
|
83
|
+
const rows = this.db
|
|
84
|
+
.query(`SELECT oid, size FROM blob_sizes WHERE oid IN (${chunk.map(() => '?').join(',')})`)
|
|
85
|
+
.all(...chunk) as Array<{ oid: string; size: number }>
|
|
86
|
+
for (const r of rows) out.set(r.oid, r.size)
|
|
87
|
+
}
|
|
88
|
+
return out
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Store blob sizes in one transaction; a memo, so a full disk skips it. */
|
|
92
|
+
rememberBlobSizes(sizes: ReadonlyMap<string, number>): void {
|
|
93
|
+
if (!this.write || sizes.size === 0) return
|
|
94
|
+
const now = Date.now()
|
|
95
|
+
const insert = this.db.prepare(
|
|
96
|
+
'INSERT OR REPLACE INTO blob_sizes(oid, size, seen_at) VALUES (?, ?, ?)',
|
|
97
|
+
)
|
|
98
|
+
try {
|
|
99
|
+
this.db.transaction(() => {
|
|
100
|
+
for (const [oid, size] of sizes) insert.run(oid, size, now)
|
|
101
|
+
})()
|
|
102
|
+
} catch (err) {
|
|
103
|
+
if (!isIndexFull(err)) throw err
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
77
107
|
/**
|
|
78
108
|
* Content-hash a file (as a git blob OID, v20) with an mtime+size
|
|
79
109
|
* fast path. If the `file_hashes` table has a row for `path` whose
|
package/src/cache/git-inputs.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// task declared. Nothing here reads a config or applies a boundary.
|
|
7
7
|
|
|
8
8
|
import path from 'node:path'
|
|
9
|
-
import { existsSync, lstatSync, realpathSync } from 'node:fs'
|
|
9
|
+
import { existsSync, lstatSync, readdirSync, realpathSync } from 'node:fs'
|
|
10
10
|
import { UserError, executablePath, gitSpawnRefusal } from '../util/index.js'
|
|
11
11
|
|
|
12
12
|
/** Three facts of the repository a directory is in, from one `git rev-parse`. */
|
|
@@ -394,7 +394,14 @@ function expandNestedRepos(
|
|
|
394
394
|
}
|
|
395
395
|
out ??= files.slice(0, i)
|
|
396
396
|
const abs = path.join(cwd, nested)
|
|
397
|
-
if (!existsSync(path.join(abs, '.git')))
|
|
397
|
+
if (!existsSync(path.join(abs, '.git'))) {
|
|
398
|
+
// No repository answers for it: a submodule never initialised (empty)
|
|
399
|
+
// or one whose `.git` was removed to vendor its files, the gitlink
|
|
400
|
+
// left in the index and `git status` silent. Its files are what the
|
|
401
|
+
// task reads, so a walk lists them; they hash by content (A-61).
|
|
402
|
+
for (const f of walkFiles(abs)) out.push(`${nested}/${f}`)
|
|
403
|
+
continue
|
|
404
|
+
}
|
|
398
405
|
const inner = runGitLsFiles(abs)
|
|
399
406
|
for (const f of inner.files) out.push(`${nested}/${f}`)
|
|
400
407
|
for (const f of inner.undecodable) undecodable.push(`${nested}/${f}`)
|
|
@@ -402,6 +409,24 @@ function expandNestedRepos(
|
|
|
402
409
|
return { files: out ?? files, undecodable }
|
|
403
410
|
}
|
|
404
411
|
|
|
412
|
+
/** Every file and symlink under `dir`, relative and `/`-separated; a `.git` is skipped, a link not followed. */
|
|
413
|
+
function walkFiles(dir: string, rel = ''): string[] {
|
|
414
|
+
let entries
|
|
415
|
+
try {
|
|
416
|
+
entries = readdirSync(path.join(dir, rel), { withFileTypes: true })
|
|
417
|
+
} catch {
|
|
418
|
+
return []
|
|
419
|
+
}
|
|
420
|
+
const out: string[] = []
|
|
421
|
+
for (const e of entries) {
|
|
422
|
+
if (e.name === '.git') continue
|
|
423
|
+
const r = rel === '' ? e.name : `${rel}/${e.name}`
|
|
424
|
+
if (e.isDirectory()) out.push(...walkFiles(dir, r))
|
|
425
|
+
else out.push(r)
|
|
426
|
+
}
|
|
427
|
+
return out
|
|
428
|
+
}
|
|
429
|
+
|
|
405
430
|
/**
|
|
406
431
|
* A file's identity as the key folds it: its blob OID, prefixed by its git
|
|
407
432
|
* mode unless that is a plain file's (100644). A blob OID holds no mode, so
|
|
@@ -451,6 +476,108 @@ function parseLsFilesOutput(out: string, undecodableRecords: ReadonlySet<string>
|
|
|
451
476
|
return { files, oids, flagged, gitlinks, undecodable }
|
|
452
477
|
}
|
|
453
478
|
|
|
479
|
+
/**
|
|
480
|
+
* `ls-files -s -v -z --debug` as the plain `-s -v -z` stream, plus the size
|
|
481
|
+
* the index records for each regular stage-0 entry's worktree file and its
|
|
482
|
+
* raw OID. With `-z`, each record's NUL is followed by its five
|
|
483
|
+
* newline-ended debug lines (ctime, mtime, dev/ino, uid/gid, size/flags),
|
|
484
|
+
* then the next record.
|
|
485
|
+
*/
|
|
486
|
+
function stripLsDebug(run: GitRun): {
|
|
487
|
+
plain: string
|
|
488
|
+
undecodable: Set<string>
|
|
489
|
+
indexed: Map<string, { oid: string; size: number }>
|
|
490
|
+
} {
|
|
491
|
+
const segs = run.stdout.split('\0')
|
|
492
|
+
const records: string[] = []
|
|
493
|
+
const undecodable = new Set<string>()
|
|
494
|
+
const indexed = new Map<string, { oid: string; size: number }>()
|
|
495
|
+
let record = segs[0]!
|
|
496
|
+
let recordRaw = record
|
|
497
|
+
for (let i = 1; i <= segs.length; i++) {
|
|
498
|
+
const seg = segs[i]
|
|
499
|
+
let rest = ''
|
|
500
|
+
let size = -1
|
|
501
|
+
if (seg !== undefined) {
|
|
502
|
+
let at = 0
|
|
503
|
+
for (let k = 0; k < 5 && at >= 0; k++) {
|
|
504
|
+
const nl = seg.indexOf('\n', at)
|
|
505
|
+
if (nl < 0) at = -1
|
|
506
|
+
else {
|
|
507
|
+
if (k === 4) size = Number(/size: (\d+)/.exec(seg.slice(at, nl))?.[1] ?? -1)
|
|
508
|
+
at = nl + 1
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
rest = at < 0 ? '' : seg.slice(at)
|
|
512
|
+
}
|
|
513
|
+
if (record.length > 0) {
|
|
514
|
+
records.push(record)
|
|
515
|
+
if (run.undecodable.has(recordRaw)) undecodable.add(record)
|
|
516
|
+
const m = LS_FILES_STAGE_RE.exec(record)
|
|
517
|
+
if (m !== null && m[4] === '0' && (m[2] === '100644' || m[2] === '100755')) {
|
|
518
|
+
indexed.set(record.slice(m[0].length), { oid: m[3]!, size })
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
record = rest
|
|
522
|
+
recordRaw = seg ?? ''
|
|
523
|
+
}
|
|
524
|
+
return { plain: records.join('\0'), undecodable, indexed }
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/** Blob sizes learned once and kept: a blob's size is fixed for its OID. `Cache` is one. */
|
|
528
|
+
export interface BlobSizeMemo {
|
|
529
|
+
knownBlobSizes(oids: readonly string[]): Map<string, number>
|
|
530
|
+
rememberBlobSizes(sizes: ReadonlyMap<string, number>): void
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* Drop each trusted OID whose blob is not the size the index recorded for
|
|
535
|
+
* the worktree file (A-60). A clean filter's blob differs from the file it
|
|
536
|
+
* was added from, and git holds a stat-clean entry clean without re-reading
|
|
537
|
+
* it once the filter is gone (`core.autocrlf` turned off, a `.gitattributes`
|
|
538
|
+
* rule removed): `status` and the filter gate, which reads today's config,
|
|
539
|
+
* both let the blob stand for bytes it does not hold. The recorded sizes
|
|
540
|
+
* cost no read of the worktree; the blob sizes come from `memo`, and only
|
|
541
|
+
* the ones it lacks from one `cat-file` (65 ms over 3,000 loose objects,
|
|
542
|
+
* the whole of the cost, measured 2026-10-02). A smudged (racy) entry
|
|
543
|
+
* records 0 and is hashed from disk. An answer that cannot be read trusts
|
|
544
|
+
* nothing.
|
|
545
|
+
*/
|
|
546
|
+
async function dropResizedOids(enumeration: GitEnumeration, memo?: BlobSizeMemo): Promise<void> {
|
|
547
|
+
const { trusted, indexed } = enumeration
|
|
548
|
+
const wanted = new Set<string>()
|
|
549
|
+
for (const rel of trusted.keys()) {
|
|
550
|
+
const entry = indexed.get(rel)
|
|
551
|
+
if (entry !== undefined) wanted.add(entry.oid)
|
|
552
|
+
}
|
|
553
|
+
if (wanted.size === 0) return
|
|
554
|
+
const sizes = memo?.knownBlobSizes([...wanted]) ?? new Map<string, number>()
|
|
555
|
+
const unknown = [...wanted].filter((oid) => !sizes.has(oid))
|
|
556
|
+
if (unknown.length > 0) {
|
|
557
|
+
const run = await enumeration.catFile(unknown.join('\n') + '\n')
|
|
558
|
+
if (run === null || run.exitCode !== 0) {
|
|
559
|
+
trusted.clear()
|
|
560
|
+
return
|
|
561
|
+
}
|
|
562
|
+
const learned = new Map<string, number>()
|
|
563
|
+
for (const line of run.stdout.split('\n')) {
|
|
564
|
+
const sp = line.indexOf(' ')
|
|
565
|
+
const n = Number(line.slice(sp + 1))
|
|
566
|
+
if (sp > 0 && line.slice(sp + 1) !== '' && Number.isInteger(n))
|
|
567
|
+
learned.set(line.slice(0, sp), n)
|
|
568
|
+
}
|
|
569
|
+
for (const [oid, n] of learned) sizes.set(oid, n)
|
|
570
|
+
memo?.rememberBlobSizes(learned)
|
|
571
|
+
}
|
|
572
|
+
for (const rel of trusted.keys()) {
|
|
573
|
+
const entry = indexed.get(rel)
|
|
574
|
+
if (entry === undefined) continue // a symlink: its blob is its target string
|
|
575
|
+
const size = sizes.get(entry.oid)
|
|
576
|
+
// The index keeps the low 32 bits of a size.
|
|
577
|
+
if (size === undefined || size % 2 ** 32 !== entry.size) trusted.delete(rel)
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
454
581
|
/** One completed `git` invocation. */
|
|
455
582
|
interface GitRun {
|
|
456
583
|
exitCode: number
|
|
@@ -825,7 +952,7 @@ export async function populateGitFilesCache(
|
|
|
825
952
|
workspaceRoot,
|
|
826
953
|
gitPathspecs(workspaceRoot, projectDirs, workspaceWide),
|
|
827
954
|
)
|
|
828
|
-
applyGitEnumeration(enumeration, workspaceRoot, projectDirs, cache, workspaceWide)
|
|
955
|
+
await applyGitEnumeration(enumeration, workspaceRoot, projectDirs, cache, workspaceWide)
|
|
829
956
|
}
|
|
830
957
|
|
|
831
958
|
/** What one workspace-wide enumeration learned, before it is partitioned per project. */
|
|
@@ -851,8 +978,15 @@ export interface GitEnumeration {
|
|
|
851
978
|
undecodable: readonly string[]
|
|
852
979
|
/** `Date.now()` before the spawns: what `trusted` says is true as of no earlier. */
|
|
853
980
|
startedAtMs: number
|
|
981
|
+
/** Each regular stage-0 entry's raw OID and the worktree size the index recorded. */
|
|
982
|
+
indexed: ReadonlyMap<string, { oid: string; size: number }>
|
|
983
|
+
/** `git cat-file --batch-check` over `stdin`'s OIDs; null when it could not spawn. */
|
|
984
|
+
catFile(stdin: string): Promise<{ exitCode: number; stdout: string } | null>
|
|
854
985
|
}
|
|
855
986
|
|
|
987
|
+
/** Above this many project dirs the enumeration walks the whole tree. */
|
|
988
|
+
export const MAX_SCOPED_PATHSPECS = 64
|
|
989
|
+
|
|
856
990
|
/**
|
|
857
991
|
* Pathspec scoping: when the run only needs a handful of projects
|
|
858
992
|
* (scoped config loading), let git scan just those dirs — 75 ms →
|
|
@@ -868,7 +1002,7 @@ export function gitPathspecs(
|
|
|
868
1002
|
const scoped =
|
|
869
1003
|
!workspaceWide &&
|
|
870
1004
|
rels.length > 0 &&
|
|
871
|
-
rels.length <=
|
|
1005
|
+
rels.length <= MAX_SCOPED_PATHSPECS &&
|
|
872
1006
|
rels.every((r) => r !== '' && r !== '.')
|
|
873
1007
|
return scoped ? rels : ['.']
|
|
874
1008
|
}
|
|
@@ -888,7 +1022,11 @@ export async function startGitEnumeration(
|
|
|
888
1022
|
// bulk-populate costs max(ls-files, status) wall time, not the sum
|
|
889
1023
|
// (status alone is ~74 ms on a 1000-project tree; serial spawning
|
|
890
1024
|
// was a measurable warm-path regression vs the pre-OID code).
|
|
891
|
-
const spawnGit = async (
|
|
1025
|
+
const spawnGit = async (
|
|
1026
|
+
args: string[],
|
|
1027
|
+
stdin?: string,
|
|
1028
|
+
env?: Record<string, string>,
|
|
1029
|
+
): Promise<GitRun | null> => {
|
|
892
1030
|
try {
|
|
893
1031
|
const proc = Bun.spawn({
|
|
894
1032
|
// `status` refreshes the index when it can take `index.lock`, so
|
|
@@ -900,6 +1038,7 @@ export async function startGitEnumeration(
|
|
|
900
1038
|
stdin: stdin === undefined ? 'ignore' : new TextEncoder().encode(stdin),
|
|
901
1039
|
stdout: 'pipe',
|
|
902
1040
|
stderr: 'pipe',
|
|
1041
|
+
...(env !== undefined ? { env: { ...process.env, ...env } } : {}),
|
|
903
1042
|
})
|
|
904
1043
|
const [stdout, stderr, exitCode] = await Promise.all([
|
|
905
1044
|
new Response(proc.stdout).bytes(),
|
|
@@ -921,8 +1060,14 @@ export async function startGitEnumeration(
|
|
|
921
1060
|
// tracked paths are dirty AND which files are untracked. Asking
|
|
922
1061
|
// `ls-files --others` for the untracked set walked the same tree a second
|
|
923
1062
|
// time (~50 ms of CPU, concurrent with status but contending with it).
|
|
1063
|
+
// `--debug` adds each entry's recorded stat, for the blob-size check
|
|
1064
|
+
// (`dropResizedOids`, A-60).
|
|
1065
|
+
const listing = spawnGit(['ls-files', '-s', '-v', '-z', '--debug', '--', ...pathspecs]).then(
|
|
1066
|
+
(run) =>
|
|
1067
|
+
run === null || run.exitCode !== 0 ? { run, debug: null } : { run, debug: stripLsDebug(run) },
|
|
1068
|
+
)
|
|
924
1069
|
const running = Promise.all([
|
|
925
|
-
|
|
1070
|
+
listing,
|
|
926
1071
|
// `--ignored=matching` names an ignored path without descending into an
|
|
927
1072
|
// ignored directory: git applies an ignored `.gitattributes` as it does
|
|
928
1073
|
// any other, and the filter gate below must see it (A-19).
|
|
@@ -962,7 +1107,7 @@ export async function startGitEnumeration(
|
|
|
962
1107
|
// at all. `--git-dir` named the per-worktree directory, so the gate
|
|
963
1108
|
// looked where the rule can never be.
|
|
964
1109
|
const facts = repoFacts(workspaceRoot)
|
|
965
|
-
const [ls, status, vars] = await running
|
|
1110
|
+
const [{ run: ls, debug }, status, vars] = await running
|
|
966
1111
|
if (ls === null) {
|
|
967
1112
|
throw gitSpawnRefusal(workspaceRoot)
|
|
968
1113
|
}
|
|
@@ -979,7 +1124,7 @@ export async function startGitEnumeration(
|
|
|
979
1124
|
flagged,
|
|
980
1125
|
gitlinks,
|
|
981
1126
|
undecodable,
|
|
982
|
-
} = parseLsFilesOutput(
|
|
1127
|
+
} = parseLsFilesOutput(debug!.plain, debug!.undecodable)
|
|
983
1128
|
// Normalize `status`'s repo-root-relative paths to workspace-relative (strip
|
|
984
1129
|
// the `--show-prefix`) so the dirty set is keyed identically to the trusted
|
|
985
1130
|
// OID map. Without this, when the workspace root is a git subdir, a modified
|
|
@@ -1057,6 +1202,13 @@ export async function startGitEnumeration(
|
|
|
1057
1202
|
untracked: dirty === null ? null : untracked,
|
|
1058
1203
|
undecodable,
|
|
1059
1204
|
startedAtMs,
|
|
1205
|
+
indexed: debug!.indexed,
|
|
1206
|
+
// A missing blob is an answer ("missing"), never a fetch from a partial
|
|
1207
|
+
// clone's promisor remote.
|
|
1208
|
+
catFile: (stdin) =>
|
|
1209
|
+
spawnGit(['cat-file', '--batch-check=%(objectname) %(objectsize)'], stdin, {
|
|
1210
|
+
GIT_NO_LAZY_FETCH: '1',
|
|
1211
|
+
}),
|
|
1060
1212
|
}
|
|
1061
1213
|
}
|
|
1062
1214
|
|
|
@@ -1086,13 +1238,15 @@ export function lazyGitEnumeration(workspaceRoot: string): LazyGitEnumeration {
|
|
|
1086
1238
|
}
|
|
1087
1239
|
|
|
1088
1240
|
/** The partition half of `populateGitFilesCache`: store per-project slices of one enumeration. */
|
|
1089
|
-
export function applyGitEnumeration(
|
|
1241
|
+
export async function applyGitEnumeration(
|
|
1090
1242
|
enumeration: GitEnumeration,
|
|
1091
1243
|
workspaceRoot: string,
|
|
1092
1244
|
projectDirs: readonly string[],
|
|
1093
1245
|
cache: GitFilesCache,
|
|
1094
1246
|
workspaceWide = false,
|
|
1095
|
-
|
|
1247
|
+
memo?: BlobSizeMemo,
|
|
1248
|
+
): Promise<void> {
|
|
1249
|
+
await dropResizedOids(enumeration, memo)
|
|
1096
1250
|
const { all, trusted } = enumeration
|
|
1097
1251
|
cache.setWorktreeDirty(enumeration.dirty)
|
|
1098
1252
|
cache.enumeratedAtMs = enumeration.startedAtMs
|
package/src/cache/index.ts
CHANGED
|
@@ -43,12 +43,14 @@ export {
|
|
|
43
43
|
ownWorkspaceOutputsSince,
|
|
44
44
|
stampOutputs,
|
|
45
45
|
stampWorkspaceOutputs,
|
|
46
|
+
stopRuntimeProbes,
|
|
46
47
|
type OutputStamp,
|
|
47
48
|
} from './inputs.js'
|
|
48
49
|
export {
|
|
49
50
|
GitFilesCache,
|
|
50
51
|
applyGitEnumeration,
|
|
51
52
|
gitPathspecs,
|
|
53
|
+
MAX_SCOPED_PATHSPECS,
|
|
52
54
|
repoRootOf,
|
|
53
55
|
startGitEnumeration,
|
|
54
56
|
lazyGitEnumeration,
|
package/src/cache/inputs.ts
CHANGED
|
@@ -448,6 +448,22 @@ async function runRuntimeCommand(
|
|
|
448
448
|
*/
|
|
449
449
|
const liveProbes = new Set<ReturnType<typeof Bun.spawn>>()
|
|
450
450
|
let probeExitHooked = false
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Kill every runtime probe still running, with its tree. A run's stop asks
|
|
454
|
+
* this: a Ctrl-C while a probe ran waited for the probe, or for the signal
|
|
455
|
+
* handler's bound, about 7 s, before vx exited (C-65). Its answer is no
|
|
456
|
+
* longer needed; its caller sees it fail and the run reads it aborted.
|
|
457
|
+
*/
|
|
458
|
+
export function stopRuntimeProbes(): void {
|
|
459
|
+
for (const p of liveProbes) {
|
|
460
|
+
try {
|
|
461
|
+
process.kill(-p.pid, 'SIGKILL')
|
|
462
|
+
} catch {
|
|
463
|
+
// the group is gone
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
451
467
|
function killProbesOnExit(): void {
|
|
452
468
|
if (probeExitHooked) return
|
|
453
469
|
probeExitHooked = true
|
|
@@ -1032,27 +1048,75 @@ function refuseOneAlternativeBrace(
|
|
|
1032
1048
|
}
|
|
1033
1049
|
}
|
|
1034
1050
|
|
|
1035
|
-
|
|
1051
|
+
/** What a `files` declaration and its task's outputs compile to, whatever the project. */
|
|
1052
|
+
interface FilesPlan {
|
|
1053
|
+
positive: string[]
|
|
1054
|
+
negative: string[]
|
|
1055
|
+
excludeGlobs: Bun.Glob[]
|
|
1056
|
+
ownOutput: (rel: string) => boolean
|
|
1057
|
+
positiveGlobs: Bun.Glob[]
|
|
1058
|
+
/** The literal entries, each naming one path (see `unmatchedLiterals`). */
|
|
1059
|
+
literals: string[]
|
|
1060
|
+
/**
|
|
1061
|
+
* Whether a project-relative path is an input by the declaration alone:
|
|
1062
|
+
* a positive glob selects it, and no exclude and no own output takes it
|
|
1063
|
+
* back. Projects repeat their relative paths (`src/index.ts`,
|
|
1064
|
+
* `package.json`), so each is matched once per plan.
|
|
1065
|
+
*/
|
|
1066
|
+
verdicts: Map<string, boolean>
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* Compiled once per declaration for the life of the process: a workspace
|
|
1071
|
+
* declares a handful of `files` lists over thousands of tasks, and
|
|
1072
|
+
* splitting, normalizing and compiling them per task was ~6 ms of a
|
|
1073
|
+
* 1,000-task warm run (I-29). A refused declaration is never stored, so it
|
|
1074
|
+
* throws for every task that carries it.
|
|
1075
|
+
*/
|
|
1076
|
+
const filesPlans = new Map<string, FilesPlan | null>()
|
|
1077
|
+
|
|
1078
|
+
function filesPlan(
|
|
1079
|
+
files: readonly string[] | undefined,
|
|
1080
|
+
ownOutputs: readonly string[],
|
|
1081
|
+
): FilesPlan | null {
|
|
1082
|
+
const key = JSON.stringify([files ?? null, ownOutputs])
|
|
1083
|
+
const memo = filesPlans.get(key)
|
|
1084
|
+
if (memo !== undefined) return memo
|
|
1036
1085
|
const positive: string[] = []
|
|
1037
1086
|
const negative: string[] = []
|
|
1038
|
-
|
|
1039
|
-
if (args.files === undefined) {
|
|
1087
|
+
if (files === undefined) {
|
|
1040
1088
|
positive.push(...DEFAULT_FILE_GLOBS)
|
|
1041
1089
|
} else {
|
|
1042
|
-
refuseOneAlternativeBrace(
|
|
1043
|
-
for (const entry of
|
|
1090
|
+
refuseOneAlternativeBrace(files, 'files')
|
|
1091
|
+
for (const entry of files) {
|
|
1044
1092
|
if (entry.startsWith('!')) negative.push(entry.slice(1))
|
|
1045
1093
|
else positive.push(entry)
|
|
1046
1094
|
}
|
|
1047
1095
|
}
|
|
1096
|
+
const plan =
|
|
1097
|
+
positive.length === 0
|
|
1098
|
+
? null
|
|
1099
|
+
: {
|
|
1100
|
+
positive,
|
|
1101
|
+
negative,
|
|
1102
|
+
excludeGlobs: [...ALWAYS_IGNORE, ...asTrees(negative)].map(globFor),
|
|
1103
|
+
// A path the task's own outputs take back with `!` is no output, so it
|
|
1104
|
+
// stays an input: a tracked file under `dist` the build reads (A-44).
|
|
1105
|
+
ownOutput: outputMatcher(ownOutputs, globFor),
|
|
1106
|
+
positiveGlobs: asTrees(positive).map(globFor),
|
|
1107
|
+
literals: positive.map(normalizeGlob).filter(isLiteralPattern).map(stripTrailingSlash),
|
|
1108
|
+
verdicts: new Map<string, boolean>(),
|
|
1109
|
+
}
|
|
1110
|
+
filesPlans.set(key, plan)
|
|
1111
|
+
return plan
|
|
1112
|
+
}
|
|
1048
1113
|
|
|
1049
|
-
|
|
1114
|
+
async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
|
|
1115
|
+
const plan = filesPlan(args.files, args.ownOutputs)
|
|
1116
|
+
if (plan === null) return []
|
|
1117
|
+
const { positive, negative, excludeGlobs, ownOutput, positiveGlobs } = plan
|
|
1050
1118
|
|
|
1051
1119
|
const nested = inNestedProject(args.projectDir, args.nestedProjectDirs)
|
|
1052
|
-
const excludeGlobs = [...ALWAYS_IGNORE, ...asTrees(negative)].map(globFor)
|
|
1053
|
-
// A path the task's own outputs take back with `!` is no output, so it
|
|
1054
|
-
// stays an input: a tracked file under `dist` the build reads (A-44).
|
|
1055
|
-
const ownOutput = outputMatcher(args.ownOutputs, globFor)
|
|
1056
1120
|
|
|
1057
1121
|
// Defer to git for the file set (Turbo / Nx parity). Nested .gitignore
|
|
1058
1122
|
// files, .git/info/exclude, and global excludes all participate
|
|
@@ -1062,7 +1126,6 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
|
|
|
1062
1126
|
// per task (build + test + lint + …). Spawning git N times for the
|
|
1063
1127
|
// same project per run is wasteful; we cache the result for the
|
|
1064
1128
|
// duration of one orchestrator run.
|
|
1065
|
-
const positiveGlobs = asTrees(positive).map(globFor)
|
|
1066
1129
|
// Everything below the snapshot that decides the result: the project, what
|
|
1067
1130
|
// it declares, what it excludes as its own outputs, and the boundaries.
|
|
1068
1131
|
const memoKey = `${args.projectDir}\0${positive.join('\u0001')}\0${negative.join('\u0001')}\0${args.ownOutputs.join('\u0001')}\0${args.nestedProjectDirs.join('\u0001')}`
|
|
@@ -1102,22 +1165,21 @@ async function resolveFiles(args: ResolveFilesArgs): Promise<string[]> {
|
|
|
1102
1165
|
// an artifact built from an older version of a file the config explicitly
|
|
1103
1166
|
// claims as an input. See the refusal below for why this is not simply
|
|
1104
1167
|
// honoured instead.
|
|
1105
|
-
const unmatchedLiterals = new Set(
|
|
1106
|
-
positive.map(normalizeGlob).filter(isLiteralPattern).map(stripTrailingSlash),
|
|
1107
|
-
)
|
|
1168
|
+
const unmatchedLiterals = new Set(plan.literals)
|
|
1108
1169
|
// First pass: glob-filter to candidate absolute paths (no I/O).
|
|
1109
1170
|
const candidates: string[] = []
|
|
1171
|
+
const verdicts = plan.verdicts
|
|
1110
1172
|
for (const rel of gitFiles) {
|
|
1111
1173
|
if (unmatchedLiterals.size > 0) settleLiterals(unmatchedLiterals, rel)
|
|
1112
|
-
let
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1174
|
+
let input = verdicts.get(rel)
|
|
1175
|
+
if (input === undefined) {
|
|
1176
|
+
input =
|
|
1177
|
+
positiveGlobs.some((g) => g.match(rel)) &&
|
|
1178
|
+
!excludeGlobs.some((g) => g.match(rel)) &&
|
|
1179
|
+
!ownOutput(rel)
|
|
1180
|
+
verdicts.set(rel, input)
|
|
1118
1181
|
}
|
|
1119
|
-
if (!
|
|
1120
|
-
if (nested(rel) || excludeGlobs.some((g) => g.match(rel)) || ownOutput(rel)) continue
|
|
1182
|
+
if (!input || nested(rel)) continue
|
|
1121
1183
|
candidates.push(path.resolve(args.projectDir, rel))
|
|
1122
1184
|
}
|
|
1123
1185
|
if (unmatchedLiterals.size > 0) {
|
package/src/cache/key-fold.ts
CHANGED
|
@@ -99,7 +99,11 @@ import type { CacheKeyInput } from './layer.js'
|
|
|
99
99
|
// v37: an entry saved while `git status` hid a deletion (paired as an
|
|
100
100
|
// unmerged path's rename source) holds output built without a file its key
|
|
101
101
|
// folds (A-59).
|
|
102
|
-
|
|
102
|
+
// v38: an entry saved while the key folded a filtered LF blob for CRLF
|
|
103
|
+
// bytes on disk, the filter since removed, holds the CRLF build (A-60).
|
|
104
|
+
// v39: an entry saved while a gitlink without `.git` held files the key
|
|
105
|
+
// never listed sits under the key of that directory empty (A-61).
|
|
106
|
+
export const CACHE_VERSION = 'vx-cache-v39'
|
|
103
107
|
|
|
104
108
|
/**
|
|
105
109
|
* The digest a key folds for an input gone between its enumeration and its
|
package/src/cache/schema.ts
CHANGED
|
@@ -115,6 +115,15 @@ export function createTables(db: Database): void {
|
|
|
115
115
|
content_hash TEXT NOT NULL,
|
|
116
116
|
seen_at INTEGER NOT NULL
|
|
117
117
|
);
|
|
118
|
+
-- The size of each index blob the enumeration checked against the
|
|
119
|
+
-- worktree size git recorded (A-60). Fixed for its OID, so a warm
|
|
120
|
+
-- run asks git for none; swept with file_hashes by seen_at, the
|
|
121
|
+
-- time the row was written.
|
|
122
|
+
CREATE TABLE IF NOT EXISTS blob_sizes (
|
|
123
|
+
oid TEXT PRIMARY KEY,
|
|
124
|
+
size INTEGER NOT NULL,
|
|
125
|
+
seen_at INTEGER NOT NULL
|
|
126
|
+
);
|
|
118
127
|
-- v16: per-output-file fingerprints, scoped by the cache entry
|
|
119
128
|
-- that produced them. Lets loadOutputFilesBatch(hashes) answer
|
|
120
129
|
-- "for entry X, what are its outputs supposed to look like?"
|
package/src/cli/show.ts
CHANGED
|
@@ -339,6 +339,12 @@ function taskBlock(taskName: string, raw: TaskConfig): string {
|
|
|
339
339
|
add('outputs.workspaceFiles', list(cache.outputs.workspaceFiles))
|
|
340
340
|
}
|
|
341
341
|
const labelW = Math.max(...rows.map(([label]) => label.length))
|
|
342
|
-
|
|
342
|
+
// A multi-line value (a generated `vx_script` wrapper, a heredoc)
|
|
343
|
+
// continues under its first line, not at column 0 where it reads as
|
|
344
|
+
// the next task's header.
|
|
345
|
+
const indent = `\n${' '.repeat(labelW + 4)}`
|
|
346
|
+
const body = rows.map(
|
|
347
|
+
([label, value]) => ` ${`${label}:`.padEnd(labelW + 1)} ${value.replaceAll('\n', indent)}`,
|
|
348
|
+
)
|
|
343
349
|
return `${taskName}\n${body.join('\n')}\n`
|
|
344
350
|
}
|
package/src/exec/index.ts
CHANGED
|
@@ -29,7 +29,13 @@ export {
|
|
|
29
29
|
thrownReason,
|
|
30
30
|
} from './sandbox-runtime.js'
|
|
31
31
|
export { type DeniedCall } from './sandbox-violations.js'
|
|
32
|
-
export {
|
|
32
|
+
export {
|
|
33
|
+
atOrUnder,
|
|
34
|
+
isMountableLiteral,
|
|
35
|
+
MOUNT_WILDCARDS,
|
|
36
|
+
sandboxReads,
|
|
37
|
+
toRealPath,
|
|
38
|
+
} from './sandbox-paths.js'
|
|
33
39
|
export { bindableWrites, punchWalls } from './sandbox-binds.js'
|
|
34
40
|
export { isLocalExecutor, localExecutor } from './local-executor.js'
|
|
35
41
|
export { holdGroups, killTree, untilGroupsGone } from './kill-tree.js'
|
|
@@ -76,3 +76,25 @@ export function unique(arr: readonly string[]): string[] {
|
|
|
76
76
|
export function localBindingOn(c: { localBinding?: boolean | readonly number[] }): boolean {
|
|
77
77
|
return c.localBinding === true || (Array.isArray(c.localBinding) && c.localBinding.length > 0)
|
|
78
78
|
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Whether a sandboxed task may read `file`: it lies under a read grant,
|
|
82
|
+
* a write grant or a baseline read, by its canonical path. A path no
|
|
83
|
+
* grant holds is not there inside the sandbox, which the shell reports
|
|
84
|
+
* as "not found" with no denial a trace can see (an `execve`).
|
|
85
|
+
*/
|
|
86
|
+
export function sandboxReads(
|
|
87
|
+
sandbox: {
|
|
88
|
+
readonly baseAllowRead: readonly string[]
|
|
89
|
+
readonly config: {
|
|
90
|
+
readonly allowRead: readonly string[]
|
|
91
|
+
readonly allowWrite: readonly string[]
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
file: string,
|
|
95
|
+
): boolean {
|
|
96
|
+
const real = toRealPath(file)
|
|
97
|
+
return [...sandbox.baseAllowRead, ...sandbox.config.allowRead, ...sandbox.config.allowWrite].some(
|
|
98
|
+
(g) => atOrUnder(real, toRealPath(g)),
|
|
99
|
+
)
|
|
100
|
+
}
|