@vzn/vx 0.0.21 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -18
- package/package.json +6 -6
- package/src/bin.ts +52 -3
- package/src/cache/archive.ts +170 -21
- package/src/cache/cache.ts +667 -288
- package/src/cache/chained-cache.ts +14 -2
- package/src/cache/config-evals.ts +29 -6
- package/src/cache/file-hashes.ts +63 -35
- package/src/cache/git-inputs.ts +398 -69
- package/src/cache/index.ts +8 -1
- package/src/cache/inputs.ts +403 -102
- package/src/cache/key-fold.ts +232 -0
- package/src/cache/layer.ts +81 -11
- package/src/cache/layered-cache.ts +199 -32
- package/src/cache/output-index.ts +109 -10
- package/src/cache/run-history.ts +8 -3
- package/src/cache/zstd.ts +53 -30
- package/src/cli/cache.ts +51 -21
- package/src/cli/format.ts +4 -15
- package/src/cli/help.ts +13 -3
- package/src/cli/index.ts +35 -9
- package/src/cli/info.ts +49 -7
- package/src/cli/init.ts +17 -5
- package/src/cli/last.ts +96 -21
- package/src/cli/lock.ts +34 -17
- package/src/cli/plan-format.ts +4 -4
- package/src/cli/plugin-commands.ts +20 -5
- package/src/cli/run.ts +55 -12
- package/src/cli/select.ts +23 -13
- package/src/cli/show.ts +9 -4
- package/src/cli/upgrade.ts +134 -14
- package/src/cli/watch.ts +611 -102
- package/src/cli/why.ts +8 -16
- package/src/cli/workspace-config.ts +21 -9
- package/src/config.ts +46 -9
- package/src/exec/executor.ts +51 -5
- package/src/exec/index.ts +10 -0
- package/src/exec/kill-tree.ts +281 -0
- package/src/exec/local-executor.ts +1 -1
- package/src/exec/runner.ts +327 -82
- package/src/exec/sandbox-binds.ts +34 -5
- package/src/exec/sandbox-paths.ts +22 -2
- package/src/exec/sandbox-runtime.ts +627 -102
- package/src/exec/sandbox-violations.ts +36 -23
- package/src/graph/dependency-spec.ts +14 -14
- package/src/graph/index.ts +12 -2
- package/src/graph/priorities.ts +130 -0
- package/src/graph/scheduler.ts +182 -134
- package/src/graph/task-graph.ts +552 -188
- package/src/index.ts +16 -2
- package/src/orchestrator/admission.ts +41 -17
- package/src/orchestrator/colors.ts +8 -3
- package/src/orchestrator/deferred-outputs.ts +22 -7
- package/src/orchestrator/doctor.ts +122 -14
- package/src/orchestrator/events.ts +71 -2
- package/src/orchestrator/excluded-keys.ts +155 -0
- package/src/orchestrator/execute-task.ts +331 -37
- package/src/orchestrator/failure-recap.ts +121 -0
- package/src/orchestrator/fingerprint-watch.ts +89 -0
- package/src/orchestrator/framed-output.ts +81 -14
- package/src/orchestrator/history.ts +6 -7
- package/src/orchestrator/hit-restore.ts +82 -18
- package/src/orchestrator/index.ts +7 -1
- package/src/orchestrator/keyed-projects.ts +89 -0
- package/src/orchestrator/local-shortcircuit.ts +83 -23
- package/src/orchestrator/lockfile-claim.ts +152 -32
- package/src/orchestrator/logger.ts +116 -20
- package/src/orchestrator/metrics.ts +156 -60
- package/src/orchestrator/miss-save.ts +106 -43
- package/src/orchestrator/options.ts +30 -4
- package/src/orchestrator/persistent.ts +41 -21
- package/src/orchestrator/placement.ts +55 -25
- package/src/orchestrator/plan.ts +24 -11
- package/src/orchestrator/plugin-host.ts +17 -23
- package/src/orchestrator/plugin.ts +31 -8
- package/src/orchestrator/prepare.ts +99 -13
- package/src/orchestrator/projects.ts +19 -8
- package/src/orchestrator/remote-prefetch.ts +21 -1
- package/src/orchestrator/run-artifacts.ts +9 -0
- package/src/orchestrator/run-context.ts +19 -4
- package/src/orchestrator/run-lock.ts +349 -0
- package/src/orchestrator/run-records.ts +6 -23
- package/src/orchestrator/run-report.ts +4 -2
- package/src/orchestrator/run.ts +264 -57
- package/src/orchestrator/sandbox-request.ts +388 -62
- package/src/orchestrator/save-lane.ts +4 -7
- package/src/orchestrator/shell-verdict.ts +117 -0
- package/src/orchestrator/signals.ts +86 -30
- package/src/orchestrator/stable-keys.ts +202 -10
- package/src/orchestrator/status-line.ts +33 -1
- package/src/orchestrator/summary.ts +49 -2
- package/src/orchestrator/task-hash.ts +87 -7
- package/src/orchestrator/task-log-buffer.ts +7 -7
- package/src/orchestrator/telemetry-host.ts +16 -7
- package/src/orchestrator/telemetry.ts +69 -28
- package/src/orchestrator/upstream.ts +91 -45
- package/src/util/bun-version.ts +43 -0
- package/src/util/errors.ts +86 -0
- package/src/util/hash.ts +10 -1
- package/src/util/index.ts +32 -3
- package/src/util/paths.ts +147 -5
- package/src/util/procfs.ts +20 -0
- package/src/util/settle.ts +4 -2
- package/src/util/size.ts +33 -0
- package/src/util/task-id.ts +20 -0
- package/src/util/timing.ts +11 -1
- package/src/util/which.ts +48 -0
- package/src/workspace/affected.ts +272 -64
- package/src/workspace/config-cache.ts +262 -32
- package/src/workspace/config-eval.ts +49 -12
- package/src/workspace/config-imports.ts +126 -5
- package/src/workspace/config-schema.ts +237 -41
- package/src/workspace/filter.ts +33 -18
- package/src/workspace/fingerprint.ts +29 -11
- package/src/workspace/index.ts +7 -0
- package/src/workspace/json-data.ts +90 -0
- package/src/workspace/load-reads.ts +32 -0
- package/src/workspace/lockfile.ts +33 -11
- package/src/workspace/migrate-scripts.ts +60 -7
- package/src/workspace/migration.ts +40 -1
- package/src/workspace/package-graph.ts +154 -16
- package/src/workspace/project-loader.ts +227 -77
- package/src/workspace/workspace.ts +173 -23
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// The end of each failed task's output, kept for the run's last lines
|
|
2
|
+
// (item 706). A failure's frame prints when the task finishes, which in a
|
|
3
|
+
// long CI log is thousands of lines above the end, and GitHub's API returns
|
|
4
|
+
// only a job log's last 5,000 lines: the summary is the one place a reader
|
|
5
|
+
// reliably looks, so the failure's own words are repeated there.
|
|
6
|
+
//
|
|
7
|
+
// The capture is a ring over the output's END: it keeps at most
|
|
8
|
+
// RECAP_TASK_BYTES characters (and a newline) however much a task prints,
|
|
9
|
+
// and counts what it evicts, so the recap can say how much it is not
|
|
10
|
+
// showing.
|
|
11
|
+
|
|
12
|
+
/** Lines of each failed task's output the recap repeats. */
|
|
13
|
+
const RECAP_LINES = 30
|
|
14
|
+
/** The byte cap on one task's repeated lines. */
|
|
15
|
+
const RECAP_TASK_BYTES = 8 * 1024
|
|
16
|
+
/**
|
|
17
|
+
* Tasks that get a tail; the rest are named by id. Five tails at the
|
|
18
|
+
* per-task cap is 40 KiB, so the recap stays under 64 KiB with no cap
|
|
19
|
+
* of its own.
|
|
20
|
+
*/
|
|
21
|
+
export const RECAP_TASKS = 5
|
|
22
|
+
|
|
23
|
+
export interface RecapRing {
|
|
24
|
+
chunks: string[]
|
|
25
|
+
chars: number
|
|
26
|
+
/** Newlines evicted from the head. */
|
|
27
|
+
cutLines: number
|
|
28
|
+
/** Bytes evicted from the line the kept text starts with; 0 when it starts a line. */
|
|
29
|
+
cutPartialBytes: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface RecapTail {
|
|
33
|
+
/** The last lines, at most RECAP_LINES of them and RECAP_TASK_BYTES bytes. */
|
|
34
|
+
text: string
|
|
35
|
+
/** Whole lines above `text`. */
|
|
36
|
+
earlierLines: number
|
|
37
|
+
/** Bytes cut from the start of `text`'s first line. */
|
|
38
|
+
cutBytes: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function createRecapRing(): RecapRing {
|
|
42
|
+
return { chunks: [], chars: 0, cutLines: 0, cutPartialBytes: 0 }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// The ring counts characters, the cap bytes: a UTF-16 unit is at least one
|
|
46
|
+
// UTF-8 byte, so the last RECAP_TASK_BYTES characters always hold the last
|
|
47
|
+
// RECAP_TASK_BYTES bytes. One more for the final newline the tail drops.
|
|
48
|
+
const RING_CHARS = RECAP_TASK_BYTES + 1
|
|
49
|
+
|
|
50
|
+
export function appendRecapRing(r: RecapRing, chunk: string): void {
|
|
51
|
+
if (chunk.length === 0) return
|
|
52
|
+
r.chunks.push(chunk)
|
|
53
|
+
r.chars += chunk.length
|
|
54
|
+
// A whole chunk goes only when what follows it still fills the ring:
|
|
55
|
+
// evicting past the cap would leave the recap fewer lines than it shows.
|
|
56
|
+
while (r.chunks.length > 1 && r.chars - r.chunks[0]!.length >= RING_CHARS) {
|
|
57
|
+
const gone = r.chunks.shift()!
|
|
58
|
+
r.chars -= gone.length
|
|
59
|
+
evict(r, gone, gone.length)
|
|
60
|
+
}
|
|
61
|
+
if (r.chars > RING_CHARS) {
|
|
62
|
+
const head = r.chunks[0]!
|
|
63
|
+
const at = r.chars - RING_CHARS
|
|
64
|
+
evict(r, head, at)
|
|
65
|
+
r.chunks[0] = head.slice(at)
|
|
66
|
+
r.chars = RING_CHARS
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function evict(r: RecapRing, s: string, end: number): void {
|
|
71
|
+
const last = s.lastIndexOf('\n', end - 1)
|
|
72
|
+
const partial = Buffer.byteLength(s.slice(last + 1, end))
|
|
73
|
+
if (last < 0) {
|
|
74
|
+
r.cutPartialBytes += partial
|
|
75
|
+
return
|
|
76
|
+
}
|
|
77
|
+
r.cutLines += countNewlines(s, last + 1)
|
|
78
|
+
r.cutPartialBytes = partial
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function countNewlines(s: string, end: number): number {
|
|
82
|
+
let n = 0
|
|
83
|
+
for (let i = s.indexOf('\n'); i !== -1 && i < end; i = s.indexOf('\n', i + 1)) n++
|
|
84
|
+
return n
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The recap's view of what the ring kept: its last RECAP_LINES lines, then
|
|
89
|
+
* its last RECAP_TASK_BYTES bytes (cut on a character boundary). The text
|
|
90
|
+
* is a fresh copy, so it holds nothing of the chunks it came from.
|
|
91
|
+
*/
|
|
92
|
+
export function recapTail(r: RecapRing): RecapTail {
|
|
93
|
+
const t = r.chunks.join('')
|
|
94
|
+
const end = t.endsWith('\n') ? t.length - 1 : t.length
|
|
95
|
+
let start = 0
|
|
96
|
+
for (let pos = end, taken = 1; ; pos = start - 1, taken++) {
|
|
97
|
+
const nl = pos === 0 ? -1 : t.lastIndexOf('\n', pos - 1)
|
|
98
|
+
if (nl < 0) {
|
|
99
|
+
start = 0
|
|
100
|
+
break
|
|
101
|
+
}
|
|
102
|
+
start = nl + 1
|
|
103
|
+
if (taken === RECAP_LINES) break
|
|
104
|
+
}
|
|
105
|
+
let earlierLines = r.cutLines + countNewlines(t, start)
|
|
106
|
+
let cutBytes = start === 0 ? r.cutPartialBytes : 0
|
|
107
|
+
const bytes = Buffer.from(t.slice(start, end), 'utf8')
|
|
108
|
+
if (bytes.length <= RECAP_TASK_BYTES)
|
|
109
|
+
return { text: bytes.toString('utf8'), earlierLines, cutBytes }
|
|
110
|
+
let at = bytes.length - RECAP_TASK_BYTES
|
|
111
|
+
while ((bytes[at]! & 0xc0) === 0x80) at++
|
|
112
|
+
const lastNl = bytes.lastIndexOf(0x0a, at - 1)
|
|
113
|
+
if (lastNl < 0) cutBytes += at
|
|
114
|
+
else {
|
|
115
|
+
for (let i = bytes.indexOf(0x0a); i !== -1 && i <= lastNl; i = bytes.indexOf(0x0a, i + 1)) {
|
|
116
|
+
earlierLines++
|
|
117
|
+
}
|
|
118
|
+
cutBytes = at - lastNl - 1
|
|
119
|
+
}
|
|
120
|
+
return { text: bytes.subarray(at).toString('utf8'), earlierLines, cutBytes }
|
|
121
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// The workspace fingerprint is read once per run, before any task, and
|
|
2
|
+
// every key folds it (and a lockfile plugin's per-project parts, taken from
|
|
3
|
+
// the same files). A task in the root project may rewrite one of those
|
|
4
|
+
// files — `pnpm install` without `--frozen-lockfile` — and every key taken
|
|
5
|
+
// before it then names a lockfile the tree no longer holds: a save under
|
|
6
|
+
// one filed bytes built against the new install under the old lockfile's
|
|
7
|
+
// key, replayed whenever the tree went back (item 750). The watch answers
|
|
8
|
+
// "has a fingerprinted file moved since the run read it", re-learning the
|
|
9
|
+
// fact only after a task that may have written one ran (`wrote`), so a run
|
|
10
|
+
// with no such task never touches the files again.
|
|
11
|
+
|
|
12
|
+
import { readFileSync, statSync } from 'node:fs'
|
|
13
|
+
import path from 'node:path'
|
|
14
|
+
import { FILE_HASH_RACY_MS } from '../cache/index.js'
|
|
15
|
+
import { WORKSPACE_FINGERPRINT_FILES, type WorkspaceFingerprints } from '../workspace/index.js'
|
|
16
|
+
import type { Logger } from './logger.js'
|
|
17
|
+
|
|
18
|
+
export class FingerprintWatch {
|
|
19
|
+
private pending = false
|
|
20
|
+
private movedFiles: string[] | undefined
|
|
21
|
+
private said = false
|
|
22
|
+
|
|
23
|
+
/** `at`: when the run started reading the files (ms since the epoch). */
|
|
24
|
+
constructor(
|
|
25
|
+
private readonly workspaceRoot: string,
|
|
26
|
+
private readonly read: WorkspaceFingerprints,
|
|
27
|
+
private readonly at: number,
|
|
28
|
+
) {}
|
|
29
|
+
|
|
30
|
+
/** A task that may rewrite a fingerprinted file (`mayWriteFingerprint`) ran a command. */
|
|
31
|
+
wrote(): void {
|
|
32
|
+
this.pending = true
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The fingerprinted files that no longer hold what the run read, or
|
|
37
|
+
* undefined. Once one has, the run stays moved: every key already taken
|
|
38
|
+
* folded the old bytes. One `stat` per table entry after each writer; a
|
|
39
|
+
* file written since the read (by ctime, with git's racy window) is read
|
|
40
|
+
* and compared. The stat follows a link as both reads do: a lockfile
|
|
41
|
+
* that is a symlink, rewritten through it, moves its target's ctime and
|
|
42
|
+
* never the link's, and an `lstat` here skipped it once the link had
|
|
43
|
+
* aged past the window — a stale hit on every later run (item 760).
|
|
44
|
+
*/
|
|
45
|
+
moved(): readonly string[] | undefined {
|
|
46
|
+
if (this.movedFiles !== undefined || !this.pending) return this.movedFiles
|
|
47
|
+
this.pending = false
|
|
48
|
+
const moved: string[] = []
|
|
49
|
+
for (const f of WORKSPACE_FINGERPRINT_FILES) {
|
|
50
|
+
const abs = path.join(this.workspaceRoot, f)
|
|
51
|
+
const before = this.read.files.get(f)
|
|
52
|
+
let ctimeMs: number
|
|
53
|
+
try {
|
|
54
|
+
ctimeMs = statSync(abs).ctimeMs
|
|
55
|
+
} catch {
|
|
56
|
+
if (before !== undefined) moved.push(f)
|
|
57
|
+
continue
|
|
58
|
+
}
|
|
59
|
+
if (before !== undefined && ctimeMs < this.at - FILE_HASH_RACY_MS) continue
|
|
60
|
+
let now: Uint8Array | undefined
|
|
61
|
+
try {
|
|
62
|
+
now = readFileSync(abs)
|
|
63
|
+
} catch {
|
|
64
|
+
// Not a file (the run-start read skipped it the same way).
|
|
65
|
+
}
|
|
66
|
+
if (
|
|
67
|
+
now === undefined ? before !== undefined : before === undefined || !sameBytes(now, before)
|
|
68
|
+
) {
|
|
69
|
+
moved.push(f)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (moved.length > 0) this.movedFiles = moved
|
|
73
|
+
return this.movedFiles
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The run's one line about it, however many tasks it withholds. */
|
|
77
|
+
say(log: Logger): void {
|
|
78
|
+
if (this.said || this.movedFiles === undefined) return
|
|
79
|
+
this.said = true
|
|
80
|
+
log.status(
|
|
81
|
+
`[vx] a task rewrote \`${this.movedFiles.join('`, `')}\` during the run, which every key ` +
|
|
82
|
+
`folds (the workspace fingerprint): nothing keyed before it is restored or saved from here on`,
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function sameBytes(a: Uint8Array, b: Uint8Array): boolean {
|
|
88
|
+
return a.length === b.length && Buffer.compare(a, b) === 0
|
|
89
|
+
}
|
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
|
|
29
29
|
import { paint, type ColorSupport } from './colors.js'
|
|
30
30
|
import { formatDuration } from './summary.js'
|
|
31
|
-
import { outcomeLabel } from './events.js'
|
|
31
|
+
import { outcomeLabel, skippedReason } from './events.js'
|
|
32
|
+
import type { RecapTail } from './failure-recap.js'
|
|
32
33
|
|
|
33
34
|
const NO_COLOR: ColorSupport = { enabled: false }
|
|
34
35
|
|
|
@@ -205,7 +206,7 @@ function paintTaskId(node: TaskNode, colors: ColorSupport, opts: { bold?: boolea
|
|
|
205
206
|
|
|
206
207
|
// ── Reported-line grid: glyph · time · status · cache · name ────────
|
|
207
208
|
// glyph SHAPE encodes the cache axis (⏺ miss · ► fresh · ⇢ local · ⇣
|
|
208
|
-
// remote; ◼ failed · ⊘ skipped
|
|
209
|
+
// remote; ◼ failed · ⊘ skipped), glyph COLOR encodes the
|
|
209
210
|
// task axis (green/red/yellow/cyan). The status and cache WORDS spell
|
|
210
211
|
// the two axes out, each in its own color. Time is right-aligned in a
|
|
211
212
|
// fixed cell so durations line up and a ticking elapsed never shifts
|
|
@@ -350,17 +351,29 @@ function formatOutcomeRow(
|
|
|
350
351
|
)
|
|
351
352
|
}
|
|
352
353
|
|
|
353
|
-
/**
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
354
|
+
/**
|
|
355
|
+
* Compact one-liner for a skipped task — it never ran (blank time). The
|
|
356
|
+
* blocker rides the row (`• blocked by lib#build`): the footer's Skipped
|
|
357
|
+
* section groups the same fact, but the row is where the reader's eye is.
|
|
358
|
+
*/
|
|
359
|
+
export function formatTaskSkippedLine(
|
|
360
|
+
node: TaskNode,
|
|
361
|
+
colors: ColorSupport = NO_COLOR,
|
|
362
|
+
blockedBy?: string,
|
|
363
|
+
): string {
|
|
364
|
+
const why = skippedReason(blockedBy)
|
|
365
|
+
const suffix = why === '' ? '' : ` ${paint('', `• ${why}`, colors, { dim: true })}`
|
|
366
|
+
return (
|
|
367
|
+
formatTaskRow(
|
|
368
|
+
paint(WARN, '\u2298', colors),
|
|
369
|
+
null,
|
|
370
|
+
'skipped',
|
|
371
|
+
WARN,
|
|
372
|
+
'',
|
|
373
|
+
'',
|
|
374
|
+
paintTaskId(node, colors),
|
|
375
|
+
colors,
|
|
376
|
+
) + suffix
|
|
364
377
|
)
|
|
365
378
|
}
|
|
366
379
|
|
|
@@ -467,6 +480,60 @@ export function formatPersistentList(
|
|
|
467
480
|
)
|
|
468
481
|
}
|
|
469
482
|
|
|
483
|
+
/** One failed task's section of the end-of-run recap. */
|
|
484
|
+
export interface RecapEntry {
|
|
485
|
+
node: TaskNode
|
|
486
|
+
outcome: TaskOutcome
|
|
487
|
+
tail: RecapTail
|
|
488
|
+
/** Characters a bounded capture (a persistent task's) dropped before the tail saw them. */
|
|
489
|
+
droppedChars: number
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* The run's last block: each failed task's last lines, under the failure's
|
|
494
|
+
* label, then the ids of the failures past the tail limit. The lines are
|
|
495
|
+
* raw, like a frame's, and `fence` wraps them where a task's text could be
|
|
496
|
+
* read as something other than text (GitHub Actions' workflow commands).
|
|
497
|
+
*/
|
|
498
|
+
export function formatFailureRecap(
|
|
499
|
+
entries: readonly RecapEntry[],
|
|
500
|
+
more: readonly string[],
|
|
501
|
+
colors: ColorSupport = NO_COLOR,
|
|
502
|
+
fence: (lines: string[]) => string[] = (lines) => lines,
|
|
503
|
+
): string[] {
|
|
504
|
+
const total = entries.length + more.length
|
|
505
|
+
const dim = (s: string) => paint('', s, colors, { dim: true })
|
|
506
|
+
const lines = [
|
|
507
|
+
'',
|
|
508
|
+
` Failed: ${total} task${total === 1 ? '' : 's'} — the last lines ${total === 1 ? 'it' : 'each one'} printed`,
|
|
509
|
+
]
|
|
510
|
+
for (const e of entries) {
|
|
511
|
+
lines.push(
|
|
512
|
+
'',
|
|
513
|
+
` ${paint(ERROR, '◼︎', colors)} ${paintTaskId(e.node, colors, { bold: true })} ${dim('—')} ${paint(ERROR, outcomeLabel(e.outcome), colors, { bold: true })}`,
|
|
514
|
+
)
|
|
515
|
+
const cut: string[] = []
|
|
516
|
+
const { earlierLines, cutBytes } = e.tail
|
|
517
|
+
if (earlierLines > 0)
|
|
518
|
+
cut.push(`${count(earlierLines)} earlier line${earlierLines === 1 ? '' : 's'}`)
|
|
519
|
+
if (cutBytes > 0) cut.push(`${count(cutBytes)} bytes cut from the start of the line below`)
|
|
520
|
+
if (e.droppedChars > 0) {
|
|
521
|
+
cut.push(`the capture dropped ${count(e.droppedChars)} characters of the task's output`)
|
|
522
|
+
}
|
|
523
|
+
if (cut.length > 0) lines.push(dim(` … ${cut.join(', and ')}`))
|
|
524
|
+
if (e.tail.text.length === 0) lines.push(dim(' (no output)'))
|
|
525
|
+
else lines.push(...fence(e.tail.text.split('\n')))
|
|
526
|
+
}
|
|
527
|
+
if (more.length > 0) {
|
|
528
|
+
lines.push('', ` … and ${more.length} more failed: ${more.join(', ')}`)
|
|
529
|
+
}
|
|
530
|
+
return lines
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function count(n: number): string {
|
|
534
|
+
return n.toLocaleString('en-US')
|
|
535
|
+
}
|
|
536
|
+
|
|
470
537
|
function formatBlockHeader(o: TaskOutcome, colors: ColorSupport): string {
|
|
471
538
|
const shortHash = o.hash ? o.hash.slice(0, 8) : ''
|
|
472
539
|
const dim = (s: string) => paint('', s, colors, { dim: true })
|
|
@@ -482,7 +549,7 @@ function formatBlockHeader(o: TaskOutcome, colors: ColorSupport): string {
|
|
|
482
549
|
// carries the outcome like every other status.
|
|
483
550
|
return paint(ERROR, word, colors, { bold: true })
|
|
484
551
|
case 'skipped':
|
|
485
|
-
return paint(WARN,
|
|
552
|
+
return paint(WARN, word, colors)
|
|
486
553
|
default:
|
|
487
554
|
return dim(word)
|
|
488
555
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
// HistoryTable + HistoryProvider — historical run data the
|
|
2
|
-
//
|
|
1
|
+
// HistoryTable + HistoryProvider — historical run data the `schedule`
|
|
2
|
+
// plugin (`@vzn/vx-schedule-history`, and its `vx history` verb) and
|
|
3
|
+
// `--dry`'s time prediction read.
|
|
3
4
|
//
|
|
4
5
|
// The data has been in cache.db.runs since schema v11; what's new is
|
|
5
6
|
// surfacing it. One SQL statement per call pulls the last N executed rows
|
|
@@ -7,10 +8,8 @@
|
|
|
7
8
|
// result becomes a read-only snapshot for the run's lifetime. Loaded once
|
|
8
9
|
// by the `schedule` stage (or `--dry`); never mutated mid-run.
|
|
9
10
|
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
// RemoteHistoryProvider — would call a service RPC; deferred to
|
|
13
|
-
// when such an RPC actually exists.
|
|
11
|
+
// One provider ships, `LocalHistoryProvider`, reading cache.db directly
|
|
12
|
+
// (zero-config); the interface is the seam a plugin fills for any other.
|
|
14
13
|
|
|
15
14
|
import type { Database } from 'bun:sqlite'
|
|
16
15
|
import { EXECUTED_RUNS_SQL } from '../cache/index.js'
|
|
@@ -94,7 +93,7 @@ export class LocalHistoryProvider implements HistoryProvider {
|
|
|
94
93
|
const floorRow = this.db
|
|
95
94
|
.query(
|
|
96
95
|
`SELECT MIN(id) AS id FROM runs WHERE run_id =
|
|
97
|
-
(SELECT run_id FROM invocations ORDER BY
|
|
96
|
+
(SELECT run_id FROM invocations ORDER BY rowid DESC LIMIT 1 OFFSET ?)`,
|
|
98
97
|
)
|
|
99
98
|
.get(this.recent - 1) as { id: number | null }
|
|
100
99
|
// Fewer invocations than the window (or none): the whole table is the window.
|
|
@@ -12,6 +12,7 @@ import path from 'node:path'
|
|
|
12
12
|
import type { CacheConfig } from '../config.js'
|
|
13
13
|
import {
|
|
14
14
|
type CacheEntry,
|
|
15
|
+
cleanOutputPaths,
|
|
15
16
|
cleanOutputs,
|
|
16
17
|
cleanWorkspaceOutputs,
|
|
17
18
|
resolveOutputs,
|
|
@@ -19,7 +20,7 @@ import {
|
|
|
19
20
|
WORKSPACE_OUTPUT_PREFIX,
|
|
20
21
|
} from '../cache/index.js'
|
|
21
22
|
import type { TaskOutcome } from '../graph/index.js'
|
|
22
|
-
import { span, wholeSubtreePrefixes } from '../util/index.js'
|
|
23
|
+
import { asTrees, span, taskGlob, wholeSubtreePrefixes } from '../util/index.js'
|
|
23
24
|
import type { ExecuteArgs } from './execute-task.js'
|
|
24
25
|
|
|
25
26
|
export interface RestoreHitArgs {
|
|
@@ -56,6 +57,18 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
56
57
|
nestedProjectDirs: args.nestedProjectDirs,
|
|
57
58
|
}
|
|
58
59
|
const wsCleanArgs = { workspaceRoot: args.workspaceRoot, outputs: wsOutputs }
|
|
60
|
+
// The two sides of an overlapping pair (item 588). An ADDITIVE task owns
|
|
61
|
+
// only its recorded rows: its glob also selects the upstream's files it
|
|
62
|
+
// adds beside, so "no strays" is not a claim it can make, and its clean
|
|
63
|
+
// takes the rows, never the glob. The UPSTREAM ignores what its
|
|
64
|
+
// dependants declare they add when it judges its own tree current — an
|
|
65
|
+
// addition below is not a stray — while its glob clean still takes them:
|
|
66
|
+
// the dependant restores or runs after it, by the edge.
|
|
67
|
+
const additive = (node.addsToOutputsOf?.length ?? 0) > 0
|
|
68
|
+
const addedGlobs = (node.outputsAddedToBy ?? [])
|
|
69
|
+
.flatMap((g) => asTrees([g]))
|
|
70
|
+
.map((g) => taskGlob(g))
|
|
71
|
+
const isAddition = (rel: string): boolean => addedGlobs.some((g) => g.match(rel))
|
|
59
72
|
|
|
60
73
|
// "Tree is already current" short-circuit — skip cleanOutputs
|
|
61
74
|
// + restoreOutputs when the on-disk state matches what this
|
|
@@ -100,6 +113,11 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
100
113
|
const endDirs = span('output dirs')
|
|
101
114
|
// Loaded with the entry (batched by getMany, or with the lazy get).
|
|
102
115
|
const dirRows = hit.outputDirRows ?? []
|
|
116
|
+
// Rows are recorded for these same prefixes under this same key
|
|
117
|
+
// (the outputs are folded into it), so `covers` cannot be false
|
|
118
|
+
// today — deleting it reddens nothing (item 638). Kept: a row
|
|
119
|
+
// set that shrank would otherwise vouch for a tree it never saw,
|
|
120
|
+
// and the failure it would let through is a skipped restore.
|
|
103
121
|
const covers = dirPrefixes.every((pre) => dirRows.some((r) => r.path === pre))
|
|
104
122
|
setKnown = covers && (await args.cache.outputDirsCurrent!(node.projectDir, dirRows))
|
|
105
123
|
endDirs()
|
|
@@ -129,13 +147,28 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
129
147
|
const expSet = new Set(exp.map((e) => e.path))
|
|
130
148
|
return actual.length === expSet.size && actual.every((r) => expSet.has(r))
|
|
131
149
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
)
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
150
|
+
// A stray a dependant's glob could have added is not a stray; a path
|
|
151
|
+
// this entry recorded is never dropped, even where that glob covers
|
|
152
|
+
// the whole tree (strapi's shape: both on `dist`).
|
|
153
|
+
const expectedRels = new Set(projExpected.map((e) => e.path))
|
|
154
|
+
const expectedWsRels = new Set(wsExpected.map((e) => e.path))
|
|
155
|
+
const actualRels = actualAbs
|
|
156
|
+
.map((p) => path.relative(node.projectDir, p).split(path.sep).join('/'))
|
|
157
|
+
.filter((rel) => expectedRels.has(rel) || !isAddition(rel))
|
|
158
|
+
const actualWsRels = actualWsAbs
|
|
159
|
+
.map((p) => path.relative(args.workspaceRoot, p).split(path.sep).join('/'))
|
|
160
|
+
.filter((rel) => expectedWsRels.has(rel) || !isAddition(rel))
|
|
161
|
+
const rowsPresent = (
|
|
162
|
+
actual: readonly string[],
|
|
163
|
+
exp: ReadonlyArray<{ path: string }>,
|
|
164
|
+
): boolean => {
|
|
165
|
+
const have = new Set(actual)
|
|
166
|
+
return exp.every((e) => have.has(e.path))
|
|
167
|
+
}
|
|
168
|
+
const treeMatches = additive
|
|
169
|
+
? rowsPresent(actualRels, projExpected) && rowsPresent(actualWsRels, wsExpected)
|
|
170
|
+
: setsMatch(actualRels, projExpected) && setsMatch(actualWsRels, wsExpected)
|
|
171
|
+
if (treeMatches) {
|
|
139
172
|
const endStat = span('output stat')
|
|
140
173
|
skipRestore =
|
|
141
174
|
(await args.cache.isOutputsCurrent(node.projectDir, projExpected)) &&
|
|
@@ -150,25 +183,56 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
150
183
|
}
|
|
151
184
|
}
|
|
152
185
|
}
|
|
186
|
+
// An entry with NO rows (its declared outputs matched nothing at save
|
|
187
|
+
// time, the warned case) is current exactly when the globs still match
|
|
188
|
+
// nothing: then the empty artifact has nothing to put back and the read
|
|
189
|
+
// of the tar is spared. A stray under the glob still falls through to
|
|
190
|
+
// the clean, as strict ownership requires (item 589).
|
|
191
|
+
if (anyOutputs && !skipRestore) {
|
|
192
|
+
const expected = hit.outputRows ?? args.cache.loadOutputFilesBatch([hash]).get(hash) ?? []
|
|
193
|
+
if (expected.length === 0) {
|
|
194
|
+
const endGlob = span('output glob')
|
|
195
|
+
const actual = await resolveOutputs({
|
|
196
|
+
projectDir: node.projectDir,
|
|
197
|
+
outputs,
|
|
198
|
+
nestedProjectDirs: args.nestedProjectDirs,
|
|
199
|
+
})
|
|
200
|
+
const actualWs = await resolveWorkspaceOutputs({
|
|
201
|
+
workspaceRoot: args.workspaceRoot,
|
|
202
|
+
outputs: wsOutputs,
|
|
203
|
+
})
|
|
204
|
+
endGlob()
|
|
205
|
+
skipRestore = actual.length === 0 && actualWs.length === 0
|
|
206
|
+
}
|
|
207
|
+
}
|
|
153
208
|
if (!skipRestore) {
|
|
154
209
|
let cleanedRels: string[] = []
|
|
155
210
|
let cleanedWsRels: string[] = []
|
|
156
|
-
if (
|
|
157
|
-
|
|
211
|
+
if (additive) {
|
|
212
|
+
cleanedRels = hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
213
|
+
await cleanOutputPaths({ projectDir: node.projectDir, rels: cleanedRels })
|
|
214
|
+
cleanedWsRels = hit.outputFiles
|
|
215
|
+
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
216
|
+
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
217
|
+
await cleanOutputPaths({ projectDir: args.workspaceRoot, rels: cleanedWsRels })
|
|
218
|
+
} else {
|
|
219
|
+
if (outputs.length > 0) cleanedRels = await cleanOutputs(cleanArgs)
|
|
220
|
+
if (wsOutputs.length > 0) cleanedWsRels = await cleanWorkspaceOutputs(wsCleanArgs)
|
|
221
|
+
}
|
|
158
222
|
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
|
|
223
|
+
// The tree now holds this entry's bytes: stamp them for the next hit
|
|
224
|
+
// (item 886).
|
|
225
|
+
args.cache.recordOutputStamps?.(hash, node.projectDir, args.workspaceRoot)
|
|
159
226
|
// The directory snapshot behind the NEXT hit's skip-restore, taken at
|
|
160
|
-
// run end
|
|
227
|
+
// run end from the run's list (as the miss path does, see
|
|
161
228
|
// miss-save.ts): the restore renamed into these directories
|
|
162
229
|
// microseconds ago, inside the snapshot's racy window, so a snapshot
|
|
163
230
|
// taken here was refused and the first warm run after every restore
|
|
164
231
|
// walked its output trees — 41 of payload's 45 tasks, 14,430 files,
|
|
165
|
-
// ~75 ms of the no-op's CPU (2026-09-11).
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
} else {
|
|
170
|
-
await args.cache.recordOutputDirs?.(hash, node.projectDir, dirPrefixes)
|
|
171
|
-
}
|
|
232
|
+
// ~75 ms of the no-op's CPU (2026-09-11). A caller with no list gets
|
|
233
|
+
// no snapshot (item 637: recording here was that same refusal).
|
|
234
|
+
if (dirPrefixes !== null && args.outputDirSnapshots !== undefined) {
|
|
235
|
+
args.outputDirSnapshots.push({ hash, projectDir: node.projectDir, prefixes: dirPrefixes })
|
|
172
236
|
}
|
|
173
237
|
// Restored outputs changed the project's tree — but on this
|
|
174
238
|
// path we know the EXACT changed paths (wiped declared
|
|
@@ -11,11 +11,16 @@ export {
|
|
|
11
11
|
type LoadProjectsArgs,
|
|
12
12
|
} from './projects.js'
|
|
13
13
|
export { computeTaskHash, createHashCache, type HashCache } from './task-hash.js'
|
|
14
|
-
export type { RunOptions, RunSummary } from './options.js'
|
|
14
|
+
export type { HeldPersistent, RunOptions, RunSummary } from './options.js'
|
|
15
15
|
// Re-surface the cache policy contract (defined in the cache module) so
|
|
16
16
|
// embedders constructing RunOptions.cache and the package façade can
|
|
17
17
|
// reach it without importing the cache module directly.
|
|
18
18
|
export { type CachePolicy, FULL_CACHE_POLICY, parseCachePolicy } from '../cache/index.js'
|
|
19
|
+
// The CLI reads a recorded exit code; the runner's convention decodes it.
|
|
20
|
+
export { exitSignal } from '../exec/index.js'
|
|
21
|
+
export { forwardedSignal } from './signals.js'
|
|
22
|
+
// `vx cache prune` waits for a run on the workspace before it evicts.
|
|
23
|
+
export { acquireRunLock } from './run-lock.js'
|
|
19
24
|
export { defaultLogger, resolveOutputView } from './logger.js'
|
|
20
25
|
export type { Logger, OutputView } from './logger.js'
|
|
21
26
|
export type { RunPlan, PlannedTask, PlanPrediction, CacheStatus } from './plan.js'
|
|
@@ -138,6 +143,7 @@ export {
|
|
|
138
143
|
explainCacheKey as explainCacheKeyQuery,
|
|
139
144
|
getInvocation,
|
|
140
145
|
getRun,
|
|
146
|
+
latestRunId,
|
|
141
147
|
listInvocations,
|
|
142
148
|
listRuns,
|
|
143
149
|
whyDidThisRerun as whyDidThisRerunQuery,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// The projects a task's cache key answers for: K(T) in
|
|
2
|
+
// docs/design/linked-sibling-reads-2026-09.md. A sandboxed task that
|
|
3
|
+
// declares `cache` is granted a sibling's `node_modules` link only when its
|
|
4
|
+
// key already moves with that sibling's files, and this is where "already"
|
|
5
|
+
// is decided — on the graph, before any hash exists, through the same
|
|
6
|
+
// selection the hash path applies (`selectFoldedDeps`).
|
|
7
|
+
|
|
8
|
+
import { isGroupTask, type TaskNode } from '../graph/index.js'
|
|
9
|
+
import { selectFoldedDeps, type FoldCandidate } from './upstream.js'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A run's keyed-set lookup: for a task, the directories of every project
|
|
13
|
+
* holding an exec task whose key the task's key folds, transitively. The
|
|
14
|
+
* task itself is not counted.
|
|
15
|
+
*
|
|
16
|
+
* The fold relation is the hash path's, computed on the graph:
|
|
17
|
+
* - an exec task folds its dependencies as its own `cache.inputs.tasks`
|
|
18
|
+
* selects them (absent, or no `cache` at all, means all);
|
|
19
|
+
* - a group folds every dependency (`computeGroupHash`) and folds no
|
|
20
|
+
* files of its own;
|
|
21
|
+
* - a persistent task has no hash on the live path, so it is folded by
|
|
22
|
+
* no one and its own dependencies reach no key through it. (The local
|
|
23
|
+
* classify pass does give it one — stable-keys.ts — so a key there can
|
|
24
|
+
* move with more than this set names; never with less.)
|
|
25
|
+
*
|
|
26
|
+
* Every exec task reached contributes its project, cached or not: a key
|
|
27
|
+
* folds the task's own project files, and a task with no `cache` declares
|
|
28
|
+
* no `inputs.files`, so it folds every file of its project (inputs.ts's
|
|
29
|
+
* `DEFAULT_FILE_GLOBS`). `tests/keyed-projects.test.ts` holds that to the
|
|
30
|
+
* key itself.
|
|
31
|
+
*
|
|
32
|
+
* Lazy and memoized by task id: only a sandboxed task that executes asks,
|
|
33
|
+
* so a run of hits walks nothing, and a shared subgraph is walked once.
|
|
34
|
+
*/
|
|
35
|
+
export function keyedProjects(
|
|
36
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
37
|
+
): (node: TaskNode) => ReadonlySet<string> {
|
|
38
|
+
const below = new Map<string, ReadonlySet<string>>()
|
|
39
|
+
// Post-order on an explicit stack: a fold is as deep as the graph, and a
|
|
40
|
+
// recursion per edge threw `RangeError` at the 50,000 the builder takes
|
|
41
|
+
// (item 737). A frame's folded dependencies are listed when it is first
|
|
42
|
+
// reached and combined once each has its set.
|
|
43
|
+
return (root) => {
|
|
44
|
+
const stack: Array<[node: TaskNode, deps: FoldCandidate[] | undefined]> = [[root, undefined]]
|
|
45
|
+
while (stack.length > 0) {
|
|
46
|
+
const frame = stack[stack.length - 1]!
|
|
47
|
+
const [node, deps] = frame
|
|
48
|
+
if (below.has(node.id)) {
|
|
49
|
+
stack.pop()
|
|
50
|
+
continue
|
|
51
|
+
}
|
|
52
|
+
if (deps === undefined) {
|
|
53
|
+
frame[1] = folded(node, nodes)
|
|
54
|
+
for (const { node: dep } of frame[1]) if (!below.has(dep.id)) stack.push([dep, undefined])
|
|
55
|
+
continue
|
|
56
|
+
}
|
|
57
|
+
stack.pop()
|
|
58
|
+
const out = new Set<string>()
|
|
59
|
+
for (const { node: dep } of deps) {
|
|
60
|
+
if (!isGroupTask(dep)) out.add(dep.projectDir)
|
|
61
|
+
for (const dir of below.get(dep.id)!) out.add(dir)
|
|
62
|
+
}
|
|
63
|
+
below.set(node.id, out)
|
|
64
|
+
}
|
|
65
|
+
return below.get(root.id)!
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The dependencies `node`'s key folds, per the hash path's rules above. */
|
|
70
|
+
function folded(node: TaskNode, nodes: ReadonlyMap<string, TaskNode>): FoldCandidate[] {
|
|
71
|
+
const candidates: FoldCandidate[] = []
|
|
72
|
+
for (const id of node.deps) {
|
|
73
|
+
const dep = nodes.get(id)!
|
|
74
|
+
if (dep.config.exec?.persistent !== undefined) continue
|
|
75
|
+
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
76
|
+
}
|
|
77
|
+
if (isGroupTask(node)) return candidates
|
|
78
|
+
return selectFoldedDeps(candidates, node.config.cache?.inputs?.tasks, node.projectName, node.id)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* What stands in for a dependency's hash in the dedup: equal exactly when
|
|
83
|
+
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
84
|
+
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
85
|
+
* with no id of its own, so two groups over the same members hash alike.
|
|
86
|
+
*/
|
|
87
|
+
function foldUnit(node: TaskNode): string {
|
|
88
|
+
return isGroupTask(node) ? `group|${[...node.deps].sort().join('|')}` : `task|${node.id}`
|
|
89
|
+
}
|