@vzn/vx 0.0.651 → 0.0.652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/package.json +5 -5
- package/schemas/show.json +14 -20
- package/src/bin.ts +12 -0
- package/src/cache/archive.ts +1 -1
- package/src/cache/cache.ts +19 -5
- package/src/cache/history-stream.ts +134 -0
- package/src/cache/history-writer.ts +76 -0
- package/src/cache/index.ts +2 -0
- package/src/cache/inputs.ts +74 -14
- package/src/cache/layer.ts +6 -6
- package/src/cache/lockfile-memo.ts +2 -1
- package/src/cache/run-history.ts +46 -17
- package/src/cache/schema.ts +4 -1
- package/src/cli/cache.ts +2 -2
- package/src/cli/help.ts +2 -2
- package/src/cli/index.ts +24 -0
- package/src/cli/info.ts +1 -1
- package/src/cli/last.ts +34 -2
- package/src/cli/plugin-templates.ts +36 -45
- package/src/cli/show.ts +7 -10
- package/src/cli/watch-judge.ts +1 -1
- package/src/cli/watch-set.ts +4 -4
- package/src/cli/watch.ts +11 -8
- package/src/config.ts +47 -30
- package/src/exec/executor.ts +40 -1
- package/src/exec/index.ts +1 -0
- package/src/exec/sandbox-paths.ts +3 -1
- package/src/exec/sandbox-runtime.ts +314 -29
- package/src/exec/task-proxy.ts +93 -0
- package/src/graph/definition.ts +16 -3
- package/src/graph/scheduler.ts +3 -1
- package/src/graph/task-graph.ts +24 -23
- package/src/history/failure-mode.ts +24 -1
- package/src/history/index.ts +1 -0
- package/src/history/metrics.ts +5 -5
- package/src/index.ts +4 -2
- package/src/orchestrator/admission.ts +1 -1
- package/src/orchestrator/command-windows.ts +1 -1
- package/src/orchestrator/deferred-outputs.ts +1 -1
- package/src/orchestrator/download-policy.ts +5 -7
- package/src/orchestrator/excluded-keys.ts +1 -1
- package/src/orchestrator/execute-task.ts +51 -18
- package/src/orchestrator/hit-restore.ts +5 -6
- package/src/orchestrator/keyed-projects.ts +28 -0
- package/src/orchestrator/local-shortcircuit.ts +2 -2
- package/src/orchestrator/miss-save.ts +27 -8
- package/src/orchestrator/persistent-task.ts +2 -0
- package/src/orchestrator/placement.ts +1 -1
- package/src/orchestrator/plan.ts +2 -2
- package/src/orchestrator/plugin-host.ts +6 -2
- package/src/orchestrator/plugin.ts +6 -0
- package/src/orchestrator/remote-prefetch.ts +1 -1
- package/src/orchestrator/run-failures.ts +61 -12
- package/src/orchestrator/run-keys.ts +1 -1
- package/src/orchestrator/run-records.ts +86 -67
- package/src/orchestrator/run-report.ts +1 -1
- package/src/orchestrator/run.ts +92 -19
- package/src/orchestrator/sandbox-request.ts +50 -7
- package/src/orchestrator/shell-verdict.ts +9 -0
- package/src/orchestrator/stable-keys.ts +17 -18
- package/src/orchestrator/task-artifacts.ts +1 -1
- package/src/orchestrator/task-hash.ts +4 -4
- package/src/orchestrator/telemetry.ts +5 -1
- package/src/report/events.ts +3 -3
- package/src/report/framed-output.ts +16 -7
- package/src/report/logger.ts +1 -1
- package/src/report/plain-output.ts +2 -0
- package/src/report/summary.ts +5 -5
- package/src/settings.ts +8 -8
- package/src/util/config-frame.ts +128 -6
- package/src/util/paths.ts +1 -1
- package/src/util/settle.ts +2 -1
- package/src/util/tail.ts +4 -1
- package/src/workspace/bracket-literal.ts +2 -2
- package/src/workspace/config-schema.ts +66 -61
- package/src/workspace/filter.ts +8 -1
- package/src/workspace/migration.ts +12 -6
package/README.md
CHANGED
|
@@ -47,9 +47,8 @@ export default defineProject({
|
|
|
47
47
|
exec: { command: 'tsc -b' },
|
|
48
48
|
dependsOn: ['^build'], // dependencies build first
|
|
49
49
|
inputs: { files: ['src/**', 'tsconfig.json'] },
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
},
|
|
50
|
+
outputs: { files: ['dist/**'] },
|
|
51
|
+
cache: true,
|
|
53
52
|
},
|
|
54
53
|
},
|
|
55
54
|
})
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.652",
|
|
4
4
|
"description": "Task orchestration you own: correct, sandboxed, extensible, with a content-addressed build cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -21,10 +21,10 @@
|
|
|
21
21
|
"node": ">=18"
|
|
22
22
|
},
|
|
23
23
|
"optionalDependencies": {
|
|
24
|
-
"@vzn/vx-linux-x64": "0.0.
|
|
25
|
-
"@vzn/vx-linux-arm64": "0.0.
|
|
26
|
-
"@vzn/vx-darwin-x64": "0.0.
|
|
27
|
-
"@vzn/vx-darwin-arm64": "0.0.
|
|
24
|
+
"@vzn/vx-linux-x64": "0.0.652",
|
|
25
|
+
"@vzn/vx-linux-arm64": "0.0.652",
|
|
26
|
+
"@vzn/vx-darwin-x64": "0.0.652",
|
|
27
|
+
"@vzn/vx-darwin-arm64": "0.0.652"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"@anthropic-ai/sandbox-runtime": "^0.0.76"
|
package/schemas/show.json
CHANGED
|
@@ -109,25 +109,8 @@
|
|
|
109
109
|
"env": { "$ref": "#/$defs/stringList" },
|
|
110
110
|
"tasks": { "$ref": "#/$defs/stringList" },
|
|
111
111
|
"runtime": { "$ref": "#/$defs/stringList" },
|
|
112
|
-
"workspaceRuntime": { "$ref": "#/$defs/stringList" }
|
|
113
|
-
|
|
114
|
-
},
|
|
115
|
-
"cache": {
|
|
116
|
-
"type": "object",
|
|
117
|
-
"additionalProperties": false,
|
|
118
|
-
"required": ["outputs"],
|
|
119
|
-
"properties": {
|
|
120
|
-
"outputs": {
|
|
121
|
-
"type": "object",
|
|
122
|
-
"additionalProperties": false,
|
|
123
|
-
"required": ["files"],
|
|
124
|
-
"properties": {
|
|
125
|
-
"files": { "$ref": "#/$defs/stringList" },
|
|
126
|
-
"workspaceFiles": { "$ref": "#/$defs/stringList" },
|
|
127
|
-
"clean": { "type": "boolean" }
|
|
128
|
-
}
|
|
129
|
-
},
|
|
130
|
-
"inputStability": {
|
|
112
|
+
"workspaceRuntime": { "$ref": "#/$defs/stringList" },
|
|
113
|
+
"stability": {
|
|
131
114
|
"type": "object",
|
|
132
115
|
"additionalProperties": false,
|
|
133
116
|
"properties": {
|
|
@@ -136,7 +119,18 @@
|
|
|
136
119
|
}
|
|
137
120
|
}
|
|
138
121
|
}
|
|
139
|
-
}
|
|
122
|
+
},
|
|
123
|
+
"outputs": {
|
|
124
|
+
"type": "object",
|
|
125
|
+
"additionalProperties": false,
|
|
126
|
+
"required": ["files"],
|
|
127
|
+
"properties": {
|
|
128
|
+
"files": { "$ref": "#/$defs/stringList" },
|
|
129
|
+
"workspaceFiles": { "$ref": "#/$defs/stringList" },
|
|
130
|
+
"clean": { "type": "boolean" }
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
"cache": { "type": "boolean" }
|
|
140
134
|
}
|
|
141
135
|
}
|
|
142
136
|
}
|
package/src/bin.ts
CHANGED
|
@@ -79,6 +79,10 @@ async function main(): Promise<void> {
|
|
|
79
79
|
err.parts,
|
|
80
80
|
)
|
|
81
81
|
ui.data('err', maskedLine(ui.render([block], ui.colorsOf('err'))))
|
|
82
|
+
} else if (cwdGone(err)) {
|
|
83
|
+
// Removed under the shell (a checkout of another branch, `rm -rf` in
|
|
84
|
+
// another): every verb starts from the cwd.
|
|
85
|
+
say('vx: the current directory no longer exists — cd into the workspace again and re-run\n')
|
|
82
86
|
} else if (isOutOfFds(err)) {
|
|
83
87
|
say(`vx: ${err.message} — ${OUT_OF_FDS_HINT}\n`)
|
|
84
88
|
} else if (isFsRefusal(err)) {
|
|
@@ -159,6 +163,14 @@ process.on('beforeExit', () => {
|
|
|
159
163
|
process.exitCode = 1
|
|
160
164
|
})
|
|
161
165
|
|
|
166
|
+
/** `process.cwd()`'s refusal of a directory removed under the process. */
|
|
167
|
+
function cwdGone(err: unknown): boolean {
|
|
168
|
+
return (
|
|
169
|
+
(err as NodeJS.ErrnoException | null)?.code === 'ENOENT' &&
|
|
170
|
+
/\buv_cwd\b/.test((err as Error).message)
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
162
174
|
// A reader that leaves is not the run's failure. `vx run build | head -1`
|
|
163
175
|
// closes the pipe after one line; every later write gets EPIPE, which Bun
|
|
164
176
|
// raises as an `error` event on the stream, and an `error` nobody listens
|
package/src/cache/archive.ts
CHANGED
|
@@ -350,7 +350,7 @@ export async function planArtifact(args: PackArgs): Promise<ArtifactPlan> {
|
|
|
350
350
|
}
|
|
351
351
|
if (!st.isFile()) {
|
|
352
352
|
throw new UserError(
|
|
353
|
-
`${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow
|
|
353
|
+
`${shown} is not a regular file: vx stores regular files and symlinks only — emit a file there, or narrow outputs.files to the files the task produces, or take it back with a '!' entry`,
|
|
354
354
|
)
|
|
355
355
|
}
|
|
356
356
|
const mode = st.mode & 0o777
|
package/src/cache/cache.ts
CHANGED
|
@@ -51,7 +51,13 @@ import {
|
|
|
51
51
|
} from 'node:fs'
|
|
52
52
|
import { chmod, lstat, mkdir, readdir, rm, stat, unlink, utimes, writeFile } from 'node:fs/promises'
|
|
53
53
|
import path from 'node:path'
|
|
54
|
-
import {
|
|
54
|
+
import {
|
|
55
|
+
createArtifactTables,
|
|
56
|
+
createTables,
|
|
57
|
+
inHashes,
|
|
58
|
+
lazyStatement,
|
|
59
|
+
SQLITE_FCNTL_PERSIST_WAL,
|
|
60
|
+
} from './schema.js'
|
|
55
61
|
import {
|
|
56
62
|
UserError,
|
|
57
63
|
DOT_VX_GITIGNORE,
|
|
@@ -275,7 +281,7 @@ function cantOpenIndex(dbFile: string, err: unknown): UserError | undefined {
|
|
|
275
281
|
// the memo simply stops answering wrongly, so an affected task's
|
|
276
282
|
// key moves from a WRONG value to the right one: it misses once,
|
|
277
283
|
// re-runs, re-caches. Self-healing, never a wrong hit.
|
|
278
|
-
// v25: runs.cached — whether the task declared
|
|
284
|
+
// v25: runs.cached — whether the task declared `cache: true`. An
|
|
279
285
|
// uncached task's row looked like any executed miss, so `vx why`
|
|
280
286
|
// headlined "cache key changed (inputs differ)" for a task that
|
|
281
287
|
// runs every time by design, and `vx last` could not mark it the
|
|
@@ -2851,7 +2857,7 @@ export class Cache implements CacheLayer {
|
|
|
2851
2857
|
if (plan.size > this.artifactCeiling) {
|
|
2852
2858
|
throw new Error(
|
|
2853
2859
|
`${args.entry.taskId} is not cached: its outputs pack to ${formatBytes(plan.size)}, past the ` +
|
|
2854
|
-
`${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow
|
|
2860
|
+
`${formatBytes(this.artifactCeiling)} artifact ceiling a restore enforces — narrow outputs.files`,
|
|
2855
2861
|
)
|
|
2856
2862
|
}
|
|
2857
2863
|
return plan
|
|
@@ -3727,8 +3733,16 @@ export class Cache implements CacheLayer {
|
|
|
3727
3733
|
}
|
|
3728
3734
|
}
|
|
3729
3735
|
|
|
3730
|
-
/**
|
|
3731
|
-
|
|
3736
|
+
/**
|
|
3737
|
+
* The files a second connection writes a run's hits through
|
|
3738
|
+
* (history-stream.ts); undefined when the store keeps no history on disk.
|
|
3739
|
+
* A function, as `indexDb` is: no plugin writes the history.
|
|
3740
|
+
*/
|
|
3741
|
+
export function historyFiles(cache: Cache): { index: string; store?: string } | undefined {
|
|
3742
|
+
if (cache['empty'] || cache['writeBlocked'] !== null || cache['inspecting']) return undefined
|
|
3743
|
+
if (cache.storeDir === undefined) return { index: cache['dbFile'] }
|
|
3744
|
+
return { index: cache['dbFile'], store: path.join(cache.storeDir, 'store.db') }
|
|
3745
|
+
}
|
|
3732
3746
|
|
|
3733
3747
|
/**
|
|
3734
3748
|
* The cache's `bun:sqlite` handle, for core's own readers (`vx last`, `vx
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// A run's hits written to the history while the graph still runs, on a
|
|
2
|
+
// thread of their own (history-writer.ts). A warm run's rows were the
|
|
3
|
+
// main thread's last ~65 ms at 8,002 hits, after the graph; here they land
|
|
4
|
+
// while it runs. What does not land (no thread, a refused batch) is
|
|
5
|
+
// handed back and goes in the run's own bundle, as before.
|
|
6
|
+
|
|
7
|
+
import type { RunRecord } from './layer.js'
|
|
8
|
+
import type { HistoryWriterIn, HistoryWriterOut } from './history-writer.js'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Rows per batch, and the hits a run has before it starts a thread: a
|
|
12
|
+
* run with fewer writes them in its bundle, as cheap as a thread's start.
|
|
13
|
+
*/
|
|
14
|
+
const BATCH = 512
|
|
15
|
+
|
|
16
|
+
/** A record of `T` whose row the stream holds. */
|
|
17
|
+
export interface Streamed<T> {
|
|
18
|
+
item: T
|
|
19
|
+
record: RunRecord
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export class HistoryStream<T> {
|
|
23
|
+
private worker: Worker | undefined
|
|
24
|
+
private buffer: Streamed<T>[] = []
|
|
25
|
+
private readonly sent = new Map<number, Streamed<T>[]>()
|
|
26
|
+
private readonly landed: Streamed<T>[] = []
|
|
27
|
+
private seq = 0
|
|
28
|
+
/** A batch was refused or the thread died: nothing more is sent. */
|
|
29
|
+
private dead = false
|
|
30
|
+
/** The thread died: no batch answers any more. */
|
|
31
|
+
private gone = false
|
|
32
|
+
private readonly refused = new Set<number>()
|
|
33
|
+
private settle: (() => void) | undefined
|
|
34
|
+
private ended: (() => void) | undefined
|
|
35
|
+
|
|
36
|
+
constructor(private readonly files: { index: string; store?: string }) {}
|
|
37
|
+
|
|
38
|
+
add(item: T, record: RunRecord): void {
|
|
39
|
+
this.buffer.push({ item, record })
|
|
40
|
+
if (this.buffer.length >= BATCH && !this.dead) this.send()
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Waits for every batch sent, closes the thread, and returns what landed
|
|
45
|
+
* and what did not (the run's bundle writes those).
|
|
46
|
+
*/
|
|
47
|
+
async finish(): Promise<{ landed: Streamed<T>[]; unlanded: Streamed<T>[] }> {
|
|
48
|
+
const w = this.worker
|
|
49
|
+
if (w !== undefined && !this.gone) {
|
|
50
|
+
// Unreferenced, its answers alone left the loop with nothing to
|
|
51
|
+
// wait on: the run ended "awaited can never settle".
|
|
52
|
+
w.ref()
|
|
53
|
+
await new Promise<void>((resolve) => {
|
|
54
|
+
this.settle = resolve
|
|
55
|
+
this.settled()
|
|
56
|
+
})
|
|
57
|
+
if (!this.gone) {
|
|
58
|
+
await new Promise<void>((resolve) => {
|
|
59
|
+
this.ended = resolve
|
|
60
|
+
w.postMessage({ end: true } satisfies HistoryWriterIn)
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
w?.terminate()
|
|
65
|
+
const unlanded = [...this.sent.values()].flat().concat(this.buffer)
|
|
66
|
+
this.sent.clear()
|
|
67
|
+
this.buffer = []
|
|
68
|
+
return { landed: this.landed, unlanded }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private send(): void {
|
|
72
|
+
const w = (this.worker ??= this.start())
|
|
73
|
+
const batch = this.buffer
|
|
74
|
+
this.buffer = []
|
|
75
|
+
const seq = this.seq++
|
|
76
|
+
this.sent.set(seq, batch)
|
|
77
|
+
// A record's spans are bigint, which JSON cannot carry: as numbers, exact
|
|
78
|
+
// below 2^53 ns (104 days into a run). A replacer function cost 20 ms
|
|
79
|
+
// of 8,002 rows, the copies 4.
|
|
80
|
+
const hits = JSON.stringify(
|
|
81
|
+
batch.map(({ record: r }) => ({
|
|
82
|
+
...r,
|
|
83
|
+
wallclockStartNs: r.wallclockStartNs === undefined ? undefined : Number(r.wallclockStartNs),
|
|
84
|
+
wallclockEndNs: r.wallclockEndNs === undefined ? undefined : Number(r.wallclockEndNs),
|
|
85
|
+
})),
|
|
86
|
+
)
|
|
87
|
+
w.postMessage({ seq, hits } satisfies HistoryWriterIn)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private start(): Worker {
|
|
91
|
+
const w = new Worker(new URL('./history-writer.ts', import.meta.url).href)
|
|
92
|
+
// A run that throws before `finish` must still exit; a batch cut off
|
|
93
|
+
// mid-write rolls back whole.
|
|
94
|
+
w.unref()
|
|
95
|
+
w.onmessage = (e: MessageEvent<HistoryWriterOut>) => {
|
|
96
|
+
const m = e.data
|
|
97
|
+
if ('ended' in m) {
|
|
98
|
+
this.ended?.()
|
|
99
|
+
return
|
|
100
|
+
}
|
|
101
|
+
const batch = this.sent.get(m.seq)
|
|
102
|
+
if (batch === undefined) return
|
|
103
|
+
if (m.error === undefined) {
|
|
104
|
+
this.sent.delete(m.seq)
|
|
105
|
+
for (const b of batch) this.landed.push(b)
|
|
106
|
+
} else {
|
|
107
|
+
// Kept in `sent`, so the bundle writes it; no later batch is sent.
|
|
108
|
+
this.refused.add(m.seq)
|
|
109
|
+
this.dead = true
|
|
110
|
+
}
|
|
111
|
+
this.settled()
|
|
112
|
+
}
|
|
113
|
+
// A thread that cannot start or dies: what it did not answer for is
|
|
114
|
+
// the bundle's.
|
|
115
|
+
w.onerror = (e) => {
|
|
116
|
+
e.preventDefault()
|
|
117
|
+
this.dead = true
|
|
118
|
+
this.gone = true
|
|
119
|
+
this.settled()
|
|
120
|
+
this.ended?.()
|
|
121
|
+
}
|
|
122
|
+
w.postMessage({ open: this.files } satisfies HistoryWriterIn)
|
|
123
|
+
return w
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Resolves `finish`'s wait once every batch sent has answered, or the thread is gone. */
|
|
127
|
+
private settled(): void {
|
|
128
|
+
if (this.settle === undefined) return
|
|
129
|
+
if (!this.gone && [...this.sent.keys()].some((seq) => !this.refused.has(seq))) return
|
|
130
|
+
const s = this.settle
|
|
131
|
+
this.settle = undefined
|
|
132
|
+
s()
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The thread that writes a run's hits to the history while the run goes on
|
|
2
|
+
// (history-stream.ts). A connection of its own on the run's cache files,
|
|
3
|
+
// opened as `Cache` opens them; each batch is one IMMEDIATE transaction,
|
|
4
|
+
// answered by its sequence number so the run knows which landed.
|
|
5
|
+
// A separate entry point: `bun build --compile` embeds it beside bin.ts.
|
|
6
|
+
|
|
7
|
+
import { Database } from 'bun:sqlite'
|
|
8
|
+
import type { RunRecord } from './layer.js'
|
|
9
|
+
import { RunHistory } from './run-history.js'
|
|
10
|
+
import { SQLITE_FCNTL_PERSIST_WAL } from './schema.js'
|
|
11
|
+
|
|
12
|
+
export type HistoryWriterIn =
|
|
13
|
+
| { open: { index: string; store?: string } }
|
|
14
|
+
| { seq: number; hits: string }
|
|
15
|
+
| { end: true }
|
|
16
|
+
|
|
17
|
+
export type HistoryWriterOut = { seq: number; error?: string } | { ended: true }
|
|
18
|
+
|
|
19
|
+
declare const self: Worker
|
|
20
|
+
|
|
21
|
+
let db: Database | undefined
|
|
22
|
+
let history: RunHistory | undefined
|
|
23
|
+
let refused: string | undefined
|
|
24
|
+
|
|
25
|
+
function open(files: { index: string; store?: string }): void {
|
|
26
|
+
// The timeout first, as `Cache` sets it: every later pragma waits under it.
|
|
27
|
+
db = new Database(files.index)
|
|
28
|
+
db.exec('PRAGMA busy_timeout = 5000')
|
|
29
|
+
db.fileControl(SQLITE_FCNTL_PERSIST_WAL, 1)
|
|
30
|
+
db.exec('PRAGMA journal_size_limit = 67108864')
|
|
31
|
+
db.exec('PRAGMA synchronous = NORMAL')
|
|
32
|
+
if (files.store !== undefined) {
|
|
33
|
+
db.prepare('ATTACH DATABASE ? AS store').run(files.store)
|
|
34
|
+
db.fileControl('store', SQLITE_FCNTL_PERSIST_WAL, 1)
|
|
35
|
+
db.exec('PRAGMA store.journal_size_limit = 67108864')
|
|
36
|
+
db.exec('PRAGMA store.synchronous = NORMAL')
|
|
37
|
+
}
|
|
38
|
+
// A streamed hit carries no forward args (history-stream.ts): nothing digests.
|
|
39
|
+
history = new RunHistory(db, () => {
|
|
40
|
+
throw new Error('a streamed hit has no value to digest')
|
|
41
|
+
})
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
self.onmessage = (e: MessageEvent<HistoryWriterIn>) => {
|
|
45
|
+
const m = e.data
|
|
46
|
+
if ('open' in m) {
|
|
47
|
+
try {
|
|
48
|
+
open(m.open)
|
|
49
|
+
} catch (err) {
|
|
50
|
+
refused = err instanceof Error ? err.message : String(err)
|
|
51
|
+
}
|
|
52
|
+
return
|
|
53
|
+
}
|
|
54
|
+
if ('end' in m) {
|
|
55
|
+
try {
|
|
56
|
+
db?.close()
|
|
57
|
+
} catch {
|
|
58
|
+
// The rows are in; a close that fails leaves the WAL to the next opener.
|
|
59
|
+
}
|
|
60
|
+
postMessage({ ended: true } satisfies HistoryWriterOut)
|
|
61
|
+
return
|
|
62
|
+
}
|
|
63
|
+
if (history === undefined) {
|
|
64
|
+
postMessage({ seq: m.seq, error: refused ?? 'not open' } satisfies HistoryWriterOut)
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
history.recordHits(JSON.parse(m.hits) as RunRecord[])
|
|
69
|
+
postMessage({ seq: m.seq } satisfies HistoryWriterOut)
|
|
70
|
+
} catch (err) {
|
|
71
|
+
postMessage({
|
|
72
|
+
seq: m.seq,
|
|
73
|
+
error: err instanceof Error ? err.message : String(err),
|
|
74
|
+
} satisfies HistoryWriterOut)
|
|
75
|
+
}
|
|
76
|
+
}
|
package/src/cache/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ export {
|
|
|
18
18
|
type CachePolicy,
|
|
19
19
|
cachesNothing,
|
|
20
20
|
EXECUTED_RUNS_SQL,
|
|
21
|
+
historyFiles,
|
|
21
22
|
indexDb,
|
|
22
23
|
formatCachePolicy,
|
|
23
24
|
FULL_CACHE_POLICY,
|
|
@@ -104,6 +105,7 @@ export {
|
|
|
104
105
|
} from './plugin-cache.js'
|
|
105
106
|
export { packRunOutputs, putRunEntry, runEntryName, writesRunStore } from './run-store.js'
|
|
106
107
|
export { branchSource } from './run-history.js'
|
|
108
|
+
export { HistoryStream } from './history-stream.js'
|
|
107
109
|
export {
|
|
108
110
|
type KeptFailure,
|
|
109
111
|
keepRunNotes,
|
package/src/cache/inputs.ts
CHANGED
|
@@ -17,7 +17,15 @@
|
|
|
17
17
|
// be a git work tree; non-git environments are not supported.
|
|
18
18
|
|
|
19
19
|
import path from 'node:path'
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
type Dirent,
|
|
22
|
+
lstatSync,
|
|
23
|
+
readdirSync,
|
|
24
|
+
realpathSync,
|
|
25
|
+
rmdirSync,
|
|
26
|
+
rmSync,
|
|
27
|
+
statSync,
|
|
28
|
+
} from 'node:fs'
|
|
21
29
|
import { rm, rmdir } from 'node:fs/promises'
|
|
22
30
|
import type { InputsConfig, TaskConfig } from '../config.js'
|
|
23
31
|
import {
|
|
@@ -837,15 +845,23 @@ export async function resolveOutputTree(args: {
|
|
|
837
845
|
projectDir: string
|
|
838
846
|
outputs: readonly string[]
|
|
839
847
|
nestedProjectDirs: string[]
|
|
840
|
-
|
|
848
|
+
/** Also list the FIFOs, sockets and devices the globs name (a save refuses them). */
|
|
849
|
+
special?: boolean
|
|
850
|
+
}): Promise<{ files: string[]; emptyDirs: string[]; special: string[] }> {
|
|
841
851
|
const emptyDirs = new Set<string>()
|
|
842
|
-
const
|
|
843
|
-
|
|
852
|
+
const special = args.special === true ? new Set<string>() : undefined
|
|
853
|
+
const { files } = await scanOutputs(args, emptyDirs, special)
|
|
854
|
+
return {
|
|
855
|
+
files,
|
|
856
|
+
emptyDirs: containedIn(args.projectDir, [...emptyDirs]).sort(),
|
|
857
|
+
special: special === undefined ? [] : containedIn(args.projectDir, [...special]).sort(),
|
|
858
|
+
}
|
|
844
859
|
}
|
|
845
860
|
|
|
846
861
|
async function scanOutputs(
|
|
847
862
|
args: { projectDir: string; outputs: readonly string[]; nestedProjectDirs: string[] },
|
|
848
863
|
emptyDirs?: Set<string>,
|
|
864
|
+
special?: Set<string>,
|
|
849
865
|
): Promise<{ files: string[] }> {
|
|
850
866
|
const { positive, negative } = splitNegations(args.outputs)
|
|
851
867
|
if (positive.length === 0) return { files: [] }
|
|
@@ -859,6 +875,7 @@ async function scanOutputs(
|
|
|
859
875
|
args.projectDir,
|
|
860
876
|
inNestedProject(args.projectDir, args.nestedProjectDirs),
|
|
861
877
|
emptyDirs,
|
|
878
|
+
special,
|
|
862
879
|
)),
|
|
863
880
|
]
|
|
864
881
|
// Containment, enforced HERE and not only at the loader. `cleanOutputs`
|
|
@@ -1258,7 +1275,7 @@ export async function stampOutputs(args: {
|
|
|
1258
1275
|
return stampFiles(await resolveOutputs(args))
|
|
1259
1276
|
}
|
|
1260
1277
|
|
|
1261
|
-
/** `stampOutputs` for root-anchored `
|
|
1278
|
+
/** `stampOutputs` for root-anchored `outputs.workspaceFiles` (A-43). */
|
|
1262
1279
|
export async function stampWorkspaceOutputs(args: {
|
|
1263
1280
|
workspaceRoot: string
|
|
1264
1281
|
outputs: readonly string[]
|
|
@@ -1298,7 +1315,7 @@ export async function ownOutputsSince(
|
|
|
1298
1315
|
return changedSince(await resolveOutputs(args), before)
|
|
1299
1316
|
}
|
|
1300
1317
|
|
|
1301
|
-
/** `ownOutputsSince` for root-anchored `
|
|
1318
|
+
/** `ownOutputsSince` for root-anchored `outputs.workspaceFiles` (A-43). */
|
|
1302
1319
|
export async function ownWorkspaceOutputsSince(
|
|
1303
1320
|
args: { workspaceRoot: string; outputs: readonly string[] },
|
|
1304
1321
|
before: ReadonlyMap<string, OutputStamp>,
|
|
@@ -1527,6 +1544,7 @@ export async function resolveWorkspaceOutputs(
|
|
|
1527
1544
|
outputs: readonly string[]
|
|
1528
1545
|
},
|
|
1529
1546
|
emptyDirs?: Set<string>,
|
|
1547
|
+
special?: Set<string>,
|
|
1530
1548
|
): Promise<string[]> {
|
|
1531
1549
|
const { positive, negative } = splitNegations(args.outputs)
|
|
1532
1550
|
if (positive.length === 0) return []
|
|
@@ -1534,7 +1552,7 @@ export async function resolveWorkspaceOutputs(
|
|
|
1534
1552
|
const trees = asTrees(positive)
|
|
1535
1553
|
refuseDirLinkedOut(args.workspaceRoot, trees, 'workspace')
|
|
1536
1554
|
const scanned = [
|
|
1537
|
-
...(await scanUnion(trees, excludeGlobs, args.workspaceRoot, undefined, emptyDirs)),
|
|
1555
|
+
...(await scanUnion(trees, excludeGlobs, args.workspaceRoot, undefined, emptyDirs, special)),
|
|
1538
1556
|
]
|
|
1539
1557
|
// Same containment as the project twin, anchored one level out. These globs
|
|
1540
1558
|
// deliberately ignore PROJECT boundaries — that is the escape hatch — but
|
|
@@ -1547,10 +1565,17 @@ export async function resolveWorkspaceOutputs(
|
|
|
1547
1565
|
export async function resolveWorkspaceOutputTree(args: {
|
|
1548
1566
|
workspaceRoot: string
|
|
1549
1567
|
outputs: readonly string[]
|
|
1550
|
-
|
|
1568
|
+
/** As `resolveOutputTree`'s. */
|
|
1569
|
+
special?: boolean
|
|
1570
|
+
}): Promise<{ files: string[]; emptyDirs: string[]; special: string[] }> {
|
|
1551
1571
|
const emptyDirs = new Set<string>()
|
|
1552
|
-
const
|
|
1553
|
-
|
|
1572
|
+
const special = args.special === true ? new Set<string>() : undefined
|
|
1573
|
+
const files = await resolveWorkspaceOutputs(args, emptyDirs, special)
|
|
1574
|
+
return {
|
|
1575
|
+
files,
|
|
1576
|
+
emptyDirs: containedIn(args.workspaceRoot, [...emptyDirs]).sort(),
|
|
1577
|
+
special: special === undefined ? [] : containedIn(args.workspaceRoot, [...special]).sort(),
|
|
1578
|
+
}
|
|
1554
1579
|
}
|
|
1555
1580
|
|
|
1556
1581
|
/**
|
|
@@ -1965,7 +1990,7 @@ export function declaresInput(
|
|
|
1965
1990
|
// A changed path under the task's own outputs is a committed file
|
|
1966
1991
|
// (`git diff` lists only what git tracks) or a stray `git status`
|
|
1967
1992
|
// lists: either way it may be an input, so it counts.
|
|
1968
|
-
const plan = filesPlan(inputs.files, task.
|
|
1993
|
+
const plan = filesPlan(inputs.files, task.outputs?.files ?? [])
|
|
1969
1994
|
if (plan !== null && (inPlan(plan, projectRel) || ownOnly(plan, projectRel))) return true
|
|
1970
1995
|
if (plan !== null && dirLink && plan.mayHold(projectRel) && !plan.isExcluded(projectRel)) {
|
|
1971
1996
|
return true
|
|
@@ -1995,8 +2020,7 @@ export function workspaceFilesReachInto(task: TaskConfig, dir: string): boolean
|
|
|
1995
2020
|
*/
|
|
1996
2021
|
export function namedOutright(task: TaskConfig): { files: string[]; workspaceFiles: string[] } {
|
|
1997
2022
|
const inputs = task.inputs
|
|
1998
|
-
const plan =
|
|
1999
|
-
inputs === undefined ? null : filesPlan(inputs.files, task.cache?.outputs.files ?? [])
|
|
2023
|
+
const plan = inputs === undefined ? null : filesPlan(inputs.files, task.outputs?.files ?? [])
|
|
2000
2024
|
const ws = inputs?.workspaceFiles ?? []
|
|
2001
2025
|
return {
|
|
2002
2026
|
files: plan === null ? [] : outright(plan),
|
|
@@ -2242,7 +2266,7 @@ function listingFor(
|
|
|
2242
2266
|
function refuseUndecodableOutput(abs: string): void {
|
|
2243
2267
|
if (!undecodableOnDisk(abs)) return
|
|
2244
2268
|
throw new UserError(
|
|
2245
|
-
`
|
|
2269
|
+
`outputs matched ${JSON.stringify(abs)}: the name is not valid UTF-8 (shown with ` +
|
|
2246
2270
|
`\ufffd), and vx cannot save a file by it. Rename it, or keep it out of the outputs.`,
|
|
2247
2271
|
)
|
|
2248
2272
|
}
|
|
@@ -2456,13 +2480,19 @@ async function scanUnion(
|
|
|
2456
2480
|
cwd: string,
|
|
2457
2481
|
nested: (rel: string) => boolean = () => false,
|
|
2458
2482
|
emptyDirs?: Set<string>,
|
|
2483
|
+
special?: Set<string>,
|
|
2459
2484
|
): Promise<Set<string>> {
|
|
2460
2485
|
const matches = new Set<string>()
|
|
2461
2486
|
const links: string[] = []
|
|
2487
|
+
// Directories the walk entered: Bun's scan skips a FIFO, socket or
|
|
2488
|
+
// device outright, so `special` is found by listing them.
|
|
2489
|
+
const walked = special === undefined ? undefined : new Set<string>()
|
|
2462
2490
|
const take = (rel: string): void => {
|
|
2463
2491
|
if (nested(rel) || excludeGlobs.some((g) => g.match(rel))) return
|
|
2464
2492
|
const abs = path.resolve(cwd, rel)
|
|
2493
|
+
walked?.add(path.dirname(abs))
|
|
2465
2494
|
const st = lstatSync(abs, { throwIfNoEntry: false })
|
|
2495
|
+
if (walked !== undefined && st?.isDirectory() === true) walked.add(abs)
|
|
2466
2496
|
if (st !== undefined && (st.isFile() || st.isSymbolicLink())) {
|
|
2467
2497
|
matches.add(abs)
|
|
2468
2498
|
if (st.isSymbolicLink()) links.push(abs)
|
|
@@ -2487,17 +2517,22 @@ async function scanUnion(
|
|
|
2487
2517
|
) {
|
|
2488
2518
|
const any = anyTaskGlob(patterns)
|
|
2489
2519
|
for (const rel of WHOLE_TREE.scanSync(opts)) if (any(rel)) take(rel)
|
|
2520
|
+
if (special !== undefined) findSpecial(walked!, cwd, any, excludeGlobs, nested, special)
|
|
2490
2521
|
return matches
|
|
2491
2522
|
}
|
|
2492
2523
|
for (const pattern of patterns) {
|
|
2493
2524
|
if (absentPrefix(cwd, pattern)) continue
|
|
2494
2525
|
const from = scanRoot(cwd, pattern)
|
|
2495
2526
|
if (from === null) continue
|
|
2527
|
+
walked?.add(from === undefined ? cwd : from.dir)
|
|
2496
2528
|
if (from === undefined) for (const rel of globFor(pattern).scanSync(opts)) take(rel)
|
|
2497
2529
|
else
|
|
2498
2530
|
for (const rel of globFor(from.rest).scanSync({ ...opts, cwd: from.dir }))
|
|
2499
2531
|
take(`${from.prefix}/${rel}`)
|
|
2500
2532
|
}
|
|
2533
|
+
if (special !== undefined) {
|
|
2534
|
+
findSpecial(walked!, cwd, anyTaskGlob(patterns), excludeGlobs, nested, special)
|
|
2535
|
+
}
|
|
2501
2536
|
// A literal names the path and its tree (`dist`, `dist/**`), and a tree
|
|
2502
2537
|
// whose head is a directory link is walked through it: the link and the
|
|
2503
2538
|
// files under it were one artifact holding `dist` as a file and a
|
|
@@ -2513,6 +2548,31 @@ async function scanUnion(
|
|
|
2513
2548
|
return matches
|
|
2514
2549
|
}
|
|
2515
2550
|
|
|
2551
|
+
/** The FIFOs, sockets and devices in `dirs` that the output globs name. */
|
|
2552
|
+
function findSpecial(
|
|
2553
|
+
dirs: ReadonlySet<string>,
|
|
2554
|
+
cwd: string,
|
|
2555
|
+
any: (rel: string) => boolean,
|
|
2556
|
+
excludeGlobs: readonly Bun.Glob[],
|
|
2557
|
+
nested: (rel: string) => boolean,
|
|
2558
|
+
into: Set<string>,
|
|
2559
|
+
): void {
|
|
2560
|
+
for (const dir of dirs) {
|
|
2561
|
+
let entries: Dirent[]
|
|
2562
|
+
try {
|
|
2563
|
+
entries = readdirSync(dir, { withFileTypes: true })
|
|
2564
|
+
} catch {
|
|
2565
|
+
continue
|
|
2566
|
+
}
|
|
2567
|
+
for (const e of entries) {
|
|
2568
|
+
if (e.isFile() || e.isDirectory() || e.isSymbolicLink()) continue
|
|
2569
|
+
const abs = path.join(dir, e.name)
|
|
2570
|
+
const rel = path.relative(cwd, abs).split(path.sep).join('/')
|
|
2571
|
+
if (any(rel) && !nested(rel) && !excludeGlobs.some((g) => g.match(rel))) into.add(abs)
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2516
2576
|
/** `pattern`'s static directory passes through a symlink under `cwd`. */
|
|
2517
2577
|
function headLinked(cwd: string, pattern: string): boolean {
|
|
2518
2578
|
const prefix = staticPrefix(pattern)
|
package/src/cache/layer.ts
CHANGED
|
@@ -199,7 +199,7 @@ export interface RunRecord {
|
|
|
199
199
|
restored?: boolean
|
|
200
200
|
attempts?: number // >1 when the task retried; the direct within-run flaky signal
|
|
201
201
|
/**
|
|
202
|
-
* Whether the task declared
|
|
202
|
+
* Whether the task declared `cache: true`. An uncached task executes
|
|
203
203
|
* on every run and its key exists only for dependents to fold, so a
|
|
204
204
|
* reader (`vx why`, `vx last`) must not present its re-run as a cache
|
|
205
205
|
* decision. Absent on rows written before the column existed.
|
|
@@ -358,7 +358,7 @@ export interface PruneResult {
|
|
|
358
358
|
|
|
359
359
|
/**
|
|
360
360
|
* Artifact + `output_files` namespace prefix for workspace-root-
|
|
361
|
-
* anchored outputs (`
|
|
361
|
+
* anchored outputs (`outputs.workspaceFiles`). Project outputs
|
|
362
362
|
* keep their bare project-relative `path` rows; workspace rows store
|
|
363
363
|
* the full `workspace-outputs/<rel-to-root>` tar entry name as the
|
|
364
364
|
* discriminator — least-invasive row format, no schema change. A
|
|
@@ -467,7 +467,7 @@ export interface CacheGetContext {
|
|
|
467
467
|
taskId: string
|
|
468
468
|
command: string
|
|
469
469
|
/**
|
|
470
|
-
* The task's declared `
|
|
470
|
+
* The task's declared `outputs`. A remote artifact is ingested
|
|
471
471
|
* only when every file it carries is one of them: bytes off the network
|
|
472
472
|
* named `outputs/src/in.txt` or `workspace-outputs/.git/hooks/…` wrote
|
|
473
473
|
* an input and a git hook on a green `cache-hit-remote` (item 942).
|
|
@@ -804,9 +804,9 @@ export interface CacheLayer {
|
|
|
804
804
|
*/
|
|
805
805
|
ingest(hash: string, body: Blob | Response, meta: IngestMeta): Promise<void>
|
|
806
806
|
/**
|
|
807
|
-
* The
|
|
808
|
-
*
|
|
809
|
-
* transaction (one fsync). Input-fingerprint rows are NOT written here — they
|
|
807
|
+
* The run-history write at a run's end, atomically: the per-task `runs`
|
|
808
|
+
* rows (less the hits a `HistoryStream` already wrote) and the one
|
|
809
|
+
* `invocations` header row — in a SINGLE transaction (one fsync). Input-fingerprint rows are NOT written here — they
|
|
810
810
|
* ride the entry-save transaction (`save`/`ingest`) so a warm
|
|
811
811
|
* all-cache-hit run writes nothing.
|
|
812
812
|
*/
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
import { Database } from 'bun:sqlite'
|
|
12
12
|
import { existsSync } from 'node:fs'
|
|
13
13
|
import path from 'node:path'
|
|
14
|
-
import { closeDb, SCHEMA_VERSION
|
|
14
|
+
import { closeDb, SCHEMA_VERSION } from './cache.js'
|
|
15
|
+
import { SQLITE_FCNTL_PERSIST_WAL } from './schema.js'
|
|
15
16
|
|
|
16
17
|
/** One claimant's parse of one lockfile's content. */
|
|
17
18
|
export interface LockfileMemo {
|