@vzn/vx 0.0.640 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.sh +9 -2
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +735 -107
- package/src/cache/cache.ts +133 -29
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +9 -2
- package/src/cache/inputs.ts +244 -108
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +9 -7
- package/src/cache/layered-cache.ts +74 -9
- package/src/cache/output-index.ts +25 -4
- package/src/cache/remote-lane.ts +27 -10
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +11 -3
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +11 -5
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/run.ts +10 -3
- package/src/cli/select.ts +2 -0
- package/src/cli/upgrade.ts +37 -2
- package/src/cli/watch-filter.ts +1 -6
- package/src/cli/watch-fs.ts +2 -1
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/handoff.ts +5 -0
- package/src/history/history.ts +43 -33
- package/src/index.ts +11 -1
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +111 -29
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/local-shortcircuit.ts +8 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +35 -5
- package/src/orchestrator/plugin.ts +29 -3
- package/src/orchestrator/prepare.ts +74 -5
- package/src/orchestrator/projects.ts +6 -1
- package/src/orchestrator/run-report.ts +30 -9
- package/src/orchestrator/run.ts +62 -38
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +46 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +58 -15
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +11 -6
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +13 -0
- package/src/util/paths.ts +58 -5
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +63 -13
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-schema.ts +37 -3
- package/src/workspace/project-loader.ts +6 -4
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2352
- package/docs/cli.md +0 -3280
- package/docs/execution.md +0 -675
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2461
- package/docs/security.md +0 -252
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
package/src/cache/key-fold.ts
CHANGED
|
@@ -110,7 +110,14 @@ import type { CacheKeyInput } from './layer.js'
|
|
|
110
110
|
// v43 stays when core stopped folding package files (2026-10-14): the
|
|
111
111
|
// `workspace:` and `pkg:` parts left the chain, so every key moved and no
|
|
112
112
|
// unchanged key holds wrong bytes.
|
|
113
|
-
|
|
113
|
+
// v44: the sidecar lists the empty directories under the outputs; a v43
|
|
114
|
+
// artifact holds none and a hit left them out (Growth's parity, 2026-10-16).
|
|
115
|
+
// v45: the sidecar lists the symlinks under the outputs and a hit makes
|
|
116
|
+
// them; a v44 artifact holds a link as a copy of its target (M, 2026-10-16).
|
|
117
|
+
// v46: vx's names are reserved only at the project root; a v45 artifact
|
|
118
|
+
// left out a build's own `dist/.git/**`, `dist/.vx/**` and nested
|
|
119
|
+
// `vx.lock.json` (Growth's parity, 2026-10-17).
|
|
120
|
+
export const CACHE_VERSION = 'vx-cache-v46'
|
|
114
121
|
|
|
115
122
|
/**
|
|
116
123
|
* The digest a key folds for an input gone between its enumeration and its
|
|
@@ -151,6 +158,87 @@ export async function foldKey(
|
|
|
151
158
|
*/
|
|
152
159
|
behind?: (link: string) => Promise<string>,
|
|
153
160
|
): Promise<string> {
|
|
161
|
+
const cap = input.captureInto
|
|
162
|
+
let h = foldHead(input, digestValue)
|
|
163
|
+
|
|
164
|
+
// `resolveFiles` already returns its paths sorted, and a copy-and-sort
|
|
165
|
+
// of three thousand strings per task is not free: 7.4 ms of a 44-task
|
|
166
|
+
// run here (2026-09-20). A linear order check costs one comparison per
|
|
167
|
+
// file instead of n log n, and a caller that hands over an unsorted set
|
|
168
|
+
// — a plugin's own input list — still gets one. `>` on strings compares
|
|
169
|
+
// UTF-16 code units, which is what a comparator-less `sort` does.
|
|
170
|
+
let sortedInputs: readonly string[] = input.inputFiles
|
|
171
|
+
for (let i = 1; i < sortedInputs.length; i++) {
|
|
172
|
+
if (sortedInputs[i - 1]! > sortedInputs[i]!) {
|
|
173
|
+
sortedInputs = [...input.inputFiles].sort()
|
|
174
|
+
break
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
h = xxh3(`inputs:${sortedInputs.length}`, h)
|
|
178
|
+
// Per-file hash source, in preference order: the caller-supplied
|
|
179
|
+
// index-OID map (clean tracked files — zero I/O), then hashFile's
|
|
180
|
+
// stat memo (no read), then a full in-process blob-OID computation.
|
|
181
|
+
// All three describe the WORKTREE bytes, which is what the task
|
|
182
|
+
// reads — the index-OID map is only populated for paths whose
|
|
183
|
+
// worktree form matches their index form, because a clean filter
|
|
184
|
+
// (`text`/`eol`/`ident`/`filter`/`working-tree-encoding`) makes the index blob a DIFFERENT sequence
|
|
185
|
+
// of bytes and folding it would let two distinct worktree contents
|
|
186
|
+
// share a key. The fold order is locked to `sortedInputs` so
|
|
187
|
+
// results are stable across runs.
|
|
188
|
+
// The common warm shape is that the caller's OID map covers every file,
|
|
189
|
+
// and then there is nothing to await: building three thousand promises
|
|
190
|
+
// per task to resolve values already in hand cost 8.4 ms of that same
|
|
191
|
+
// run. One pass fills the array, and the first gap falls back to the
|
|
192
|
+
// awaited form for the whole list.
|
|
193
|
+
let fileHashes: readonly string[] | undefined
|
|
194
|
+
const provided = input.fileHashes
|
|
195
|
+
if (provided !== undefined) {
|
|
196
|
+
const out: string[] = []
|
|
197
|
+
for (const file of sortedInputs) {
|
|
198
|
+
const oid = provided.get(file)
|
|
199
|
+
if (oid === undefined) break
|
|
200
|
+
out.push(oid)
|
|
201
|
+
}
|
|
202
|
+
if (out.length === sortedInputs.length) fileHashes = out
|
|
203
|
+
}
|
|
204
|
+
fileHashes ??= await Promise.all(
|
|
205
|
+
sortedInputs.map((f) => provided?.get(f) ?? hashFile(f).catch(absentOr)),
|
|
206
|
+
)
|
|
207
|
+
// Links are rare, so a list without one awaits nothing more.
|
|
208
|
+
let behindLinks: Map<number, string> | undefined
|
|
209
|
+
if (behind !== undefined) {
|
|
210
|
+
const links: number[] = []
|
|
211
|
+
for (let i = 0; i < fileHashes.length; i++)
|
|
212
|
+
if (fileHashes[i]!.startsWith('120000:')) links.push(i)
|
|
213
|
+
if (links.length > 0) {
|
|
214
|
+
const digests = await Promise.all(links.map((i) => behind(sortedInputs[i]!)))
|
|
215
|
+
behindLinks = new Map(links.map((i, n) => [i, digests[n]!]))
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
for (let i = 0; i < sortedInputs.length; i++) {
|
|
219
|
+
const file = sortedInputs[i]!
|
|
220
|
+
const rel = relOf(file)
|
|
221
|
+
const oid = fileHashes[i]!
|
|
222
|
+
const linked = behindLinks?.get(i)
|
|
223
|
+
// A link folds what it points to in its own step, so a key without one is unchanged.
|
|
224
|
+
h = xxh3(linked === undefined ? `${rel}\0${oid}` : `${rel}\0${oid}\0${linked}`, h)
|
|
225
|
+
// The OID is already awaited — file capture is zero extra I/O.
|
|
226
|
+
if (cap) cap.push({ kind: 'file', name: rel, hash: oid })
|
|
227
|
+
if (linked !== undefined && cap) cap.push({ kind: 'linked', name: rel, hash: xxh3hex(linked) })
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return h.toString(16).padStart(16, '0')
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* The fold up to the input files, which come last: what a task's key is
|
|
235
|
+
* once its files are folded onto it. `computeTaskHash` asks it alone when
|
|
236
|
+
* a remembered fold of the same files answers the rest (`FoldMemo`).
|
|
237
|
+
*/
|
|
238
|
+
export function foldHead(
|
|
239
|
+
input: CacheKeyInput,
|
|
240
|
+
digestValue: (value: string) => string = xxh3hex,
|
|
241
|
+
): bigint {
|
|
154
242
|
// Seed-chained xxHash3: each step folds one field into the
|
|
155
243
|
// running digest via `xxh3(part, prevDigest)`. Equivalent to the
|
|
156
244
|
// old CryptoHasher.update() pattern, no intermediate buffer.
|
|
@@ -230,72 +318,5 @@ export async function foldKey(
|
|
|
230
318
|
for (const [n, v] of pluginParts) cap.push({ kind: 'plugin', name: n, hash: digestValue(v) })
|
|
231
319
|
}
|
|
232
320
|
}
|
|
233
|
-
|
|
234
|
-
// `resolveFiles` already returns its paths sorted, and a copy-and-sort
|
|
235
|
-
// of three thousand strings per task is not free: 7.4 ms of a 44-task
|
|
236
|
-
// run here (2026-09-20). A linear order check costs one comparison per
|
|
237
|
-
// file instead of n log n, and a caller that hands over an unsorted set
|
|
238
|
-
// — a plugin's own input list — still gets one. `>` on strings compares
|
|
239
|
-
// UTF-16 code units, which is what a comparator-less `sort` does.
|
|
240
|
-
let sortedInputs: readonly string[] = input.inputFiles
|
|
241
|
-
for (let i = 1; i < sortedInputs.length; i++) {
|
|
242
|
-
if (sortedInputs[i - 1]! > sortedInputs[i]!) {
|
|
243
|
-
sortedInputs = [...input.inputFiles].sort()
|
|
244
|
-
break
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
h = xxh3(`inputs:${sortedInputs.length}`, h)
|
|
248
|
-
// Per-file hash source, in preference order: the caller-supplied
|
|
249
|
-
// index-OID map (clean tracked files — zero I/O), then hashFile's
|
|
250
|
-
// stat memo (no read), then a full in-process blob-OID computation.
|
|
251
|
-
// All three describe the WORKTREE bytes, which is what the task
|
|
252
|
-
// reads — the index-OID map is only populated for paths whose
|
|
253
|
-
// worktree form matches their index form, because a clean filter
|
|
254
|
-
// (`text`/`eol`/`ident`/`filter`/`working-tree-encoding`) makes the index blob a DIFFERENT sequence
|
|
255
|
-
// of bytes and folding it would let two distinct worktree contents
|
|
256
|
-
// share a key. The fold order is locked to `sortedInputs` so
|
|
257
|
-
// results are stable across runs.
|
|
258
|
-
// The common warm shape is that the caller's OID map covers every file,
|
|
259
|
-
// and then there is nothing to await: building three thousand promises
|
|
260
|
-
// per task to resolve values already in hand cost 8.4 ms of that same
|
|
261
|
-
// run. One pass fills the array, and the first gap falls back to the
|
|
262
|
-
// awaited form for the whole list.
|
|
263
|
-
let fileHashes: readonly string[] | undefined
|
|
264
|
-
const provided = input.fileHashes
|
|
265
|
-
if (provided !== undefined) {
|
|
266
|
-
const out: string[] = []
|
|
267
|
-
for (const file of sortedInputs) {
|
|
268
|
-
const oid = provided.get(file)
|
|
269
|
-
if (oid === undefined) break
|
|
270
|
-
out.push(oid)
|
|
271
|
-
}
|
|
272
|
-
if (out.length === sortedInputs.length) fileHashes = out
|
|
273
|
-
}
|
|
274
|
-
fileHashes ??= await Promise.all(
|
|
275
|
-
sortedInputs.map((f) => provided?.get(f) ?? hashFile(f).catch(absentOr)),
|
|
276
|
-
)
|
|
277
|
-
// Links are rare, so a list without one awaits nothing more.
|
|
278
|
-
let behindLinks: Map<number, string> | undefined
|
|
279
|
-
if (behind !== undefined) {
|
|
280
|
-
const links: number[] = []
|
|
281
|
-
for (let i = 0; i < fileHashes.length; i++)
|
|
282
|
-
if (fileHashes[i]!.startsWith('120000:')) links.push(i)
|
|
283
|
-
if (links.length > 0) {
|
|
284
|
-
const digests = await Promise.all(links.map((i) => behind(sortedInputs[i]!)))
|
|
285
|
-
behindLinks = new Map(links.map((i, n) => [i, digests[n]!]))
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
for (let i = 0; i < sortedInputs.length; i++) {
|
|
289
|
-
const file = sortedInputs[i]!
|
|
290
|
-
const rel = relOf(file)
|
|
291
|
-
const oid = fileHashes[i]!
|
|
292
|
-
const linked = behindLinks?.get(i)
|
|
293
|
-
// A link folds what it points to in its own step, so a key without one is unchanged.
|
|
294
|
-
h = xxh3(linked === undefined ? `${rel}\0${oid}` : `${rel}\0${oid}\0${linked}`, h)
|
|
295
|
-
// The OID is already awaited — file capture is zero extra I/O.
|
|
296
|
-
if (cap) cap.push({ kind: 'file', name: rel, hash: oid })
|
|
297
|
-
if (linked !== undefined && cap) cap.push({ kind: 'linked', name: rel, hash: xxh3hex(linked) })
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
return h.toString(16).padStart(16, '0')
|
|
321
|
+
return h
|
|
301
322
|
}
|
package/src/cache/layer.ts
CHANGED
|
@@ -677,7 +677,13 @@ export interface CacheLayer {
|
|
|
677
677
|
* into the workspace root. Callers restoring entries that may carry
|
|
678
678
|
* workspace outputs must pass `workspaceRoot`.
|
|
679
679
|
*/
|
|
680
|
-
restoreOutputs(
|
|
680
|
+
restoreOutputs(
|
|
681
|
+
hash: string,
|
|
682
|
+
projectDir: string,
|
|
683
|
+
workspaceRoot?: string,
|
|
684
|
+
/** The task's declared outputs: an entry outside them fails the restore, nothing written. */
|
|
685
|
+
outputs?: DeclaredOutputs,
|
|
686
|
+
): Promise<void>
|
|
681
687
|
save(args: {
|
|
682
688
|
hash: string
|
|
683
689
|
/**
|
|
@@ -692,6 +698,8 @@ export interface CacheLayer {
|
|
|
692
698
|
entry: Omit<CacheEntry, 'hash' | 'storedAt' | 'outputFiles' | 'exitCode'>
|
|
693
699
|
projectDir: string
|
|
694
700
|
outputFiles: string[]
|
|
701
|
+
/** The empty directories under the outputs (absolute), kept so a hit makes them. */
|
|
702
|
+
outputDirs?: string[]
|
|
695
703
|
/**
|
|
696
704
|
* Resolved `outputs.workspaceFiles` (absolute paths) + the root
|
|
697
705
|
* they're relative to. Packed under `workspace-outputs/<rel>`.
|
|
@@ -708,12 +716,6 @@ export interface CacheLayer {
|
|
|
708
716
|
* Only computed on the miss/save path — never on a hit.
|
|
709
717
|
*/
|
|
710
718
|
inputComponents?: readonly TaskInputRow[]
|
|
711
|
-
/**
|
|
712
|
-
* For a sandboxed task: whether it could read a canonical path. A
|
|
713
|
-
* symlinked output is packed as its target's bytes, outside the
|
|
714
|
-
* sandbox, so one whose target the task could not read is refused.
|
|
715
|
-
*/
|
|
716
|
-
readable?: ((real: string) => boolean) | undefined
|
|
717
719
|
}): Promise<void>
|
|
718
720
|
/**
|
|
719
721
|
* Adopt an artifact produced elsewhere — the remote-hit path. Streams
|
|
@@ -21,6 +21,7 @@ import type { Cache } from './cache.js'
|
|
|
21
21
|
import type {
|
|
22
22
|
CacheEntry,
|
|
23
23
|
CacheGetContext,
|
|
24
|
+
DeclaredOutputs,
|
|
24
25
|
CacheKeyInput,
|
|
25
26
|
CacheLayer,
|
|
26
27
|
CacheStats,
|
|
@@ -35,6 +36,44 @@ import type {
|
|
|
35
36
|
} from './cache.js'
|
|
36
37
|
import type { OutputDirRow } from './cache.js'
|
|
37
38
|
import { RemoteLane, type ResolvedRemote } from './remote-lane.js'
|
|
39
|
+
import type { PluginSecrets } from '../util/index.js'
|
|
40
|
+
|
|
41
|
+
const STOPPED: unique symbol = Symbol('stopped')
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* A response body that errors once the run stops, so the ingest reading it
|
|
45
|
+
* fails at once, removes its temp, and releases the connection; the cache
|
|
46
|
+
* then closes under no write.
|
|
47
|
+
*/
|
|
48
|
+
function endsOnStop(body: Blob | Response, stopped: Promise<typeof STOPPED>): Blob | Response {
|
|
49
|
+
if (body instanceof Blob || body.body === null) return body
|
|
50
|
+
const reader = body.body.getReader()
|
|
51
|
+
let stop = false
|
|
52
|
+
// Once, not raced per chunk: a cancel ends the pending read as `done`,
|
|
53
|
+
// which the flag turns into an error rather than a short body.
|
|
54
|
+
void stopped.then(() => {
|
|
55
|
+
stop = true
|
|
56
|
+
void reader.cancel().catch(() => undefined)
|
|
57
|
+
})
|
|
58
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
59
|
+
async pull(controller) {
|
|
60
|
+
const next = await reader.read()
|
|
61
|
+
if (stop) controller.error(new Error('the run was stopped'))
|
|
62
|
+
else if (next.done) controller.close()
|
|
63
|
+
else controller.enqueue(next.value)
|
|
64
|
+
},
|
|
65
|
+
cancel: (reason) => reader.cancel(reason),
|
|
66
|
+
})
|
|
67
|
+
return new Response(stream, { headers: body.headers })
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** What a remote's `open` is handed: the run's CI verdict and its plugin's keychain. */
|
|
71
|
+
export interface RemoteOpenContext extends PluginSecrets {
|
|
72
|
+
/** The run is on CI: nobody may be asked. */
|
|
73
|
+
readonly ci: boolean
|
|
74
|
+
/** One line naming the plugin, for a fact that leaves the remote on. */
|
|
75
|
+
warn(message: string): void
|
|
76
|
+
}
|
|
38
77
|
|
|
39
78
|
/**
|
|
40
79
|
* What a remote cache layer must provide — THE plugin seam for remote
|
|
@@ -62,10 +101,16 @@ export interface RemoteCacheLayer {
|
|
|
62
101
|
*/
|
|
63
102
|
readonly endpoint?: string
|
|
64
103
|
/**
|
|
65
|
-
*
|
|
66
|
-
*
|
|
104
|
+
* Optional: make the layer ready (read a credential, ask for one).
|
|
105
|
+
* Called once per run, in declared order, before the status line and the
|
|
106
|
+
* first task, and only when the run's policy lets this remote read or
|
|
107
|
+
* write: a `--cache-policy=remote:` run, `--no-cache` and a plan
|
|
108
|
+
* (`--dry`, `vx show`) never call it, so the other methods may be asked
|
|
109
|
+
* without it. A throw is a warning naming the plugin and turns the
|
|
110
|
+
* remote off for the run; the run goes on. `ctx.secret` reads the
|
|
111
|
+
* plugin's keychain and may ask on the terminal.
|
|
67
112
|
*/
|
|
68
|
-
|
|
113
|
+
open?(ctx: RemoteOpenContext): Promise<void>
|
|
69
114
|
/** Existence probe: `--dry` and `--force` ask it per key when the remote cannot batch. */
|
|
70
115
|
has(hash: string): Promise<boolean>
|
|
71
116
|
/**
|
|
@@ -103,7 +148,6 @@ export class LayeredCache implements CacheLayer {
|
|
|
103
148
|
readonly hasRemote = true
|
|
104
149
|
|
|
105
150
|
private readonly lanes: readonly RemoteLane[]
|
|
106
|
-
private readonly readable: readonly RemoteLane[]
|
|
107
151
|
|
|
108
152
|
/**
|
|
109
153
|
* In-flight remote pulls keyed by hash. `prefetch` and `get` both go
|
|
@@ -134,7 +178,11 @@ export class LayeredCache implements CacheLayer {
|
|
|
134
178
|
remotes: readonly ResolvedRemote[],
|
|
135
179
|
) {
|
|
136
180
|
this.lanes = remotes.map((r) => new RemoteLane(r, local.traffic))
|
|
137
|
-
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Asked, not kept: a remote whose `open` failed leaves the run after this is built.
|
|
184
|
+
private get readable(): RemoteLane[] {
|
|
185
|
+
return this.lanes.filter((l) => l.read)
|
|
138
186
|
}
|
|
139
187
|
|
|
140
188
|
key(input: CacheKeyInput): Promise<string> {
|
|
@@ -284,7 +332,17 @@ export class LayeredCache implements CacheLayer {
|
|
|
284
332
|
|
|
285
333
|
private async pullFrom(lane: RemoteLane, hash: string, ctx?: CacheGetContext): Promise<boolean> {
|
|
286
334
|
const fetchStart = performance.now()
|
|
287
|
-
|
|
335
|
+
// A stopped run waits on no read, as on no upload: a lookup or download
|
|
336
|
+
// in flight held a Ctrl-C until its last byte (M, 2026-10-17). The
|
|
337
|
+
// plugin's request is left to the exit, and its body is cancelled.
|
|
338
|
+
const stopped = lane.whenStopped.then((): typeof STOPPED => STOPPED)
|
|
339
|
+
const pending = lane.get(hash)
|
|
340
|
+
const remoteBody = await Promise.race([pending, stopped])
|
|
341
|
+
if (remoteBody === STOPPED) {
|
|
342
|
+
// A late answer's body would hold its connection to the exit.
|
|
343
|
+
void pending.then((r) => (r?.body instanceof Response ? r.body.body?.cancel() : undefined))
|
|
344
|
+
return false
|
|
345
|
+
}
|
|
288
346
|
if (remoteBody === null) return false
|
|
289
347
|
|
|
290
348
|
// Ingest the remote body into local using the caller-supplied
|
|
@@ -305,7 +363,9 @@ export class LayeredCache implements CacheLayer {
|
|
|
305
363
|
...(ctx?.outputs !== undefined ? { outputs: ctx.outputs } : {}),
|
|
306
364
|
}
|
|
307
365
|
try {
|
|
308
|
-
|
|
366
|
+
const ingest = this.local.ingest(hash, endsOnStop(remoteBody.body, stopped), meta)
|
|
367
|
+
ingest.catch(() => undefined)
|
|
368
|
+
if ((await Promise.race([ingest, stopped])) === STOPPED) return false
|
|
309
369
|
} catch (err) {
|
|
310
370
|
// The bytes came off the network — a corrupt/truncated remote
|
|
311
371
|
// artifact must degrade to a cache miss (task re-executes), not
|
|
@@ -330,8 +390,13 @@ export class LayeredCache implements CacheLayer {
|
|
|
330
390
|
return this.local.hashFile(filePath)
|
|
331
391
|
}
|
|
332
392
|
|
|
333
|
-
async restoreOutputs(
|
|
334
|
-
|
|
393
|
+
async restoreOutputs(
|
|
394
|
+
hash: string,
|
|
395
|
+
projectDir: string,
|
|
396
|
+
workspaceRoot?: string,
|
|
397
|
+
outputs?: DeclaredOutputs,
|
|
398
|
+
): Promise<void> {
|
|
399
|
+
await this.local.restoreOutputs(hash, projectDir, workspaceRoot, outputs)
|
|
335
400
|
}
|
|
336
401
|
|
|
337
402
|
async save(args: SaveArgs): Promise<void> {
|
|
@@ -17,7 +17,8 @@ const FILE_PREFIX_MTIME = -2
|
|
|
17
17
|
|
|
18
18
|
import type { Database } from 'bun:sqlite'
|
|
19
19
|
import { inHashes, lazyStatement } from './schema.js'
|
|
20
|
-
import {
|
|
20
|
+
import { modeMatches } from './umask.js'
|
|
21
|
+
import { lstatSync, readlinkSync, statSync } from 'node:fs'
|
|
21
22
|
import { lstat, readdir } from 'node:fs/promises'
|
|
22
23
|
import path from 'node:path'
|
|
23
24
|
import {
|
|
@@ -30,6 +31,19 @@ import {
|
|
|
30
31
|
WORKSPACE_OUTPUT_PREFIX,
|
|
31
32
|
} from './layer.js'
|
|
32
33
|
|
|
34
|
+
/**
|
|
35
|
+
* The mode a symlink output's row carries (S_IFLNK): its size is the
|
|
36
|
+
* target's byte length and its mtime the target's CRC-32.
|
|
37
|
+
*/
|
|
38
|
+
export const LINK_MODE = 0o120000
|
|
39
|
+
|
|
40
|
+
/** A link row is current while a link with that target stands there; never followed. */
|
|
41
|
+
function linkCurrent(abs: string, e: { size: number; mtimeMs: number }): boolean {
|
|
42
|
+
if (!lstatSync(abs).isSymbolicLink()) return false
|
|
43
|
+
const to = readlinkSync(abs, 'buffer')
|
|
44
|
+
return to.byteLength === e.size && Bun.hash.crc32(to) === e.mtimeMs
|
|
45
|
+
}
|
|
46
|
+
|
|
33
47
|
export class OutputIndex {
|
|
34
48
|
private readonly insertOutputFile: ReturnType<Database['prepare']>
|
|
35
49
|
private readonly deleteOutputFiles: ReturnType<Database['prepare']>
|
|
@@ -206,10 +220,13 @@ export class OutputIndex {
|
|
|
206
220
|
// graph stage 100 → 54 ms, the whole process 422 → 368 ms.
|
|
207
221
|
const results = expected.map((e) => {
|
|
208
222
|
try {
|
|
223
|
+
if (e.mode === LINK_MODE) return linkCurrent(path.join(projectDir, e.path), e)
|
|
209
224
|
const s = statSync(path.join(projectDir, e.path))
|
|
225
|
+
// An empty output directory: current while a directory stands there.
|
|
226
|
+
if (e.path.endsWith('/')) return s.isDirectory()
|
|
210
227
|
return (
|
|
211
228
|
s.size === e.size &&
|
|
212
|
-
(s.mode
|
|
229
|
+
modeMatches(s.mode, e.mode) &&
|
|
213
230
|
// MILLISECOND comparison (sub-ms tolerance for the float
|
|
214
231
|
// round-trip through utimes). Save rows carry stat-ms and
|
|
215
232
|
// restoreOutputs re-syncs restored files to the row value,
|
|
@@ -324,6 +341,7 @@ export class OutputIndex {
|
|
|
324
341
|
const rows = this.rowsOf(hash)
|
|
325
342
|
const stamps: Array<[string, number, number]> = []
|
|
326
343
|
for (const e of rows) {
|
|
344
|
+
if (e.path.endsWith('/') || e.mode === LINK_MODE) continue
|
|
327
345
|
const ws = e.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
|
|
328
346
|
const abs = ws
|
|
329
347
|
? path.join(workspaceRoot, e.path.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
@@ -332,7 +350,7 @@ export class OutputIndex {
|
|
|
332
350
|
const s = statSync(abs)
|
|
333
351
|
if (
|
|
334
352
|
s.size === e.size &&
|
|
335
|
-
(s.mode
|
|
353
|
+
modeMatches(s.mode, e.mode) &&
|
|
336
354
|
Math.abs(s.mtimeMs - e.mtimeMs) < 1
|
|
337
355
|
) {
|
|
338
356
|
stamps.push([e.path, s.ino, Math.floor(s.ctimeMs)])
|
|
@@ -415,10 +433,13 @@ export class OutputIndex {
|
|
|
415
433
|
const results = rows.map((r) => {
|
|
416
434
|
let st
|
|
417
435
|
try {
|
|
418
|
-
|
|
436
|
+
// No throw for a missing dir: every restore after a clean asks
|
|
437
|
+
// about one, and an exception per hit cost more than the stat.
|
|
438
|
+
st = lstatSync(path.join(projectDir, r.path), { throwIfNoEntry: false })
|
|
419
439
|
} catch {
|
|
420
440
|
return r.mtimeMs === ABSENT_DIR_MTIME
|
|
421
441
|
}
|
|
442
|
+
if (st === undefined) return r.mtimeMs === ABSENT_DIR_MTIME
|
|
422
443
|
if (r.mtimeMs === ABSENT_DIR_MTIME) return false
|
|
423
444
|
if (r.mtimeMs === FILE_PREFIX_MTIME) return st.isFile()
|
|
424
445
|
return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
|
package/src/cache/remote-lane.ts
CHANGED
|
@@ -10,8 +10,11 @@ import type { RemoteCacheLayer } from './layered-cache.js'
|
|
|
10
10
|
/** One remote, resolved: its wire and what the run's policy and its `access` let it do. */
|
|
11
11
|
export interface ResolvedRemote {
|
|
12
12
|
readonly remote: RemoteCacheLayer
|
|
13
|
-
|
|
14
|
-
readonly
|
|
13
|
+
/** The plugin that supplied it, whose keychain its `open` reads; none for an injected layer. */
|
|
14
|
+
readonly plugin?: string
|
|
15
|
+
/** Cleared, with `write`, when the remote's `open` fails: off for the rest of the run. */
|
|
16
|
+
read: boolean
|
|
17
|
+
write: boolean
|
|
15
18
|
/** Told of each failure this remote degrades to a miss; the default writes stderr. */
|
|
16
19
|
readonly onError?: (err: Error) => void
|
|
17
20
|
}
|
|
@@ -89,8 +92,7 @@ function batchOf(hashes: readonly string[]): { batch: string } {
|
|
|
89
92
|
}
|
|
90
93
|
|
|
91
94
|
export class RemoteLane {
|
|
92
|
-
readonly
|
|
93
|
-
readonly write: boolean
|
|
95
|
+
private readonly resolved: ResolvedRemote
|
|
94
96
|
private readonly remote: RemoteCacheLayer
|
|
95
97
|
private readonly endpoint: string | undefined
|
|
96
98
|
private readonly onError: ((err: Error) => void) | undefined
|
|
@@ -132,13 +134,20 @@ export class RemoteLane {
|
|
|
132
134
|
resolved: ResolvedRemote,
|
|
133
135
|
private readonly traffic: RemoteTraffic,
|
|
134
136
|
) {
|
|
137
|
+
this.resolved = resolved
|
|
135
138
|
this.remote = resolved.remote
|
|
136
|
-
this.read = resolved.read
|
|
137
|
-
this.write = resolved.write
|
|
138
139
|
this.onError = resolved.onError
|
|
139
140
|
this.endpoint = printableEndpoint(resolved.remote.endpoint)
|
|
140
141
|
}
|
|
141
142
|
|
|
143
|
+
get read(): boolean {
|
|
144
|
+
return this.resolved.read
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
get write(): boolean {
|
|
148
|
+
return this.resolved.write
|
|
149
|
+
}
|
|
150
|
+
|
|
142
151
|
/**
|
|
143
152
|
* Existence probe; a failure is a miss. A key a `hasMany` this run
|
|
144
153
|
* answered for is not asked again: `--dry` batches its keys first.
|
|
@@ -248,6 +257,7 @@ export class RemoteLane {
|
|
|
248
257
|
tally: { count: number; bytes: number; ms: number; failed: number },
|
|
249
258
|
): void {
|
|
250
259
|
const key = this.trafficOf(hash)
|
|
260
|
+
const counted = this.traffic.aliases.get(hash) ?? hash
|
|
251
261
|
key.uploading++
|
|
252
262
|
this.traffic.inFlight++
|
|
253
263
|
this.uploadQueue.push(async () => {
|
|
@@ -274,12 +284,17 @@ export class RemoteLane {
|
|
|
274
284
|
// Nothing sent and nothing failed (the remote held it, or the run
|
|
275
285
|
// stopped first): no traffic, so the footer counts no upload.
|
|
276
286
|
if (key.uploading === 0 && !key.landed && key.failures.length === 0)
|
|
277
|
-
this.traffic.keys.delete(
|
|
287
|
+
this.traffic.keys.delete(counted)
|
|
278
288
|
}
|
|
279
289
|
})
|
|
280
290
|
this.pump()
|
|
281
291
|
}
|
|
282
292
|
|
|
293
|
+
/** Settles when the run is stopped. */
|
|
294
|
+
get whenStopped(): Promise<void> {
|
|
295
|
+
return this.stopping
|
|
296
|
+
}
|
|
297
|
+
|
|
283
298
|
/** Resolves once every queued and in-flight upload has settled, or the run was stopped. */
|
|
284
299
|
async drain(): Promise<void> {
|
|
285
300
|
// A probe no upload waited on still runs: settled before the run tears
|
|
@@ -292,7 +307,8 @@ export class RemoteLane {
|
|
|
292
307
|
/**
|
|
293
308
|
* The run was stopped (Ctrl-C, SIGTERM, an embedder's abort): queued
|
|
294
309
|
* uploads never start and the drain stops waiting on the ones in flight,
|
|
295
|
-
* which a plugin cannot be asked to cancel.
|
|
310
|
+
* which a plugin cannot be asked to cancel. Reads in flight are given up
|
|
311
|
+
* the same way (`LayeredCache.pullFrom`). A slow remote held a stopped
|
|
296
312
|
* run past the signal's exit bound, which then exited with no footer.
|
|
297
313
|
*/
|
|
298
314
|
stop(): void {
|
|
@@ -343,10 +359,11 @@ export class RemoteLane {
|
|
|
343
359
|
}
|
|
344
360
|
|
|
345
361
|
private trafficOf(hash: string): RemoteKeyTraffic {
|
|
346
|
-
|
|
362
|
+
const counted = this.traffic.aliases.get(hash) ?? hash
|
|
363
|
+
let key = this.traffic.keys.get(counted)
|
|
347
364
|
if (key === undefined) {
|
|
348
365
|
key = { uploading: 0, landed: false, failures: [] }
|
|
349
|
-
this.traffic.keys.set(
|
|
366
|
+
this.traffic.keys.set(counted, key)
|
|
350
367
|
}
|
|
351
368
|
return key
|
|
352
369
|
}
|
package/src/cache/schema.ts
CHANGED
|
@@ -140,6 +140,14 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
140
140
|
content_hash TEXT NOT NULL,
|
|
141
141
|
seen_at INTEGER NOT NULL
|
|
142
142
|
);
|
|
143
|
+
-- v42: each task's last key by its fold up to the files and a token of
|
|
144
|
+
-- the files' git records (fold-memo.ts). A memo: the KEY is unchanged.
|
|
145
|
+
CREATE TABLE IF NOT EXISTS main.fold_memo (
|
|
146
|
+
task_id TEXT PRIMARY KEY,
|
|
147
|
+
probe TEXT NOT NULL,
|
|
148
|
+
key TEXT NOT NULL,
|
|
149
|
+
seen_at INTEGER NOT NULL
|
|
150
|
+
);
|
|
143
151
|
-- The size of each index blob the enumeration checked against the
|
|
144
152
|
-- worktree size git recorded (A-60). Fixed for its OID, so a warm
|
|
145
153
|
-- run asks git for none; swept with file_hashes by seen_at, the
|
|
@@ -317,8 +325,10 @@ export function createTables(db: Database, store: 'main' | 'store' = 'main'): vo
|
|
|
317
325
|
* the artifact is the record and the entries only its inventory, so a reset
|
|
318
326
|
* drops the rows and keeps these, as it keeps the files. `artifacts_meta`
|
|
319
327
|
* holds the table's own layout version (`'layout'`), the one thing that may
|
|
320
|
-
* drop it. `at` is the last use a row-less sweep judges, as a file's mtime
|
|
321
|
-
*
|
|
328
|
+
* drop it. `at` is the last use a row-less sweep judges, as a file's mtime.
|
|
329
|
+
* A rowid table, not WITHOUT ROWID: there the blob sits in the key's b-tree,
|
|
330
|
+
* so every probe of a hash reads overflow pages; 10 000 lookups took 312 ms
|
|
331
|
+
* against 8.5 (a2, 2026-10-17).
|
|
322
332
|
*/
|
|
323
333
|
export function createArtifactTables(db: Database, store: 'main' | 'store'): void {
|
|
324
334
|
db.exec(artifactTables(store))
|
|
@@ -331,7 +341,7 @@ function artifactTables(store: 'main' | 'store'): string {
|
|
|
331
341
|
hash TEXT PRIMARY KEY,
|
|
332
342
|
at INTEGER NOT NULL,
|
|
333
343
|
bytes BLOB NOT NULL
|
|
334
|
-
)
|
|
344
|
+
);
|
|
335
345
|
CREATE TABLE IF NOT EXISTS ${store}.artifacts_meta (
|
|
336
346
|
key TEXT PRIMARY KEY,
|
|
337
347
|
value TEXT NOT NULL
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
CacheEntry,
|
|
9
9
|
CacheGetContext,
|
|
10
10
|
CacheLayer,
|
|
11
|
+
DeclaredOutputs,
|
|
11
12
|
IngestMeta,
|
|
12
13
|
InvocationRecord,
|
|
13
14
|
OutputDirRow,
|
|
@@ -60,7 +61,9 @@ export class ScopedCache implements CacheLayer {
|
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
private own(hash: string): string {
|
|
63
|
-
|
|
64
|
+
const key = scopedKey(this.scope, hash)
|
|
65
|
+
this.local?.traffic.aliases.set(key, hash)
|
|
66
|
+
return key
|
|
64
67
|
}
|
|
65
68
|
|
|
66
69
|
private keyOf(hash: string): string {
|
|
@@ -194,8 +197,13 @@ export class ScopedCache implements CacheLayer {
|
|
|
194
197
|
return (await this.inner.outputDirsCurrent?.(projectDir, rows)) ?? false
|
|
195
198
|
}
|
|
196
199
|
|
|
197
|
-
restoreOutputs(
|
|
198
|
-
|
|
200
|
+
restoreOutputs(
|
|
201
|
+
hash: string,
|
|
202
|
+
projectDir: string,
|
|
203
|
+
workspaceRoot?: string,
|
|
204
|
+
outputs?: DeclaredOutputs,
|
|
205
|
+
): Promise<void> {
|
|
206
|
+
return this.inner.restoreOutputs(this.keyOf(hash), projectDir, workspaceRoot, outputs)
|
|
199
207
|
}
|
|
200
208
|
|
|
201
209
|
save(args: SaveArgs): Promise<void> {
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// The process umask, read when a restore starts: Bun reads it by setting 0
|
|
2
|
+
// and putting it back in one synchronous call (project-loader.ts
|
|
3
|
+
// `currentUmask`), so no spawn on this thread sees the 0, and a run
|
|
4
|
+
// evaluates no config in a worker. Through globalThis: the playground's
|
|
5
|
+
// shim has no umask.
|
|
6
|
+
const proc = globalThis.process as { umask?: () => number } | undefined
|
|
7
|
+
let umask = 0
|
|
8
|
+
|
|
9
|
+
/** Read the umask again: a restore's modes follow the process's as it is now. */
|
|
10
|
+
export function readUmask(): void {
|
|
11
|
+
if (typeof proc?.umask === 'function') umask = proc.umask()
|
|
12
|
+
}
|
|
13
|
+
readUmask()
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The permission bits a restore gives a stored `mode`: minus the umask, as
|
|
17
|
+
* tar gives a non-root user and Turbo does. A hit under `umask 077` gave
|
|
18
|
+
* back 0644 / 0755 where the build left 0600 / 0700, readable by every
|
|
19
|
+
* user on the machine (Growth's parity run, 2026-10-16).
|
|
20
|
+
*/
|
|
21
|
+
export function restoredMode(mode: number): number {
|
|
22
|
+
return mode & 0o777 & ~umask
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** A file on disk holds the stored `mode`: as saved, or as a restore here leaves it. */
|
|
26
|
+
export function modeMatches(disk: number, stored: number): boolean {
|
|
27
|
+
const d = disk & 0o777
|
|
28
|
+
return d === (stored & 0o777) || d === restoredMode(stored)
|
|
29
|
+
}
|