@vzn/vx 0.0.640 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.sh +9 -2
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +735 -107
- package/src/cache/cache.ts +133 -29
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +9 -2
- package/src/cache/inputs.ts +244 -108
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +9 -7
- package/src/cache/layered-cache.ts +74 -9
- package/src/cache/output-index.ts +25 -4
- package/src/cache/remote-lane.ts +27 -10
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +11 -3
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +11 -5
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/run.ts +10 -3
- package/src/cli/select.ts +2 -0
- package/src/cli/upgrade.ts +37 -2
- package/src/cli/watch-filter.ts +1 -6
- package/src/cli/watch-fs.ts +2 -1
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/handoff.ts +5 -0
- package/src/history/history.ts +43 -33
- package/src/index.ts +11 -1
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +111 -29
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/local-shortcircuit.ts +8 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +35 -5
- package/src/orchestrator/plugin.ts +29 -3
- package/src/orchestrator/prepare.ts +74 -5
- package/src/orchestrator/projects.ts +6 -1
- package/src/orchestrator/run-report.ts +30 -9
- package/src/orchestrator/run.ts +62 -38
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +46 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +58 -15
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +11 -6
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +13 -0
- package/src/util/paths.ts +58 -5
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +63 -13
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-schema.ts +37 -3
- package/src/workspace/project-loader.ts +6 -4
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2352
- package/docs/cli.md +0 -3280
- package/docs/execution.md +0 -675
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2461
- package/docs/security.md +0 -252
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
|
@@ -61,7 +61,6 @@ import {
|
|
|
61
61
|
type Placeholder,
|
|
62
62
|
reachedWithheld,
|
|
63
63
|
sandboxRequestFor,
|
|
64
|
-
sandboxReadable,
|
|
65
64
|
sweepPlaceholders,
|
|
66
65
|
undeclaredWriteReach,
|
|
67
66
|
untouchedPlaceholderLine,
|
|
@@ -233,7 +232,7 @@ export interface ExecuteArgs {
|
|
|
233
232
|
*/
|
|
234
233
|
outputDirSnapshots?: OutputDirSnapshot[]
|
|
235
234
|
/** The run's save lane: a miss's cache save runs off the execution slot (save-lane.ts). */
|
|
236
|
-
deferSave?: (save: () => Promise<void>) => Promise<
|
|
235
|
+
deferSave?: (save: () => Promise<void>) => Promise<boolean>
|
|
237
236
|
/**
|
|
238
237
|
* Where a deferred save's `landed` promise is parked by task id, for the
|
|
239
238
|
* in-flight join (admission.ts): a duplicate of this task in another run
|
|
@@ -1278,6 +1277,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1278
1277
|
} else if (
|
|
1279
1278
|
effectiveExitCode === 0 &&
|
|
1280
1279
|
willSave &&
|
|
1280
|
+
nothingLeftRunning() &&
|
|
1281
1281
|
(await keyStillTrue()) &&
|
|
1282
1282
|
(own = await ownOutputs()) !== undefined
|
|
1283
1283
|
) {
|
|
@@ -1302,19 +1302,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1302
1302
|
deferSave: args.deferSave,
|
|
1303
1303
|
// A sink listens: say what the save cost.
|
|
1304
1304
|
measure: args.explainMiss !== undefined,
|
|
1305
|
-
...(userSandbox
|
|
1306
|
-
? {
|
|
1307
|
-
readable: sandboxReadable(
|
|
1308
|
-
node,
|
|
1309
|
-
cfg.exec!.sandbox!,
|
|
1310
|
-
args.workspaceRoot,
|
|
1311
|
-
args.nestedProjectDirs,
|
|
1312
|
-
),
|
|
1313
|
-
}
|
|
1314
|
-
: {}),
|
|
1315
1305
|
})
|
|
1316
|
-
|
|
1317
|
-
|
|
1306
|
+
// A run that settles the outcome on `landed` learns there whether the
|
|
1307
|
+
// entry is in; without one, the save is taken as done.
|
|
1308
|
+
if (args.deferredSaves !== undefined) args.deferredSaves.set(node.id, landed)
|
|
1309
|
+
else saved = true
|
|
1318
1310
|
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants !== true) {
|
|
1319
1311
|
// Ran here and saves nothing — failed, a read-only policy, a tainted
|
|
1320
1312
|
// upstream, a key that no longer held — yet it wrote what a save would
|
|
@@ -1362,6 +1354,20 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1362
1354
|
return mine
|
|
1363
1355
|
}
|
|
1364
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* A process the shell left behind still held its output when the drain
|
|
1359
|
+
* bound cut it, so it may still write outputs: a save now would file a
|
|
1360
|
+
* tree the run does not leave, and a hit elsewhere would replay it.
|
|
1361
|
+
*/
|
|
1362
|
+
function nothingLeftRunning(): boolean {
|
|
1363
|
+
if (result.leftRunning !== true) return true
|
|
1364
|
+
warnOf(log)(
|
|
1365
|
+
`[vx] ${node.id}: a process it left running may still write its outputs — ` +
|
|
1366
|
+
`the result stands, but is not saved; wait for it in the command (\`wait\`)`,
|
|
1367
|
+
)
|
|
1368
|
+
return false
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1365
1371
|
/**
|
|
1366
1372
|
* The input that no longer holds what the key folded, or one the key did
|
|
1367
1373
|
* not fold that is there now: `null` when the key re-derived just before
|
|
@@ -8,15 +8,15 @@
|
|
|
8
8
|
// which bytes a green run replays. Moved out of execute-task.ts
|
|
9
9
|
// 2026-09-10 as pure code motion, the mirror of miss-save.ts.
|
|
10
10
|
|
|
11
|
+
import path from 'node:path'
|
|
11
12
|
import type { CacheConfig } from '../config.js'
|
|
12
13
|
import {
|
|
13
14
|
type CacheEntry,
|
|
14
15
|
cleanOutputPaths,
|
|
15
|
-
cleanOutputs,
|
|
16
|
-
cleanWorkspaceOutputs,
|
|
17
16
|
committedFiles,
|
|
18
17
|
resolveOutputs,
|
|
19
18
|
resolveWorkspaceOutputs,
|
|
19
|
+
sweepRestoreTemps,
|
|
20
20
|
WORKSPACE_OUTPUT_PREFIX,
|
|
21
21
|
} from '../cache/index.js'
|
|
22
22
|
import type { TaskOutcome } from '../graph/index.js'
|
|
@@ -167,7 +167,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
167
167
|
actual: readonly string[],
|
|
168
168
|
exp: ReadonlyArray<{ path: string }>,
|
|
169
169
|
): boolean => {
|
|
170
|
-
const expSet = new Set(exp.map((e) => e.path))
|
|
170
|
+
const expSet = new Set(exp.map((e) => e.path).filter((p) => !p.endsWith('/')))
|
|
171
171
|
return actual.length === expSet.size && actual.every((r) => expSet.has(r))
|
|
172
172
|
}
|
|
173
173
|
// A stray a dependant's glob could have added is not a stray; a path
|
|
@@ -186,7 +186,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
186
186
|
exp: ReadonlyArray<{ path: string }>,
|
|
187
187
|
): boolean => {
|
|
188
188
|
const have = new Set(actual)
|
|
189
|
-
return exp.every((e) => have.has(e.path))
|
|
189
|
+
return exp.every((e) => e.path.endsWith('/') || have.has(e.path))
|
|
190
190
|
}
|
|
191
191
|
treeMatches = additive
|
|
192
192
|
? rowsPresent(actualRels, projExpected) && rowsPresent(actualWsRels, wsExpected)
|
|
@@ -232,34 +232,81 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
232
232
|
}
|
|
233
233
|
}
|
|
234
234
|
if (!skipRestore) {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
235
|
+
const heldRels = hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
236
|
+
const heldWsRels = hit.outputFiles
|
|
237
|
+
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
238
|
+
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
239
|
+
// In place, never clean-then-restore: each file is renamed over the one
|
|
240
|
+
// it replaces, so a reader of the tree never finds a declared output
|
|
241
|
+
// missing, and a restore that fails before its renames (a full disk,
|
|
242
|
+
// a damaged entry) leaves the outputs it found as they were. Only what blocks a rename
|
|
243
|
+
// goes first; what the entry does not hold goes after. The clean used
|
|
244
|
+
// to run first: a reader polling a 64 MB output while ten hits restored
|
|
245
|
+
// met ENOENT on 153,336 of 204,411 reads (2026-10-16). An ADDITIVE task
|
|
246
|
+
// owns only its rows (item 588), and the restore rewrites exactly those.
|
|
247
|
+
let foundRels: string[] = []
|
|
248
|
+
let foundWsRels: string[] = []
|
|
249
|
+
let staleRels: string[] = []
|
|
250
|
+
let staleWsRels: string[] = []
|
|
251
|
+
if (!additive) {
|
|
252
|
+
sweepRestoreTemps(node.projectDir, outputs)
|
|
253
|
+
sweepRestoreTemps(args.workspaceRoot, wsOutputs)
|
|
254
|
+
if (outputs.length > 0) {
|
|
255
|
+
const abs = walked?.proj ?? (await resolveOutputs(cleanArgs))
|
|
256
|
+
foundRels = abs.map((f) => relPosix(node.projectDir, f))
|
|
257
|
+
const blocking = inTheWay(foundRels, heldRels)
|
|
258
|
+
staleRels = notHeld(foundRels, heldRels).filter((f) => !blocking.has(f))
|
|
259
|
+
if (blocking.size > 0) {
|
|
260
|
+
await cleanOutputPaths({
|
|
261
|
+
projectDir: node.projectDir,
|
|
262
|
+
rels: [...blocking],
|
|
263
|
+
outputs,
|
|
264
|
+
committed: cleanArgs.committed,
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
if (wsOutputs.length > 0) {
|
|
269
|
+
const abs = walked?.ws ?? (await resolveWorkspaceOutputs(wsCleanArgs))
|
|
270
|
+
foundWsRels = abs.map((f) => relPosix(args.workspaceRoot, f))
|
|
271
|
+
const blocking = inTheWay(foundWsRels, heldWsRels)
|
|
272
|
+
staleWsRels = notHeld(foundWsRels, heldWsRels).filter((f) => !blocking.has(f))
|
|
273
|
+
if (blocking.size > 0) {
|
|
274
|
+
await cleanOutputPaths({
|
|
275
|
+
projectDir: args.workspaceRoot,
|
|
276
|
+
rels: [...blocking],
|
|
277
|
+
outputs: wsOutputs,
|
|
278
|
+
committed: wsCleanArgs.committed,
|
|
279
|
+
})
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot, {
|
|
284
|
+
files: outputs,
|
|
285
|
+
workspaceFiles: wsOutputs,
|
|
286
|
+
})
|
|
287
|
+
if (staleRels.length > 0) {
|
|
239
288
|
await cleanOutputPaths({
|
|
240
289
|
projectDir: node.projectDir,
|
|
241
|
-
rels:
|
|
290
|
+
rels: staleRels,
|
|
242
291
|
outputs,
|
|
243
292
|
committed: cleanArgs.committed,
|
|
293
|
+
// A stale file in an output directory the entry holds empty takes
|
|
294
|
+
// only itself: the directory stays, as the miss left it.
|
|
295
|
+
keep: new Set(
|
|
296
|
+
heldRels
|
|
297
|
+
.filter((r) => r.endsWith('/'))
|
|
298
|
+
.map((r) => path.join(node.projectDir, r.slice(0, -1))),
|
|
299
|
+
),
|
|
244
300
|
})
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length))
|
|
301
|
+
}
|
|
302
|
+
if (staleWsRels.length > 0) {
|
|
248
303
|
await cleanOutputPaths({
|
|
249
304
|
projectDir: args.workspaceRoot,
|
|
250
|
-
rels:
|
|
305
|
+
rels: staleWsRels,
|
|
251
306
|
outputs: wsOutputs,
|
|
252
307
|
committed: wsCleanArgs.committed,
|
|
253
308
|
})
|
|
254
|
-
} else {
|
|
255
|
-
if (outputs.length > 0) {
|
|
256
|
-
cleanedRels = await cleanOutputs({ ...cleanArgs, resolved: walked?.proj })
|
|
257
|
-
}
|
|
258
|
-
if (wsOutputs.length > 0) {
|
|
259
|
-
cleanedWsRels = await cleanWorkspaceOutputs({ ...wsCleanArgs, resolved: walked?.ws })
|
|
260
|
-
}
|
|
261
309
|
}
|
|
262
|
-
await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
|
|
263
310
|
// The tree now holds this entry's bytes: stamp them for the next hit
|
|
264
311
|
// (item 886).
|
|
265
312
|
args.cache.recordOutputStamps?.(hash, node.projectDir, args.workspaceRoot)
|
|
@@ -288,17 +335,12 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
288
335
|
// unconditional drop (an executed task may write undeclared
|
|
289
336
|
// files only git can see).
|
|
290
337
|
if (outputs.length > 0) {
|
|
291
|
-
args.gitFilesCache?.markOutputsChanged(node.projectDir, [
|
|
292
|
-
...cleanedRels,
|
|
293
|
-
...hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX)),
|
|
294
|
-
])
|
|
338
|
+
args.gitFilesCache?.markOutputsChanged(node.projectDir, [...foundRels, ...heldRels])
|
|
295
339
|
}
|
|
296
340
|
if (wsOutputs.length > 0) {
|
|
297
341
|
args.gitFilesCache?.markWorkspaceOutputsChanged(args.workspaceRoot, [
|
|
298
|
-
...
|
|
299
|
-
...
|
|
300
|
-
.filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
|
|
301
|
-
.map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length)),
|
|
342
|
+
...foundWsRels,
|
|
343
|
+
...heldWsRels,
|
|
302
344
|
])
|
|
303
345
|
}
|
|
304
346
|
}
|
|
@@ -361,3 +403,43 @@ function dirPrefixesOf(outputs: readonly string[]): readonly string[] | null {
|
|
|
361
403
|
}
|
|
362
404
|
|
|
363
405
|
const prefixMemo = new Map<string, readonly string[] | null>()
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* On a case-insensitive volume (macOS's default) a rename onto `out.txt`
|
|
409
|
+
* replaces `Out.txt`: the two are one file, and removing the found name
|
|
410
|
+
* after the restore would remove the restored one.
|
|
411
|
+
*/
|
|
412
|
+
const fold: (rel: string) => string =
|
|
413
|
+
process.platform === 'darwin' ? (rel) => rel.normalize('NFC').toLowerCase() : (rel) => rel
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* The found outputs a rename cannot replace: a file where the entry needs a
|
|
417
|
+
* directory (`dist/out` for `dist/out/a.js`) and a file under a path the
|
|
418
|
+
* entry holds as a file (`dist/out/a.js` for `dist/out`), whose directory
|
|
419
|
+
* the prune then takes.
|
|
420
|
+
*/
|
|
421
|
+
function inTheWay(found: readonly string[], held: readonly string[]): Set<string> {
|
|
422
|
+
if (found.length === 0 || held.length === 0) return new Set()
|
|
423
|
+
const files = new Set<string>()
|
|
424
|
+
const dirs = new Set<string>()
|
|
425
|
+
for (const h of held) {
|
|
426
|
+
const key = fold(h)
|
|
427
|
+
files.add(key)
|
|
428
|
+
for (let i = key.indexOf('/'); i !== -1; i = key.indexOf('/', i + 1)) dirs.add(key.slice(0, i))
|
|
429
|
+
}
|
|
430
|
+
const out = new Set<string>()
|
|
431
|
+
for (const f of found) {
|
|
432
|
+
const key = fold(f)
|
|
433
|
+
if (dirs.has(key)) out.add(f)
|
|
434
|
+
for (let i = key.indexOf('/'); i !== -1; i = key.indexOf('/', i + 1)) {
|
|
435
|
+
if (files.has(key.slice(0, i))) out.add(f)
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
return out
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/** The found outputs the entry does not hold: the stale ones, once it has restored. */
|
|
442
|
+
function notHeld(found: readonly string[], held: readonly string[]): string[] {
|
|
443
|
+
const keep = new Set(held.map(fold))
|
|
444
|
+
return found.filter((f) => !keep.has(fold(f)))
|
|
445
|
+
}
|
|
@@ -60,6 +60,12 @@ export interface ShortCircuitArgs {
|
|
|
60
60
|
hashCache: HashCache
|
|
61
61
|
/** Cap on concurrent probes — the run's concurrency. */
|
|
62
62
|
concurrency: number
|
|
63
|
+
/**
|
|
64
|
+
* `cache` is the local store in front of a remote: a hit is certain, a
|
|
65
|
+
* miss is not (the remote may hold it), so only hits are recorded and
|
|
66
|
+
* execute probes every other task through the layers.
|
|
67
|
+
*/
|
|
68
|
+
localOnly?: boolean | undefined
|
|
63
69
|
}
|
|
64
70
|
|
|
65
71
|
/** Per-task result of the up-front probe. `hit` null = a confirmed
|
|
@@ -159,6 +165,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
159
165
|
endProbe()
|
|
160
166
|
for (const { hash, node } of candidates) {
|
|
161
167
|
const hit = hits.get(hash) ?? null
|
|
168
|
+
if (hit === null && args.localOnly === true) continue
|
|
162
169
|
preProbed.set(node.id, { hash, hit })
|
|
163
170
|
if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
|
|
164
171
|
}
|
|
@@ -175,6 +182,7 @@ export async function startLocalShortCircuit(args: ShortCircuitArgs): Promise<Sh
|
|
|
175
182
|
const { hash, node } = candidates[next++]!
|
|
176
183
|
try {
|
|
177
184
|
const hit = await args.cache.get(hash, getContext(node, node.config.exec?.command ?? ''))
|
|
185
|
+
if (hit === null && args.localOnly === true) continue
|
|
178
186
|
preProbed.set(node.id, { hash, hit })
|
|
179
187
|
if (hit !== null && !keptOut.has(node.id)) restoreTier.add(node.id)
|
|
180
188
|
} catch {
|
|
@@ -39,7 +39,9 @@ export interface LockfileClaimOptions {
|
|
|
39
39
|
* the lockfile does not pin (bun.lock's `patchedDependencies` names a patch
|
|
40
40
|
* file and records no hash of it). Their content hashes ('' when absent)
|
|
41
41
|
* reach `digest` as `files` and join the memo's identity, so an edited
|
|
42
|
-
* patch moves the key (item 1014).
|
|
42
|
+
* patch moves the key (item 1014). A directory (a `file:` dependency the
|
|
43
|
+
* install copies) hashes every file under it but `node_modules` and
|
|
44
|
+
* `.git`. Called only when the memo misses.
|
|
43
45
|
*/
|
|
44
46
|
readonly extraFiles?: (text: string) => readonly string[]
|
|
45
47
|
/** Folded into the memo's identity: bump when `digest` folds differently. */
|
|
@@ -109,13 +111,33 @@ interface Memo {
|
|
|
109
111
|
/** A root-relative extra file's content hash, '' when absent; a path leaving the root is never read. */
|
|
110
112
|
async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
111
113
|
if (path.isAbsolute(rel) || rel.split(/[\\/]/).includes('..')) return 'outside-root'
|
|
114
|
+
const abs = path.join(workspaceRoot, rel)
|
|
112
115
|
try {
|
|
113
|
-
return xxh3hex(await Bun.file(
|
|
114
|
-
} catch {
|
|
116
|
+
return xxh3hex(await Bun.file(abs).bytes())
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if ((err as NodeJS.ErrnoException).code === 'EISDIR') return hashDir(abs)
|
|
115
119
|
return ''
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Every file under `dir` but its `node_modules` and `.git`, as `rel\0hash`
|
|
125
|
+
* lines in sorted order. Links are followed: a linked file is what a task
|
|
126
|
+
* reads, and the scan skipped it, so an edit to its target moved no key.
|
|
127
|
+
*/
|
|
128
|
+
async function hashDir(dir: string): Promise<string> {
|
|
129
|
+
const rels: string[] = []
|
|
130
|
+
const scan = { cwd: dir, dot: true, onlyFiles: true, followSymlinks: true }
|
|
131
|
+
for await (const rel of new Bun.Glob('**').scan(scan)) {
|
|
132
|
+
if (!rel.split('/').some((s) => s === 'node_modules' || s === '.git')) rels.push(rel)
|
|
133
|
+
}
|
|
134
|
+
rels.sort()
|
|
135
|
+
const lines = await Promise.all(
|
|
136
|
+
rels.map(async (rel) => `${rel}\0${xxh3hex(await Bun.file(path.join(dir, rel)).bytes())}`),
|
|
137
|
+
)
|
|
138
|
+
return `dir\0${xxh3hex(lines.join('\n'))}`
|
|
139
|
+
}
|
|
140
|
+
|
|
119
141
|
/**
|
|
120
142
|
* A file's identity for the in-process gate, as `Cache.hashFile`'s memo
|
|
121
143
|
* keys one: size and mtime alone kept the digests of a lockfile or patch
|
|
@@ -126,9 +148,16 @@ function identityOf(st: { size: number; mtimeMs: number; ctimeMs: number; ino: n
|
|
|
126
148
|
return `${st.size}:${st.mtimeMs}:${st.ctimeMs}:${st.ino}`
|
|
127
149
|
}
|
|
128
150
|
|
|
151
|
+
// A directory's own stat moves only when an entry is added or removed, not
|
|
152
|
+
// when a file in it is edited: its identity is its content hash.
|
|
129
153
|
async function statsOf(workspaceRoot: string, rels: readonly string[]): Promise<string[]> {
|
|
130
154
|
return Promise.all(
|
|
131
|
-
rels.map((rel) =>
|
|
155
|
+
rels.map((rel) =>
|
|
156
|
+
stat(path.join(workspaceRoot, rel)).then(
|
|
157
|
+
(st) => (st.isDirectory() ? hashExtra(workspaceRoot, rel) : identityOf(st)),
|
|
158
|
+
() => 'gone',
|
|
159
|
+
),
|
|
160
|
+
),
|
|
132
161
|
)
|
|
133
162
|
}
|
|
134
163
|
|
|
@@ -262,15 +291,29 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
262
291
|
// lockfile is broken" and "the base commit's is" — a migration
|
|
263
292
|
// commit hits the second, and the bare parser message named
|
|
264
293
|
// neither (2026-09-20).
|
|
265
|
-
|
|
294
|
+
// A changed extra file is folded by its content on each side; one
|
|
295
|
+
// left out is the same on both, so it moves no digest either way.
|
|
296
|
+
const extrasOf = (pick: 'before' | 'after'): Map<string, string> => {
|
|
297
|
+
const out = new Map<string, string>()
|
|
298
|
+
for (const [rel, sides] of change.extras ?? []) {
|
|
299
|
+
const bytes = sides[pick]
|
|
300
|
+
out.set(rel, bytes === null ? '' : xxh3hex(bytes))
|
|
301
|
+
}
|
|
302
|
+
return out
|
|
303
|
+
}
|
|
304
|
+
const sideOf = (
|
|
305
|
+
bytes: Uint8Array,
|
|
306
|
+
pick: 'before' | 'after',
|
|
307
|
+
side: string,
|
|
308
|
+
): ReadonlyMap<string, string> => {
|
|
266
309
|
try {
|
|
267
|
-
return digest(decode(bytes),
|
|
310
|
+
return digest(decode(bytes), extrasOf(pick))
|
|
268
311
|
} catch (err) {
|
|
269
312
|
throw new Error(`${err instanceof Error ? err.message : String(err)} (${side})`)
|
|
270
313
|
}
|
|
271
314
|
}
|
|
272
|
-
const before = sideOf(change.before, `as of the base ref`)
|
|
273
|
-
const after = sideOf(change.after, `in the working tree`)
|
|
315
|
+
const before = sideOf(change.before, 'before', `as of the base ref`)
|
|
316
|
+
const after = sideOf(change.after, 'after', `in the working tree`)
|
|
274
317
|
const names: string[] = []
|
|
275
318
|
for (const p of ctx.projects) {
|
|
276
319
|
const importer = importerOf(ctx.workspaceRoot, p.dir)
|
|
@@ -278,6 +321,9 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
278
321
|
}
|
|
279
322
|
return names
|
|
280
323
|
},
|
|
324
|
+
...(options.extraFiles !== undefined
|
|
325
|
+
? { extraFiles: (_file: string, bytes: Uint8Array) => options.extraFiles!(decode(bytes)) }
|
|
326
|
+
: {}),
|
|
281
327
|
},
|
|
282
328
|
async key(task, ctx) {
|
|
283
329
|
const digests = await loadOnce(ctx)
|
|
@@ -12,6 +12,7 @@ import path from 'node:path'
|
|
|
12
12
|
import {
|
|
13
13
|
type CacheLayer,
|
|
14
14
|
type GitFilesCache,
|
|
15
|
+
resolveOutputTree,
|
|
15
16
|
resolveOutputs,
|
|
16
17
|
resolveWorkspaceOutputs,
|
|
17
18
|
WORKSPACE_OUTPUT_PREFIX,
|
|
@@ -51,7 +52,9 @@ export function entryHolds(
|
|
|
51
52
|
): (files: readonly string[], rows: ReadonlyArray<{ path: string }>) => boolean {
|
|
52
53
|
return (files, rows) => {
|
|
53
54
|
const expected = new Set(
|
|
54
|
-
rows
|
|
55
|
+
rows
|
|
56
|
+
.map((r) => r.path)
|
|
57
|
+
.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX) && !p.endsWith('/')),
|
|
55
58
|
)
|
|
56
59
|
const seen = new Set(files)
|
|
57
60
|
if ((node.addsToOutputsOf?.length ?? 0) > 0) return [...expected].every((p) => seen.has(p))
|
|
@@ -111,8 +114,6 @@ export interface SaveMissArgs {
|
|
|
111
114
|
ownOutputFiles?: string[] | undefined
|
|
112
115
|
/** `ownOutputFiles` for `cache.outputs.workspaceFiles` (A-43). */
|
|
113
116
|
ownWsOutputFiles?: string[] | undefined
|
|
114
|
-
/** A sandboxed task's reach: a symlinked output it could not read is refused (`sandboxReadable`). */
|
|
115
|
-
readable?: ((real: string) => boolean) | undefined
|
|
116
117
|
/** The Tier-3 input fingerprint rows captured by the pre-exec describe. */
|
|
117
118
|
captured: readonly TaskInputComponent[]
|
|
118
119
|
command: string
|
|
@@ -129,7 +130,7 @@ export interface SaveMissArgs {
|
|
|
129
130
|
* (a same-project downstream task reads both), and the pack + write +
|
|
130
131
|
* index go to the run's save lane (save-lane.ts).
|
|
131
132
|
*/
|
|
132
|
-
deferSave?: ((save: () => Promise<void>) => Promise<
|
|
133
|
+
deferSave?: ((save: () => Promise<void>) => Promise<boolean>) | undefined
|
|
133
134
|
/** Time the save and size its artifact, for a telemetry sink: one stat. */
|
|
134
135
|
measure?: boolean | undefined
|
|
135
136
|
}
|
|
@@ -138,6 +139,8 @@ export interface SaveMissArgs {
|
|
|
138
139
|
export interface SaveFacts {
|
|
139
140
|
saveMs?: number
|
|
140
141
|
artifactBytes?: number
|
|
142
|
+
/** The entry landed: the outcome's `saved`, set only once it has. */
|
|
143
|
+
saved?: true
|
|
141
144
|
}
|
|
142
145
|
|
|
143
146
|
/**
|
|
@@ -147,13 +150,16 @@ export interface SaveFacts {
|
|
|
147
150
|
export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveFacts> }> {
|
|
148
151
|
const { node, cache, log } = a
|
|
149
152
|
const endResolve = span('miss: resolve outputs')
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
153
|
+
// An additive task owns only the files it wrote (item 588), no directory.
|
|
154
|
+
const tree =
|
|
155
|
+
a.ownOutputFiles !== undefined
|
|
156
|
+
? { files: a.ownOutputFiles, emptyDirs: [] }
|
|
157
|
+
: await resolveOutputTree({
|
|
158
|
+
projectDir: node.projectDir,
|
|
159
|
+
outputs: a.outputs,
|
|
160
|
+
nestedProjectDirs: a.nestedProjectDirs,
|
|
161
|
+
})
|
|
162
|
+
const outputFiles = tree.files
|
|
157
163
|
const wsOutputFiles =
|
|
158
164
|
a.ownWsOutputFiles ??
|
|
159
165
|
(await resolveWorkspaceOutputs({
|
|
@@ -204,11 +210,11 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
204
210
|
hash: a.hash,
|
|
205
211
|
projectDir: node.projectDir,
|
|
206
212
|
outputFiles,
|
|
207
|
-
...(
|
|
208
|
-
|
|
209
|
-
|
|
213
|
+
...(tree.emptyDirs.length > 0 ? { outputDirs: tree.emptyDirs } : {}),
|
|
214
|
+
// The root an output symlink's target must stay inside, outputs.workspaceFiles or not.
|
|
215
|
+
workspaceRoot: a.workspaceRoot,
|
|
216
|
+
...(wsOutputFiles.length > 0 ? { workspaceOutputFiles: wsOutputFiles } : {}),
|
|
210
217
|
inputComponents: a.captured.map((c) => ({ entryHash: a.hash, ...c })),
|
|
211
|
-
readable: a.readable,
|
|
212
218
|
// No `exitCode`: the save only runs under `effectiveExitCode === 0`, and
|
|
213
219
|
// the contract no longer accepts one — so the invariant is enforced by
|
|
214
220
|
// the type rather than by every call site remembering the gate.
|
|
@@ -258,9 +264,13 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
258
264
|
// Off the slot when the run keeps a lane (the save lane bounds and
|
|
259
265
|
// drains it); in the slot otherwise — an embedder without a lane gets
|
|
260
266
|
// the entry before the outcome.
|
|
261
|
-
if (a.deferSave !== undefined)
|
|
267
|
+
if (a.deferSave !== undefined) {
|
|
268
|
+
return {
|
|
269
|
+
landed: a.deferSave(save).then((stored) => (stored ? { ...facts, saved: true } : facts)),
|
|
270
|
+
}
|
|
271
|
+
}
|
|
262
272
|
await save()
|
|
263
|
-
return { landed: Promise.resolve(facts) }
|
|
273
|
+
return { landed: Promise.resolve({ ...facts, saved: true }) }
|
|
264
274
|
}
|
|
265
275
|
|
|
266
276
|
/** What `markUnsaved` reads of a miss: where its declared outputs are. */
|
|
@@ -7,7 +7,15 @@ import type { Logger } from '../report/index.js'
|
|
|
7
7
|
import type { OutputContext, VxPlugin } from './plugin.js'
|
|
8
8
|
|
|
9
9
|
const REQUIRED = ['status', 'taskStdout', 'taskStderr', 'taskComplete'] as const
|
|
10
|
-
const OPTIONAL = [
|
|
10
|
+
const OPTIONAL = [
|
|
11
|
+
'runStart',
|
|
12
|
+
'taskStart',
|
|
13
|
+
'runEnd',
|
|
14
|
+
'stepStart',
|
|
15
|
+
'stepEnd',
|
|
16
|
+
'runStop',
|
|
17
|
+
'runDone',
|
|
18
|
+
] as const
|
|
11
19
|
|
|
12
20
|
/** Reject anything that is not Logger-shaped, naming what is missing. */
|
|
13
21
|
function checkLogger(value: unknown): Logger {
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -146,6 +146,10 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
146
146
|
const outcomes = await runGraph({
|
|
147
147
|
nodes: args.nodes,
|
|
148
148
|
concurrency: 1,
|
|
149
|
+
// A plan's keys and probes do not depend on the order it walks in, so
|
|
150
|
+
// it ranks nothing: the default ranking's closure over every task was
|
|
151
|
+
// ~30 ms of a 1,000-project `--dry`.
|
|
152
|
+
strategy: 'ready-order',
|
|
149
153
|
execute: async (node, live) => {
|
|
150
154
|
const upstream = keyUpstream(node, live)
|
|
151
155
|
const hashArgs = {
|
|
@@ -381,6 +381,36 @@ export async function claimedAffected(
|
|
|
381
381
|
return names
|
|
382
382
|
}
|
|
383
383
|
|
|
384
|
+
/**
|
|
385
|
+
* The files a claimed file names that its plugin keys too
|
|
386
|
+
* (`FingerprintClaim.extraFiles`); `undefined` when it names none it can
|
|
387
|
+
* answer for, and a plugin that throws is refused by its name.
|
|
388
|
+
*/
|
|
389
|
+
export function claimedExtraFiles(
|
|
390
|
+
plugin: VxPlugin,
|
|
391
|
+
file: string,
|
|
392
|
+
bytes: Uint8Array,
|
|
393
|
+
): readonly string[] | undefined {
|
|
394
|
+
const read = plugin.fingerprint?.extraFiles
|
|
395
|
+
if (read === undefined) return undefined
|
|
396
|
+
let names: unknown
|
|
397
|
+
try {
|
|
398
|
+
names = read.call(plugin.fingerprint, file, bytes)
|
|
399
|
+
} catch (err) {
|
|
400
|
+
throw new UserError(
|
|
401
|
+
maskedLine(
|
|
402
|
+
`plugin '${plugin.name}' failed in fingerprint: ${err instanceof Error ? err.message : String(err)}`,
|
|
403
|
+
),
|
|
404
|
+
)
|
|
405
|
+
}
|
|
406
|
+
if (!Array.isArray(names) || names.some((n) => typeof n !== 'string')) {
|
|
407
|
+
throw new UserError(
|
|
408
|
+
`plugin '${plugin.name}' failed in fingerprint: extraFiles returned ${describeValue(names)}, not a list of paths`,
|
|
409
|
+
)
|
|
410
|
+
}
|
|
411
|
+
return names as string[]
|
|
412
|
+
}
|
|
413
|
+
|
|
384
414
|
/** What a run hands the `schedule` stage: a step per plugin, and where refined weights go. */
|
|
385
415
|
export interface ScheduleRun {
|
|
386
416
|
readonly step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
@@ -531,8 +561,8 @@ export interface CacheHost extends CacheContext {
|
|
|
531
561
|
* Collect every plugin's remote (`cache` hook) in declaration order and
|
|
532
562
|
* layer each over the local store: a lookup asks the local store first,
|
|
533
563
|
* then each remote in order (owner, 2026-10-13), and a save reaches every
|
|
534
|
-
* remote. The run's policy
|
|
535
|
-
*
|
|
564
|
+
* remote. The run's policy alone gates each remote (M, 2026-10-16: a
|
|
565
|
+
* plugin moves bytes, it never decides reads or writes); a remote's throw is a miss and a warning naming its plugin. No
|
|
536
566
|
* remote → the local store alone.
|
|
537
567
|
*/
|
|
538
568
|
export async function resolveCache(
|
|
@@ -546,11 +576,11 @@ export async function resolveCache(
|
|
|
546
576
|
const remote = await safe(plugin, 'cache', () => plugin.cache!(ctx))
|
|
547
577
|
if (remote === undefined) continue
|
|
548
578
|
assertShape(plugin, 'cache', remote, REMOTE_CACHE_METHODS, 'a remote cache layer')
|
|
549
|
-
const access = remote.access
|
|
550
579
|
const entry: ResolvedRemote = {
|
|
551
580
|
remote,
|
|
552
|
-
|
|
553
|
-
|
|
581
|
+
plugin: plugin.name,
|
|
582
|
+
read: policy.remoteRead,
|
|
583
|
+
write: policy.remoteWrite,
|
|
554
584
|
onError: (err) => ctx.warn(`[vx] plugin '${plugin.name}': ${err.message}`),
|
|
555
585
|
}
|
|
556
586
|
resolved.push(entry)
|