@vzn/vx 0.0.641 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/cache/archive.ts +525 -104
- package/src/cache/cache.ts +92 -12
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +6 -2
- package/src/cache/inputs.ts +169 -79
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +2 -6
- package/src/cache/layered-cache.ts +68 -4
- package/src/cache/output-index.ts +25 -4
- package/src/cache/remote-lane.ts +27 -10
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +3 -1
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +4 -4
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/run.ts +10 -3
- package/src/cli/select.ts +2 -0
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/history/history.ts +43 -33
- package/src/index.ts +10 -0
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +10 -2
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +31 -0
- package/src/orchestrator/plugin.ts +29 -3
- package/src/orchestrator/prepare.ts +74 -5
- package/src/orchestrator/projects.ts +6 -1
- package/src/orchestrator/run.ts +37 -15
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +46 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +53 -13
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +5 -2
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +12 -2
- package/src/util/paths.ts +33 -14
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +63 -13
- package/src/workspace/config-schema.ts +37 -3
- package/src/workspace/project-loader.ts +1 -0
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2376
- package/docs/cli.md +0 -3285
- package/docs/execution.md +0 -676
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2468
- package/docs/security.md +0 -253
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
|
@@ -1344,7 +1344,7 @@ async function wrapIn(
|
|
|
1344
1344
|
if (process.platform === 'darwin') {
|
|
1345
1345
|
const rules = [
|
|
1346
1346
|
...macProfileRules(args.config),
|
|
1347
|
-
...
|
|
1347
|
+
...darwinGrantRules(args.config, baselines.allowRead, baselines.denyRead),
|
|
1348
1348
|
]
|
|
1349
1349
|
try {
|
|
1350
1350
|
if (rules.length > 0) wrapped = injectProfileRules(wrapped, rules)
|
|
@@ -2270,6 +2270,7 @@ async function runSandboxedOnce(
|
|
|
2270
2270
|
violations,
|
|
2271
2271
|
...(proc.signalCode ? { signal: proc.signalCode } : {}),
|
|
2272
2272
|
...(timeout.timedOut() ? { timedOut: true } : {}),
|
|
2273
|
+
...(cut ? { leftRunning: true } : {}),
|
|
2273
2274
|
// Linux: the task's CPU and peak never reach this wait. bwrap runs it
|
|
2274
2275
|
// in a pid namespace (`--unshare-pid`), and what the namespace's
|
|
2275
2276
|
// processes used is not folded into bwrap's usage: a 500 ms busy loop
|
|
@@ -2570,6 +2571,85 @@ export function wallsGlobsReach(grants: readonly string[], walls: readonly strin
|
|
|
2570
2571
|
return walls.filter((w) => heads.some((h) => atOrUnder(w, h)))
|
|
2571
2572
|
}
|
|
2572
2573
|
|
|
2574
|
+
/**
|
|
2575
|
+
* macOS: the grant rules vx appends. The listings come first so a wall's
|
|
2576
|
+
* deny, later, wins: SBPL is last-match-wins.
|
|
2577
|
+
*/
|
|
2578
|
+
export function darwinGrantRules(
|
|
2579
|
+
c: Pick<ResolvedSandboxConfig, 'wallsReached' | 'allowRead' | 'allowWrite'>,
|
|
2580
|
+
baseAllowRead: readonly string[],
|
|
2581
|
+
anchors: readonly string[],
|
|
2582
|
+
): string[] {
|
|
2583
|
+
return [...darwinAncestorRules(c, anchors), ...darwinWallRules(c, baseAllowRead)]
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
/**
|
|
2587
|
+
* macOS: each directory above a grant may be listed, as on Linux. bwrap
|
|
2588
|
+
* makes every directory above a mount, so a task granted
|
|
2589
|
+
* `../vx-*\/package.json` could glob it from `..`; seatbelt matches paths
|
|
2590
|
+
* and refused the listing (EPERM), so `@vzn/vx-docs#import` failed on
|
|
2591
|
+
* every Mac (2026-10-17). Above the glob that is each literal directory
|
|
2592
|
+
* strictly inside a deny anchor (outside one, all is readable already);
|
|
2593
|
+
* a directory the glob itself names (`vx-a/`) is matched as a regex, and
|
|
2594
|
+
* under a `**` any directory is. The rules grant directories only, never
|
|
2595
|
+
* a file, and come before the wall denies, which still win.
|
|
2596
|
+
*
|
|
2597
|
+
* A listing shows every name in the directory, where Linux's scaffold
|
|
2598
|
+
* shows only what it mounted: names, never a file's data. Not the anchor
|
|
2599
|
+
* itself: Bun, shown the workspace root's `package.json` it
|
|
2600
|
+
* may not read, failed with "Cannot read file" (EPERM).
|
|
2601
|
+
*/
|
|
2602
|
+
export function darwinAncestorRules(
|
|
2603
|
+
c: Pick<ResolvedSandboxConfig, 'allowRead' | 'allowWrite'>,
|
|
2604
|
+
anchors: readonly string[],
|
|
2605
|
+
): string[] {
|
|
2606
|
+
const literals = new Set<string>()
|
|
2607
|
+
const patterns = new Set<string>()
|
|
2608
|
+
// Grants are canonical already (`resolveSandboxConfig`), and so is each
|
|
2609
|
+
// directory above one: no realpath here, at every spawn.
|
|
2610
|
+
const inside = (d: string): boolean => anchors.some((a) => d !== a && atOrUnder(d, a))
|
|
2611
|
+
for (const g of [...c.allowRead, ...c.allowWrite]) {
|
|
2612
|
+
const named = namedPath(g)
|
|
2613
|
+
const head = g.slice(0, g.search(MOUNT_WILDCARDS))
|
|
2614
|
+
const cut = head.lastIndexOf('/')
|
|
2615
|
+
const top = named !== undefined ? path.dirname(named) : head.slice(0, cut) || '/'
|
|
2616
|
+
for (let d = top; inside(d); d = path.dirname(d)) literals.add(d)
|
|
2617
|
+
if (named !== undefined || g.includes('\\')) continue
|
|
2618
|
+
let re = top === '/' ? '' : regexLiteral(top)
|
|
2619
|
+
for (const part of g
|
|
2620
|
+
.slice(cut + 1)
|
|
2621
|
+
.split('/')
|
|
2622
|
+
.slice(0, -1)) {
|
|
2623
|
+
if (part === '**') {
|
|
2624
|
+
patterns.add(`^${re}(/.*)?$`)
|
|
2625
|
+
break
|
|
2626
|
+
}
|
|
2627
|
+
re += `/${part
|
|
2628
|
+
.replace(/[.^$+{}()|]/g, '[$&]')
|
|
2629
|
+
.replaceAll('*', '[^/]*')
|
|
2630
|
+
.replaceAll('?', '[^/]')}`
|
|
2631
|
+
patterns.add(`^${re}$`)
|
|
2632
|
+
}
|
|
2633
|
+
}
|
|
2634
|
+
// A name seatbelt's quoted string cannot hold keeps today's refusal
|
|
2635
|
+
// rather than failing the task (`sbplResolvedPath` would throw).
|
|
2636
|
+
const spellable = (p: string): boolean => !UNSPELLABLE.test(p)
|
|
2637
|
+
return [
|
|
2638
|
+
...[...literals].filter(spellable).map((d) => `(allow file-read-data (literal "${d}"))`),
|
|
2639
|
+
...[...patterns]
|
|
2640
|
+
.filter(spellable)
|
|
2641
|
+
.map((r) => `(allow file-read-data (require-all (vnode-type DIRECTORY) (regex #"${r}")))`),
|
|
2642
|
+
]
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
// eslint-disable-next-line no-control-regex -- the control range is the point
|
|
2646
|
+
const UNSPELLABLE = /["'\\\x00-\x1f\x7f]/
|
|
2647
|
+
|
|
2648
|
+
/** A real path as a regex matching only itself: each special in a class of one. */
|
|
2649
|
+
function regexLiteral(p: string): string {
|
|
2650
|
+
return p.replace(/[.^$+{}()|*?[\]]/g, (ch) => `[${ch}]`)
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2573
2653
|
/**
|
|
2574
2654
|
* macOS: the walls a glob grant reaches, denied at the profile's tail
|
|
2575
2655
|
* (SBPL is last-match-wins). SRT re-emits a wall's deny after the allows
|
package/src/graph/priorities.ts
CHANGED
|
@@ -23,7 +23,20 @@ export function tieredReverseDepCount(
|
|
|
23
23
|
restoreTier: ReadonlySet<string>,
|
|
24
24
|
): Map<string, number> {
|
|
25
25
|
const exec = new Map<string, TaskNode>()
|
|
26
|
-
|
|
26
|
+
let runsAny = false
|
|
27
|
+
for (const [id, node] of nodes) {
|
|
28
|
+
if (restoreTier.has(id)) continue
|
|
29
|
+
exec.set(id, node)
|
|
30
|
+
if (node.config.exec !== undefined) runsAny = true
|
|
31
|
+
}
|
|
32
|
+
// Nothing outside the restore tier runs a command (a warm run: restores
|
|
33
|
+
// and groups), so no order finishes any work sooner: everything ranks 0.
|
|
34
|
+
// The weights below were 10 ms of a 1,000-project warm run.
|
|
35
|
+
if (!runsAny) {
|
|
36
|
+
const zeros = new Map<string, number>()
|
|
37
|
+
for (const id of nodes.keys()) zeros.set(id, 0)
|
|
38
|
+
return zeros
|
|
39
|
+
}
|
|
27
40
|
// An exec tier with no edge inside it (a warm run's groups) counts 0
|
|
28
41
|
// everywhere: no closure to build.
|
|
29
42
|
let linked = false
|
package/src/history/history.ts
CHANGED
|
@@ -263,56 +263,66 @@ export class LocalHistoryProvider implements HistoryProvider {
|
|
|
263
263
|
* newest imported timing (`importTimings`), else the median on any
|
|
264
264
|
* branch, else none. The timings are the shared store's `task_durations`, so every
|
|
265
265
|
* worktree of the repo reads them and a new one forecasts on its first
|
|
266
|
-
* run.
|
|
267
|
-
* however old the rows, where the
|
|
268
|
-
* invocations.
|
|
266
|
+
* run. One statement per tier, an index probe per task on `(project,
|
|
267
|
+
* task, source, id)` inside it: exact however old the rows, where the
|
|
268
|
+
* runs window holds only the newest invocations.
|
|
269
269
|
*/
|
|
270
270
|
private forecasts(wanted: ReadonlySet<string>): Map<string, number> {
|
|
271
271
|
const out = new Map<string, number>()
|
|
272
272
|
this.lackedBranchHistory = false
|
|
273
273
|
if (wanted.size === 0) return out
|
|
274
|
-
// Tiers, first answer wins
|
|
275
|
-
|
|
274
|
+
// Tiers, first answer wins: a WHERE over `task_durations`, its params,
|
|
275
|
+
// and how many of the newest rows it reads.
|
|
276
|
+
type Tier = { where: string; params: string[]; limit: number }
|
|
276
277
|
const tiers: Tier[] = []
|
|
277
|
-
const newest = (where: string) =>
|
|
278
|
-
this.db.query(
|
|
279
|
-
`SELECT duration_ms AS ms FROM task_durations WHERE project = ? AND task = ? AND ${where}
|
|
280
|
-
ORDER BY id DESC LIMIT ${FORECAST_RUNS}`,
|
|
281
|
-
)
|
|
282
278
|
if (this.branch !== undefined) {
|
|
283
|
-
|
|
284
|
-
const q = newest('source = ?')
|
|
285
|
-
tiers.push((p, t) => q.all(p, t, source) as Array<{ ms: number }>)
|
|
279
|
+
tiers.push({ where: 'source = ?', params: [branchSource(this.branch)], limit: FORECAST_RUNS })
|
|
286
280
|
}
|
|
287
281
|
// One probe on the partial index first: a store with no import costs
|
|
288
|
-
// no
|
|
282
|
+
// no tier.
|
|
289
283
|
if (
|
|
290
284
|
this.db.query(`SELECT 1 FROM task_durations WHERE source = 'remote' LIMIT 1`).get() !== null
|
|
291
285
|
) {
|
|
292
|
-
|
|
293
|
-
`SELECT duration_ms AS ms FROM task_durations WHERE project = ? AND task = ? AND source = 'remote'
|
|
294
|
-
ORDER BY id DESC LIMIT 1`,
|
|
295
|
-
)
|
|
296
|
-
tiers.push((p, t) => remote.all(p, t) as Array<{ ms: number }>)
|
|
286
|
+
tiers.push({ where: `source = 'remote'`, params: [], limit: 1 })
|
|
297
287
|
}
|
|
298
|
-
|
|
299
|
-
tiers.push((p, t) => any.all(p, t) as Array<{ ms: number }>)
|
|
288
|
+
tiers.push({ where: `source LIKE 'branch:%'`, params: [], limit: FORECAST_RUNS })
|
|
300
289
|
const own = this.branch !== undefined ? tiers[0] : tiers.at(-1)
|
|
301
|
-
let
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
290
|
+
let pending = [...wanted]
|
|
291
|
+
let ownAnswered = 0
|
|
292
|
+
for (const tier of tiers) {
|
|
293
|
+
if (pending.length === 0) break
|
|
294
|
+
// One statement per tier over every task it is asked for: an index
|
|
295
|
+
// probe per task inside SQLite, where a query per task from here was
|
|
296
|
+
// 31-40 ms of 2,000 tasks against 5-8.
|
|
297
|
+
const rows = this.db
|
|
298
|
+
.query(
|
|
299
|
+
`SELECT j.key AS i, (SELECT group_concat(ms) FROM (
|
|
300
|
+
SELECT duration_ms AS ms FROM task_durations
|
|
301
|
+
WHERE project = j.value ->> 0 AND task = j.value ->> 1 AND ${tier.where}
|
|
302
|
+
ORDER BY id DESC LIMIT ${tier.limit})) AS ds
|
|
303
|
+
FROM json_each(?) AS j`,
|
|
304
|
+
)
|
|
305
|
+
.all(...tier.params, JSON.stringify(pending.map(splitTaskId))) as {
|
|
306
|
+
i: number
|
|
307
|
+
ds: string | null
|
|
308
|
+
}[]
|
|
309
|
+
const left: string[] = []
|
|
310
|
+
for (const r of rows) {
|
|
311
|
+
const id = pending[r.i]!
|
|
312
|
+
if (r.ds === null) {
|
|
313
|
+
left.push(id)
|
|
314
|
+
continue
|
|
315
|
+
}
|
|
316
|
+
const durations = r.ds
|
|
317
|
+
.split(',')
|
|
318
|
+
.map(Number)
|
|
319
|
+
.sort((x, y) => x - y)
|
|
309
320
|
out.set(id, pickPercentile(durations, 0.5))
|
|
310
|
-
|
|
311
|
-
break
|
|
321
|
+
if (tier === own) ownAnswered++
|
|
312
322
|
}
|
|
313
|
-
|
|
323
|
+
pending = left
|
|
314
324
|
}
|
|
315
|
-
this.lackedBranchHistory =
|
|
325
|
+
this.lackedBranchHistory = ownAnswered < wanted.size
|
|
316
326
|
return out
|
|
317
327
|
}
|
|
318
328
|
|
package/src/index.ts
CHANGED
|
@@ -83,9 +83,19 @@ export type {
|
|
|
83
83
|
CacheLayer,
|
|
84
84
|
PluginCache,
|
|
85
85
|
RemoteCacheLayer,
|
|
86
|
+
RemoteOpenContext,
|
|
86
87
|
RunRecord,
|
|
87
88
|
InvocationRecord,
|
|
88
89
|
} from './cache/index.js'
|
|
90
|
+
// A plugin's keychain (`ctx.secret` on `setup` and a remote's `open`), and
|
|
91
|
+
// the guard that tells a missing secret from a fault.
|
|
92
|
+
export { isSecretUnavailable } from './util/index.js'
|
|
93
|
+
export type {
|
|
94
|
+
PluginSecrets,
|
|
95
|
+
SecretRequest,
|
|
96
|
+
SecretUnavailable,
|
|
97
|
+
SecretUnavailableReason,
|
|
98
|
+
} from './util/index.js'
|
|
89
99
|
|
|
90
100
|
// Workspace discovery + the project/config catalog surface — an
|
|
91
101
|
// out-of-process service/CLI needs these. `loadProjectConfig` is the RAW
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import {
|
|
16
16
|
cleanOutputs,
|
|
17
17
|
committedFiles,
|
|
18
|
-
|
|
18
|
+
resolveOutputTree,
|
|
19
19
|
type CacheLayer,
|
|
20
20
|
type GitFilesCache,
|
|
21
21
|
} from '../cache/index.js'
|
|
@@ -144,7 +144,7 @@ export class DeferredOutputs {
|
|
|
144
144
|
`could not fetch deferred outputs of ${taskId}: ${msg} — re-run it with --force, or use --cache-download=all`,
|
|
145
145
|
)
|
|
146
146
|
}
|
|
147
|
-
const outputFiles = await
|
|
147
|
+
const { files: outputFiles, emptyDirs } = await resolveOutputTree(cleanArgs)
|
|
148
148
|
this.args.gitFilesCache?.markOutputsChanged(
|
|
149
149
|
producer.projectDir,
|
|
150
150
|
outputFiles.map((f) => f.slice(producer.projectDir.length + 1)),
|
|
@@ -157,6 +157,7 @@ export class DeferredOutputs {
|
|
|
157
157
|
hash: entry.hash,
|
|
158
158
|
projectDir: producer.projectDir,
|
|
159
159
|
outputFiles,
|
|
160
|
+
...(emptyDirs.length > 0 ? { outputDirs: emptyDirs } : {}),
|
|
160
161
|
entry: entry.entry,
|
|
161
162
|
})
|
|
162
163
|
}
|
|
@@ -61,7 +61,6 @@ import {
|
|
|
61
61
|
type Placeholder,
|
|
62
62
|
reachedWithheld,
|
|
63
63
|
sandboxRequestFor,
|
|
64
|
-
sandboxReadable,
|
|
65
64
|
sweepPlaceholders,
|
|
66
65
|
undeclaredWriteReach,
|
|
67
66
|
untouchedPlaceholderLine,
|
|
@@ -233,7 +232,7 @@ export interface ExecuteArgs {
|
|
|
233
232
|
*/
|
|
234
233
|
outputDirSnapshots?: OutputDirSnapshot[]
|
|
235
234
|
/** The run's save lane: a miss's cache save runs off the execution slot (save-lane.ts). */
|
|
236
|
-
deferSave?: (save: () => Promise<void>) => Promise<
|
|
235
|
+
deferSave?: (save: () => Promise<void>) => Promise<boolean>
|
|
237
236
|
/**
|
|
238
237
|
* Where a deferred save's `landed` promise is parked by task id, for the
|
|
239
238
|
* in-flight join (admission.ts): a duplicate of this task in another run
|
|
@@ -1278,6 +1277,7 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1278
1277
|
} else if (
|
|
1279
1278
|
effectiveExitCode === 0 &&
|
|
1280
1279
|
willSave &&
|
|
1280
|
+
nothingLeftRunning() &&
|
|
1281
1281
|
(await keyStillTrue()) &&
|
|
1282
1282
|
(own = await ownOutputs()) !== undefined
|
|
1283
1283
|
) {
|
|
@@ -1302,19 +1302,11 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1302
1302
|
deferSave: args.deferSave,
|
|
1303
1303
|
// A sink listens: say what the save cost.
|
|
1304
1304
|
measure: args.explainMiss !== undefined,
|
|
1305
|
-
...(userSandbox
|
|
1306
|
-
? {
|
|
1307
|
-
readable: sandboxReadable(
|
|
1308
|
-
node,
|
|
1309
|
-
cfg.exec!.sandbox!,
|
|
1310
|
-
args.workspaceRoot,
|
|
1311
|
-
args.nestedProjectDirs,
|
|
1312
|
-
),
|
|
1313
|
-
}
|
|
1314
|
-
: {}),
|
|
1315
1305
|
})
|
|
1316
|
-
|
|
1317
|
-
|
|
1306
|
+
// A run that settles the outcome on `landed` learns there whether the
|
|
1307
|
+
// entry is in; without one, the save is taken as done.
|
|
1308
|
+
if (args.deferredSaves !== undefined) args.deferredSaves.set(node.id, landed)
|
|
1309
|
+
else saved = true
|
|
1318
1310
|
} else if (cfgCacheable && !remoteOnly && !deferralRequested && args.noDependants !== true) {
|
|
1319
1311
|
// Ran here and saves nothing — failed, a read-only policy, a tainted
|
|
1320
1312
|
// upstream, a key that no longer held — yet it wrote what a save would
|
|
@@ -1362,6 +1354,20 @@ async function executeCachedTask(args: ExecuteArgs): Promise<TaskOutcome> {
|
|
|
1362
1354
|
return mine
|
|
1363
1355
|
}
|
|
1364
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* A process the shell left behind still held its output when the drain
|
|
1359
|
+
* bound cut it, so it may still write outputs: a save now would file a
|
|
1360
|
+
* tree the run does not leave, and a hit elsewhere would replay it.
|
|
1361
|
+
*/
|
|
1362
|
+
function nothingLeftRunning(): boolean {
|
|
1363
|
+
if (result.leftRunning !== true) return true
|
|
1364
|
+
warnOf(log)(
|
|
1365
|
+
`[vx] ${node.id}: a process it left running may still write its outputs — ` +
|
|
1366
|
+
`the result stands, but is not saved; wait for it in the command (\`wait\`)`,
|
|
1367
|
+
)
|
|
1368
|
+
return false
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1365
1371
|
/**
|
|
1366
1372
|
* The input that no longer holds what the key folded, or one the key did
|
|
1367
1373
|
* not fold that is there now: `null` when the key re-derived just before
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// which bytes a green run replays. Moved out of execute-task.ts
|
|
9
9
|
// 2026-09-10 as pure code motion, the mirror of miss-save.ts.
|
|
10
10
|
|
|
11
|
+
import path from 'node:path'
|
|
11
12
|
import type { CacheConfig } from '../config.js'
|
|
12
13
|
import {
|
|
13
14
|
type CacheEntry,
|
|
@@ -166,7 +167,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
166
167
|
actual: readonly string[],
|
|
167
168
|
exp: ReadonlyArray<{ path: string }>,
|
|
168
169
|
): boolean => {
|
|
169
|
-
const expSet = new Set(exp.map((e) => e.path))
|
|
170
|
+
const expSet = new Set(exp.map((e) => e.path).filter((p) => !p.endsWith('/')))
|
|
170
171
|
return actual.length === expSet.size && actual.every((r) => expSet.has(r))
|
|
171
172
|
}
|
|
172
173
|
// A stray a dependant's glob could have added is not a stray; a path
|
|
@@ -185,7 +186,7 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
185
186
|
exp: ReadonlyArray<{ path: string }>,
|
|
186
187
|
): boolean => {
|
|
187
188
|
const have = new Set(actual)
|
|
188
|
-
return exp.every((e) => have.has(e.path))
|
|
189
|
+
return exp.every((e) => e.path.endsWith('/') || have.has(e.path))
|
|
189
190
|
}
|
|
190
191
|
treeMatches = additive
|
|
191
192
|
? rowsPresent(actualRels, projExpected) && rowsPresent(actualWsRels, wsExpected)
|
|
@@ -289,6 +290,13 @@ export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome>
|
|
|
289
290
|
rels: staleRels,
|
|
290
291
|
outputs,
|
|
291
292
|
committed: cleanArgs.committed,
|
|
293
|
+
// A stale file in an output directory the entry holds empty takes
|
|
294
|
+
// only itself: the directory stays, as the miss left it.
|
|
295
|
+
keep: new Set(
|
|
296
|
+
heldRels
|
|
297
|
+
.filter((r) => r.endsWith('/'))
|
|
298
|
+
.map((r) => path.join(node.projectDir, r.slice(0, -1))),
|
|
299
|
+
),
|
|
292
300
|
})
|
|
293
301
|
}
|
|
294
302
|
if (staleWsRels.length > 0) {
|
|
@@ -39,7 +39,9 @@ export interface LockfileClaimOptions {
|
|
|
39
39
|
* the lockfile does not pin (bun.lock's `patchedDependencies` names a patch
|
|
40
40
|
* file and records no hash of it). Their content hashes ('' when absent)
|
|
41
41
|
* reach `digest` as `files` and join the memo's identity, so an edited
|
|
42
|
-
* patch moves the key (item 1014).
|
|
42
|
+
* patch moves the key (item 1014). A directory (a `file:` dependency the
|
|
43
|
+
* install copies) hashes every file under it but `node_modules` and
|
|
44
|
+
* `.git`. Called only when the memo misses.
|
|
43
45
|
*/
|
|
44
46
|
readonly extraFiles?: (text: string) => readonly string[]
|
|
45
47
|
/** Folded into the memo's identity: bump when `digest` folds differently. */
|
|
@@ -109,13 +111,33 @@ interface Memo {
|
|
|
109
111
|
/** A root-relative extra file's content hash, '' when absent; a path leaving the root is never read. */
|
|
110
112
|
async function hashExtra(workspaceRoot: string, rel: string): Promise<string> {
|
|
111
113
|
if (path.isAbsolute(rel) || rel.split(/[\\/]/).includes('..')) return 'outside-root'
|
|
114
|
+
const abs = path.join(workspaceRoot, rel)
|
|
112
115
|
try {
|
|
113
|
-
return xxh3hex(await Bun.file(
|
|
114
|
-
} catch {
|
|
116
|
+
return xxh3hex(await Bun.file(abs).bytes())
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if ((err as NodeJS.ErrnoException).code === 'EISDIR') return hashDir(abs)
|
|
115
119
|
return ''
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Every file under `dir` but its `node_modules` and `.git`, as `rel\0hash`
|
|
125
|
+
* lines in sorted order. Links are followed: a linked file is what a task
|
|
126
|
+
* reads, and the scan skipped it, so an edit to its target moved no key.
|
|
127
|
+
*/
|
|
128
|
+
async function hashDir(dir: string): Promise<string> {
|
|
129
|
+
const rels: string[] = []
|
|
130
|
+
const scan = { cwd: dir, dot: true, onlyFiles: true, followSymlinks: true }
|
|
131
|
+
for await (const rel of new Bun.Glob('**').scan(scan)) {
|
|
132
|
+
if (!rel.split('/').some((s) => s === 'node_modules' || s === '.git')) rels.push(rel)
|
|
133
|
+
}
|
|
134
|
+
rels.sort()
|
|
135
|
+
const lines = await Promise.all(
|
|
136
|
+
rels.map(async (rel) => `${rel}\0${xxh3hex(await Bun.file(path.join(dir, rel)).bytes())}`),
|
|
137
|
+
)
|
|
138
|
+
return `dir\0${xxh3hex(lines.join('\n'))}`
|
|
139
|
+
}
|
|
140
|
+
|
|
119
141
|
/**
|
|
120
142
|
* A file's identity for the in-process gate, as `Cache.hashFile`'s memo
|
|
121
143
|
* keys one: size and mtime alone kept the digests of a lockfile or patch
|
|
@@ -126,9 +148,16 @@ function identityOf(st: { size: number; mtimeMs: number; ctimeMs: number; ino: n
|
|
|
126
148
|
return `${st.size}:${st.mtimeMs}:${st.ctimeMs}:${st.ino}`
|
|
127
149
|
}
|
|
128
150
|
|
|
151
|
+
// A directory's own stat moves only when an entry is added or removed, not
|
|
152
|
+
// when a file in it is edited: its identity is its content hash.
|
|
129
153
|
async function statsOf(workspaceRoot: string, rels: readonly string[]): Promise<string[]> {
|
|
130
154
|
return Promise.all(
|
|
131
|
-
rels.map((rel) =>
|
|
155
|
+
rels.map((rel) =>
|
|
156
|
+
stat(path.join(workspaceRoot, rel)).then(
|
|
157
|
+
(st) => (st.isDirectory() ? hashExtra(workspaceRoot, rel) : identityOf(st)),
|
|
158
|
+
() => 'gone',
|
|
159
|
+
),
|
|
160
|
+
),
|
|
132
161
|
)
|
|
133
162
|
}
|
|
134
163
|
|
|
@@ -262,15 +291,29 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
262
291
|
// lockfile is broken" and "the base commit's is" — a migration
|
|
263
292
|
// commit hits the second, and the bare parser message named
|
|
264
293
|
// neither (2026-09-20).
|
|
265
|
-
|
|
294
|
+
// A changed extra file is folded by its content on each side; one
|
|
295
|
+
// left out is the same on both, so it moves no digest either way.
|
|
296
|
+
const extrasOf = (pick: 'before' | 'after'): Map<string, string> => {
|
|
297
|
+
const out = new Map<string, string>()
|
|
298
|
+
for (const [rel, sides] of change.extras ?? []) {
|
|
299
|
+
const bytes = sides[pick]
|
|
300
|
+
out.set(rel, bytes === null ? '' : xxh3hex(bytes))
|
|
301
|
+
}
|
|
302
|
+
return out
|
|
303
|
+
}
|
|
304
|
+
const sideOf = (
|
|
305
|
+
bytes: Uint8Array,
|
|
306
|
+
pick: 'before' | 'after',
|
|
307
|
+
side: string,
|
|
308
|
+
): ReadonlyMap<string, string> => {
|
|
266
309
|
try {
|
|
267
|
-
return digest(decode(bytes),
|
|
310
|
+
return digest(decode(bytes), extrasOf(pick))
|
|
268
311
|
} catch (err) {
|
|
269
312
|
throw new Error(`${err instanceof Error ? err.message : String(err)} (${side})`)
|
|
270
313
|
}
|
|
271
314
|
}
|
|
272
|
-
const before = sideOf(change.before, `as of the base ref`)
|
|
273
|
-
const after = sideOf(change.after, `in the working tree`)
|
|
315
|
+
const before = sideOf(change.before, 'before', `as of the base ref`)
|
|
316
|
+
const after = sideOf(change.after, 'after', `in the working tree`)
|
|
274
317
|
const names: string[] = []
|
|
275
318
|
for (const p of ctx.projects) {
|
|
276
319
|
const importer = importerOf(ctx.workspaceRoot, p.dir)
|
|
@@ -278,6 +321,9 @@ export function lockfileClaim(options: LockfileClaimOptions): LockfileClaimHooks
|
|
|
278
321
|
}
|
|
279
322
|
return names
|
|
280
323
|
},
|
|
324
|
+
...(options.extraFiles !== undefined
|
|
325
|
+
? { extraFiles: (_file: string, bytes: Uint8Array) => options.extraFiles!(decode(bytes)) }
|
|
326
|
+
: {}),
|
|
281
327
|
},
|
|
282
328
|
async key(task, ctx) {
|
|
283
329
|
const digests = await loadOnce(ctx)
|
|
@@ -12,6 +12,7 @@ import path from 'node:path'
|
|
|
12
12
|
import {
|
|
13
13
|
type CacheLayer,
|
|
14
14
|
type GitFilesCache,
|
|
15
|
+
resolveOutputTree,
|
|
15
16
|
resolveOutputs,
|
|
16
17
|
resolveWorkspaceOutputs,
|
|
17
18
|
WORKSPACE_OUTPUT_PREFIX,
|
|
@@ -51,7 +52,9 @@ export function entryHolds(
|
|
|
51
52
|
): (files: readonly string[], rows: ReadonlyArray<{ path: string }>) => boolean {
|
|
52
53
|
return (files, rows) => {
|
|
53
54
|
const expected = new Set(
|
|
54
|
-
rows
|
|
55
|
+
rows
|
|
56
|
+
.map((r) => r.path)
|
|
57
|
+
.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX) && !p.endsWith('/')),
|
|
55
58
|
)
|
|
56
59
|
const seen = new Set(files)
|
|
57
60
|
if ((node.addsToOutputsOf?.length ?? 0) > 0) return [...expected].every((p) => seen.has(p))
|
|
@@ -111,8 +114,6 @@ export interface SaveMissArgs {
|
|
|
111
114
|
ownOutputFiles?: string[] | undefined
|
|
112
115
|
/** `ownOutputFiles` for `cache.outputs.workspaceFiles` (A-43). */
|
|
113
116
|
ownWsOutputFiles?: string[] | undefined
|
|
114
|
-
/** A sandboxed task's reach: a symlinked output it could not read is refused (`sandboxReadable`). */
|
|
115
|
-
readable?: ((real: string) => boolean) | undefined
|
|
116
117
|
/** The Tier-3 input fingerprint rows captured by the pre-exec describe. */
|
|
117
118
|
captured: readonly TaskInputComponent[]
|
|
118
119
|
command: string
|
|
@@ -129,7 +130,7 @@ export interface SaveMissArgs {
|
|
|
129
130
|
* (a same-project downstream task reads both), and the pack + write +
|
|
130
131
|
* index go to the run's save lane (save-lane.ts).
|
|
131
132
|
*/
|
|
132
|
-
deferSave?: ((save: () => Promise<void>) => Promise<
|
|
133
|
+
deferSave?: ((save: () => Promise<void>) => Promise<boolean>) | undefined
|
|
133
134
|
/** Time the save and size its artifact, for a telemetry sink: one stat. */
|
|
134
135
|
measure?: boolean | undefined
|
|
135
136
|
}
|
|
@@ -138,6 +139,8 @@ export interface SaveMissArgs {
|
|
|
138
139
|
export interface SaveFacts {
|
|
139
140
|
saveMs?: number
|
|
140
141
|
artifactBytes?: number
|
|
142
|
+
/** The entry landed: the outcome's `saved`, set only once it has. */
|
|
143
|
+
saved?: true
|
|
141
144
|
}
|
|
142
145
|
|
|
143
146
|
/**
|
|
@@ -147,13 +150,16 @@ export interface SaveFacts {
|
|
|
147
150
|
export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveFacts> }> {
|
|
148
151
|
const { node, cache, log } = a
|
|
149
152
|
const endResolve = span('miss: resolve outputs')
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
153
|
+
// An additive task owns only the files it wrote (item 588), no directory.
|
|
154
|
+
const tree =
|
|
155
|
+
a.ownOutputFiles !== undefined
|
|
156
|
+
? { files: a.ownOutputFiles, emptyDirs: [] }
|
|
157
|
+
: await resolveOutputTree({
|
|
158
|
+
projectDir: node.projectDir,
|
|
159
|
+
outputs: a.outputs,
|
|
160
|
+
nestedProjectDirs: a.nestedProjectDirs,
|
|
161
|
+
})
|
|
162
|
+
const outputFiles = tree.files
|
|
157
163
|
const wsOutputFiles =
|
|
158
164
|
a.ownWsOutputFiles ??
|
|
159
165
|
(await resolveWorkspaceOutputs({
|
|
@@ -204,11 +210,11 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
204
210
|
hash: a.hash,
|
|
205
211
|
projectDir: node.projectDir,
|
|
206
212
|
outputFiles,
|
|
207
|
-
...(
|
|
208
|
-
|
|
209
|
-
|
|
213
|
+
...(tree.emptyDirs.length > 0 ? { outputDirs: tree.emptyDirs } : {}),
|
|
214
|
+
// The root an output symlink's target must stay inside, outputs.workspaceFiles or not.
|
|
215
|
+
workspaceRoot: a.workspaceRoot,
|
|
216
|
+
...(wsOutputFiles.length > 0 ? { workspaceOutputFiles: wsOutputFiles } : {}),
|
|
210
217
|
inputComponents: a.captured.map((c) => ({ entryHash: a.hash, ...c })),
|
|
211
|
-
readable: a.readable,
|
|
212
218
|
// No `exitCode`: the save only runs under `effectiveExitCode === 0`, and
|
|
213
219
|
// the contract no longer accepts one — so the invariant is enforced by
|
|
214
220
|
// the type rather than by every call site remembering the gate.
|
|
@@ -258,9 +264,13 @@ export async function saveMiss(a: SaveMissArgs): Promise<{ landed: Promise<SaveF
|
|
|
258
264
|
// Off the slot when the run keeps a lane (the save lane bounds and
|
|
259
265
|
// drains it); in the slot otherwise — an embedder without a lane gets
|
|
260
266
|
// the entry before the outcome.
|
|
261
|
-
if (a.deferSave !== undefined)
|
|
267
|
+
if (a.deferSave !== undefined) {
|
|
268
|
+
return {
|
|
269
|
+
landed: a.deferSave(save).then((stored) => (stored ? { ...facts, saved: true } : facts)),
|
|
270
|
+
}
|
|
271
|
+
}
|
|
262
272
|
await save()
|
|
263
|
-
return { landed: Promise.resolve(facts) }
|
|
273
|
+
return { landed: Promise.resolve({ ...facts, saved: true }) }
|
|
264
274
|
}
|
|
265
275
|
|
|
266
276
|
/** What `markUnsaved` reads of a miss: where its declared outputs are. */
|
|
@@ -7,7 +7,15 @@ import type { Logger } from '../report/index.js'
|
|
|
7
7
|
import type { OutputContext, VxPlugin } from './plugin.js'
|
|
8
8
|
|
|
9
9
|
const REQUIRED = ['status', 'taskStdout', 'taskStderr', 'taskComplete'] as const
|
|
10
|
-
const OPTIONAL = [
|
|
10
|
+
const OPTIONAL = [
|
|
11
|
+
'runStart',
|
|
12
|
+
'taskStart',
|
|
13
|
+
'runEnd',
|
|
14
|
+
'stepStart',
|
|
15
|
+
'stepEnd',
|
|
16
|
+
'runStop',
|
|
17
|
+
'runDone',
|
|
18
|
+
] as const
|
|
11
19
|
|
|
12
20
|
/** Reject anything that is not Logger-shaped, naming what is missing. */
|
|
13
21
|
function checkLogger(value: unknown): Logger {
|
package/src/orchestrator/plan.ts
CHANGED
|
@@ -146,6 +146,10 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
|
|
|
146
146
|
const outcomes = await runGraph({
|
|
147
147
|
nodes: args.nodes,
|
|
148
148
|
concurrency: 1,
|
|
149
|
+
// A plan's keys and probes do not depend on the order it walks in, so
|
|
150
|
+
// it ranks nothing: the default ranking's closure over every task was
|
|
151
|
+
// ~30 ms of a 1,000-project `--dry`.
|
|
152
|
+
strategy: 'ready-order',
|
|
149
153
|
execute: async (node, live) => {
|
|
150
154
|
const upstream = keyUpstream(node, live)
|
|
151
155
|
const hashArgs = {
|