@vzn/vx 0.0.640 → 0.0.642
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/launcher.sh +9 -2
- package/package.json +5 -12
- package/skills/vx/SKILL.md +4 -4
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +735 -107
- package/src/cache/cache.ts +133 -29
- package/src/cache/fold-memo.ts +58 -0
- package/src/cache/git-inputs.ts +213 -31
- package/src/cache/index.ts +9 -2
- package/src/cache/inputs.ts +244 -108
- package/src/cache/key-fold.ts +90 -69
- package/src/cache/layer.ts +9 -7
- package/src/cache/layered-cache.ts +74 -9
- package/src/cache/output-index.ts +25 -4
- package/src/cache/remote-lane.ts +27 -10
- package/src/cache/schema.ts +13 -3
- package/src/cache/scoped-cache.ts +11 -3
- package/src/cache/umask.ts +29 -0
- package/src/cli/docs.ts +71 -7
- package/src/cli/help.ts +11 -5
- package/src/cli/plan-format.ts +4 -1
- package/src/cli/run.ts +10 -3
- package/src/cli/select.ts +2 -0
- package/src/cli/upgrade.ts +37 -2
- package/src/cli/watch-filter.ts +1 -6
- package/src/cli/watch-fs.ts +2 -1
- package/src/exec/runner.ts +5 -0
- package/src/exec/sandbox-runtime.ts +81 -1
- package/src/graph/priorities.ts +14 -1
- package/src/handoff.ts +5 -0
- package/src/history/history.ts +43 -33
- package/src/index.ts +11 -1
- package/src/orchestrator/deferred-outputs.ts +3 -2
- package/src/orchestrator/execute-task.ts +20 -14
- package/src/orchestrator/hit-restore.ts +111 -29
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/local-shortcircuit.ts +8 -0
- package/src/orchestrator/lockfile-claim.ts +54 -8
- package/src/orchestrator/miss-save.ts +27 -17
- package/src/orchestrator/output-host.ts +9 -1
- package/src/orchestrator/plan.ts +4 -0
- package/src/orchestrator/plugin-host.ts +35 -5
- package/src/orchestrator/plugin.ts +29 -3
- package/src/orchestrator/prepare.ts +74 -5
- package/src/orchestrator/projects.ts +6 -1
- package/src/orchestrator/run-report.ts +30 -9
- package/src/orchestrator/run.ts +62 -38
- package/src/orchestrator/sandbox-request.ts +1 -48
- package/src/orchestrator/save-lane.ts +17 -10
- package/src/orchestrator/signals.ts +7 -10
- package/src/orchestrator/stable-keys.ts +83 -25
- package/src/orchestrator/task-hash.ts +46 -12
- package/src/report/events.ts +12 -0
- package/src/report/framed-output.ts +2 -1
- package/src/report/logger.ts +58 -15
- package/src/report/status-line.ts +45 -4
- package/src/report/summary.ts +11 -6
- package/src/util/errors.ts +1 -1
- package/src/util/hidden-prompt.ts +74 -0
- package/src/util/index.ts +13 -0
- package/src/util/paths.ts +58 -5
- package/src/util/secrets.ts +150 -0
- package/src/workspace/affected.ts +63 -13
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-schema.ts +37 -3
- package/src/workspace/project-loader.ts +6 -4
- package/src/workspace/workspace.ts +17 -11
- package/docs/caching.md +0 -2352
- package/docs/cli.md +0 -3280
- package/docs/execution.md +0 -675
- package/docs/features.md +0 -178
- package/docs/patterns.md +0 -219
- package/docs/schema.md +0 -2461
- package/docs/security.md +0 -252
- package/src/cli/docs-corpus.ts +0 -21
- package/src/cli/docs-text.d.ts +0 -5
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import { PLUGIN_PACKAGE, type ProjectConfig, type WorkspaceConfig } from '../config.js'
|
|
21
21
|
import type { TaskExecutor } from '../exec/index.js'
|
|
22
22
|
import type { TaskNode, TaskOutcome, TaskStatus } from '../graph/index.js'
|
|
23
|
-
import { maskedLine, nearest, UserError } from '../util/index.js'
|
|
23
|
+
import { maskedLine, nearest, type PluginSecrets, SecretBroker, UserError } from '../util/index.js'
|
|
24
24
|
import type { ProjectMeta } from '../workspace/index.js'
|
|
25
25
|
import type { EventBus, Logger, RunStartInfo } from '../report/index.js'
|
|
26
26
|
import type { CiContext } from './run-context.js'
|
|
@@ -285,8 +285,12 @@ interface BaseContext {
|
|
|
285
285
|
warn(message: string): void
|
|
286
286
|
}
|
|
287
287
|
|
|
288
|
-
/**
|
|
289
|
-
|
|
288
|
+
/**
|
|
289
|
+
* What `setup` receives: the run's lifecycle, observe-only, and the
|
|
290
|
+
* plugin's keychain (`secret`, `forgetSecret`): setup runs before the
|
|
291
|
+
* status line, so a prompt there is seen.
|
|
292
|
+
*/
|
|
293
|
+
export interface PluginSetupContext extends BaseContext, PluginSecrets {
|
|
290
294
|
/** This plugin's entries in the workspace's cache (`PluginCache`). */
|
|
291
295
|
readonly cache: PluginCache
|
|
292
296
|
/** The run is on CI (`CI` set, or a provider core detects). */
|
|
@@ -419,6 +423,15 @@ export interface FingerprintClaim {
|
|
|
419
423
|
change: FingerprintChange,
|
|
420
424
|
ctx: FingerprintContext,
|
|
421
425
|
): Iterable<string> | undefined | Promise<Iterable<string> | undefined>
|
|
426
|
+
/**
|
|
427
|
+
* The workspace-relative files a claimed file's bytes name that the
|
|
428
|
+
* plugin's key reads too: `bun.lock`'s patch files, which it names by
|
|
429
|
+
* path and hashes nowhere, or a `file:` dependency's directory. A change
|
|
430
|
+
* to one, or under one, asks `affected` about the file naming it, with
|
|
431
|
+
* `change.extras`. Without it, a `bun.lock` patch edit selects every
|
|
432
|
+
* project.
|
|
433
|
+
*/
|
|
434
|
+
extraFiles?(file: string, bytes: Uint8Array): readonly string[]
|
|
422
435
|
}
|
|
423
436
|
|
|
424
437
|
export interface FingerprintChange {
|
|
@@ -426,6 +439,15 @@ export interface FingerprintChange {
|
|
|
426
439
|
readonly file: string
|
|
427
440
|
readonly before: Uint8Array | null
|
|
428
441
|
readonly after: Uint8Array | null
|
|
442
|
+
/**
|
|
443
|
+
* Each changed `extraFiles` entry, as the claimant names it, at the base
|
|
444
|
+
* ref and in the working tree. A directory's sides are not its bytes:
|
|
445
|
+
* they differ because a file under it changed.
|
|
446
|
+
*/
|
|
447
|
+
readonly extras?: ReadonlyMap<
|
|
448
|
+
string,
|
|
449
|
+
{ readonly before: Uint8Array | null; readonly after: Uint8Array | null }
|
|
450
|
+
>
|
|
429
451
|
}
|
|
430
452
|
|
|
431
453
|
export interface FingerprintContext extends BaseContext {
|
|
@@ -695,6 +717,8 @@ export interface InstallPluginsArgs {
|
|
|
695
717
|
ci?: boolean
|
|
696
718
|
/** Default null. */
|
|
697
719
|
branch?: string | null
|
|
720
|
+
/** The run's secrets; default: the OS keychain and this terminal. */
|
|
721
|
+
secrets?: SecretBroker
|
|
698
722
|
/** Each setup as the step `<plugin>:setup`; default: untimed. */
|
|
699
723
|
step?: <T>(id: string, work: () => Promise<T>) => Promise<T>
|
|
700
724
|
}
|
|
@@ -726,6 +750,7 @@ export class PluginSetupError extends UserError {
|
|
|
726
750
|
export async function installPlugins(args: InstallPluginsArgs): Promise<() => void> {
|
|
727
751
|
const { plugins, bus, workspaceRoot, cacheDir } = args
|
|
728
752
|
const warn = args.warn ?? ((m) => console.error(m))
|
|
753
|
+
const secrets = args.secrets ?? new SecretBroker(args.ci ?? false, warn)
|
|
729
754
|
const disposers: Array<() => void> = []
|
|
730
755
|
// By object, not name: one package can export two plugins.
|
|
731
756
|
const disabled = new Set<Plugin>()
|
|
@@ -754,6 +779,7 @@ export async function installPlugins(args: InstallPluginsArgs): Promise<() => vo
|
|
|
754
779
|
cacheDir,
|
|
755
780
|
warn,
|
|
756
781
|
cache: args.cache?.(plugin.name) ?? absentPluginCache,
|
|
782
|
+
...secrets.for(plugin.name),
|
|
757
783
|
ci: args.ci ?? false,
|
|
758
784
|
branch: args.branch ?? null,
|
|
759
785
|
// A subscription made past the hooks leaves with the run like a
|
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// closes at the bottom of execution, `planRun()` does so via
|
|
8
8
|
// try/finally around its plan() call.
|
|
9
9
|
|
|
10
|
+
import { existsSync } from 'node:fs'
|
|
10
11
|
import path from 'node:path'
|
|
11
12
|
import type { WorkspaceConfig } from '../config.js'
|
|
12
|
-
import { beginRun, mark, nearest, UserError } from '../util/index.js'
|
|
13
|
+
import { beginRun, mark, nearest, relPosix, type SecretBroker, UserError } from '../util/index.js'
|
|
13
14
|
import {
|
|
14
15
|
Cache,
|
|
15
16
|
type CacheLayer,
|
|
@@ -40,6 +41,7 @@ import {
|
|
|
40
41
|
cacheScopeEnvError,
|
|
41
42
|
isCacheScope,
|
|
42
43
|
type LoadReads,
|
|
44
|
+
PROJECT_CONFIG_FILENAMES,
|
|
43
45
|
readClaimedFiles,
|
|
44
46
|
resolveCacheDir,
|
|
45
47
|
resolveStoreRoot,
|
|
@@ -111,6 +113,8 @@ export interface PreparedRun {
|
|
|
111
113
|
hasRemoteLayer: boolean
|
|
112
114
|
/** `--cache-policy` with the workspace's `cacheScope` applied: what the layers were built under. */
|
|
113
115
|
cachePolicy: CachePolicy
|
|
116
|
+
/** The run's remotes in declared order, as the policy gates each; `openRemotes` turns one off. */
|
|
117
|
+
remotes: ResolvedRemote[]
|
|
114
118
|
/**
|
|
115
119
|
* Scheduling priorities from plugins' `schedule` stage (task id → weight,
|
|
116
120
|
* merged over the structural baseline by the scheduler). Empty when no
|
|
@@ -325,16 +329,18 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
325
329
|
const readOnly =
|
|
326
330
|
options.remoteReadOnlyByDefault === true &&
|
|
327
331
|
!(options.cache === undefined && namesRemote(workspaceConfig?.cache?.policy))
|
|
328
|
-
const
|
|
332
|
+
const wanted: CachePolicy = readOnly ? { ...asked, remoteWrite: false } : asked
|
|
329
333
|
const localCache = new Cache(
|
|
330
334
|
cacheDir,
|
|
331
|
-
{ read:
|
|
335
|
+
{ read: wanted.localRead, write: wanted.localWrite },
|
|
332
336
|
workspaceRoot,
|
|
333
337
|
options.artifactCeiling,
|
|
334
338
|
'open',
|
|
335
339
|
await storeRoot,
|
|
336
340
|
)
|
|
337
|
-
localCache.
|
|
341
|
+
const unwritable = localCache.readOnlyNotice()
|
|
342
|
+
if (unwritable !== null) warnOf(log)(`[vx] ${unwritable}`)
|
|
343
|
+
const policy: CachePolicy = unwritable === null ? wanted : { ...wanted, localWrite: false }
|
|
338
344
|
// The root files the plugins claim (`VxPlugin.fingerprint`), whose bytes
|
|
339
345
|
// their claimants fold: read now so a task that rewrites one is caught.
|
|
340
346
|
const fingerprintsAt = Date.now()
|
|
@@ -405,6 +411,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
405
411
|
// A dependant's `^build` still reaches it, and `lib#build` names it.
|
|
406
412
|
const requested = expandRequested(tasks, candidateProjects, projects, isDefaultBuild)
|
|
407
413
|
let unresolvedTasks = unresolvedRequests(tasks, candidateProjects, projects, isDefaultBuild)
|
|
414
|
+
refuseUnlockedConfig(tasks, projects, projectMetas, workspaceRoot)
|
|
408
415
|
if (options.selectedByDiff === true && unresolvedTasks.some((t) => !t.includes('#'))) {
|
|
409
416
|
// A name a loaded project declares is no typo, whether or not the load
|
|
410
417
|
// was whole: an unaffected dependency loaded for the closure declared
|
|
@@ -508,7 +515,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
508
515
|
// cycle under `vx watch` (item 1029). run() owns both only once this
|
|
509
516
|
// returns.
|
|
510
517
|
try {
|
|
511
|
-
const hashCache = createHashCache()
|
|
518
|
+
const hashCache = createHashCache(localCache.foldMemo())
|
|
512
519
|
const fingerprintWatch = new FingerprintWatch(
|
|
513
520
|
workspaceRoot,
|
|
514
521
|
claimed,
|
|
@@ -527,6 +534,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
527
534
|
localCache,
|
|
528
535
|
hasRemoteLayer,
|
|
529
536
|
cachePolicy: policy,
|
|
537
|
+
remotes,
|
|
530
538
|
keepRunEntry: undefined,
|
|
531
539
|
priorities: () => Promise.resolve(new Map()),
|
|
532
540
|
branch,
|
|
@@ -775,6 +783,7 @@ export async function prepareRun(options: RunOptions, log: Logger): Promise<Prep
|
|
|
775
783
|
localCache,
|
|
776
784
|
hasRemoteLayer,
|
|
777
785
|
cachePolicy: policy,
|
|
786
|
+
remotes,
|
|
778
787
|
priorities,
|
|
779
788
|
branch,
|
|
780
789
|
cacheFor,
|
|
@@ -906,3 +915,63 @@ function workspaceCachePolicy(config: WorkspaceConfig | null | undefined): Cache
|
|
|
906
915
|
)
|
|
907
916
|
}
|
|
908
917
|
}
|
|
918
|
+
|
|
919
|
+
/**
|
|
920
|
+
* A `project#build` naming the default `build` of a project the lock holds
|
|
921
|
+
* no config for, while its directory holds one, asks for that config's task, not
|
|
922
|
+
* one that runs nothing: ran, `c#build` was 0 tasks, exit 0, where a
|
|
923
|
+
* renamed task gets "run `vx lock`" (Growth, lock drift).
|
|
924
|
+
*/
|
|
925
|
+
function refuseUnlockedConfig(
|
|
926
|
+
tasks: readonly string[],
|
|
927
|
+
projects: ReadonlyMap<string, ProjectEntry>,
|
|
928
|
+
metas: ReadonlyArray<{ name: string; configPath?: string | null }>,
|
|
929
|
+
workspaceRoot: string,
|
|
930
|
+
): void {
|
|
931
|
+
for (const spec of tasks) {
|
|
932
|
+
const at = spec.indexOf('#')
|
|
933
|
+
if (at < 0) continue
|
|
934
|
+
const name = spec.slice(0, at)
|
|
935
|
+
const entry = projects.get(name)
|
|
936
|
+
if (entry === undefined || !isDefaultBuild(entry.config.tasks?.[spec.slice(at + 1)])) continue
|
|
937
|
+
const locked = metas.find((m) => m.name === name)?.configPath
|
|
938
|
+
if (typeof locked === 'string' && locked !== '') continue
|
|
939
|
+
const config = PROJECT_CONFIG_FILENAMES.find((n) => existsSync(path.join(entry.dir, n)))
|
|
940
|
+
if (config === undefined) continue
|
|
941
|
+
throw new UserError(
|
|
942
|
+
`${spec}: ${relPosix(workspaceRoot, path.join(entry.dir, config))} is not in the lock yet — run \`vx lock\``,
|
|
943
|
+
)
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* Each remote's `open`, in declared order, for the ones the policy lets read
|
|
949
|
+
* or write. One at a time: two that ask on the terminal must not interleave.
|
|
950
|
+
* A throw warns through the remote's `onError` (naming its plugin) and
|
|
951
|
+
* turns that remote off; the run goes on, as after any remote failure.
|
|
952
|
+
* True when that left the run no remote at all.
|
|
953
|
+
*/
|
|
954
|
+
export async function openRemotes(
|
|
955
|
+
remotes: readonly ResolvedRemote[],
|
|
956
|
+
ci: boolean,
|
|
957
|
+
secrets: SecretBroker,
|
|
958
|
+
): Promise<boolean> {
|
|
959
|
+
let closed = false
|
|
960
|
+
for (const entry of remotes) {
|
|
961
|
+
if (entry.remote.open === undefined || !(entry.read || entry.write)) continue
|
|
962
|
+
try {
|
|
963
|
+
await entry.remote.open({
|
|
964
|
+
ci,
|
|
965
|
+
warn: (message) => entry.onError?.(new Error(message)),
|
|
966
|
+
...secrets.for(entry.plugin ?? 'vx'),
|
|
967
|
+
})
|
|
968
|
+
} catch (err) {
|
|
969
|
+
entry.read = false
|
|
970
|
+
entry.write = false
|
|
971
|
+
const e = err instanceof Error ? err : new Error(String(err))
|
|
972
|
+
entry.onError?.(e)
|
|
973
|
+
closed = true
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return closed && !remotes.some((r) => r.read || r.write)
|
|
977
|
+
}
|
|
@@ -127,11 +127,16 @@ export async function discoverProjects(
|
|
|
127
127
|
(e) => e.changed,
|
|
128
128
|
() => null,
|
|
129
129
|
)
|
|
130
|
+
const byDir = new Map(projects.map((p) => [p.dir, p]))
|
|
130
131
|
const edges = await applyDiscoverHooks(
|
|
131
132
|
plugins,
|
|
132
133
|
projects,
|
|
133
134
|
{ workspaceRoot, cacheDir, warn, worktreeChanges },
|
|
134
|
-
(entry, plugin) =>
|
|
135
|
+
async (entry, plugin) => {
|
|
136
|
+
const found = await discoveredProject(workspaceRoot, byDir, entry, plugin.name)
|
|
137
|
+
if (found.added) byDir.set(found.meta.dir, found.meta)
|
|
138
|
+
return found
|
|
139
|
+
},
|
|
135
140
|
)
|
|
136
141
|
checkProjectNames(workspaceRoot, projects)
|
|
137
142
|
linkDiscoveredEdges(workspaceRoot, projects, edges)
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
// The report every plugin's teardown reads, and the run's own bytes kept
|
|
2
2
|
// in a remote that takes its writes (owner 2026-10-15): a failed task's
|
|
3
3
|
// log and the files its outputs left (`cache.upload.failures`), and the
|
|
4
|
-
// log of a task that saved no entry
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// log of a task that saved no entry, on every run (M, 2026-10-16: "Always
|
|
5
|
+
// keep"). Each task's bytes leave when it ends, off its path, like its
|
|
6
|
+
// entry's upload (owner 2026-10-16); the run only waits for them where it
|
|
7
|
+
// waits for every upload, and the footer counts them as the task's remote
|
|
8
|
+
// traffic.
|
|
7
9
|
|
|
8
10
|
import {
|
|
11
|
+
type RemoteTraffic,
|
|
9
12
|
packRunOutputs,
|
|
10
13
|
resolveOutputs,
|
|
11
14
|
resolveWorkspaceOutputs,
|
|
@@ -27,6 +30,8 @@ export interface RunKeeperArgs {
|
|
|
27
30
|
readonly keep: KeepRunEntry
|
|
28
31
|
/** Whether a failed task's log and outputs are kept (`cache.upload.failures`). */
|
|
29
32
|
readonly failures: boolean
|
|
33
|
+
/** The footer's remote traffic: each kept task counts under its id. */
|
|
34
|
+
readonly traffic: RemoteTraffic
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
/** Whether a run on CI or off it keeps failures, by `cache.upload.failures` (default `'ci'`). */
|
|
@@ -81,8 +86,10 @@ export class RunKeeper {
|
|
|
81
86
|
if (text.length > 0) {
|
|
82
87
|
const name = runEntryName(runId, id, 'log')
|
|
83
88
|
const bytes = new TextEncoder().encode(text)
|
|
84
|
-
this.enqueue(async () => {
|
|
85
|
-
if (await this.args.keep(name, bytes))
|
|
89
|
+
this.enqueue(id, async () => {
|
|
90
|
+
if (!(await this.args.keep(name, bytes))) return false
|
|
91
|
+
slot().log = name
|
|
92
|
+
return true
|
|
86
93
|
})
|
|
87
94
|
}
|
|
88
95
|
}
|
|
@@ -94,9 +101,11 @@ export class RunKeeper {
|
|
|
94
101
|
this.args.nestedDirsByProject.get(o.node.projectName) ?? [],
|
|
95
102
|
).then((paths) => packRunOutputs(workspaceRoot, paths))
|
|
96
103
|
const name = runEntryName(runId, id, 'outputs')
|
|
97
|
-
this.enqueue(async () => {
|
|
104
|
+
this.enqueue(id, async () => {
|
|
98
105
|
const bytes = await packed
|
|
99
|
-
if (bytes
|
|
106
|
+
if (bytes === null || !(await this.args.keep(name, bytes))) return false
|
|
107
|
+
slot().outputs = name
|
|
108
|
+
return true
|
|
100
109
|
})
|
|
101
110
|
}
|
|
102
111
|
}
|
|
@@ -117,16 +126,28 @@ export class RunKeeper {
|
|
|
117
126
|
return this.kept.get(id)
|
|
118
127
|
}
|
|
119
128
|
|
|
120
|
-
|
|
129
|
+
/** Queue `job`, which writes `id`'s bytes; a write that lands counts as `id`'s upload. */
|
|
130
|
+
private enqueue(id: string, job: () => Promise<boolean>): void {
|
|
121
131
|
let done!: () => void
|
|
122
132
|
const p = new Promise<void>((resolve) => (done = resolve))
|
|
123
133
|
this.pending.add(p)
|
|
134
|
+
const { traffic } = this.args
|
|
135
|
+
let key = traffic.keys.get(id)
|
|
136
|
+
if (key === undefined)
|
|
137
|
+
traffic.keys.set(id, (key = { uploading: 0, landed: false, failures: [] }))
|
|
138
|
+
const counted = key
|
|
139
|
+
counted.uploading++
|
|
140
|
+
traffic.inFlight++
|
|
124
141
|
this.queue.push(async () => {
|
|
125
142
|
try {
|
|
126
|
-
if (!this.stopped
|
|
143
|
+
if (!this.stopped && (await job())) counted.landed = true
|
|
127
144
|
} catch {
|
|
128
145
|
// a record, never the run's verdict
|
|
129
146
|
} finally {
|
|
147
|
+
counted.uploading--
|
|
148
|
+
traffic.inFlight--
|
|
149
|
+
if (counted.uploading === 0 && !counted.landed && counted.failures.length === 0)
|
|
150
|
+
traffic.keys.delete(id)
|
|
130
151
|
this.pending.delete(p)
|
|
131
152
|
done()
|
|
132
153
|
}
|
package/src/orchestrator/run.ts
CHANGED
|
@@ -8,11 +8,11 @@ import { realpathSync } from 'node:fs'
|
|
|
8
8
|
import path from 'node:path'
|
|
9
9
|
import {
|
|
10
10
|
Cache,
|
|
11
|
-
type CacheLayer,
|
|
12
11
|
type CachePolicy,
|
|
13
12
|
cachesNothing,
|
|
14
13
|
formatCachePolicy,
|
|
15
14
|
stopRuntimeProbes,
|
|
15
|
+
sweepStaging,
|
|
16
16
|
} from '../cache/index.js'
|
|
17
17
|
import { VERSION } from '../version.js'
|
|
18
18
|
import {
|
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
isUserError,
|
|
50
50
|
detectColors,
|
|
51
51
|
type ColorSupport,
|
|
52
|
+
SecretBroker,
|
|
52
53
|
} from '../util/index.js'
|
|
53
54
|
import { keyedProjects } from './keyed-projects.js'
|
|
54
55
|
import { isDefaultBuild } from './projects.js'
|
|
@@ -87,7 +88,7 @@ import { isPassStatus } from '../graph/index.js'
|
|
|
87
88
|
import type { RunContextRecord } from './telemetry.js'
|
|
88
89
|
import { plan, type RunPlan } from './plan.js'
|
|
89
90
|
import type { AffectedReason } from './affected-tasks.js'
|
|
90
|
-
import { prepareRun, type PreparedRun } from './prepare.js'
|
|
91
|
+
import { openRemotes, prepareRun, type PreparedRun } from './prepare.js'
|
|
91
92
|
import { acquireRunLock } from './run-lock.js'
|
|
92
93
|
import { terminateChildren } from './signals.js'
|
|
93
94
|
import { runStop } from './run-stop.js'
|
|
@@ -162,17 +163,14 @@ function effectiveCachePolicy(requested: CachePolicy, hasRemoteLayer: boolean):
|
|
|
162
163
|
/**
|
|
163
164
|
* The perf firewall for the local short-circuit. Always-on; the only
|
|
164
165
|
* gates are correctness/no-op gates:
|
|
165
|
-
* - LOCAL-ONLY cache. Behind a remote layer, `cache.get` is a remote
|
|
166
|
-
* READ-THROUGH — the up-front classify is awaited before scheduling,
|
|
167
|
-
* so N remote GETs would land on the critical path before any task
|
|
168
|
-
* starts. Remote runs are owned by `startRemotePrefetch`, which
|
|
169
|
-
* overlaps the GETs with execution instead (fire-and-forget) and
|
|
170
|
-
* ingests hits into local for execute-task's lazy probe. The gate asks
|
|
171
|
-
* the layer (`hasRemote`), not `instanceof LayeredCache` — a
|
|
172
|
-
* third-party remote layer stalls on the classify just as hard.
|
|
173
166
|
* - the policy reads locally (a `--no-cache`/`--force`/`--cache-policy=local:`
|
|
174
167
|
* run reads nothing locally → nothing to restore → skip);
|
|
175
168
|
* - there is at least one task.
|
|
169
|
+
* A remote layer does not close it: the local cache sits in front of every
|
|
170
|
+
* remote (#3619), so the classify probes that local store alone and a local
|
|
171
|
+
* hit runs ahead of its dependencies as it does with no remote; the remote
|
|
172
|
+
* GETs stay on `startRemotePrefetch`'s fire-and-forget path (M's solid
|
|
173
|
+
* run, 2026-10-16: a local hit waited 18 s on a dependency it never read).
|
|
176
174
|
* A flat graph has no ordering to bypass, but the classify pass is worth
|
|
177
175
|
* paying for anyway: it probes every stable key through ONE batched
|
|
178
176
|
* `getMany` instead of a `cache.get` per task inside the run, which
|
|
@@ -180,12 +178,7 @@ function effectiveCachePolicy(requested: CachePolicy, hasRemoteLayer: boolean):
|
|
|
180
178
|
* (2026-09-03, interleaved arms). The per-task correctness gates (stable
|
|
181
179
|
* key, no workspace-outputs in graph) live in `startLocalShortCircuit`.
|
|
182
180
|
*/
|
|
183
|
-
export function shouldShortCircuit(
|
|
184
|
-
nodes: Map<string, TaskNode>,
|
|
185
|
-
policy: CachePolicy,
|
|
186
|
-
cache: CacheLayer,
|
|
187
|
-
): boolean {
|
|
188
|
-
if (cache.hasRemote === true) return false
|
|
181
|
+
export function shouldShortCircuit(nodes: Map<string, TaskNode>, policy: CachePolicy): boolean {
|
|
189
182
|
if (!policy.localRead) return false
|
|
190
183
|
return nodes.size > 0
|
|
191
184
|
}
|
|
@@ -450,6 +443,8 @@ async function runOnBus(
|
|
|
450
443
|
// carries it so analytics queries can group by invocation; plugins get it
|
|
451
444
|
// in their teardown's `RunReport`.
|
|
452
445
|
const runId = ulid()
|
|
446
|
+
// A killed run's restore staging (`.vx/tmp/<pid>-0`), once per run.
|
|
447
|
+
sweepStaging(prepared.workspaceRoot)
|
|
453
448
|
const head = captureGitContext(prepared.workspaceRoot)
|
|
454
449
|
const ciContext = detectCi(process.env)
|
|
455
450
|
const keepRun =
|
|
@@ -470,7 +465,17 @@ async function runOnBus(
|
|
|
470
465
|
nestedDirsByProject: prepared.nestedDirsByProject,
|
|
471
466
|
keep: prepared.keepRunEntry,
|
|
472
467
|
failures: keepRun.failures,
|
|
468
|
+
traffic: prepared.localCache.traffic,
|
|
473
469
|
})
|
|
470
|
+
// Before setup and the status line: a remote's `open` may ask on the terminal.
|
|
471
|
+
const secrets = new SecretBroker(
|
|
472
|
+
ciContext.ci,
|
|
473
|
+
warnOf(log),
|
|
474
|
+
undefined,
|
|
475
|
+
undefined,
|
|
476
|
+
terminal === null ? undefined : () => terminal.holdStatus(),
|
|
477
|
+
)
|
|
478
|
+
const remotesClosed = await openRemotes(prepared.remotes, ciContext.ci, secrets)
|
|
474
479
|
try {
|
|
475
480
|
disposePlugins = await installPlugins({
|
|
476
481
|
plugins: prepared.plugins,
|
|
@@ -481,6 +486,7 @@ async function runOnBus(
|
|
|
481
486
|
cache: prepared.cacheFor,
|
|
482
487
|
ci: ciContext.ci,
|
|
483
488
|
branch: prepared.branch(),
|
|
489
|
+
secrets,
|
|
484
490
|
step,
|
|
485
491
|
})
|
|
486
492
|
} catch (err) {
|
|
@@ -597,6 +603,7 @@ async function runOnBus(
|
|
|
597
603
|
handsTerminal: holders.size > 0,
|
|
598
604
|
teardowns: prepared.plugins.filter((p) => p.teardown).length,
|
|
599
605
|
onStop: () => {
|
|
606
|
+
log.runStop?.()
|
|
600
607
|
stopRuntimeProbes(hashCache.runtime, hashCache.workspaceRuntime)
|
|
601
608
|
cache.stopUploads?.()
|
|
602
609
|
keeper?.stop()
|
|
@@ -639,13 +646,15 @@ async function runOnBus(
|
|
|
639
646
|
try {
|
|
640
647
|
const runStartHrTimeNs = process.hrtime.bigint()
|
|
641
648
|
const endedAtMsAtStart = Date.now()
|
|
642
|
-
|
|
649
|
+
// Every remote turned off by its `open` is no remote: a task must not clean
|
|
650
|
+
// its outputs for a save that goes nowhere.
|
|
651
|
+
const remoteCacheEnabled = prepared.hasRemoteLayer && !remotesClosed
|
|
643
652
|
// Normalised ONCE so every consumer — execute-task, the dedup
|
|
644
653
|
// predicate, the recorded invocation row — reads the policy that
|
|
645
654
|
// actually governed the run. Reading the raw request here made tasks
|
|
646
655
|
// clean their outputs before every exec for a save that never
|
|
647
656
|
// happened.
|
|
648
|
-
const policy: CachePolicy = effectiveCachePolicy(prepared.cachePolicy,
|
|
657
|
+
const policy: CachePolicy = effectiveCachePolicy(prepared.cachePolicy, remoteCacheEnabled)
|
|
649
658
|
if (options.remoteRequested === true && !prepared.hasRemoteLayer) {
|
|
650
659
|
warnOf(log)(
|
|
651
660
|
'vx: --cache-policy named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run',
|
|
@@ -799,6 +808,7 @@ async function runOnBus(
|
|
|
799
808
|
},
|
|
800
809
|
concurrency,
|
|
801
810
|
workspaceProjectCount,
|
|
811
|
+
runId: undefined as string | undefined,
|
|
802
812
|
}
|
|
803
813
|
|
|
804
814
|
// Asked only when a plugin declares `output`: a plain run pays nothing.
|
|
@@ -815,6 +825,8 @@ async function runOnBus(
|
|
|
815
825
|
|
|
816
826
|
// Lifecycle hooks drive the default logger's dynamic status line
|
|
817
827
|
// (TTY-only); custom loggers may ignore them.
|
|
828
|
+
// The status line starts here: a prompt now would be drawn over.
|
|
829
|
+
secrets.closePrompts()
|
|
818
830
|
log.runStart?.({
|
|
819
831
|
total: taskCount,
|
|
820
832
|
concurrency,
|
|
@@ -822,18 +834,22 @@ async function runOnBus(
|
|
|
822
834
|
context: runContext,
|
|
823
835
|
startedAtMs: endedAtMsAtStart,
|
|
824
836
|
})
|
|
825
|
-
// The live region's
|
|
826
|
-
// region is still up a second in, so a warm run
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
+
// The live region's forecast: the graph's from the first free row, and
|
|
838
|
+
// history's only if the region is still up a second in, so a warm run
|
|
839
|
+
// never reads history.
|
|
840
|
+
terminal?.forecast(
|
|
841
|
+
() => createForecast(nodes, new Map(), concurrency),
|
|
842
|
+
async () => {
|
|
843
|
+
const ids: string[] = []
|
|
844
|
+
for (const n of nodes.values()) if (!isGroupTask(n)) ids.push(n.id)
|
|
845
|
+
const p50s = await new LocalHistoryProvider(
|
|
846
|
+
prepared.localCache.dbHandle(),
|
|
847
|
+
undefined,
|
|
848
|
+
prepared.branch(),
|
|
849
|
+
).p50sFor(ids)
|
|
850
|
+
return createForecast(nodes, p50s, concurrency)
|
|
851
|
+
},
|
|
852
|
+
)
|
|
837
853
|
|
|
838
854
|
// Remote-only: kick off background prefetches so remote-GET latency
|
|
839
855
|
// overlaps execution. Fire-and-forget — execution starts on the next
|
|
@@ -873,16 +889,22 @@ async function runOnBus(
|
|
|
873
889
|
// An empty local cache answers every probe with a miss: each task
|
|
874
890
|
// derives its key as it starts instead (~100 ms before a cold 1,090-
|
|
875
891
|
// package run's first spawn), and every sandboxed task runs.
|
|
892
|
+
const remote = cache.hasRemote === true
|
|
876
893
|
const coldLocal =
|
|
877
|
-
cache === prepared.localCache
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
894
|
+
shouldShortCircuit(nodes, policy) && (remote || cache === prepared.localCache)
|
|
895
|
+
? prepared.localCache.holdsNothing()
|
|
896
|
+
: false
|
|
897
|
+
// A remote may still hold what an empty local store does not: the
|
|
898
|
+
// prefetch asks it, and no confirmed miss warms the sandbox early.
|
|
899
|
+
if (coldLocal && !remote) prewarm()
|
|
900
|
+
else if (!coldLocal && shouldShortCircuit(nodes, policy)) {
|
|
882
901
|
shortCircuit = await step('vx:keys', () =>
|
|
883
902
|
startLocalShortCircuit({
|
|
884
903
|
nodes,
|
|
885
|
-
|
|
904
|
+
// Behind a remote, the local store alone: its hits are certain
|
|
905
|
+
// now, and a local miss may still be a remote hit.
|
|
906
|
+
cache: remote ? prepared.localCache : cache,
|
|
907
|
+
localOnly: remote,
|
|
886
908
|
workspaceRoot,
|
|
887
909
|
claimedFiles: fingerprintWatch.names,
|
|
888
910
|
forwardArgs: options.forwardArgs,
|
|
@@ -893,7 +915,8 @@ async function runOnBus(
|
|
|
893
915
|
}),
|
|
894
916
|
)
|
|
895
917
|
// A confirmed miss runs too.
|
|
896
|
-
if (sandboxed.some((n) => shortCircuit.preProbed.get(n.id)?.hit === null))
|
|
918
|
+
if (!remote && sandboxed.some((n) => shortCircuit.preProbed.get(n.id)?.hit === null))
|
|
919
|
+
prewarm()
|
|
897
920
|
}
|
|
898
921
|
|
|
899
922
|
// Whether this task runs behind a failure (`continueMode: 'always'`
|
|
@@ -1267,6 +1290,7 @@ async function runOnBus(
|
|
|
1267
1290
|
})
|
|
1268
1291
|
}
|
|
1269
1292
|
mark('record history')
|
|
1293
|
+
runContext.runId = runId
|
|
1270
1294
|
// Drain any still-in-flight background prefetches before closing the
|
|
1271
1295
|
// cache handle — a prefetch ingesting into a closed SQLite DB would
|
|
1272
1296
|
// throw. Tasks that resolved as local hits never awaited their
|
|
@@ -1605,7 +1629,7 @@ async function planRestorable(
|
|
|
1605
1629
|
const serves = [...nodes.values()].some(
|
|
1606
1630
|
(n) => n.config.exec?.persistent !== undefined && !n.requested && n.surfaced !== true,
|
|
1607
1631
|
)
|
|
1608
|
-
if (!serves || !shouldShortCircuit(nodes, policy
|
|
1632
|
+
if (!serves || !shouldShortCircuit(nodes, policy)) return {}
|
|
1609
1633
|
const keptOut = restoreTierExclusions(nodes, prepared.workspaceRoot)
|
|
1610
1634
|
const stable = await deriveStableKeys({
|
|
1611
1635
|
nodes,
|
|
@@ -7,7 +7,7 @@ import { existsSync } from 'node:fs'
|
|
|
7
7
|
import { lstat, mkdir, readdir, realpath, rm, writeFile } from 'node:fs/promises'
|
|
8
8
|
import { homedir } from 'node:os'
|
|
9
9
|
import path from 'node:path'
|
|
10
|
-
import type { ExecConfig
|
|
10
|
+
import type { ExecConfig } from '../config.js'
|
|
11
11
|
import {
|
|
12
12
|
bindableWrites,
|
|
13
13
|
initSandbox,
|
|
@@ -289,53 +289,6 @@ export async function sandboxRequestFor(
|
|
|
289
289
|
return { sandbox: request, placeholders, withheld: links.withheld }
|
|
290
290
|
}
|
|
291
291
|
|
|
292
|
-
/**
|
|
293
|
-
* Whether a sandboxed task could read `real` (a canonical path inside its
|
|
294
|
-
* project): under one of its read or write grants or its `node_modules`,
|
|
295
|
-
* and not behind a wall. vx packs a symlinked output as its target's
|
|
296
|
-
* bytes, outside the sandbox, so a link the task planted to a file it was
|
|
297
|
-
* denied (`dist/x -> ../.env` under `read: ['src/']`) packed that file
|
|
298
|
-
* into the cache wherever creating the link went unseen: seatbelt, or
|
|
299
|
-
* Linux untraced. The grants are matched as written, not expanded: a
|
|
300
|
-
* glob's hit made by the task itself counts.
|
|
301
|
-
*/
|
|
302
|
-
export function sandboxReadable(
|
|
303
|
-
node: TaskNode,
|
|
304
|
-
sandbox: SandboxConfig,
|
|
305
|
-
workspaceRoot: string,
|
|
306
|
-
nested: readonly string[] = [],
|
|
307
|
-
): (real: string) => boolean {
|
|
308
|
-
const projectReal = toRealPath(node.projectDir)
|
|
309
|
-
const at = (p: string): string =>
|
|
310
|
-
p.startsWith('~')
|
|
311
|
-
? path.join(homedir(), p.slice(1))
|
|
312
|
-
: path.isAbsolute(p)
|
|
313
|
-
? p
|
|
314
|
-
: path.join(projectReal, p)
|
|
315
|
-
const grants = [...(sandbox.allow?.read ?? []), ...(sandbox.allow?.write ?? [])].map(at)
|
|
316
|
-
const literals = [
|
|
317
|
-
...grants.filter(isMountableLiteral).map((g) => toRealPath(g.replace(/\/+$/, '') || '/')),
|
|
318
|
-
toRealPath(path.join(node.projectDir, 'node_modules')),
|
|
319
|
-
toRealPath(path.join(workspaceRoot, 'node_modules')),
|
|
320
|
-
]
|
|
321
|
-
const globs = grants.filter((g) => !isMountableLiteral(g)).map((g) => new Bun.Glob(g))
|
|
322
|
-
const walls = [
|
|
323
|
-
...nested,
|
|
324
|
-
path.join(workspaceRoot, '.git'),
|
|
325
|
-
path.join(workspaceRoot, '.vx', 'cache'),
|
|
326
|
-
].map(toRealPath)
|
|
327
|
-
return (real) => {
|
|
328
|
-
if (walls.some((w) => atOrUnder(real, w))) return false
|
|
329
|
-
if (literals.some((l) => atOrUnder(real, l))) return true
|
|
330
|
-
// A directory a glob matches is granted whole.
|
|
331
|
-
for (let p = real; atOrUnder(p, projectReal); p = path.dirname(p)) {
|
|
332
|
-
if (globs.some((g) => g.match(p))) return true
|
|
333
|
-
if (p === projectReal) break
|
|
334
|
-
}
|
|
335
|
-
return false
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
|
|
339
292
|
/**
|
|
340
293
|
* The sandbox half of the request, plus the empty files vx created so a
|
|
341
294
|
* literal write grant had something to bind (`placeholders`). They are
|
|
@@ -12,22 +12,29 @@
|
|
|
12
12
|
export interface SaveLane {
|
|
13
13
|
/**
|
|
14
14
|
* Start `save` now if the lane has room, else when one finishes. Resolves
|
|
15
|
-
* once the save has settled
|
|
16
|
-
*
|
|
17
|
-
* same task in another run
|
|
15
|
+
* once the save has settled, true when the entry landed — a failure is
|
|
16
|
+
* reported and resolves false — for the readers that must know: the
|
|
17
|
+
* task's outcome, and a duplicate of the same task in another run
|
|
18
|
+
* (admission's in-flight join).
|
|
18
19
|
*/
|
|
19
|
-
defer(save: () => Promise<void>): Promise<
|
|
20
|
+
defer(save: () => Promise<void>): Promise<boolean>
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
export function createSaveLane(cap: number, onError: (err: unknown) => void): SaveLane {
|
|
23
|
-
const queue: Array<{ save: () => Promise<void>; settle: () => void }> = []
|
|
24
|
+
const queue: Array<{ save: () => Promise<void>; settle: (stored: boolean) => void }> = []
|
|
24
25
|
let running = 0
|
|
25
|
-
const start = (save: () => Promise<void>, settle: () => void): void => {
|
|
26
|
+
const start = (save: () => Promise<void>, settle: (stored: boolean) => void): void => {
|
|
26
27
|
void save()
|
|
27
|
-
.
|
|
28
|
-
|
|
28
|
+
.then(
|
|
29
|
+
() => true,
|
|
30
|
+
(err: unknown) => {
|
|
31
|
+
onError(err)
|
|
32
|
+
return false
|
|
33
|
+
},
|
|
34
|
+
)
|
|
35
|
+
.then((stored) => {
|
|
29
36
|
running--
|
|
30
|
-
settle()
|
|
37
|
+
settle(stored)
|
|
31
38
|
const next = queue.shift()
|
|
32
39
|
if (next !== undefined) start(next.save, next.settle)
|
|
33
40
|
})
|
|
@@ -35,7 +42,7 @@ export function createSaveLane(cap: number, onError: (err: unknown) => void): Sa
|
|
|
35
42
|
}
|
|
36
43
|
return {
|
|
37
44
|
defer(save) {
|
|
38
|
-
return new Promise<
|
|
45
|
+
return new Promise<boolean>((settle) => {
|
|
39
46
|
if (running < cap) start(save, settle)
|
|
40
47
|
else queue.push({ save, settle })
|
|
41
48
|
})
|