@vzn/vx 0.0.639 → 0.0.641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/caching.md +158 -106
- package/docs/cli.md +80 -45
- package/docs/execution.md +11 -10
- package/docs/features.md +9 -8
- package/docs/patterns.md +1 -1
- package/docs/schema.md +252 -190
- package/docs/security.md +5 -4
- package/launcher.sh +9 -2
- package/package.json +5 -5
- package/schemas/show.json +14 -14
- package/skills/vx/SKILL.md +4 -3
- package/src/bin.ts +8 -6
- package/src/cache/archive.ts +248 -41
- package/src/cache/cache.ts +46 -19
- package/src/cache/file-hashes.ts +7 -12
- package/src/cache/fs-clock.ts +62 -0
- package/src/cache/git-inputs.ts +2 -2
- package/src/cache/index.ts +6 -0
- package/src/cache/inputs.ts +228 -111
- package/src/cache/key-fold.ts +23 -3
- package/src/cache/layer.ts +15 -8
- package/src/cache/layered-cache.ts +8 -7
- package/src/cache/linked-inputs.ts +87 -0
- package/src/cache/scoped-cache.ts +8 -2
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/help.ts +7 -1
- package/src/cli/init.ts +23 -11
- package/src/cli/last.ts +24 -1
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/select.ts +16 -6
- package/src/cli/show.ts +9 -6
- package/src/cli/upgrade.ts +132 -21
- package/src/cli/watch-filter.ts +23 -12
- package/src/cli/watch-fs.ts +2 -1
- package/src/cli/watch-judge.ts +49 -5
- package/src/cli/watch-set.ts +18 -8
- package/src/cli/watch.ts +74 -2
- package/src/cli/why.ts +5 -3
- package/src/config.ts +19 -15
- package/src/exec/executor.ts +4 -4
- package/src/graph/dependency-spec.ts +2 -2
- package/src/graph/folds.ts +236 -0
- package/src/graph/index.ts +8 -1
- package/src/graph/scheduler.ts +41 -15
- package/src/graph/task-graph.ts +135 -73
- package/src/handoff.ts +5 -0
- package/src/index.ts +2 -2
- package/src/orchestrator/affected-tasks.ts +20 -19
- package/src/orchestrator/download-policy.ts +16 -22
- package/src/orchestrator/excluded-keys.ts +4 -5
- package/src/orchestrator/execute-task.ts +29 -25
- package/src/orchestrator/hit-restore.ts +101 -27
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/keyed-projects.ts +12 -51
- package/src/orchestrator/local-shortcircuit.ts +9 -1
- package/src/orchestrator/persistent-task.ts +3 -3
- package/src/orchestrator/placement.ts +2 -2
- package/src/orchestrator/plan.ts +1 -1
- package/src/orchestrator/plugin-host.ts +66 -27
- package/src/orchestrator/plugin.ts +12 -2
- package/src/orchestrator/prepare.ts +12 -5
- package/src/orchestrator/projects.ts +7 -10
- package/src/orchestrator/run-report.ts +145 -46
- package/src/orchestrator/run-steps.ts +57 -0
- package/src/orchestrator/run.ts +76 -44
- package/src/orchestrator/sandbox-fix.ts +1 -1
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/stable-keys.ts +59 -26
- package/src/orchestrator/steps.ts +26 -6
- package/src/orchestrator/task-hash.ts +31 -9
- package/src/orchestrator/telemetry-host.ts +3 -3
- package/src/orchestrator/telemetry.ts +9 -4
- package/src/orchestrator/upstream.ts +4 -181
- package/src/report/framed-output.ts +1 -1
- package/src/report/logger.ts +5 -2
- package/src/report/status-line.ts +7 -3
- package/src/report/summary.ts +6 -4
- package/src/settings.ts +11 -17
- package/src/util/index.ts +3 -0
- package/src/util/paths.ts +39 -5
- package/src/workspace/affected.ts +81 -2
- package/src/workspace/config-eval.ts +2 -2
- package/src/workspace/config-imports.ts +1 -1
- package/src/workspace/config-schema.ts +142 -143
- package/src/workspace/lockfile.ts +1 -1
- package/src/workspace/migration.ts +19 -3
- package/src/workspace/project-loader.ts +5 -4
package/src/cli/watch.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
//
|
|
8
8
|
// The cache does the heavy lifting: re-runs are typically cache hits
|
|
9
9
|
// (input hash matches), so spurious events from edits outside any
|
|
10
|
-
// task's `
|
|
10
|
+
// task's `inputs.files` cost ~tens of ms of orchestrator
|
|
11
11
|
// overhead. We deliberately don't try to filter events through the
|
|
12
12
|
// per-task input globs — the cache key is the source of truth, and
|
|
13
13
|
// filtering would mean re-doing the glob/boundary work on every event.
|
|
@@ -279,6 +279,7 @@ export async function watchCmd(args: readonly string[]): Promise<number> {
|
|
|
279
279
|
outputs: swept.outputs,
|
|
280
280
|
inputs: swept.inputs,
|
|
281
281
|
uncached: swept.uncached,
|
|
282
|
+
namedIgnored: swept.namedIgnored,
|
|
282
283
|
configImports: swept.configImports,
|
|
283
284
|
workspaceConfigImports: swept.workspaceConfigImports,
|
|
284
285
|
fenceDirs: fenceDirs(allProjects),
|
|
@@ -301,6 +302,7 @@ export async function watchCmd(args: readonly string[]): Promise<number> {
|
|
|
301
302
|
outputs: sweep.outputs,
|
|
302
303
|
inputs: sweep.inputs,
|
|
303
304
|
uncached: sweep.uncached,
|
|
305
|
+
namedIgnored: sweep.namedIgnored,
|
|
304
306
|
configImports: sweep.configImports,
|
|
305
307
|
workspaceConfigImports: sweep.workspaceConfigImports,
|
|
306
308
|
fenceDirs: fenceDirs(all),
|
|
@@ -351,6 +353,8 @@ interface WatchLoopArgs {
|
|
|
351
353
|
inputs: ReadonlyMap<string, ReadonlyArray<readonly string[]>>
|
|
352
354
|
/** Projects whose uncached task may read a git-ignored file: such a path there is still an edit. */
|
|
353
355
|
uncached: ReadonlySet<string>
|
|
356
|
+
/** Paths a task's inputs name outright: a git-ignored one there is still an edit. */
|
|
357
|
+
namedIgnored: readonly string[]
|
|
354
358
|
/** Files the configs import: an edit to one outside the watched projects is a cycle that re-reads them. */
|
|
355
359
|
configImports: readonly string[]
|
|
356
360
|
/** Files the workspace config imports: loaded once per process, so an edit is named, not run. */
|
|
@@ -379,6 +383,7 @@ interface Rediscovered {
|
|
|
379
383
|
outputs: ReadonlyMap<string, readonly string[]>
|
|
380
384
|
inputs: ReadonlyMap<string, ReadonlyArray<readonly string[]>>
|
|
381
385
|
uncached: ReadonlySet<string>
|
|
386
|
+
namedIgnored: readonly string[]
|
|
382
387
|
configImports: readonly string[]
|
|
383
388
|
workspaceConfigImports: readonly string[]
|
|
384
389
|
fenceDirs: readonly string[]
|
|
@@ -394,6 +399,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
394
399
|
let outputs = args.outputs
|
|
395
400
|
let inputs = args.inputs
|
|
396
401
|
let uncached = args.uncached
|
|
402
|
+
let namedIgnored = args.namedIgnored
|
|
397
403
|
let claimedRootFiles = args.claimedRootFiles
|
|
398
404
|
let configImportFiles = args.configImports
|
|
399
405
|
let wsConfigImportFiles = args.workspaceConfigImports
|
|
@@ -567,6 +573,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
567
573
|
armedAt,
|
|
568
574
|
held: () => held !== undefined,
|
|
569
575
|
uncached: () => uncached,
|
|
576
|
+
namedIgnored: () => namedIgnored,
|
|
570
577
|
fenced: (ownDir, abs) => fenced(ownDir, abs),
|
|
571
578
|
...(atArm !== undefined ? { existedAtArm: atArm.listed, trackedAtArm: atArm.tracked } : {}),
|
|
572
579
|
})
|
|
@@ -642,7 +649,8 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
642
649
|
if (isIgnoredPath(proj.dir, filename) || fenced(proj.dir, abs)) return
|
|
643
650
|
// A module the config imports from inside the project is the config
|
|
644
651
|
// too: its edit can add an output or a `workspaceFiles` input.
|
|
645
|
-
if (shapesWatchedSet(filename) || configImportFiles.includes(abs)
|
|
652
|
+
if (shapesWatchedSet(filename) || configImportFiles.includes(abs) || linkChanged(abs))
|
|
653
|
+
reread = true
|
|
646
654
|
trigger(`${proj.name} ${filename}`, abs)
|
|
647
655
|
}
|
|
648
656
|
const armProject = (proj: ProjectMeta): void => {
|
|
@@ -816,6 +824,65 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
816
824
|
armAncestors()
|
|
817
825
|
}
|
|
818
826
|
|
|
827
|
+
/**
|
|
828
|
+
* What each symlink under a watched project points to: the key reads it,
|
|
829
|
+
* wherever it lies (cache/linked-inputs.ts), and the project's arm hears
|
|
830
|
+
* only the link. A file target's directory, non-recursive, or a directory
|
|
831
|
+
* target, recursive; each event names the link. Links are read again when
|
|
832
|
+
* one comes or goes (`projectEvent`).
|
|
833
|
+
*/
|
|
834
|
+
const linkArms = new Map<string, { target: string; handle: WatchHandle }>()
|
|
835
|
+
let links = atArm?.links ?? new Set<string>()
|
|
836
|
+
let watched = projects
|
|
837
|
+
/** A link came, went or moved: the set `armLinks` reads is stale. */
|
|
838
|
+
const linkChanged = (abs: string): boolean => {
|
|
839
|
+
if (links.has(abs)) return true
|
|
840
|
+
try {
|
|
841
|
+
return fs.lstatSync(abs).isSymbolicLink()
|
|
842
|
+
} catch {
|
|
843
|
+
return false
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
const armLinks = (): void => {
|
|
847
|
+
const want = new Map<string, { proj: ProjectMeta; real: string; dir: boolean }>()
|
|
848
|
+
for (const link of links) {
|
|
849
|
+
let proj: ProjectMeta | undefined
|
|
850
|
+
for (const p of watched)
|
|
851
|
+
if (
|
|
852
|
+
link.startsWith(p.dir + path.sep) &&
|
|
853
|
+
(proj === undefined || p.dir.length > proj.dir.length)
|
|
854
|
+
)
|
|
855
|
+
proj = p
|
|
856
|
+
if (proj === undefined) continue
|
|
857
|
+
try {
|
|
858
|
+
const real = fs.realpathSync(link)
|
|
859
|
+
want.set(link, { proj, real, dir: fs.statSync(real).isDirectory() })
|
|
860
|
+
} catch {
|
|
861
|
+
// Dangling: its target string is all the key reads, and the link's own arm hears that.
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
for (const [link, armed] of linkArms) {
|
|
865
|
+
if (want.get(link)?.real === armed.target) continue
|
|
866
|
+
armed.handle.close()
|
|
867
|
+
linkArms.delete(link)
|
|
868
|
+
}
|
|
869
|
+
for (const [link, { proj, real, dir }] of want) {
|
|
870
|
+
if (linkArms.has(link)) continue
|
|
871
|
+
const name = path.basename(real)
|
|
872
|
+
const label = `${proj.name} ${path.relative(proj.dir, link)}`
|
|
873
|
+
try {
|
|
874
|
+
const handle = arm(dir ? real : path.dirname(real), dir, (filename) => {
|
|
875
|
+
if (!dir && filename !== name) return
|
|
876
|
+
trigger(label, link)
|
|
877
|
+
})
|
|
878
|
+
linkArms.set(link, { target: real, handle })
|
|
879
|
+
changes.seed(link)
|
|
880
|
+
} catch (err) {
|
|
881
|
+
sayCannot(`cannot watch ${real}`, err)
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
819
886
|
// Said once, when the loop first arms: a re-arm repeated it after every
|
|
820
887
|
// cycle that re-read the watched set, between task rows. `debug.watch`
|
|
821
888
|
// says each arm on stderr, for whoever needs to see the set change.
|
|
@@ -843,6 +910,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
843
910
|
outputs = next.outputs
|
|
844
911
|
inputs = next.inputs
|
|
845
912
|
uncached = next.uncached
|
|
913
|
+
namedIgnored = next.namedIgnored
|
|
846
914
|
claimedRootFiles = next.claimedRootFiles
|
|
847
915
|
configImportFiles = next.configImports
|
|
848
916
|
wsConfigImportFiles = next.workspaceConfigImports
|
|
@@ -876,6 +944,9 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
876
944
|
for (const proj of own) if (!perProject.has(proj.dir)) armProject(proj)
|
|
877
945
|
}
|
|
878
946
|
armImports()
|
|
947
|
+
watched = next.projects
|
|
948
|
+
links = gitFiles(workspaceRoot)?.links ?? links
|
|
949
|
+
armLinks()
|
|
879
950
|
// A new arm proves delivery like the first ones: an edit in the new
|
|
880
951
|
// project right after this cycle is seen, not lost in the gap.
|
|
881
952
|
await pool.proved()
|
|
@@ -958,6 +1029,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
958
1029
|
}
|
|
959
1030
|
armAncestors()
|
|
960
1031
|
armImports()
|
|
1032
|
+
armLinks()
|
|
961
1033
|
|
|
962
1034
|
// The orchestrator's own writes never kick the loop: `makeWatchIgnore`
|
|
963
1035
|
// closes over the RESOLVED cache dir (relocated or not) and the tasks'
|
package/src/cli/why.ts
CHANGED
|
@@ -36,8 +36,10 @@ interface WhyArgs {
|
|
|
36
36
|
* `tests/why.test.ts`.
|
|
37
37
|
*/
|
|
38
38
|
export const WHAT_TO_DO: Readonly<Record<string, string>> = {
|
|
39
|
-
file: 'an edit re-runs by design; a file the task does not read belongs out of
|
|
40
|
-
|
|
39
|
+
file: 'an edit re-runs by design; a file the task does not read belongs out of inputs.files',
|
|
40
|
+
linked:
|
|
41
|
+
'what the symlink points to changed (the file, or a file under the directory); an edit there re-runs by design',
|
|
42
|
+
env: "the variable's value moved; one the task does not read belongs out of inputs.env",
|
|
41
43
|
runtime: 'the command printed something else; drop timestamps and paths from its output',
|
|
42
44
|
'ws-runtime':
|
|
43
45
|
'the workspace command printed something else; drop timestamps and paths from its output',
|
|
@@ -372,4 +374,4 @@ export async function whyCmd(args: readonly string[]): Promise<number> {
|
|
|
372
374
|
const ROOTS_SHOWN = 5
|
|
373
375
|
|
|
374
376
|
const WRITTEN_TODO =
|
|
375
|
-
'a task that writes a file a key reads re-keys it every run; give that write its own task with the file as an output, or keep the file out of
|
|
377
|
+
'a task that writes a file a key reads re-keys it every run; give that write its own task with the file as an output, or keep the file out of inputs.files'
|
package/src/config.ts
CHANGED
|
@@ -110,6 +110,12 @@ export interface CacheSettings {
|
|
|
110
110
|
inputStability?: InputStability
|
|
111
111
|
/** The `cache.outputs.clean` of every task that does not set its own. */
|
|
112
112
|
outputs?: { clean?: boolean }
|
|
113
|
+
/**
|
|
114
|
+
* Which runs keep a failed task's log and the files its outputs left in
|
|
115
|
+
* the remote: CI runs (`'ci'`, the default), runs off CI (`'local'`), or
|
|
116
|
+
* both (`'always'`). Only a run whose policy writes a remote keeps any.
|
|
117
|
+
*/
|
|
118
|
+
upload?: { failures?: 'ci' | 'local' | 'always' }
|
|
113
119
|
}
|
|
114
120
|
|
|
115
121
|
/** `WorkspaceConfig.lock`. */
|
|
@@ -137,12 +143,6 @@ export interface WorkspaceRules {
|
|
|
137
143
|
* overlapping tasks with no edge are refused either way.
|
|
138
144
|
*/
|
|
139
145
|
exclusiveOutputs?: boolean
|
|
140
|
-
/**
|
|
141
|
-
* Refuse a task whose input globs can match another task's declared
|
|
142
|
-
* outputs, so every key is derived before any task runs. Off, such a key
|
|
143
|
-
* waits for its producer (`caching.md` § Local restore tier).
|
|
144
|
-
*/
|
|
145
|
-
upfrontKeys?: boolean
|
|
146
146
|
/**
|
|
147
147
|
* Which workspace projects a project may depend on, checked by `vx lock`
|
|
148
148
|
* over each project's direct dependencies (its manifest's and its tasks'
|
|
@@ -298,12 +298,18 @@ export interface TaskConfig {
|
|
|
298
298
|
* - `'pkg#name'` — specific package's `name` task; not key-checked.
|
|
299
299
|
*/
|
|
300
300
|
dependsOn?: readonly string[]
|
|
301
|
+
/**
|
|
302
|
+
* What the task's key folds, whether or not it caches: its dependents
|
|
303
|
+
* fold that key. A task with `exec` and no `inputs` has no key, so every
|
|
304
|
+
* task that folds it runs and saves nothing. A group with `inputs` is a
|
|
305
|
+
* keyed group: its inputs are a key its dependents fold, with no process.
|
|
306
|
+
*/
|
|
307
|
+
inputs?: InputsConfig
|
|
301
308
|
/**
|
|
302
309
|
* Caching configuration. **Caching is opt-in.** If this field is omitted,
|
|
303
310
|
* the task always runs and nothing is read from or written to the cache.
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
* runs.
|
|
311
|
+
* Requires `exec` and `inputs`: the key is the task's inputs, and a group
|
|
312
|
+
* runs nothing to cache.
|
|
307
313
|
*/
|
|
308
314
|
cache?: CacheConfig
|
|
309
315
|
}
|
|
@@ -324,7 +330,7 @@ export interface TaskConfig {
|
|
|
324
330
|
* `node_modules`, and through it the real path of every workspace
|
|
325
331
|
* package linked there.
|
|
326
332
|
*
|
|
327
|
-
* `cache` grants no access in either direction: `
|
|
333
|
+
* `cache` grants no access in either direction: `inputs` says what
|
|
328
334
|
* INVALIDATES a task, `allow` says what it may TOUCH, and deriving one
|
|
329
335
|
* from the other coupled them both ways — a declaration added for
|
|
330
336
|
* caching silently widened the sandbox, and a path the task needed had
|
|
@@ -398,7 +404,7 @@ export interface SandboxIgnore {
|
|
|
398
404
|
export interface SandboxGrants {
|
|
399
405
|
/**
|
|
400
406
|
* Readable paths or globs, added to the baseline — which is
|
|
401
|
-
* `node_modules`, NOT `
|
|
407
|
+
* `node_modules`, NOT `inputs.files`. A task that reads its own
|
|
402
408
|
* source in the sandbox says so here.
|
|
403
409
|
*/
|
|
404
410
|
read?: readonly string[]
|
|
@@ -573,8 +579,6 @@ export interface ExecEnv {
|
|
|
573
579
|
}
|
|
574
580
|
|
|
575
581
|
export interface CacheConfig {
|
|
576
|
-
/** What participates in the cache key. */
|
|
577
|
-
inputs: CacheInputs
|
|
578
582
|
/** What this task produces. Captured for restore on a cache hit. */
|
|
579
583
|
outputs: CacheOutputs
|
|
580
584
|
/**
|
|
@@ -634,7 +638,7 @@ export interface CacheOutputs {
|
|
|
634
638
|
clean?: boolean
|
|
635
639
|
}
|
|
636
640
|
|
|
637
|
-
export interface
|
|
641
|
+
export interface InputsConfig {
|
|
638
642
|
/**
|
|
639
643
|
* Project-relative globs. Each entry is a positive glob, or a negation
|
|
640
644
|
* prefixed with `!`. Required: pass a recursive glob to mean "all
|
|
@@ -773,8 +777,8 @@ export function defineProject<const T extends ProjectConfig>(
|
|
|
773
777
|
[K in keyof NonNullable<T['tasks']>]?: Known<NonNullable<T['tasks']>[K], TaskConfig> & {
|
|
774
778
|
dependsOn?: readonly DependsOnEntry<Extract<keyof NonNullable<T['tasks']>, string>>[]
|
|
775
779
|
exec?: Known<At<NonNullable<T['tasks']>[K], 'exec'>, ExecConfig>
|
|
780
|
+
inputs?: Known<At<NonNullable<T['tasks']>[K], 'inputs'>, InputsConfig>
|
|
776
781
|
cache?: Known<At<NonNullable<T['tasks']>[K], 'cache'>, CacheConfig> & {
|
|
777
|
-
inputs?: Known<At<At<NonNullable<T['tasks']>[K], 'cache'>, 'inputs'>, CacheInputs>
|
|
778
782
|
outputs?: Known<At<At<NonNullable<T['tasks']>[K], 'cache'>, 'outputs'>, CacheOutputs>
|
|
779
783
|
}
|
|
780
784
|
}
|
package/src/exec/executor.ts
CHANGED
|
@@ -53,13 +53,13 @@ export interface InputFile {
|
|
|
53
53
|
* secrets (cache.db stores only their digests).
|
|
54
54
|
*/
|
|
55
55
|
export interface TaskInputs {
|
|
56
|
-
/** Declared `
|
|
56
|
+
/** Declared `inputs.files` + `workspaceFiles` (own outputs excluded), sorted by path. */
|
|
57
57
|
readonly files: readonly InputFile[]
|
|
58
|
-
/** Declared `
|
|
58
|
+
/** Declared `inputs.env` names with their resolved values; `undefined` when unset. */
|
|
59
59
|
readonly env: ReadonlyArray<{ readonly name: string; readonly value: string | undefined }>
|
|
60
|
-
/** `
|
|
60
|
+
/** `inputs.runtime` commands and the output that was folded — a toolchain expectation. */
|
|
61
61
|
readonly runtime: ReadonlyArray<{ readonly command: string; readonly output: string }>
|
|
62
|
-
/** `
|
|
62
|
+
/** `inputs.workspaceRuntime`, same shape. */
|
|
63
63
|
readonly workspaceRuntime: ReadonlyArray<{ readonly command: string; readonly output: string }>
|
|
64
64
|
/**
|
|
65
65
|
* Dependencies whose cache keys are folded. `outputs` lists each one's
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Parser for the Turbo/Nx-style task-edge micro-syntax used in
|
|
2
|
-
// `dependsOn` and `
|
|
2
|
+
// `dependsOn` and `inputs.tasks`. Returns a small discriminated
|
|
3
3
|
// union the graph builder + cache filter both consume.
|
|
4
4
|
|
|
5
5
|
export type DependencySpec =
|
|
@@ -32,7 +32,7 @@ export function compileTaskPattern(pattern: string): RegExp {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
/**
|
|
35
|
-
* Parse one `dependsOn` / `
|
|
35
|
+
* Parse one `dependsOn` / `inputs.tasks` entry.
|
|
36
36
|
*
|
|
37
37
|
* `name` → self.name
|
|
38
38
|
* `^name` → every dep workspace's `name` task
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// Which dependencies a task's key folds (`inputs.tasks`), on the graph
|
|
2
|
+
// alone: the hash path, the sandbox's keyed set and `vx lock`'s
|
|
3
|
+
// input-overlap rule read this one selection, so they cannot drift.
|
|
4
|
+
|
|
5
|
+
import { UserError } from '../util/index.js'
|
|
6
|
+
import {
|
|
7
|
+
DependencySpecError,
|
|
8
|
+
compileTaskPattern,
|
|
9
|
+
isTaskPattern,
|
|
10
|
+
parseDependencySpec,
|
|
11
|
+
type DependencySpec,
|
|
12
|
+
} from './dependency-spec.js'
|
|
13
|
+
import type { TaskNode } from './task-graph.js'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A task is a "group" if it has no `exec` — it exists only to chain
|
|
17
|
+
* `dependsOn` (an umbrella for `vx run ci`). Group tasks:
|
|
18
|
+
* - never spawn a process,
|
|
19
|
+
* - never read/write the cache (it's a config error to declare one),
|
|
20
|
+
* - never appear in the run summary or `runs` analytics table,
|
|
21
|
+
* - render no framed block in the live output.
|
|
22
|
+
* Six call sites used to repeat `node.config.exec === undefined`;
|
|
23
|
+
* centralising the predicate prevents that check from drifting.
|
|
24
|
+
*/
|
|
25
|
+
export function isGroupTask(node: TaskNode): boolean {
|
|
26
|
+
return node.config.exec === undefined
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `node.deps` less its ordering-only edges: the dependencies a key may fold. */
|
|
30
|
+
export function keyedDeps(node: TaskNode): readonly string[] {
|
|
31
|
+
const order = node.orderOnly
|
|
32
|
+
return order === undefined ? node.deps : node.deps.filter((d) => !order.includes(d))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* A filter selects among the task's own dependencies, never through a
|
|
37
|
+
* group: `dependsOn: ['build']` over a group `build` of `build.a` and
|
|
38
|
+
* `build.b` offers the filter `build`, not its members. A pattern that
|
|
39
|
+
* matches only a member (`build.*`) therefore selected nothing, and the key stopped moving
|
|
40
|
+
* with that member's inputs: a stale hit under a green run. Such a spec
|
|
41
|
+
* is refused, once, over the graph every key site reads. One that matches
|
|
42
|
+
* nothing anywhere stays legal (a preset's pattern needn't match in every
|
|
43
|
+
* project). An exact name is refused earlier, at load, unless `dependsOn`
|
|
44
|
+
* names it (`assertFilterNamesDeclaredDeps`).
|
|
45
|
+
*/
|
|
46
|
+
export function refuseFiltersThroughGroups(nodes: ReadonlyMap<string, TaskNode>): void {
|
|
47
|
+
for (const node of nodes.values()) {
|
|
48
|
+
const filter = node.config.inputs?.tasks
|
|
49
|
+
if (filter === undefined || filter.length === 0) continue
|
|
50
|
+
const deps = keyedDeps(node).flatMap((id) => nodes.get(id) ?? [])
|
|
51
|
+
const groups = deps.filter(isGroupTask)
|
|
52
|
+
if (groups.length === 0) continue
|
|
53
|
+
for (const raw of filter) {
|
|
54
|
+
const spec = parseSpec(raw, node.id)
|
|
55
|
+
if (spec.negated) continue
|
|
56
|
+
const matches = specMatcher(spec)
|
|
57
|
+
const hit = (n: TaskNode): boolean => matches(n, n.projectName === node.projectName)
|
|
58
|
+
if (deps.some(hit)) continue
|
|
59
|
+
for (const group of groups) {
|
|
60
|
+
const member = groupMembers(group, nodes).find(hit)
|
|
61
|
+
if (member === undefined) continue
|
|
62
|
+
throw new UserError(
|
|
63
|
+
`${node.id}: inputs.tasks: '${raw}' matches ${member.id} only through the group ${group.id}, so it selects nothing; name '${group.taskName}' or depend on ${member.id} directly`,
|
|
64
|
+
)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function parseSpec(raw: string, selfTaskId: string): DependencySpec {
|
|
71
|
+
try {
|
|
72
|
+
return parseDependencySpec(raw)
|
|
73
|
+
} catch (err) {
|
|
74
|
+
if (err instanceof DependencySpecError) {
|
|
75
|
+
throw new UserError(`${selfTaskId}: inputs.tasks: ${err.message}`)
|
|
76
|
+
}
|
|
77
|
+
throw err
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Every task a group stands for, through nested groups, each once. */
|
|
82
|
+
function groupMembers(group: TaskNode, nodes: ReadonlyMap<string, TaskNode>): TaskNode[] {
|
|
83
|
+
const out: TaskNode[] = []
|
|
84
|
+
const seen = new Set<string>([group.id])
|
|
85
|
+
const stack = group.deps.toReversed()
|
|
86
|
+
while (stack.length > 0) {
|
|
87
|
+
const id = stack.pop()!
|
|
88
|
+
if (seen.has(id)) continue
|
|
89
|
+
seen.add(id)
|
|
90
|
+
const n = nodes.get(id)
|
|
91
|
+
if (n === undefined) continue
|
|
92
|
+
out.push(n)
|
|
93
|
+
if (isGroupTask(n)) stack.push(...n.deps.toReversed())
|
|
94
|
+
}
|
|
95
|
+
return out
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* One dependency as the key fold sees it: the task, and the value the
|
|
100
|
+
* fold dedups by — its hash on the hash path, a structural stand-in for
|
|
101
|
+
* the hash on the graph (`foldUnit` in keyed-projects.ts).
|
|
102
|
+
*/
|
|
103
|
+
export interface FoldCandidate {
|
|
104
|
+
node: TaskNode
|
|
105
|
+
unit: string
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The dependencies a task's key folds, per its `inputs.tasks`.
|
|
110
|
+
*
|
|
111
|
+
* Patterns (Turbo/Nx micro-syntax + filter extensions):
|
|
112
|
+
* '*' all same-project upstream
|
|
113
|
+
* '^*' all dep-workspace upstream
|
|
114
|
+
* 'name' same-project task `name`
|
|
115
|
+
* '^name' `name` task in every dep workspace
|
|
116
|
+
* 'pkg#name' specific package's `name` task
|
|
117
|
+
* 'name.*' patterns — EITHER half of any form above may contain `*`
|
|
118
|
+
* (same glob as dependsOn patterns), including the project
|
|
119
|
+
* half of `pkg#name`. A filter that matched literally here
|
|
120
|
+
* would silently select ZERO upstream hashes and decouple the
|
|
121
|
+
* task from its dependencies — a stale-hit trap, so every
|
|
122
|
+
* name is matched through the shared glob.
|
|
123
|
+
* '!<form>' exclude — any of the above with a leading `!`
|
|
124
|
+
*
|
|
125
|
+
* Patterns are applied in order; last write wins, so
|
|
126
|
+
* `['*', '^*', '!^noisy']` reads as "all minus deps' noisy".
|
|
127
|
+
*
|
|
128
|
+
* Defaults:
|
|
129
|
+
* - `filter === undefined` → every candidate, as given.
|
|
130
|
+
* - `filter === []` → none (fully decoupled).
|
|
131
|
+
*
|
|
132
|
+
* Otherwise deduped by `unit`, first candidate kept. The unit is the
|
|
133
|
+
* fold's own identity, not the task's: two groups over the same members
|
|
134
|
+
* share one hash, so excluding either excludes both, and a selection by
|
|
135
|
+
* task id would keep the other while the key folded neither.
|
|
136
|
+
*/
|
|
137
|
+
export function selectFoldedDeps(
|
|
138
|
+
deps: readonly FoldCandidate[],
|
|
139
|
+
filter: readonly string[] | undefined,
|
|
140
|
+
selfProjectName: string,
|
|
141
|
+
selfTaskId: string,
|
|
142
|
+
): FoldCandidate[] {
|
|
143
|
+
if (filter === undefined) return [...deps]
|
|
144
|
+
|
|
145
|
+
const specs: DependencySpec[] = filter.map((raw) => parseSpec(raw, selfTaskId))
|
|
146
|
+
|
|
147
|
+
// Per-spec predicate, compiled once (exact compares + patterns).
|
|
148
|
+
const matchers = specs.map((spec) => specMatcher(spec))
|
|
149
|
+
|
|
150
|
+
const selected = new Map<string, FoldCandidate>()
|
|
151
|
+
for (let i = 0; i < specs.length; i++) {
|
|
152
|
+
const spec = specs[i]!
|
|
153
|
+
const matches = matchers[i]!
|
|
154
|
+
for (const d of deps) {
|
|
155
|
+
const isSelf = d.node.projectName === selfProjectName
|
|
156
|
+
if (!matches(d.node, isSelf)) continue
|
|
157
|
+
if (spec.negated) selected.delete(d.unit)
|
|
158
|
+
else if (!selected.has(d.unit)) selected.set(d.unit, d)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return [...selected.values()]
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Exact-name compare, or the shared `*`-glob when the name is a pattern. */
|
|
165
|
+
function nameMatcher(name: string): (candidate: string) => boolean {
|
|
166
|
+
if (isTaskPattern(name)) {
|
|
167
|
+
const re = compileTaskPattern(name)
|
|
168
|
+
return (candidate) => re.test(candidate)
|
|
169
|
+
}
|
|
170
|
+
return (candidate) => candidate === name
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Compile one spec into an upstream predicate. BOTH halves of a `pkg#task`
|
|
175
|
+
* form glob: a filter only ever selects from upstreams that already exist,
|
|
176
|
+
* so a package pattern is unambiguous here — unlike dependsOn, which must
|
|
177
|
+
* materialize concrete edges and therefore rejects it.
|
|
178
|
+
*/
|
|
179
|
+
function specMatcher(spec: DependencySpec): (n: TaskNode, isSelf: boolean) => boolean {
|
|
180
|
+
switch (spec.kind) {
|
|
181
|
+
case 'wildcardSelf':
|
|
182
|
+
return (_n, isSelf) => isSelf
|
|
183
|
+
case 'wildcardDeps':
|
|
184
|
+
return (_n, isSelf) => !isSelf
|
|
185
|
+
case 'self': {
|
|
186
|
+
const task = nameMatcher(spec.task)
|
|
187
|
+
return (n, isSelf) => isSelf && task(n.taskName)
|
|
188
|
+
}
|
|
189
|
+
case 'deps': {
|
|
190
|
+
const task = nameMatcher(spec.task)
|
|
191
|
+
return (n, isSelf) => !isSelf && task(n.taskName)
|
|
192
|
+
}
|
|
193
|
+
case 'cross': {
|
|
194
|
+
const project = nameMatcher(spec.project)
|
|
195
|
+
const task = nameMatcher(spec.task)
|
|
196
|
+
return (n) => project(n.projectName) && task(n.taskName)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The dependencies `node`'s key folds, per the hash path's rules above, as
|
|
203
|
+
* `keyUpstream` hands them to the key: the scheduled ones less the edges
|
|
204
|
+
* `--exclude-dependencies` left for order alone, plus the dropped ones it
|
|
205
|
+
* keys all the same (`excludedUpstream`), whose own dependencies the walk
|
|
206
|
+
* reads from the run's `keyOnly` map. `node.deps` held the order-only
|
|
207
|
+
* edges and lacked the dropped ones: the sandbox granted a sibling the key
|
|
208
|
+
* does not answer for.
|
|
209
|
+
*/
|
|
210
|
+
export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): FoldCandidate[] {
|
|
211
|
+
const candidates: FoldCandidate[] = []
|
|
212
|
+
for (const id of keyedDeps(node)) {
|
|
213
|
+
const dep = nodeOf(id)
|
|
214
|
+
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
215
|
+
}
|
|
216
|
+
for (const { node: dep } of node.excludedUpstream ?? []) {
|
|
217
|
+
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
218
|
+
}
|
|
219
|
+
if (isGroupTask(node) && node.config.inputs === undefined) return candidates
|
|
220
|
+
return selectFoldedDeps(candidates, node.config.inputs?.tasks, node.projectName, node.id)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* What stands in for a dependency's hash in the dedup: equal exactly when
|
|
225
|
+
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
226
|
+
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
227
|
+
* with no id of its own, so two groups over the same members hash alike.
|
|
228
|
+
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
229
|
+
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
230
|
+
* dropped ones, and two groups the key told apart shared a unit.
|
|
231
|
+
*/
|
|
232
|
+
function foldUnit(node: TaskNode): string {
|
|
233
|
+
if (!isGroupTask(node) || node.config.inputs !== undefined) return `task|${node.id}`
|
|
234
|
+
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
235
|
+
return `group|${members.sort().join('|')}`
|
|
236
|
+
}
|
package/src/graph/index.ts
CHANGED
|
@@ -8,6 +8,14 @@ export {
|
|
|
8
8
|
parseDependencySpec,
|
|
9
9
|
type DependencySpec,
|
|
10
10
|
} from './dependency-spec.js'
|
|
11
|
+
export {
|
|
12
|
+
type FoldCandidate,
|
|
13
|
+
foldedDeps,
|
|
14
|
+
isGroupTask,
|
|
15
|
+
keyedDeps,
|
|
16
|
+
refuseFiltersThroughGroups,
|
|
17
|
+
selectFoldedDeps,
|
|
18
|
+
} from './folds.js'
|
|
11
19
|
export { computeReverseDepCount } from './priorities.js'
|
|
12
20
|
export {
|
|
13
21
|
type ContinueMode,
|
|
@@ -33,7 +41,6 @@ export {
|
|
|
33
41
|
excludeDependencies,
|
|
34
42
|
expandRequested,
|
|
35
43
|
type GraphRules,
|
|
36
|
-
isGroupTask,
|
|
37
44
|
markSurfacedDeps,
|
|
38
45
|
outputsOverlap,
|
|
39
46
|
outputTakenBack,
|
package/src/graph/scheduler.ts
CHANGED
|
@@ -12,7 +12,8 @@ import {
|
|
|
12
12
|
tieredReverseDepCount,
|
|
13
13
|
type ScheduleStrategy,
|
|
14
14
|
} from './priorities.js'
|
|
15
|
-
import { isGroupTask
|
|
15
|
+
import { isGroupTask } from './folds.js'
|
|
16
|
+
import type { TaskNode } from './task-graph.js'
|
|
16
17
|
import type { TaskStatus } from './task-status.js'
|
|
17
18
|
|
|
18
19
|
/**
|
|
@@ -114,6 +115,12 @@ export interface TaskOutcome {
|
|
|
114
115
|
* so a dependant that folds `hash` saves nothing either (A-12).
|
|
115
116
|
*/
|
|
116
117
|
unkeyed?: true
|
|
118
|
+
/**
|
|
119
|
+
* This task's key folds a task with no `inputs` (or it is one): no run
|
|
120
|
+
* can derive that key again, so it neither read nor saved an entry, and a
|
|
121
|
+
* dependant that folds `hash` does neither.
|
|
122
|
+
*/
|
|
123
|
+
keyless?: true
|
|
117
124
|
/**
|
|
118
125
|
* The run's cache policy read and wrote nothing (`--no-cache`,
|
|
119
126
|
* `--cache-policy=local:`): no cache answered for this task, so a run of it is
|
|
@@ -219,7 +226,7 @@ export interface TaskOutcome {
|
|
|
219
226
|
/** Credential paths denied and left out of the grants: an agent applies the fix as given. */
|
|
220
227
|
withheld?: string[]
|
|
221
228
|
/**
|
|
222
|
-
* Reads of the project the task's `
|
|
229
|
+
* Reads of the project the task's `inputs.files` does not match:
|
|
223
230
|
* add them there too when the output depends on them, or a hit replays
|
|
224
231
|
* output built from an older copy. Cached tasks only.
|
|
225
232
|
*/
|
|
@@ -301,6 +308,12 @@ export interface ScheduleOptions {
|
|
|
301
308
|
* heuristic (the default and historically the only behavior).
|
|
302
309
|
*/
|
|
303
310
|
priorities?: ReadonlyMap<string, number>
|
|
311
|
+
/**
|
|
312
|
+
* Called once as the run starts with a function that replaces
|
|
313
|
+
* `priorities`: the ready queues re-rank at once, over the same baseline.
|
|
314
|
+
* A `schedule` plugin's later, better weights land through it.
|
|
315
|
+
*/
|
|
316
|
+
reprioritize?: (apply: (priorities: ReadonlyMap<string, number>) => void) => void
|
|
304
317
|
/**
|
|
305
318
|
* The baseline ranking under `priorities` (`schedule` in .vx/workspace.ts).
|
|
306
319
|
* Undefined or `most-work` is the reverse-deps count above.
|
|
@@ -371,10 +384,28 @@ class ReadyHeap {
|
|
|
371
384
|
// was two per heap level on every push and pop.
|
|
372
385
|
private readonly prio: number[] = []
|
|
373
386
|
private next = 0
|
|
374
|
-
constructor(private
|
|
387
|
+
constructor(private priority: ReadonlyMap<string, number>) {}
|
|
375
388
|
get size(): number {
|
|
376
389
|
return this.ids.length
|
|
377
390
|
}
|
|
391
|
+
/** Re-rank every queued id under `priority`, and every later push. */
|
|
392
|
+
reprioritize(priority: ReadonlyMap<string, number>): void {
|
|
393
|
+
this.priority = priority
|
|
394
|
+
for (let i = 0; i < this.ids.length; i++) this.prio[i] = priority.get(this.ids[i]!) ?? 0
|
|
395
|
+
for (let i = (this.ids.length >> 1) - 1; i >= 0; i--) this.down(i)
|
|
396
|
+
}
|
|
397
|
+
private down(i: number): void {
|
|
398
|
+
for (;;) {
|
|
399
|
+
const l = 2 * i + 1
|
|
400
|
+
const r = 2 * i + 2
|
|
401
|
+
let best = i
|
|
402
|
+
if (l < this.ids.length && this.higher(l, best)) best = l
|
|
403
|
+
if (r < this.ids.length && this.higher(r, best)) best = r
|
|
404
|
+
if (best === i) return
|
|
405
|
+
this.swap(i, best)
|
|
406
|
+
i = best
|
|
407
|
+
}
|
|
408
|
+
}
|
|
378
409
|
/** True if slot i outranks slot j (higher priority, or equal priority + earlier seq). */
|
|
379
410
|
private higher(i: number, j: number): boolean {
|
|
380
411
|
const pi = this.prio[i]!
|
|
@@ -434,17 +465,7 @@ class ReadyHeap {
|
|
|
434
465
|
this.ids[0] = lastId
|
|
435
466
|
this.seq[0] = lastSeq
|
|
436
467
|
this.prio[0] = lastPrio
|
|
437
|
-
|
|
438
|
-
for (;;) {
|
|
439
|
-
const l = 2 * i + 1
|
|
440
|
-
const r = 2 * i + 2
|
|
441
|
-
let best = i
|
|
442
|
-
if (l < this.ids.length && this.higher(l, best)) best = l
|
|
443
|
-
if (r < this.ids.length && this.higher(r, best)) best = r
|
|
444
|
-
if (best === i) break
|
|
445
|
-
this.swap(i, best)
|
|
446
|
-
i = best
|
|
447
|
-
}
|
|
468
|
+
this.down(0)
|
|
448
469
|
}
|
|
449
470
|
return top
|
|
450
471
|
}
|
|
@@ -553,6 +574,11 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
553
574
|
let settledIdle = 0
|
|
554
575
|
const execReady = new ReadyHeap(priority)
|
|
555
576
|
const restoreReady = new ReadyHeap(priority)
|
|
577
|
+
options.reprioritize?.((weights) => {
|
|
578
|
+
const merged = mergePriorities(baseline, weights)
|
|
579
|
+
execReady.reprioritize(merged)
|
|
580
|
+
restoreReady.reprioritize(merged)
|
|
581
|
+
})
|
|
556
582
|
// A restore-tier task is dep-independent (a stable hit's restore needs
|
|
557
583
|
// none of its deps' output), so it's ready immediately; its `pending`
|
|
558
584
|
// decrements still happen (in finishOne) but never re-enqueue it.
|
|
@@ -1008,7 +1034,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
|
|
|
1008
1034
|
return
|
|
1009
1035
|
}
|
|
1010
1036
|
// A UserError is a config/input failure (e.g. a failed
|
|
1011
|
-
// `
|
|
1037
|
+
// `inputs.runtime` command), not a vx bug — report it
|
|
1012
1038
|
// plainly, never as an "internal error".
|
|
1013
1039
|
let line: string
|
|
1014
1040
|
// Out of descriptors is the environment's limit too: a stack
|