@vzn/vx 0.0.639 → 0.0.640
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/caching.md +111 -83
- package/docs/cli.md +70 -40
- package/docs/execution.md +4 -4
- package/docs/features.md +9 -8
- package/docs/patterns.md +1 -1
- package/docs/schema.md +235 -180
- package/docs/security.md +1 -1
- package/package.json +5 -5
- package/schemas/show.json +14 -14
- package/skills/vx/SKILL.md +4 -3
- package/src/cache/cache.ts +4 -1
- package/src/cache/file-hashes.ts +7 -12
- package/src/cache/fs-clock.ts +62 -0
- package/src/cache/git-inputs.ts +2 -2
- package/src/cache/index.ts +3 -0
- package/src/cache/inputs.ts +154 -83
- package/src/cache/key-fold.ts +23 -3
- package/src/cache/layer.ts +8 -7
- package/src/cache/linked-inputs.ts +87 -0
- package/src/cli/foreign-flags.ts +1 -1
- package/src/cli/init.ts +23 -11
- package/src/cli/last.ts +24 -1
- package/src/cli/plugin-templates.ts +2 -2
- package/src/cli/select.ts +16 -6
- package/src/cli/show.ts +9 -6
- package/src/cli/upgrade.ts +95 -19
- package/src/cli/watch-filter.ts +22 -6
- package/src/cli/watch-judge.ts +49 -5
- package/src/cli/watch-set.ts +18 -8
- package/src/cli/watch.ts +74 -2
- package/src/cli/why.ts +5 -3
- package/src/config.ts +19 -15
- package/src/exec/executor.ts +4 -4
- package/src/graph/dependency-spec.ts +2 -2
- package/src/graph/folds.ts +236 -0
- package/src/graph/index.ts +8 -1
- package/src/graph/scheduler.ts +41 -15
- package/src/graph/task-graph.ts +135 -73
- package/src/index.ts +1 -1
- package/src/orchestrator/affected-tasks.ts +20 -19
- package/src/orchestrator/download-policy.ts +16 -22
- package/src/orchestrator/excluded-keys.ts +4 -5
- package/src/orchestrator/execute-task.ts +29 -25
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/keyed-projects.ts +12 -51
- package/src/orchestrator/local-shortcircuit.ts +1 -1
- package/src/orchestrator/persistent-task.ts +3 -3
- package/src/orchestrator/placement.ts +2 -2
- package/src/orchestrator/plan.ts +1 -1
- package/src/orchestrator/plugin-host.ts +62 -22
- package/src/orchestrator/plugin.ts +12 -2
- package/src/orchestrator/prepare.ts +12 -5
- package/src/orchestrator/projects.ts +7 -10
- package/src/orchestrator/run-report.ts +124 -46
- package/src/orchestrator/run-steps.ts +57 -0
- package/src/orchestrator/run.ts +51 -21
- package/src/orchestrator/sandbox-fix.ts +1 -1
- package/src/orchestrator/sandbox-request.ts +2 -2
- package/src/orchestrator/stable-keys.ts +59 -26
- package/src/orchestrator/steps.ts +26 -6
- package/src/orchestrator/task-hash.ts +31 -9
- package/src/orchestrator/telemetry-host.ts +3 -3
- package/src/orchestrator/telemetry.ts +9 -4
- package/src/orchestrator/upstream.ts +4 -181
- package/src/report/framed-output.ts +1 -1
- package/src/report/status-line.ts +7 -3
- package/src/settings.ts +11 -17
- package/src/util/paths.ts +1 -1
- package/src/workspace/affected.ts +81 -2
- package/src/workspace/config-imports.ts +1 -1
- package/src/workspace/config-schema.ts +142 -143
- package/src/workspace/lockfile.ts +1 -1
- package/src/workspace/migration.ts +19 -3
package/src/cli/watch-set.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import path from 'node:path'
|
|
7
7
|
import type { ProjectConfig } from '../config.js'
|
|
8
|
+
import { namedOutright } from '../cache/index.js'
|
|
8
9
|
import {
|
|
9
10
|
buildPackageGraph,
|
|
10
11
|
lockKey,
|
|
@@ -64,6 +65,8 @@ export interface ConfigSweep {
|
|
|
64
65
|
inputs: Map<string, string[][]>
|
|
65
66
|
/** Projects with a task that has a command and no cache: it reads what it likes, git-ignored files included. */
|
|
66
67
|
uncached: Set<string>
|
|
68
|
+
/** Paths a task's inputs name outright (`namedOutright`), absolute: a git-ignored one is an input. */
|
|
69
|
+
namedIgnored: string[]
|
|
67
70
|
/** Files the project configs import by relative path (item 949). */
|
|
68
71
|
configImports: string[]
|
|
69
72
|
/** Files the workspace config imports by relative path: loaded once per process (item 949). */
|
|
@@ -115,6 +118,7 @@ export async function sweepConfigs(
|
|
|
115
118
|
addTo(outputs, dir, globs)
|
|
116
119
|
const workspaceInputs = new Set<string>()
|
|
117
120
|
const uncached = new Set<string>()
|
|
121
|
+
const namedIgnored = new Set<string>()
|
|
118
122
|
const folded: [string, ProjectConfig][] = []
|
|
119
123
|
const fold = (dir: string, config: ProjectConfig): void => {
|
|
120
124
|
folded.push([dir, config])
|
|
@@ -131,15 +135,20 @@ export async function sweepConfigs(
|
|
|
131
135
|
add(dir, task.cache?.outputs?.files)
|
|
132
136
|
add(workspaceRoot, task.cache?.outputs?.workspaceFiles)
|
|
133
137
|
if (names !== null && !names.has(name)) continue
|
|
134
|
-
if (
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
addInputs(
|
|
139
|
-
workspaceRoot,
|
|
140
|
-
task.cache?.inputs?.workspaceFiles,
|
|
141
|
-
task.cache?.outputs?.workspaceFiles,
|
|
138
|
+
if (
|
|
139
|
+
task.inputs === undefined &&
|
|
140
|
+
task.exec !== undefined &&
|
|
141
|
+
task.exec.persistent === undefined
|
|
142
142
|
)
|
|
143
|
+
uncached.add(dir)
|
|
144
|
+
if (task.inputs !== undefined && task.exec !== undefined) {
|
|
145
|
+
const named = namedOutright(task)
|
|
146
|
+
for (const rel of named.files) namedIgnored.add(path.join(dir, rel))
|
|
147
|
+
for (const rel of named.workspaceFiles) namedIgnored.add(path.join(workspaceRoot, rel))
|
|
148
|
+
}
|
|
149
|
+
for (const g of task.inputs?.workspaceFiles ?? []) workspaceInputs.add(g)
|
|
150
|
+
addInputs(dir, task.inputs?.files, task.cache?.outputs?.files)
|
|
151
|
+
addInputs(workspaceRoot, task.inputs?.workspaceFiles, task.cache?.outputs?.workspaceFiles)
|
|
143
152
|
}
|
|
144
153
|
}
|
|
145
154
|
const result = (staged: Map<string, ProjectEntry> | null): ConfigSweep => ({
|
|
@@ -148,6 +157,7 @@ export async function sweepConfigs(
|
|
|
148
157
|
outputs,
|
|
149
158
|
inputs,
|
|
150
159
|
uncached,
|
|
160
|
+
namedIgnored: [...namedIgnored],
|
|
151
161
|
configImports: [...projectImports],
|
|
152
162
|
workspaceConfigImports: [...wsImports],
|
|
153
163
|
staged,
|
package/src/cli/watch.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
//
|
|
8
8
|
// The cache does the heavy lifting: re-runs are typically cache hits
|
|
9
9
|
// (input hash matches), so spurious events from edits outside any
|
|
10
|
-
// task's `
|
|
10
|
+
// task's `inputs.files` cost ~tens of ms of orchestrator
|
|
11
11
|
// overhead. We deliberately don't try to filter events through the
|
|
12
12
|
// per-task input globs — the cache key is the source of truth, and
|
|
13
13
|
// filtering would mean re-doing the glob/boundary work on every event.
|
|
@@ -279,6 +279,7 @@ export async function watchCmd(args: readonly string[]): Promise<number> {
|
|
|
279
279
|
outputs: swept.outputs,
|
|
280
280
|
inputs: swept.inputs,
|
|
281
281
|
uncached: swept.uncached,
|
|
282
|
+
namedIgnored: swept.namedIgnored,
|
|
282
283
|
configImports: swept.configImports,
|
|
283
284
|
workspaceConfigImports: swept.workspaceConfigImports,
|
|
284
285
|
fenceDirs: fenceDirs(allProjects),
|
|
@@ -301,6 +302,7 @@ export async function watchCmd(args: readonly string[]): Promise<number> {
|
|
|
301
302
|
outputs: sweep.outputs,
|
|
302
303
|
inputs: sweep.inputs,
|
|
303
304
|
uncached: sweep.uncached,
|
|
305
|
+
namedIgnored: sweep.namedIgnored,
|
|
304
306
|
configImports: sweep.configImports,
|
|
305
307
|
workspaceConfigImports: sweep.workspaceConfigImports,
|
|
306
308
|
fenceDirs: fenceDirs(all),
|
|
@@ -351,6 +353,8 @@ interface WatchLoopArgs {
|
|
|
351
353
|
inputs: ReadonlyMap<string, ReadonlyArray<readonly string[]>>
|
|
352
354
|
/** Projects whose uncached task may read a git-ignored file: such a path there is still an edit. */
|
|
353
355
|
uncached: ReadonlySet<string>
|
|
356
|
+
/** Paths a task's inputs name outright: a git-ignored one there is still an edit. */
|
|
357
|
+
namedIgnored: readonly string[]
|
|
354
358
|
/** Files the configs import: an edit to one outside the watched projects is a cycle that re-reads them. */
|
|
355
359
|
configImports: readonly string[]
|
|
356
360
|
/** Files the workspace config imports: loaded once per process, so an edit is named, not run. */
|
|
@@ -379,6 +383,7 @@ interface Rediscovered {
|
|
|
379
383
|
outputs: ReadonlyMap<string, readonly string[]>
|
|
380
384
|
inputs: ReadonlyMap<string, ReadonlyArray<readonly string[]>>
|
|
381
385
|
uncached: ReadonlySet<string>
|
|
386
|
+
namedIgnored: readonly string[]
|
|
382
387
|
configImports: readonly string[]
|
|
383
388
|
workspaceConfigImports: readonly string[]
|
|
384
389
|
fenceDirs: readonly string[]
|
|
@@ -394,6 +399,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
394
399
|
let outputs = args.outputs
|
|
395
400
|
let inputs = args.inputs
|
|
396
401
|
let uncached = args.uncached
|
|
402
|
+
let namedIgnored = args.namedIgnored
|
|
397
403
|
let claimedRootFiles = args.claimedRootFiles
|
|
398
404
|
let configImportFiles = args.configImports
|
|
399
405
|
let wsConfigImportFiles = args.workspaceConfigImports
|
|
@@ -567,6 +573,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
567
573
|
armedAt,
|
|
568
574
|
held: () => held !== undefined,
|
|
569
575
|
uncached: () => uncached,
|
|
576
|
+
namedIgnored: () => namedIgnored,
|
|
570
577
|
fenced: (ownDir, abs) => fenced(ownDir, abs),
|
|
571
578
|
...(atArm !== undefined ? { existedAtArm: atArm.listed, trackedAtArm: atArm.tracked } : {}),
|
|
572
579
|
})
|
|
@@ -642,7 +649,8 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
642
649
|
if (isIgnoredPath(proj.dir, filename) || fenced(proj.dir, abs)) return
|
|
643
650
|
// A module the config imports from inside the project is the config
|
|
644
651
|
// too: its edit can add an output or a `workspaceFiles` input.
|
|
645
|
-
if (shapesWatchedSet(filename) || configImportFiles.includes(abs)
|
|
652
|
+
if (shapesWatchedSet(filename) || configImportFiles.includes(abs) || linkChanged(abs))
|
|
653
|
+
reread = true
|
|
646
654
|
trigger(`${proj.name} ${filename}`, abs)
|
|
647
655
|
}
|
|
648
656
|
const armProject = (proj: ProjectMeta): void => {
|
|
@@ -816,6 +824,65 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
816
824
|
armAncestors()
|
|
817
825
|
}
|
|
818
826
|
|
|
827
|
+
/**
|
|
828
|
+
* What each symlink under a watched project points to: the key reads it,
|
|
829
|
+
* wherever it lies (cache/linked-inputs.ts), and the project's arm hears
|
|
830
|
+
* only the link. A file target's directory, non-recursive, or a directory
|
|
831
|
+
* target, recursive; each event names the link. Links are read again when
|
|
832
|
+
* one comes or goes (`projectEvent`).
|
|
833
|
+
*/
|
|
834
|
+
const linkArms = new Map<string, { target: string; handle: WatchHandle }>()
|
|
835
|
+
let links = atArm?.links ?? new Set<string>()
|
|
836
|
+
let watched = projects
|
|
837
|
+
/** A link came, went or moved: the set `armLinks` reads is stale. */
|
|
838
|
+
const linkChanged = (abs: string): boolean => {
|
|
839
|
+
if (links.has(abs)) return true
|
|
840
|
+
try {
|
|
841
|
+
return fs.lstatSync(abs).isSymbolicLink()
|
|
842
|
+
} catch {
|
|
843
|
+
return false
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
const armLinks = (): void => {
|
|
847
|
+
const want = new Map<string, { proj: ProjectMeta; real: string; dir: boolean }>()
|
|
848
|
+
for (const link of links) {
|
|
849
|
+
let proj: ProjectMeta | undefined
|
|
850
|
+
for (const p of watched)
|
|
851
|
+
if (
|
|
852
|
+
link.startsWith(p.dir + path.sep) &&
|
|
853
|
+
(proj === undefined || p.dir.length > proj.dir.length)
|
|
854
|
+
)
|
|
855
|
+
proj = p
|
|
856
|
+
if (proj === undefined) continue
|
|
857
|
+
try {
|
|
858
|
+
const real = fs.realpathSync(link)
|
|
859
|
+
want.set(link, { proj, real, dir: fs.statSync(real).isDirectory() })
|
|
860
|
+
} catch {
|
|
861
|
+
// Dangling: its target string is all the key reads, and the link's own arm hears that.
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
for (const [link, armed] of linkArms) {
|
|
865
|
+
if (want.get(link)?.real === armed.target) continue
|
|
866
|
+
armed.handle.close()
|
|
867
|
+
linkArms.delete(link)
|
|
868
|
+
}
|
|
869
|
+
for (const [link, { proj, real, dir }] of want) {
|
|
870
|
+
if (linkArms.has(link)) continue
|
|
871
|
+
const name = path.basename(real)
|
|
872
|
+
const label = `${proj.name} ${path.relative(proj.dir, link)}`
|
|
873
|
+
try {
|
|
874
|
+
const handle = arm(dir ? real : path.dirname(real), dir, (filename) => {
|
|
875
|
+
if (!dir && filename !== name) return
|
|
876
|
+
trigger(label, link)
|
|
877
|
+
})
|
|
878
|
+
linkArms.set(link, { target: real, handle })
|
|
879
|
+
changes.seed(link)
|
|
880
|
+
} catch (err) {
|
|
881
|
+
sayCannot(`cannot watch ${real}`, err)
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
819
886
|
// Said once, when the loop first arms: a re-arm repeated it after every
|
|
820
887
|
// cycle that re-read the watched set, between task rows. `debug.watch`
|
|
821
888
|
// says each arm on stderr, for whoever needs to see the set change.
|
|
@@ -843,6 +910,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
843
910
|
outputs = next.outputs
|
|
844
911
|
inputs = next.inputs
|
|
845
912
|
uncached = next.uncached
|
|
913
|
+
namedIgnored = next.namedIgnored
|
|
846
914
|
claimedRootFiles = next.claimedRootFiles
|
|
847
915
|
configImportFiles = next.configImports
|
|
848
916
|
wsConfigImportFiles = next.workspaceConfigImports
|
|
@@ -876,6 +944,9 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
876
944
|
for (const proj of own) if (!perProject.has(proj.dir)) armProject(proj)
|
|
877
945
|
}
|
|
878
946
|
armImports()
|
|
947
|
+
watched = next.projects
|
|
948
|
+
links = gitFiles(workspaceRoot)?.links ?? links
|
|
949
|
+
armLinks()
|
|
879
950
|
// A new arm proves delivery like the first ones: an edit in the new
|
|
880
951
|
// project right after this cycle is seen, not lost in the gap.
|
|
881
952
|
await pool.proved()
|
|
@@ -958,6 +1029,7 @@ async function runWatchLoop(args: WatchLoopArgs): Promise<void> {
|
|
|
958
1029
|
}
|
|
959
1030
|
armAncestors()
|
|
960
1031
|
armImports()
|
|
1032
|
+
armLinks()
|
|
961
1033
|
|
|
962
1034
|
// The orchestrator's own writes never kick the loop: `makeWatchIgnore`
|
|
963
1035
|
// closes over the RESOLVED cache dir (relocated or not) and the tasks'
|
package/src/cli/why.ts
CHANGED
|
@@ -36,8 +36,10 @@ interface WhyArgs {
|
|
|
36
36
|
* `tests/why.test.ts`.
|
|
37
37
|
*/
|
|
38
38
|
export const WHAT_TO_DO: Readonly<Record<string, string>> = {
|
|
39
|
-
file: 'an edit re-runs by design; a file the task does not read belongs out of
|
|
40
|
-
|
|
39
|
+
file: 'an edit re-runs by design; a file the task does not read belongs out of inputs.files',
|
|
40
|
+
linked:
|
|
41
|
+
'what the symlink points to changed (the file, or a file under the directory); an edit there re-runs by design',
|
|
42
|
+
env: "the variable's value moved; one the task does not read belongs out of inputs.env",
|
|
41
43
|
runtime: 'the command printed something else; drop timestamps and paths from its output',
|
|
42
44
|
'ws-runtime':
|
|
43
45
|
'the workspace command printed something else; drop timestamps and paths from its output',
|
|
@@ -372,4 +374,4 @@ export async function whyCmd(args: readonly string[]): Promise<number> {
|
|
|
372
374
|
const ROOTS_SHOWN = 5
|
|
373
375
|
|
|
374
376
|
const WRITTEN_TODO =
|
|
375
|
-
'a task that writes a file a key reads re-keys it every run; give that write its own task with the file as an output, or keep the file out of
|
|
377
|
+
'a task that writes a file a key reads re-keys it every run; give that write its own task with the file as an output, or keep the file out of inputs.files'
|
package/src/config.ts
CHANGED
|
@@ -110,6 +110,12 @@ export interface CacheSettings {
|
|
|
110
110
|
inputStability?: InputStability
|
|
111
111
|
/** The `cache.outputs.clean` of every task that does not set its own. */
|
|
112
112
|
outputs?: { clean?: boolean }
|
|
113
|
+
/**
|
|
114
|
+
* Which runs keep a failed task's log and the files its outputs left in
|
|
115
|
+
* the remote: CI runs (`'ci'`, the default), runs off CI (`'local'`), or
|
|
116
|
+
* both (`'always'`). Only a run whose policy writes a remote keeps any.
|
|
117
|
+
*/
|
|
118
|
+
upload?: { failures?: 'ci' | 'local' | 'always' }
|
|
113
119
|
}
|
|
114
120
|
|
|
115
121
|
/** `WorkspaceConfig.lock`. */
|
|
@@ -137,12 +143,6 @@ export interface WorkspaceRules {
|
|
|
137
143
|
* overlapping tasks with no edge are refused either way.
|
|
138
144
|
*/
|
|
139
145
|
exclusiveOutputs?: boolean
|
|
140
|
-
/**
|
|
141
|
-
* Refuse a task whose input globs can match another task's declared
|
|
142
|
-
* outputs, so every key is derived before any task runs. Off, such a key
|
|
143
|
-
* waits for its producer (`caching.md` § Local restore tier).
|
|
144
|
-
*/
|
|
145
|
-
upfrontKeys?: boolean
|
|
146
146
|
/**
|
|
147
147
|
* Which workspace projects a project may depend on, checked by `vx lock`
|
|
148
148
|
* over each project's direct dependencies (its manifest's and its tasks'
|
|
@@ -298,12 +298,18 @@ export interface TaskConfig {
|
|
|
298
298
|
* - `'pkg#name'` — specific package's `name` task; not key-checked.
|
|
299
299
|
*/
|
|
300
300
|
dependsOn?: readonly string[]
|
|
301
|
+
/**
|
|
302
|
+
* What the task's key folds, whether or not it caches: its dependents
|
|
303
|
+
* fold that key. A task with `exec` and no `inputs` has no key, so every
|
|
304
|
+
* task that folds it runs and saves nothing. A group with `inputs` is a
|
|
305
|
+
* keyed group: its inputs are a key its dependents fold, with no process.
|
|
306
|
+
*/
|
|
307
|
+
inputs?: InputsConfig
|
|
301
308
|
/**
|
|
302
309
|
* Caching configuration. **Caching is opt-in.** If this field is omitted,
|
|
303
310
|
* the task always runs and nothing is read from or written to the cache.
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
* runs.
|
|
311
|
+
* Requires `exec` and `inputs`: the key is the task's inputs, and a group
|
|
312
|
+
* runs nothing to cache.
|
|
307
313
|
*/
|
|
308
314
|
cache?: CacheConfig
|
|
309
315
|
}
|
|
@@ -324,7 +330,7 @@ export interface TaskConfig {
|
|
|
324
330
|
* `node_modules`, and through it the real path of every workspace
|
|
325
331
|
* package linked there.
|
|
326
332
|
*
|
|
327
|
-
* `cache` grants no access in either direction: `
|
|
333
|
+
* `cache` grants no access in either direction: `inputs` says what
|
|
328
334
|
* INVALIDATES a task, `allow` says what it may TOUCH, and deriving one
|
|
329
335
|
* from the other coupled them both ways — a declaration added for
|
|
330
336
|
* caching silently widened the sandbox, and a path the task needed had
|
|
@@ -398,7 +404,7 @@ export interface SandboxIgnore {
|
|
|
398
404
|
export interface SandboxGrants {
|
|
399
405
|
/**
|
|
400
406
|
* Readable paths or globs, added to the baseline — which is
|
|
401
|
-
* `node_modules`, NOT `
|
|
407
|
+
* `node_modules`, NOT `inputs.files`. A task that reads its own
|
|
402
408
|
* source in the sandbox says so here.
|
|
403
409
|
*/
|
|
404
410
|
read?: readonly string[]
|
|
@@ -573,8 +579,6 @@ export interface ExecEnv {
|
|
|
573
579
|
}
|
|
574
580
|
|
|
575
581
|
export interface CacheConfig {
|
|
576
|
-
/** What participates in the cache key. */
|
|
577
|
-
inputs: CacheInputs
|
|
578
582
|
/** What this task produces. Captured for restore on a cache hit. */
|
|
579
583
|
outputs: CacheOutputs
|
|
580
584
|
/**
|
|
@@ -634,7 +638,7 @@ export interface CacheOutputs {
|
|
|
634
638
|
clean?: boolean
|
|
635
639
|
}
|
|
636
640
|
|
|
637
|
-
export interface
|
|
641
|
+
export interface InputsConfig {
|
|
638
642
|
/**
|
|
639
643
|
* Project-relative globs. Each entry is a positive glob, or a negation
|
|
640
644
|
* prefixed with `!`. Required: pass a recursive glob to mean "all
|
|
@@ -773,8 +777,8 @@ export function defineProject<const T extends ProjectConfig>(
|
|
|
773
777
|
[K in keyof NonNullable<T['tasks']>]?: Known<NonNullable<T['tasks']>[K], TaskConfig> & {
|
|
774
778
|
dependsOn?: readonly DependsOnEntry<Extract<keyof NonNullable<T['tasks']>, string>>[]
|
|
775
779
|
exec?: Known<At<NonNullable<T['tasks']>[K], 'exec'>, ExecConfig>
|
|
780
|
+
inputs?: Known<At<NonNullable<T['tasks']>[K], 'inputs'>, InputsConfig>
|
|
776
781
|
cache?: Known<At<NonNullable<T['tasks']>[K], 'cache'>, CacheConfig> & {
|
|
777
|
-
inputs?: Known<At<At<NonNullable<T['tasks']>[K], 'cache'>, 'inputs'>, CacheInputs>
|
|
778
782
|
outputs?: Known<At<At<NonNullable<T['tasks']>[K], 'cache'>, 'outputs'>, CacheOutputs>
|
|
779
783
|
}
|
|
780
784
|
}
|
package/src/exec/executor.ts
CHANGED
|
@@ -53,13 +53,13 @@ export interface InputFile {
|
|
|
53
53
|
* secrets (cache.db stores only their digests).
|
|
54
54
|
*/
|
|
55
55
|
export interface TaskInputs {
|
|
56
|
-
/** Declared `
|
|
56
|
+
/** Declared `inputs.files` + `workspaceFiles` (own outputs excluded), sorted by path. */
|
|
57
57
|
readonly files: readonly InputFile[]
|
|
58
|
-
/** Declared `
|
|
58
|
+
/** Declared `inputs.env` names with their resolved values; `undefined` when unset. */
|
|
59
59
|
readonly env: ReadonlyArray<{ readonly name: string; readonly value: string | undefined }>
|
|
60
|
-
/** `
|
|
60
|
+
/** `inputs.runtime` commands and the output that was folded — a toolchain expectation. */
|
|
61
61
|
readonly runtime: ReadonlyArray<{ readonly command: string; readonly output: string }>
|
|
62
|
-
/** `
|
|
62
|
+
/** `inputs.workspaceRuntime`, same shape. */
|
|
63
63
|
readonly workspaceRuntime: ReadonlyArray<{ readonly command: string; readonly output: string }>
|
|
64
64
|
/**
|
|
65
65
|
* Dependencies whose cache keys are folded. `outputs` lists each one's
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Parser for the Turbo/Nx-style task-edge micro-syntax used in
|
|
2
|
-
// `dependsOn` and `
|
|
2
|
+
// `dependsOn` and `inputs.tasks`. Returns a small discriminated
|
|
3
3
|
// union the graph builder + cache filter both consume.
|
|
4
4
|
|
|
5
5
|
export type DependencySpec =
|
|
@@ -32,7 +32,7 @@ export function compileTaskPattern(pattern: string): RegExp {
|
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
/**
|
|
35
|
-
* Parse one `dependsOn` / `
|
|
35
|
+
* Parse one `dependsOn` / `inputs.tasks` entry.
|
|
36
36
|
*
|
|
37
37
|
* `name` → self.name
|
|
38
38
|
* `^name` → every dep workspace's `name` task
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// Which dependencies a task's key folds (`inputs.tasks`), on the graph
|
|
2
|
+
// alone: the hash path, the sandbox's keyed set and `vx lock`'s
|
|
3
|
+
// input-overlap rule read this one selection, so they cannot drift.
|
|
4
|
+
|
|
5
|
+
import { UserError } from '../util/index.js'
|
|
6
|
+
import {
|
|
7
|
+
DependencySpecError,
|
|
8
|
+
compileTaskPattern,
|
|
9
|
+
isTaskPattern,
|
|
10
|
+
parseDependencySpec,
|
|
11
|
+
type DependencySpec,
|
|
12
|
+
} from './dependency-spec.js'
|
|
13
|
+
import type { TaskNode } from './task-graph.js'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A task is a "group" if it has no `exec` — it exists only to chain
|
|
17
|
+
* `dependsOn` (an umbrella for `vx run ci`). Group tasks:
|
|
18
|
+
* - never spawn a process,
|
|
19
|
+
* - never read/write the cache (it's a config error to declare one),
|
|
20
|
+
* - never appear in the run summary or `runs` analytics table,
|
|
21
|
+
* - render no framed block in the live output.
|
|
22
|
+
* Six call sites used to repeat `node.config.exec === undefined`;
|
|
23
|
+
* centralising the predicate prevents that check from drifting.
|
|
24
|
+
*/
|
|
25
|
+
export function isGroupTask(node: TaskNode): boolean {
|
|
26
|
+
return node.config.exec === undefined
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `node.deps` less its ordering-only edges: the dependencies a key may fold. */
|
|
30
|
+
export function keyedDeps(node: TaskNode): readonly string[] {
|
|
31
|
+
const order = node.orderOnly
|
|
32
|
+
return order === undefined ? node.deps : node.deps.filter((d) => !order.includes(d))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* A filter selects among the task's own dependencies, never through a
|
|
37
|
+
* group: `dependsOn: ['build']` over a group `build` of `build.a` and
|
|
38
|
+
* `build.b` offers the filter `build`, not its members. A pattern that
|
|
39
|
+
* matches only a member (`build.*`) therefore selected nothing, and the key stopped moving
|
|
40
|
+
* with that member's inputs: a stale hit under a green run. Such a spec
|
|
41
|
+
* is refused, once, over the graph every key site reads. One that matches
|
|
42
|
+
* nothing anywhere stays legal (a preset's pattern needn't match in every
|
|
43
|
+
* project). An exact name is refused earlier, at load, unless `dependsOn`
|
|
44
|
+
* names it (`assertFilterNamesDeclaredDeps`).
|
|
45
|
+
*/
|
|
46
|
+
export function refuseFiltersThroughGroups(nodes: ReadonlyMap<string, TaskNode>): void {
|
|
47
|
+
for (const node of nodes.values()) {
|
|
48
|
+
const filter = node.config.inputs?.tasks
|
|
49
|
+
if (filter === undefined || filter.length === 0) continue
|
|
50
|
+
const deps = keyedDeps(node).flatMap((id) => nodes.get(id) ?? [])
|
|
51
|
+
const groups = deps.filter(isGroupTask)
|
|
52
|
+
if (groups.length === 0) continue
|
|
53
|
+
for (const raw of filter) {
|
|
54
|
+
const spec = parseSpec(raw, node.id)
|
|
55
|
+
if (spec.negated) continue
|
|
56
|
+
const matches = specMatcher(spec)
|
|
57
|
+
const hit = (n: TaskNode): boolean => matches(n, n.projectName === node.projectName)
|
|
58
|
+
if (deps.some(hit)) continue
|
|
59
|
+
for (const group of groups) {
|
|
60
|
+
const member = groupMembers(group, nodes).find(hit)
|
|
61
|
+
if (member === undefined) continue
|
|
62
|
+
throw new UserError(
|
|
63
|
+
`${node.id}: inputs.tasks: '${raw}' matches ${member.id} only through the group ${group.id}, so it selects nothing; name '${group.taskName}' or depend on ${member.id} directly`,
|
|
64
|
+
)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function parseSpec(raw: string, selfTaskId: string): DependencySpec {
|
|
71
|
+
try {
|
|
72
|
+
return parseDependencySpec(raw)
|
|
73
|
+
} catch (err) {
|
|
74
|
+
if (err instanceof DependencySpecError) {
|
|
75
|
+
throw new UserError(`${selfTaskId}: inputs.tasks: ${err.message}`)
|
|
76
|
+
}
|
|
77
|
+
throw err
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Every task a group stands for, through nested groups, each once. */
|
|
82
|
+
function groupMembers(group: TaskNode, nodes: ReadonlyMap<string, TaskNode>): TaskNode[] {
|
|
83
|
+
const out: TaskNode[] = []
|
|
84
|
+
const seen = new Set<string>([group.id])
|
|
85
|
+
const stack = group.deps.toReversed()
|
|
86
|
+
while (stack.length > 0) {
|
|
87
|
+
const id = stack.pop()!
|
|
88
|
+
if (seen.has(id)) continue
|
|
89
|
+
seen.add(id)
|
|
90
|
+
const n = nodes.get(id)
|
|
91
|
+
if (n === undefined) continue
|
|
92
|
+
out.push(n)
|
|
93
|
+
if (isGroupTask(n)) stack.push(...n.deps.toReversed())
|
|
94
|
+
}
|
|
95
|
+
return out
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* One dependency as the key fold sees it: the task, and the value the
|
|
100
|
+
* fold dedups by — its hash on the hash path, a structural stand-in for
|
|
101
|
+
* the hash on the graph (`foldUnit` in keyed-projects.ts).
|
|
102
|
+
*/
|
|
103
|
+
export interface FoldCandidate {
|
|
104
|
+
node: TaskNode
|
|
105
|
+
unit: string
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The dependencies a task's key folds, per its `inputs.tasks`.
|
|
110
|
+
*
|
|
111
|
+
* Patterns (Turbo/Nx micro-syntax + filter extensions):
|
|
112
|
+
* '*' all same-project upstream
|
|
113
|
+
* '^*' all dep-workspace upstream
|
|
114
|
+
* 'name' same-project task `name`
|
|
115
|
+
* '^name' `name` task in every dep workspace
|
|
116
|
+
* 'pkg#name' specific package's `name` task
|
|
117
|
+
* 'name.*' patterns — EITHER half of any form above may contain `*`
|
|
118
|
+
* (same glob as dependsOn patterns), including the project
|
|
119
|
+
* half of `pkg#name`. A filter that matched literally here
|
|
120
|
+
* would silently select ZERO upstream hashes and decouple the
|
|
121
|
+
* task from its dependencies — a stale-hit trap, so every
|
|
122
|
+
* name is matched through the shared glob.
|
|
123
|
+
* '!<form>' exclude — any of the above with a leading `!`
|
|
124
|
+
*
|
|
125
|
+
* Patterns are applied in order; last write wins, so
|
|
126
|
+
* `['*', '^*', '!^noisy']` reads as "all minus deps' noisy".
|
|
127
|
+
*
|
|
128
|
+
* Defaults:
|
|
129
|
+
* - `filter === undefined` → every candidate, as given.
|
|
130
|
+
* - `filter === []` → none (fully decoupled).
|
|
131
|
+
*
|
|
132
|
+
* Otherwise deduped by `unit`, first candidate kept. The unit is the
|
|
133
|
+
* fold's own identity, not the task's: two groups over the same members
|
|
134
|
+
* share one hash, so excluding either excludes both, and a selection by
|
|
135
|
+
* task id would keep the other while the key folded neither.
|
|
136
|
+
*/
|
|
137
|
+
export function selectFoldedDeps(
|
|
138
|
+
deps: readonly FoldCandidate[],
|
|
139
|
+
filter: readonly string[] | undefined,
|
|
140
|
+
selfProjectName: string,
|
|
141
|
+
selfTaskId: string,
|
|
142
|
+
): FoldCandidate[] {
|
|
143
|
+
if (filter === undefined) return [...deps]
|
|
144
|
+
|
|
145
|
+
const specs: DependencySpec[] = filter.map((raw) => parseSpec(raw, selfTaskId))
|
|
146
|
+
|
|
147
|
+
// Per-spec predicate, compiled once (exact compares + patterns).
|
|
148
|
+
const matchers = specs.map((spec) => specMatcher(spec))
|
|
149
|
+
|
|
150
|
+
const selected = new Map<string, FoldCandidate>()
|
|
151
|
+
for (let i = 0; i < specs.length; i++) {
|
|
152
|
+
const spec = specs[i]!
|
|
153
|
+
const matches = matchers[i]!
|
|
154
|
+
for (const d of deps) {
|
|
155
|
+
const isSelf = d.node.projectName === selfProjectName
|
|
156
|
+
if (!matches(d.node, isSelf)) continue
|
|
157
|
+
if (spec.negated) selected.delete(d.unit)
|
|
158
|
+
else if (!selected.has(d.unit)) selected.set(d.unit, d)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return [...selected.values()]
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Exact-name compare, or the shared `*`-glob when the name is a pattern. */
|
|
165
|
+
function nameMatcher(name: string): (candidate: string) => boolean {
|
|
166
|
+
if (isTaskPattern(name)) {
|
|
167
|
+
const re = compileTaskPattern(name)
|
|
168
|
+
return (candidate) => re.test(candidate)
|
|
169
|
+
}
|
|
170
|
+
return (candidate) => candidate === name
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Compile one spec into an upstream predicate. BOTH halves of a `pkg#task`
|
|
175
|
+
* form glob: a filter only ever selects from upstreams that already exist,
|
|
176
|
+
* so a package pattern is unambiguous here — unlike dependsOn, which must
|
|
177
|
+
* materialize concrete edges and therefore rejects it.
|
|
178
|
+
*/
|
|
179
|
+
function specMatcher(spec: DependencySpec): (n: TaskNode, isSelf: boolean) => boolean {
|
|
180
|
+
switch (spec.kind) {
|
|
181
|
+
case 'wildcardSelf':
|
|
182
|
+
return (_n, isSelf) => isSelf
|
|
183
|
+
case 'wildcardDeps':
|
|
184
|
+
return (_n, isSelf) => !isSelf
|
|
185
|
+
case 'self': {
|
|
186
|
+
const task = nameMatcher(spec.task)
|
|
187
|
+
return (n, isSelf) => isSelf && task(n.taskName)
|
|
188
|
+
}
|
|
189
|
+
case 'deps': {
|
|
190
|
+
const task = nameMatcher(spec.task)
|
|
191
|
+
return (n, isSelf) => !isSelf && task(n.taskName)
|
|
192
|
+
}
|
|
193
|
+
case 'cross': {
|
|
194
|
+
const project = nameMatcher(spec.project)
|
|
195
|
+
const task = nameMatcher(spec.task)
|
|
196
|
+
return (n) => project(n.projectName) && task(n.taskName)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The dependencies `node`'s key folds, per the hash path's rules above, as
|
|
203
|
+
* `keyUpstream` hands them to the key: the scheduled ones less the edges
|
|
204
|
+
* `--exclude-dependencies` left for order alone, plus the dropped ones it
|
|
205
|
+
* keys all the same (`excludedUpstream`), whose own dependencies the walk
|
|
206
|
+
* reads from the run's `keyOnly` map. `node.deps` held the order-only
|
|
207
|
+
* edges and lacked the dropped ones: the sandbox granted a sibling the key
|
|
208
|
+
* does not answer for.
|
|
209
|
+
*/
|
|
210
|
+
export function foldedDeps(node: TaskNode, nodeOf: (id: string) => TaskNode): FoldCandidate[] {
|
|
211
|
+
const candidates: FoldCandidate[] = []
|
|
212
|
+
for (const id of keyedDeps(node)) {
|
|
213
|
+
const dep = nodeOf(id)
|
|
214
|
+
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
215
|
+
}
|
|
216
|
+
for (const { node: dep } of node.excludedUpstream ?? []) {
|
|
217
|
+
candidates.push({ node: dep, unit: foldUnit(dep) })
|
|
218
|
+
}
|
|
219
|
+
if (isGroupTask(node) && node.config.inputs === undefined) return candidates
|
|
220
|
+
return selectFoldedDeps(candidates, node.config.inputs?.tasks, node.projectName, node.id)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* What stands in for a dependency's hash in the dedup: equal exactly when
|
|
225
|
+
* the hashes are. An exec task's key folds its own id (`task:<id>`), so its
|
|
226
|
+
* id is enough; a group's hash is a roll-up of its members' ids and hashes
|
|
227
|
+
* with no id of its own, so two groups over the same members hash alike.
|
|
228
|
+
* The members are `keyUpstream`'s, which `computeGroupHash` is handed: under
|
|
229
|
+
* `--exclude-dependencies` `node.deps` holds order-only edges and lacks the
|
|
230
|
+
* dropped ones, and two groups the key told apart shared a unit.
|
|
231
|
+
*/
|
|
232
|
+
function foldUnit(node: TaskNode): string {
|
|
233
|
+
if (!isGroupTask(node) || node.config.inputs !== undefined) return `task|${node.id}`
|
|
234
|
+
const members = [...keyedDeps(node), ...(node.excludedUpstream ?? []).map((u) => u.node.id)]
|
|
235
|
+
return `group|${members.sort().join('|')}`
|
|
236
|
+
}
|
package/src/graph/index.ts
CHANGED
|
@@ -8,6 +8,14 @@ export {
|
|
|
8
8
|
parseDependencySpec,
|
|
9
9
|
type DependencySpec,
|
|
10
10
|
} from './dependency-spec.js'
|
|
11
|
+
export {
|
|
12
|
+
type FoldCandidate,
|
|
13
|
+
foldedDeps,
|
|
14
|
+
isGroupTask,
|
|
15
|
+
keyedDeps,
|
|
16
|
+
refuseFiltersThroughGroups,
|
|
17
|
+
selectFoldedDeps,
|
|
18
|
+
} from './folds.js'
|
|
11
19
|
export { computeReverseDepCount } from './priorities.js'
|
|
12
20
|
export {
|
|
13
21
|
type ContinueMode,
|
|
@@ -33,7 +41,6 @@ export {
|
|
|
33
41
|
excludeDependencies,
|
|
34
42
|
expandRequested,
|
|
35
43
|
type GraphRules,
|
|
36
|
-
isGroupTask,
|
|
37
44
|
markSurfacedDeps,
|
|
38
45
|
outputsOverlap,
|
|
39
46
|
outputTakenBack,
|