@vzn/vx 0.0.652 → 0.0.653

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +2 -2
  2. package/launcher.sh +2 -1
  3. package/package.json +5 -5
  4. package/schemas/error.json +1 -1
  5. package/schemas/show.json +1 -0
  6. package/skills/vx/SKILL.md +3 -3
  7. package/src/bin.ts +25 -3
  8. package/src/cache/archive.ts +196 -31
  9. package/src/cache/cache.ts +863 -225
  10. package/src/cache/checkpointer.ts +102 -0
  11. package/src/cache/file-hashes.ts +2 -1
  12. package/src/cache/fold-memo.ts +24 -4
  13. package/src/cache/fs-clock.ts +26 -0
  14. package/src/cache/git-inputs.ts +188 -22
  15. package/src/cache/history-stream.ts +22 -5
  16. package/src/cache/index.ts +10 -1
  17. package/src/cache/inputs.ts +573 -152
  18. package/src/cache/key-fold.ts +1 -1
  19. package/src/cache/layer.ts +23 -10
  20. package/src/cache/layered-cache.ts +28 -7
  21. package/src/cache/output-index.ts +106 -44
  22. package/src/cache/plugin-cache.ts +2 -2
  23. package/src/cache/policy.ts +6 -2
  24. package/src/cache/remote-lane.ts +3 -1
  25. package/src/cache/run-history.ts +50 -10
  26. package/src/cache/schema.ts +42 -9
  27. package/src/cli/cache.ts +147 -34
  28. package/src/cli/completions.ts +2 -2
  29. package/src/cli/core-alias.ts +9 -2
  30. package/src/cli/docs.ts +4 -2
  31. package/src/cli/help.ts +25 -2
  32. package/src/cli/index.ts +38 -7
  33. package/src/cli/info.ts +4 -3
  34. package/src/cli/init.ts +55 -20
  35. package/src/cli/last.ts +184 -41
  36. package/src/cli/lock.ts +187 -24
  37. package/src/cli/plan-format.ts +24 -21
  38. package/src/cli/plugin-commands.ts +6 -1
  39. package/src/cli/plugin-templates.ts +1 -1
  40. package/src/cli/run.ts +68 -34
  41. package/src/cli/select.ts +160 -64
  42. package/src/cli/show.ts +63 -18
  43. package/src/cli/task-verb.ts +12 -7
  44. package/src/cli/upgrade.ts +210 -76
  45. package/src/cli/watch-filter.ts +88 -15
  46. package/src/cli/watch-fs.ts +34 -47
  47. package/src/cli/watch-judge.ts +72 -8
  48. package/src/cli/watch-set.ts +6 -4
  49. package/src/cli/watch.ts +179 -25
  50. package/src/cli/why.ts +107 -33
  51. package/src/cli/workspace-config.ts +11 -1
  52. package/src/config.ts +9 -0
  53. package/src/exec/env.ts +3 -1
  54. package/src/exec/executor.ts +76 -15
  55. package/src/exec/kill-tree.ts +7 -7
  56. package/src/exec/runner.ts +270 -108
  57. package/src/exec/sandbox-binds.ts +109 -18
  58. package/src/exec/sandbox-runtime.ts +134 -37
  59. package/src/exec/sandbox-violations.ts +250 -46
  60. package/src/exec/spawn-off.ts +10 -5
  61. package/src/exec/warm-shell.ts +29 -7
  62. package/src/graph/folds.ts +89 -16
  63. package/src/graph/index.ts +8 -0
  64. package/src/graph/priorities.ts +8 -5
  65. package/src/graph/scheduler.ts +173 -38
  66. package/src/graph/stand-ins.ts +36 -0
  67. package/src/graph/task-graph.ts +327 -141
  68. package/src/graph/task-status.ts +5 -1
  69. package/src/handoff.ts +45 -10
  70. package/src/history/failure-mode.ts +15 -5
  71. package/src/history/history.ts +103 -30
  72. package/src/history/index.ts +1 -0
  73. package/src/history/metrics.ts +189 -69
  74. package/src/index.ts +2 -1
  75. package/src/orchestrator/admission.ts +13 -9
  76. package/src/orchestrator/affected-tasks.ts +35 -13
  77. package/src/orchestrator/command-windows.ts +8 -5
  78. package/src/orchestrator/deferred-outputs.ts +41 -12
  79. package/src/orchestrator/doctor.ts +21 -9
  80. package/src/orchestrator/download-policy.ts +3 -3
  81. package/src/orchestrator/excluded-keys.ts +14 -2
  82. package/src/orchestrator/execute-args.ts +9 -2
  83. package/src/orchestrator/execute-task.ts +564 -123
  84. package/src/orchestrator/graph-memo.ts +26 -4
  85. package/src/orchestrator/graph-stage.ts +27 -4
  86. package/src/orchestrator/held-servers.ts +36 -14
  87. package/src/orchestrator/hit-restore.ts +136 -87
  88. package/src/orchestrator/index.ts +4 -1
  89. package/src/orchestrator/keyed-projects.ts +25 -0
  90. package/src/orchestrator/local-floor.ts +21 -3
  91. package/src/orchestrator/lockfile-claim.ts +30 -4
  92. package/src/orchestrator/miss-save.ts +33 -53
  93. package/src/orchestrator/options.ts +13 -0
  94. package/src/orchestrator/output-host.ts +16 -4
  95. package/src/orchestrator/output-log.ts +1 -54
  96. package/src/orchestrator/persistent-task.ts +31 -17
  97. package/src/orchestrator/persistent.ts +152 -15
  98. package/src/orchestrator/placement.ts +81 -15
  99. package/src/orchestrator/plan.ts +37 -15
  100. package/src/orchestrator/plugin-host.ts +232 -108
  101. package/src/orchestrator/plugin.ts +53 -27
  102. package/src/orchestrator/prepare.ts +194 -38
  103. package/src/orchestrator/projects.ts +89 -10
  104. package/src/orchestrator/read-only.ts +115 -0
  105. package/src/orchestrator/run-context.ts +6 -2
  106. package/src/orchestrator/run-failures.ts +43 -14
  107. package/src/orchestrator/run-id.ts +13 -2
  108. package/src/orchestrator/run-keys.ts +87 -28
  109. package/src/orchestrator/run-live.ts +16 -13
  110. package/src/orchestrator/run-lock.ts +27 -4
  111. package/src/orchestrator/run-records.ts +12 -10
  112. package/src/orchestrator/run-report.ts +2 -1
  113. package/src/orchestrator/run-stop.ts +17 -9
  114. package/src/orchestrator/run.ts +443 -107
  115. package/src/orchestrator/sandbox-fix.ts +40 -12
  116. package/src/orchestrator/sandbox-request.ts +286 -59
  117. package/src/orchestrator/save-lane.ts +24 -13
  118. package/src/orchestrator/shell-verdict.ts +20 -1
  119. package/src/orchestrator/signals.ts +103 -8
  120. package/src/orchestrator/stable-keys.ts +17 -9
  121. package/src/orchestrator/task-artifacts.ts +15 -2
  122. package/src/orchestrator/task-hash.ts +187 -30
  123. package/src/orchestrator/telemetry-host.ts +25 -6
  124. package/src/orchestrator/telemetry.ts +80 -12
  125. package/src/orchestrator/upstream.ts +3 -1
  126. package/src/report/events.ts +20 -3
  127. package/src/report/forecast.ts +103 -37
  128. package/src/report/framed-output.ts +165 -157
  129. package/src/report/index.ts +2 -1
  130. package/src/report/logger.ts +107 -30
  131. package/src/report/plain-output.ts +13 -9
  132. package/src/report/status-line.ts +28 -55
  133. package/src/report/summary.ts +57 -14
  134. package/src/settings.ts +42 -6
  135. package/src/ui/components.ts +36 -42
  136. package/src/ui/index.ts +3 -1
  137. package/src/ui/live-region.ts +67 -8
  138. package/src/ui/terminal.ts +12 -0
  139. package/src/ui/theme.ts +16 -1
  140. package/src/util/config-frame.ts +17 -3
  141. package/src/util/edit-distance.ts +17 -1
  142. package/src/util/errors.ts +23 -7
  143. package/src/util/foreign-cache-dir.ts +60 -0
  144. package/src/util/git-ignored.ts +59 -5
  145. package/src/util/index.ts +15 -1
  146. package/src/util/paths.ts +50 -6
  147. package/src/util/plain-text.ts +19 -0
  148. package/src/util/secret-mask.ts +57 -1
  149. package/src/util/settle.ts +16 -0
  150. package/src/util/size.ts +12 -2
  151. package/src/util/write-file.ts +86 -9
  152. package/src/workspace/affected.ts +225 -27
  153. package/src/workspace/config-bytecode.ts +23 -4
  154. package/src/workspace/config-eval.ts +90 -16
  155. package/src/workspace/config-imports.ts +96 -41
  156. package/src/workspace/config-schema.ts +126 -25
  157. package/src/workspace/dependencies-lock.ts +7 -14
  158. package/src/workspace/filter.ts +10 -1
  159. package/src/workspace/index.ts +3 -1
  160. package/src/workspace/json-data.ts +34 -8
  161. package/src/workspace/lockfile.ts +86 -37
  162. package/src/workspace/migration.ts +46 -25
  163. package/src/workspace/module-state.ts +62 -0
  164. package/src/workspace/nested-dirs.ts +8 -0
  165. package/src/workspace/project-loader.ts +276 -32
  166. package/src/workspace/repo-id.ts +29 -6
  167. package/src/workspace/workspace.ts +190 -30
@@ -29,6 +29,7 @@ import type { PackageGraph, ProjectEntry } from '../workspace/index.js'
29
29
  import { isDefaultBuild } from './default-build.js'
30
30
  import { foldedDeps, isGroupTask } from './folds.js'
31
31
  import type { TaskOutcome } from './scheduler.js'
32
+ import { addStandIn } from './stand-ins.js'
32
33
 
33
34
  // Re-exported so existing importers keep working while the type's home
34
35
  // moves to workspace (it's the joint product of discovery + loading).
@@ -77,6 +78,11 @@ export interface TaskNode {
77
78
  * set by placement, folded into its key.
78
79
  */
79
80
  platform?: string
81
+ /**
82
+ * The worker type its executor runs it on (`TaskExecutor.workers`), set
83
+ * by placement when that executor names types: placement only, unkeyed.
84
+ */
85
+ worker?: string
80
86
  /**
81
87
  * Ids of upstream tasks whose declared outputs this task's outputs
82
88
  * overlap: this task ADDS to their trees (twenty's `build:individual`
@@ -303,11 +309,21 @@ export interface BuildGraphOptions {
303
309
  /** The workspace's `run.rules`; each is on unless set to `false`. */
304
310
  rules?: GraphRules | undefined
305
311
  /**
306
- * By project name, every project rooted inside its dir (absolute dirs).
312
+ * By project name, every project rooted inside its dir (absolute dirs),
313
+ * and every nested workspace (`workspace`, named by its root-relative dir).
307
314
  * Given with `workspaceRoot` and unlocked (`vx lock`), an `inputs.files`
308
- * entry rooted inside one is refused (`refuseNestedInputs`).
315
+ * or `outputs.files` entry rooted inside a project is refused, and an
316
+ * `outputs.files` one inside a workspace (`refuseNestedGlobs`).
317
+ */
318
+ nestedProjects?: ReadonlyMap<
319
+ string,
320
+ ReadonlyArray<{ name: string; dir: string; workspace?: boolean }>
321
+ >
322
+ /**
323
+ * Every project's config file (absolute). Given with `workspaceRoot`, an
324
+ * `outputs.workspaceFiles` glob covering one is refused (`refuseConfigOutputs`).
309
325
  */
310
- nestedProjects?: ReadonlyMap<string, ReadonlyArray<{ name: string; dir: string }>>
326
+ projectConfigs?: ReadonlyArray<{ name: string; configPath: string }>
311
327
  }
312
328
 
313
329
  /**
@@ -511,20 +527,33 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
511
527
  // packages below it, passing through the ones `visited` holds and the
512
528
  // ones that declare no `name`, and on past a default build on a cycle.
513
529
  // True when it found a holder.
514
- function walkHolders(frame: Frame, name: string, visited: Set<string>, start: string): boolean {
515
- const frontier = [...packageGraph.directDeps(start)]
530
+ // `past` holds the edges already gone past: each edge the walk adds
531
+ // beyond them stands in for them (`standInsOf`).
532
+ function walkHolders(
533
+ frame: Frame,
534
+ name: string,
535
+ visited: Set<string>,
536
+ start: string,
537
+ past: readonly string[],
538
+ ): boolean {
539
+ const frontier: Array<[project: string, past: readonly string[]]> = packageGraph
540
+ .directDeps(start)
541
+ .map((d) => [d, past])
516
542
  let held = false
517
543
  while (frontier.length > 0) {
518
- const target = frontier.pop()!
544
+ const [target, via] = frontier.pop()!
519
545
  if (visited.has(target)) continue
520
546
  visited.add(target)
521
547
  if (visit(frame, target, name, false)) {
522
548
  held = true
549
+ const id = taskId(target, name)
550
+ if (via.length > 0) addStandIn(frame.node, id, via)
523
551
  const task = declaredTask(projects.get(target)!.config, name)!
524
552
  if (isChainGroup(task, name) && onCycle(target)) {
525
- frontier.push(...packageGraph.directDeps(target))
553
+ const next = [...via, id]
554
+ for (const d of packageGraph.directDeps(target)) frontier.push([d, next])
526
555
  }
527
- } else frontier.push(...packageGraph.directDeps(target))
556
+ } else for (const d of packageGraph.directDeps(target)) frontier.push([d, via])
528
557
  }
529
558
  return held
530
559
  }
@@ -536,7 +565,8 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
536
565
  // and hit after `b` changed.
537
566
  function goPast(frame: Frame, project: string, name: string, visited: Set<string>): void {
538
567
  const task = declaredTask(projects.get(project)!.config, name)!
539
- if (isChainGroup(task, name) && onCycle(project)) walkHolders(frame, name, visited, project)
568
+ if (isChainGroup(task, name) && onCycle(project))
569
+ walkHolders(frame, name, visited, project, [taskId(project, name)])
540
570
  }
541
571
 
542
572
  // Resolves one `dependsOn` entry of `frame`'s task into its edges.
@@ -645,7 +675,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
645
675
  }
646
676
  const visited = new Set<string>([projectName])
647
677
  if (!isTaskPattern(spec.task)) {
648
- const held = walkHolders(frame, spec.task, visited, projectName)
678
+ const held = walkHolders(frame, spec.task, visited, projectName, [])
649
679
  if (!held && !declaredAnywhere(projectName, spec.task)) {
650
680
  if (undeclaredDeps === undefined) throw undeclaredDepsError(id, spec.task, declared ?? [])
651
681
  undeclaredDeps(id, spec.task)
@@ -682,8 +712,16 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
682
712
  'VX_E_DEPENDS_ON',
683
713
  )
684
714
  const meant = nearestTarget(projects, spec.project, spec.task)
715
+ // Which half is missing: a project that exists was told "no such
716
+ // project or task", though only its task was.
717
+ const owner = projects.get(spec.project)
718
+ const tasks = Object.keys(owner?.config.tasks ?? {})
719
+ const what =
720
+ owner === undefined
721
+ ? `no project is named ${spec.project}`
722
+ : `${spec.project} declares no task ${spec.task}${meant === undefined && tasks.length > 0 ? ` (its tasks: ${tasks.join(', ')})` : ''}`
685
723
  throw new UserError(
686
- `${id}: dependsOn ${taskId(spec.project, spec.task)} but no such project or task is declared${meant === undefined ? '' : `. Did you mean ${meant}?`}`,
724
+ `${id}: dependsOn ${taskId(spec.project, spec.task)}, but ${what}${meant === undefined ? '' : `. Did you mean ${meant}?`}`,
687
725
  'VX_E_DEPENDS_ON',
688
726
  )
689
727
  }
@@ -739,14 +777,19 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
739
777
  const frame: Frame = { node: x, entry: 0, added: false, pending: null, next: 0 }
740
778
  let reaching: Set<string> | null = null
741
779
  const visited = new Set<string>([x.projectName])
742
- const frontier = [...packageGraph.directDeps(x.projectName)]
780
+ const frontier: Array<[project: string, past: readonly string[]]> = packageGraph
781
+ .directDeps(x.projectName)
782
+ .map((d) => [d, []])
783
+ const push = (from: string, via: readonly string[]): void => {
784
+ for (const d of packageGraph.directDeps(from)) frontier.push([d, via])
785
+ }
743
786
  while (frontier.length > 0) {
744
- const target = frontier.pop()!
787
+ const [target, via] = frontier.pop()!
745
788
  if (visited.has(target)) continue
746
789
  visited.add(target)
747
790
  const task = declaredTask(projects.get(target)?.config, name)
748
791
  if (task === undefined) {
749
- frontier.push(...packageGraph.directDeps(target))
792
+ push(target, via)
750
793
  continue
751
794
  }
752
795
  const to = taskId(target, name)
@@ -758,7 +801,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
758
801
  reaching = null
759
802
  }
760
803
  if (isChainGroup(task, name) || (reaching ??= reachingTo(nodes, x.id)).has(to)) {
761
- frontier.push(...packageGraph.directDeps(target))
804
+ push(target, via)
762
805
  continue
763
806
  }
764
807
  }
@@ -767,8 +810,8 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
767
810
  visit(frame, target, name, false)
768
811
  drain()
769
812
  if (nodes.size !== size) reaching = null
770
- if (isChainGroup(task, name) && onCycle(target))
771
- frontier.push(...packageGraph.directDeps(target))
813
+ if (via.length > 0) addStandIn(x, to, via)
814
+ if (isChainGroup(task, name) && onCycle(target)) push(target, [...via, to])
772
815
  }
773
816
  x.deps = [...new Set(x.deps)].sort()
774
817
  }
@@ -787,44 +830,111 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
787
830
 
788
831
  checkGraph(nodes, options.workspaceRoot, options.rules)
789
832
  if (options.nestedProjects !== undefined && options.rules?.locked !== true)
790
- refuseNestedInputs(nodes, options.nestedProjects, options.workspaceRoot)
833
+ refuseNestedGlobs(nodes, options.nestedProjects, options.workspaceRoot)
834
+ if (options.projectConfigs !== undefined && options.workspaceRoot !== undefined)
835
+ refuseConfigOutputs(nodes, options.projectConfigs, options.workspaceRoot)
791
836
  return nodes
792
837
  }
793
838
 
794
839
  /**
795
- * Refuse an `inputs.files` entry rooted inside a project nested in the
796
- * task's own: every file it can match belongs to that project, which a
797
- * project's globs never reach, so the entry keys nothing and an edit there
798
- * hits (a `dist/**` over a build output holding its own config was dropped
799
- * from the key in silence). Judged by each brace alternative's fixed
800
- * prefix: `**` or a sibling reaches past the nested dir and is fine.
840
+ * Refuse an `outputs.workspaceFiles` glob that covers a project's config,
841
+ * the task's own or another's: vx cleans a task's outputs before it runs,
842
+ * so `packages/b/**` deleted b's untracked files, and its config with them
843
+ * once untracked. `outputs.files` refuses its own config at load; a
844
+ * root-anchored glob can reach any project, so it is judged here, where
845
+ * every config is known.
801
846
  */
802
- function refuseNestedInputs(
847
+ function refuseConfigOutputs(
803
848
  nodes: ReadonlyMap<string, TaskNode>,
804
- nested: ReadonlyMap<string, ReadonlyArray<{ name: string; dir: string }>>,
849
+ configs: ReadonlyArray<{ name: string; configPath: string }>,
850
+ workspaceRoot: string,
851
+ ): void {
852
+ let rels: Array<{ name: string; rel: string }> | undefined
853
+ for (const n of nodes.values()) {
854
+ const ws = n.config.outputs?.workspaceFiles
855
+ if (ws === undefined || ws.length === 0) continue
856
+ const { positive, negative } = splitNegations(ws)
857
+ rels ??= configs.map((c) => ({ name: c.name, rel: relPosix(workspaceRoot, c.configPath) }))
858
+ const back = asTrees(negative)
859
+ for (const g of positive) {
860
+ // Only a config under the glob's fixed prefix can match it.
861
+ const root = rootless(staticPrefix(g))
862
+ const trees = asTrees([g])
863
+ const hit = rels.find(
864
+ (c) =>
865
+ (root === '' || c.rel === root || c.rel.startsWith(`${root}/`)) &&
866
+ trees.some((t) => taskGlob(t).match(c.rel)) &&
867
+ !back.some((t) => taskGlob(t).match(c.rel)),
868
+ )
869
+ if (hit === undefined) continue
870
+ throw new UserError(
871
+ `${n.id}: outputs.workspaceFiles ${JSON.stringify(g)} covers ${hit.rel}, the config of ` +
872
+ `the project ${hit.name} — vx deletes a task's outputs before it runs and restores ` +
873
+ `them on a hit, so the project would lose its config. Name the directory the task ` +
874
+ `writes, or take the file back with "!${hit.rel}".`,
875
+ )
876
+ }
877
+ }
878
+ }
879
+
880
+ /**
881
+ * Refuse an `inputs.files` or `outputs.files` entry rooted inside a project
882
+ * nested in the task's own, or an `outputs.files` one inside a nested
883
+ * workspace: every file it can match belongs to that project, which a
884
+ * project's globs never reach. An input keys nothing, so an edit there hits
885
+ * (a `dist/**` over a build output holding its own config was dropped from
886
+ * the key in silence); an output saves nothing, and the nested project's
887
+ * own artifact there could hold the outer task's bytes. An input still
888
+ * reaches into a nested workspace, so one rooted there keys its files.
889
+ * Judged by each brace alternative's fixed prefix: `**` or a sibling
890
+ * reaches past the nested dir and is fine.
891
+ */
892
+ function refuseNestedGlobs(
893
+ nodes: ReadonlyMap<string, TaskNode>,
894
+ nested: ReadonlyMap<string, ReadonlyArray<{ name: string; dir: string; workspace?: boolean }>>,
805
895
  workspaceRoot: string | undefined,
806
896
  ): void {
807
897
  for (const n of nodes.values()) {
808
898
  const inner = nested.get(n.projectName)
809
899
  if (inner === undefined || inner.length === 0 || isGroupTask(n)) continue
810
- for (const g of positives(n.config.inputs?.files)) {
811
- for (const alt of braceAlternatives(g)) {
812
- const prefix = staticPrefix(alt)
813
- if (prefix === '.' || prefix === '') continue
814
- const abs = path.join(n.projectDir, prefix)
815
- const hit = inner.find((p) => abs === p.dir || abs.startsWith(p.dir + path.sep))
816
- if (hit === undefined) continue
817
- const rel =
818
- workspaceRoot === undefined
819
- ? hit.name
820
- : relPosix(workspaceRoot, path.join(n.projectDir, alt.replace(/^\.\//, '')))
821
- throw new UserError(
822
- `${n.id} reads ${JSON.stringify(g)} in inputs.files, which lies inside the project ` +
823
- `${hit.name}: a project's globs never reach another project's files, so it keys ` +
824
- `nothing. Depend on a ${hit.name} task, or name the files in inputs.workspaceFiles ` +
825
- `(${JSON.stringify(rel)}).`,
826
- 'VX_E_NESTED_INPUT',
827
- )
900
+ for (const side of ['inputs', 'outputs'] as const) {
901
+ for (const g of positives(n.config[side]?.files)) {
902
+ for (const alt of braceAlternatives(g)) {
903
+ const prefix = staticPrefix(alt)
904
+ if (prefix === '.' || prefix === '') continue
905
+ const abs = path.join(n.projectDir, prefix)
906
+ const hit = inner.find(
907
+ (p) =>
908
+ (side === 'outputs' || p.workspace !== true) &&
909
+ (abs === p.dir || abs.startsWith(p.dir + path.sep)),
910
+ )
911
+ if (hit === undefined) continue
912
+ const rel =
913
+ workspaceRoot === undefined
914
+ ? hit.name
915
+ : relPosix(workspaceRoot, path.join(n.projectDir, alt.replace(/^\.\//, '')))
916
+ if (hit.workspace === true)
917
+ throw new UserError(
918
+ `${n.id} writes ${JSON.stringify(g)} in outputs.files, which lies inside the ` +
919
+ `workspace ${hit.name} (it holds its own .vx/workspace.*): a project's outputs ` +
920
+ `never reach another workspace's files, so it saves nothing. Let that ` +
921
+ `workspace's own tasks write them, or name them in outputs.workspaceFiles ` +
922
+ `(${JSON.stringify(rel)}).`,
923
+ 'VX_E_NESTED_OUTPUT',
924
+ )
925
+ throw new UserError(
926
+ side === 'inputs'
927
+ ? `${n.id} reads ${JSON.stringify(g)} in inputs.files, which lies inside the project ` +
928
+ `${hit.name}: a project's globs never reach another project's files, so it keys ` +
929
+ `nothing. Depend on a ${hit.name} task, or name the files in inputs.workspaceFiles ` +
930
+ `(${JSON.stringify(rel)}).`
931
+ : `${n.id} writes ${JSON.stringify(g)} in outputs.files, which lies inside the project ` +
932
+ `${hit.name}: a project's globs never reach another project's files, so it saves ` +
933
+ `nothing. Let a ${hit.name} task write them, or name them in outputs.workspaceFiles ` +
934
+ `(${JSON.stringify(rel)}).`,
935
+ side === 'inputs' ? 'VX_E_NESTED_INPUT' : 'VX_E_NESTED_OUTPUT',
936
+ )
937
+ }
828
938
  }
829
939
  }
830
940
  }
@@ -853,7 +963,7 @@ function reachingTo(nodes: Map<string, TaskNode>, target: string): Set<string> {
853
963
 
854
964
  /**
855
965
  * What a task graph must hold before it is scheduled: each node under its
856
- * own id, every dep a node, no cycle, and no two tasks deleting each
966
+ * own id, every dep a node named once, no cycle, and no two tasks deleting each
857
967
  * other's outputs. The builder's output holds the first two by
858
968
  * construction; the `graph` stage runs this again after its plugins, which
859
969
  * could move a node to another key (the scheduler then died on a raw
@@ -876,10 +986,16 @@ export function checkGraph(
876
986
  throw new Error(`${node.id} depends on '${dep}', which is not a task in this run's graph`)
877
987
  }
878
988
  }
989
+ // A plugin's repeated edge, as the builder's own: folded into the key
990
+ // twice, and owed twice by a server held on the remote's answer.
991
+ if (node.deps.length > 1) {
992
+ const unique = new Set(node.deps)
993
+ if (unique.size < node.deps.length) node.deps = [...unique]
994
+ }
879
995
  delete node.addsToOutputsOf
880
996
  delete node.outputsAddedToBy
881
997
  }
882
- for (const node of nodes.values()) refuseSelfClean(node)
998
+ for (const node of nodes.values()) refuseSelfClean(node, workspaceRoot)
883
999
  detectCycle(nodes)
884
1000
  const exclusive = rules?.exclusiveOutputs !== false
885
1001
  const locked = rules?.locked === true
@@ -1331,10 +1447,14 @@ function detectOutputCollisions(
1331
1447
  entries.push({ node, globs, rel })
1332
1448
  }
1333
1449
  }
1334
- for (const [i, j] of overlapCandidates(entries, (e) => e.globs)) {
1450
+ for (const [i, j] of overlapCandidates(
1451
+ entries,
1452
+ (e) => e.globs,
1453
+ (e) => e.rel !== undefined,
1454
+ )) {
1335
1455
  const x = entries[i]!
1336
1456
  const y = entries[j]!
1337
- if ((x.rel === undefined) === (y.rel === undefined) || x.node === y.node) continue
1457
+ if (x.node === y.node) continue
1338
1458
  const [ws, files] = x.rel === undefined ? [x, y] : [y, x]
1339
1459
  collideAcross(ws.node, ws.globs, files.node, files.globs, files.rel!, reaches, exclusive)
1340
1460
  }
@@ -1485,21 +1605,20 @@ function mayOverlap<T>(
1485
1605
 
1486
1606
  /**
1487
1607
  * The pairs `[i, j]`, `i < j`, of `tasks` whose declared outputs CAN
1488
- * overlap, ascending by `i` then `j`: a superset of the pairs
1608
+ * overlap, yielded ascending by `i` then `j`: a superset of the pairs
1489
1609
  * `outputsOverlap` accepts, so `collide` over them decides exactly what it
1490
1610
  * decided over all pairs, and in the same order (the first refusal named,
1491
1611
  * and the addition marks pushed, are all-pairs' own).
1492
1612
  *
1493
- * `outputsOverlap` compares the `asTrees` forms three ways. Two equal
1494
- * globs are one lookup. Two equal literals are too, since each literal
1495
- * carries its `/**` twin. And a glob matches a literal only when the
1613
+ * `outputsOverlap` compares the `asTrees` forms three ways. Two globs,
1614
+ * equal or not, meet only when one's literal prefix lies at or under the
1615
+ * other's. Two equal literals do too, since each literal carries its `/**`
1616
+ * twin. And a glob matches a literal only when the
1496
1617
  * literal lies under the directory the glob's literal head names
1497
1618
  * (`dist/sub/**` → `dist/sub`; `*.js` → the root, so a wildcard in the
1498
1619
  * first segment meets every literal): a literal meets the globs filed at
1499
1620
  * each of its ancestors. The fuzz above found no glob matching the
1500
- * directory its own head names. A whole subtree meets the globs whose
1501
- * literal prefix lies at or under it, found the same way from the glob's
1502
- * side (item 941).
1621
+ * directory its own head names.
1503
1622
  *
1504
1623
  * Given `sideOf`, only the pairs whose members sit on different sides are
1505
1624
  * wanted (a reader and a writer, `detectInputOverlaps`), and each index is
@@ -1511,85 +1630,120 @@ function overlapCandidates<T>(
1511
1630
  tasks: readonly T[],
1512
1631
  globsOf: (n: T) => readonly string[] | undefined,
1513
1632
  sideOf?: (n: T) => boolean,
1514
- ): Array<[number, number]> {
1515
- if (!mayOverlap(tasks, globsOf, sideOf)) return []
1633
+ ): Iterable<[number, number]> {
1634
+ // Nearly every bucket stops here, so it starts no generator.
1635
+ return mayOverlap(tasks, globsOf, sideOf) ? indexedCandidates(tasks, globsOf, sideOf) : []
1636
+ }
1637
+
1638
+ /** `overlapCandidates` past `mayOverlap`. */
1639
+ function* indexedCandidates<T>(
1640
+ tasks: readonly T[],
1641
+ globsOf: (n: T) => readonly string[] | undefined,
1642
+ sideOf?: (n: T) => boolean,
1643
+ ): Generator<[number, number]> {
1516
1644
  const sides = sideOf === undefined ? 1 : 2
1517
1645
  // The index a member of side `s` looks up: its own side's, or the other's.
1518
1646
  const across = (s: number): number => (sides === 1 ? s : 1 - s)
1519
1647
  const perSide = (): Array<Map<string, number[]>> =>
1520
1648
  Array.from({ length: sides }, () => new Map<string, number[]>())
1521
- const globs = perSide()
1522
- const globsUnder = perSide()
1523
- const literals: Array<[task: number, path: string]> = []
1524
- // `covers` (item 941): a whole subtree `P/**` meets every glob whose
1525
- // literal prefix is P or under it, so each glob looks up the subtrees
1526
- // filed at its prefix and at each of that prefix's ancestors.
1527
- const subtrees = perSide()
1528
- const prefixes: Array<[task: number, prefix: string]> = []
1529
1649
  const file = (index: Map<string, number[]>, key: string, i: number): void => {
1530
1650
  const list = index.get(key)
1531
1651
  if (list === undefined) index.set(key, [i])
1532
1652
  else list.push(i)
1533
1653
  }
1654
+ const fileAncestors = (index: Map<string, number[]>, key: string, i: number): void => {
1655
+ for (let sep = key.indexOf('/'); sep !== -1; sep = key.indexOf('/', sep + 1)) {
1656
+ file(index, key.slice(0, sep), i)
1657
+ }
1658
+ }
1659
+ // Each relation below is filed from both of its ends, so a member finds
1660
+ // every partner by its own lookups and the pairs are yielded one `i` at a
1661
+ // time. 20,000 tasks sharing `dist/**` are 200 million pairs, and holding
1662
+ // them all before the first was compared ran out of memory (hunt); the
1663
+ // caller refuses at the first, so only that `i`'s partners are built.
1664
+ //
1665
+ // Two globs meet only when one's literal prefix is the other's or under
1666
+ // it: the rule of `witnessed`, and of `covers` (item 941: a whole subtree
1667
+ // `P/**` has the prefix P), and two equal globs share one. Each glob is
1668
+ // filed at its prefix (found from it and from under it), and at the
1669
+ // prefix, the root and each ancestor (found from above).
1670
+ const byPrefix = perSide()
1671
+ const overPrefix = perSide()
1672
+ // A literal meets the globs filed at each of its ancestors.
1673
+ const globsUnder = perSide()
1674
+ const literals = perSide()
1675
+ interface Tree {
1676
+ /** Its literal prefix, `''` at the root. */
1677
+ prefix: string
1678
+ /** The directory its literal head names. */
1679
+ head: string
1680
+ }
1681
+ const trees: Tree[][] = []
1682
+ const paths: string[][] = []
1534
1683
  const side: number[] = []
1535
1684
  for (let i = 0; i < tasks.length; i++) {
1536
1685
  const s = sideOf?.(tasks[i]!) === true ? 1 : 0
1537
1686
  side.push(s)
1687
+ const own: Tree[] = []
1688
+ const lits: string[] = []
1538
1689
  // Each brace alternative, as `outputsOverlap` reads them.
1539
1690
  for (const tree of asTrees((globsOf(tasks[i]!) ?? []).flatMap(braceAlternatives))) {
1540
1691
  const lit = literalPath(tree)
1541
1692
  if (lit !== null) {
1542
- literals.push([i, lit])
1693
+ lits.push(lit)
1694
+ file(literals[s]!, '', i)
1695
+ fileAncestors(literals[s]!, lit, i)
1543
1696
  continue
1544
1697
  }
1545
- file(globs[s]!, tree, i)
1546
- const dir = wholeSubtreePrefixes([tree])?.[0]
1547
- if (dir !== undefined) file(subtrees[s]!, dir, i)
1548
- prefixes.push([i, staticPrefix(tree)])
1549
- const head = tree.slice(0, tree.search(GLOB_HEAD_END))
1550
- file(globsUnder[s]!, head.slice(0, Math.max(0, head.lastIndexOf('/'))), i)
1698
+ const prefix = rootless(staticPrefix(tree))
1699
+ file(byPrefix[s]!, prefix, i)
1700
+ file(overPrefix[s]!, prefix, i)
1701
+ if (prefix !== '') file(overPrefix[s]!, '', i)
1702
+ fileAncestors(overPrefix[s]!, prefix, i)
1703
+ const lead = tree.slice(0, tree.search(GLOB_HEAD_END))
1704
+ const head = lead.slice(0, Math.max(0, lead.lastIndexOf('/')))
1705
+ file(globsUnder[s]!, head, i)
1706
+ own.push({ prefix, head })
1551
1707
  }
1708
+ trees.push(own)
1709
+ paths.push(lits)
1552
1710
  }
1553
1711
  const n = tasks.length
1554
- const pairs = new Set<number>()
1555
- const pair = (x: number, y: number): void => {
1556
- if (x !== y) pairs.add(x < y ? x * n + y : y * n + x)
1557
- }
1558
- for (const [tree, list] of globs[0]!) {
1559
- const other = sides === 1 ? list : (globs[1]!.get(tree) ?? [])
1560
- for (let x = 0; x < list.length; x++) {
1561
- for (let y = sides === 1 ? x + 1 : 0; y < other.length; y++) pair(list[x]!, other[y]!)
1712
+ // The `i` each partner was last taken for, so one found twice is kept once.
1713
+ const takenFor = new Int32Array(n).fill(-1)
1714
+ let found: number[] = []
1715
+ const take = (i: number, list: readonly number[] | undefined): void => {
1716
+ if (list === undefined) return
1717
+ for (const j of list) {
1718
+ if (j > i && takenFor[j] !== i) {
1719
+ takenFor[j] = i
1720
+ found.push(j)
1721
+ }
1562
1722
  }
1563
1723
  }
1564
- // `witnessed`: two globs meet only when one's literal prefix is the
1565
- // other's or under it, so each looks up the globs filed at its prefix
1566
- // and at each ancestor.
1567
- const byPrefix = perSide()
1568
- for (const [i, prefix] of prefixes) file(byPrefix[side[i]!]!, rootless(prefix), i)
1569
- for (const [i, raw] of prefixes) {
1570
- const index = byPrefix[across(side[i]!)]!
1571
- const prefix = rootless(raw)
1572
- for (const j of index.get(prefix) ?? []) pair(i, j)
1573
- if (prefix !== '') for (const j of index.get('') ?? []) pair(i, j)
1574
- for (let sep = prefix.indexOf('/'); sep !== -1; sep = prefix.indexOf('/', sep + 1)) {
1575
- for (const j of index.get(prefix.slice(0, sep)) ?? []) pair(i, j)
1724
+ const takeAncestors = (i: number, index: Map<string, number[]>, key: string): void => {
1725
+ for (let sep = key.indexOf('/'); sep !== -1; sep = key.indexOf('/', sep + 1)) {
1726
+ take(i, index.get(key.slice(0, sep)))
1576
1727
  }
1577
1728
  }
1578
- for (const [i, prefix] of prefixes) {
1579
- const index = subtrees[across(side[i]!)]!
1580
- for (const j of index.get(prefix) ?? []) pair(i, j)
1581
- for (let sep = prefix.indexOf('/'); sep !== -1; sep = prefix.indexOf('/', sep + 1)) {
1582
- for (const j of index.get(prefix.slice(0, sep)) ?? []) pair(i, j)
1729
+ for (let i = 0; i < n; i++) {
1730
+ const o = across(side[i]!)
1731
+ for (const { prefix, head } of trees[i]!) {
1732
+ take(i, byPrefix[o]!.get(prefix))
1733
+ if (prefix !== '') take(i, byPrefix[o]!.get(''))
1734
+ takeAncestors(i, byPrefix[o]!, prefix)
1735
+ take(i, overPrefix[o]!.get(prefix))
1736
+ take(i, literals[o]!.get(head))
1583
1737
  }
1584
- }
1585
- for (const [i, path] of literals) {
1586
- const index = globsUnder[across(side[i]!)]!
1587
- for (const j of index.get('') ?? []) pair(i, j)
1588
- for (let sep = path.indexOf('/'); sep !== -1; sep = path.indexOf('/', sep + 1)) {
1589
- for (const j of index.get(path.slice(0, sep)) ?? []) pair(i, j)
1738
+ for (const path of paths[i]!) {
1739
+ take(i, globsUnder[o]!.get(''))
1740
+ takeAncestors(i, globsUnder[o]!, path)
1590
1741
  }
1742
+ if (found.length === 0) continue
1743
+ const partners = found.sort((x, y) => x - y)
1744
+ found = []
1745
+ for (const j of partners) yield [i, j]
1591
1746
  }
1592
- return [...pairs].sort((x, y) => x - y).map((p) => [Math.floor(p / n), p % n])
1593
1747
  }
1594
1748
 
1595
1749
  /**
@@ -1703,39 +1857,65 @@ export function outputTakenBack(output: string, negatives: readonly string[]): b
1703
1857
  * sources: vx removes a task's outputs before it runs, so a formatter
1704
1858
  * declaring `src/**` as both deleted every committed file under `src`,
1705
1859
  * and its key read nothing (hunt 8). Always refused, whatever the rules.
1860
+ * The two fields are compared across too, `files` rebased to the root:
1861
+ * `src/**` read as `files` and written as the root's `app/src/**` deleted
1862
+ * an untracked source.
1706
1863
  */
1707
- function refuseSelfClean(node: TaskNode): void {
1708
- const declaredOutputs = node.config.outputs
1709
- const declared = node.config.inputs
1710
- if (declaredOutputs === undefined || declared === undefined) return
1711
- for (const field of ['files', 'workspaceFiles'] as const) {
1712
- if (!coversAny(declared[field], declaredOutputs[field])) continue
1713
- const inputs = splitNegations(declared[field] ?? [])
1714
- const out = splitNegations(declaredOutputs[field] ?? [])
1715
- const outputs = out.positive
1716
- if (outputs.length === 0) continue
1717
- const roots = outputs.map((g) => rootless(staticPrefix(g)))
1718
- for (const gi of inputs.positive) {
1719
- // Only an output rooted at or above the input's prefix can cover it;
1720
- // the prefix test keeps the glob matchers off the common disjoint pair.
1721
- const pi = rootless(staticPrefix(gi))
1722
- const near = outputs.filter((_, k) => {
1723
- const r = roots[k]!
1724
- return r === '' || pi === r || pi.startsWith(`${r}/`)
1725
- })
1726
- // Taken back by the task's own `!` inputs, or by an output `!` (no
1727
- // output, so never cleaned: A-44), the entry reads what it says.
1728
- if (near.length === 0 || outputTakenBack(gi, inputs.negative)) continue
1729
- if (outputTakenBack(gi, out.negative)) continue
1730
- if (!outputTakenBack(gi, near)) continue
1731
- throw new UserError(
1732
- `${node.id}: every file ${JSON.stringify(gi)} in inputs.${field} selects is also ` +
1733
- `its own output — vx removes a task's outputs before it runs, so the task would ` +
1734
- `delete its own sources. A task that rewrites files in place (a formatter) declares ` +
1735
- `no outputs.`,
1736
- 'VX_E_INPUT_IS_OUTPUT',
1737
- )
1738
- }
1864
+ function refuseSelfClean(node: TaskNode, workspaceRoot: string | undefined): void {
1865
+ const outputs = node.config.outputs
1866
+ const inputs = node.config.inputs
1867
+ if (outputs === undefined || inputs === undefined) return
1868
+ selfClean(node, 'files', inputs.files, outputs.files)
1869
+ selfClean(node, 'workspaceFiles', inputs.workspaceFiles, outputs.workspaceFiles)
1870
+ const wsOut = (outputs.workspaceFiles?.length ?? 0) > 0
1871
+ const wsIn = (inputs.workspaceFiles?.length ?? 0) > 0
1872
+ if (workspaceRoot === undefined || (!wsOut && !wsIn)) return
1873
+ const rel = relPosix(workspaceRoot, node.projectDir)
1874
+ const rebased = (globs: readonly string[] | undefined) =>
1875
+ globs?.map((g) => {
1876
+ const neg = g.startsWith('!') ? '!' : ''
1877
+ const body = g.slice(neg.length).replace(/^\.\//, '')
1878
+ return rel === '' ? g : `${neg}${rel}/${body}`
1879
+ })
1880
+ if (wsOut) selfClean(node, 'files', rebased(inputs.files), outputs.workspaceFiles, inputs.files)
1881
+ if (wsIn) selfClean(node, 'workspaceFiles', inputs.workspaceFiles, rebased(outputs.files))
1882
+ }
1883
+
1884
+ /** Throws when an output in `out` takes back an entry of `ins` whole. `said` is `ins` as declared. */
1885
+ function selfClean(
1886
+ node: TaskNode,
1887
+ field: 'files' | 'workspaceFiles',
1888
+ ins: readonly string[] | undefined,
1889
+ out: readonly string[] | undefined,
1890
+ said: readonly string[] | undefined = ins,
1891
+ ): void {
1892
+ if (!coversAny(ins, out)) return
1893
+ const inputs = splitNegations(ins ?? [])
1894
+ const declared = said === ins ? inputs : splitNegations(said ?? [])
1895
+ const taken = splitNegations(out ?? [])
1896
+ const outputs = taken.positive
1897
+ if (outputs.length === 0) return
1898
+ const roots = outputs.map((g) => rootless(staticPrefix(g)))
1899
+ for (const [i, gi] of inputs.positive.entries()) {
1900
+ // Only an output rooted at or above the input's prefix can cover it;
1901
+ // the prefix test keeps the glob matchers off the common disjoint pair.
1902
+ const pi = rootless(staticPrefix(gi))
1903
+ const near = outputs.filter((_, k) => {
1904
+ const r = roots[k]!
1905
+ return r === '' || pi === r || pi.startsWith(`${r}/`)
1906
+ })
1907
+ // Taken back by the task's own `!` inputs, or by an output `!` (no
1908
+ // output, so never cleaned: A-44), the entry reads what it says.
1909
+ if (near.length === 0 || outputTakenBack(gi, inputs.negative)) continue
1910
+ if (outputTakenBack(gi, taken.negative)) continue
1911
+ if (!outputTakenBack(gi, near)) continue
1912
+ throw new UserError(
1913
+ `${node.id}: every file ${JSON.stringify(declared.positive[i])} in inputs.${field} selects ` +
1914
+ `is also its own output — vx removes a task's outputs before it runs, so the task would ` +
1915
+ `delete its own sources. A task that rewrites files in place (a formatter) declares ` +
1916
+ `no outputs.`,
1917
+ 'VX_E_INPUT_IS_OUTPUT',
1918
+ )
1739
1919
  }
1740
1920
  }
1741
1921
 
@@ -2077,11 +2257,17 @@ function readsOutputs(
2077
2257
  refuse: () =>
2078
2258
  new UserError(
2079
2259
  `${reader.node.id} reads ${JSON.stringify(gi.slice(at.length))} in inputs.${field}, which matches ` +
2080
- `${writer.node.id}'s output ${JSON.stringify(shown)} — a task's key must not read ` +
2081
- `the outputs of a task it does not depend on, or whose key it already includes. ` +
2082
- `Exclude it: add ${JSON.stringify(`!${out}`)} to ${reader.node.id}'s ` +
2083
- `inputs.${field}, or depend on ${writer.node.id} and leave it out of ` +
2084
- `${reader.node.id}'s inputs.tasks, so its key hashes those bytes instead.`,
2260
+ `${writer.node.id}'s output ${JSON.stringify(shown)} — ` +
2261
+ // "or depend on it" told a task that already did to do it.
2262
+ (reader.node.deps.includes(writer.node.id)
2263
+ ? `it depends on ${writer.node.id}, whose key its own already includes. ` +
2264
+ `Exclude it: add ${JSON.stringify(`!${out}`)} to ` +
2265
+ `${reader.node.id}'s inputs.${field}, or leave ${writer.node.id} out of ` +
2266
+ `${reader.node.id}'s inputs.tasks, so its key hashes those bytes instead.`
2267
+ : `a task's key must not read the outputs of a task it does not depend on, or whose key it already includes. ` +
2268
+ `Exclude it: add ${JSON.stringify(`!${out}`)} to ${reader.node.id}'s ` +
2269
+ `inputs.${field}, or depend on ${writer.node.id} and leave it out of ` +
2270
+ `${reader.node.id}'s inputs.tasks, so its key hashes those bytes instead.`),
2085
2271
  'VX_E_INPUT_IS_OUTPUT',
2086
2272
  ),
2087
2273
  })