@pnpm/workspace.task-scheduler 1100.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/LICENSE +22 -0
- package/README.md +3 -0
- package/lib/index.d.ts +2 -0
- package/lib/index.js +3 -0
- package/lib/taskGraph.d.ts +115 -0
- package/lib/taskGraph.js +245 -0
- package/lib/taskScheduler.d.ts +57 -0
- package/lib/taskScheduler.js +221 -0
- package/package.json +57 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# @pnpm/workspace.task-scheduler
|
|
2
|
+
|
|
3
|
+
## 1100.0.0
|
|
4
|
+
|
|
5
|
+
### Major Changes
|
|
6
|
+
|
|
7
|
+
- Workspace install, rebuild, pack, publish, stage, and lifecycle work now starts as soon as its dependencies finish instead of waiting for an unrelated topological group.
|
|
8
|
+
|
|
9
|
+
### Minor Changes
|
|
10
|
+
|
|
11
|
+
- Persist completed recursive tasks so `--resume-from` skips exactly the work that passed during a matching interrupted or failed `pnpm -r run` / `pnpm -r exec` invocation. When no compatible state exists, pnpm retains its graph-based resume behavior.
|
|
12
|
+
|
|
13
|
+
- Added per-task concurrency limits to workspace task orchestration. Set `tasks.<name>.concurrency` in `pnpm-workspace.yaml` to limit how many instances of that task may run across workspace projects at once:
|
|
14
|
+
|
|
15
|
+
```yaml
|
|
16
|
+
tasks:
|
|
17
|
+
build:
|
|
18
|
+
concurrency: 2
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
### Patch Changes
|
|
22
|
+
|
|
23
|
+
- Published the workspace task graph and scheduler as `@pnpm/workspace.task-scheduler` so other workspace commands can use the same dependency-aware scheduling as recursive run and exec.
|
|
24
|
+
|
|
25
|
+
- Updated dependencies:
|
|
26
|
+
- @pnpm/deps.graph-sequencer@1101.0.0
|
|
27
|
+
- @pnpm/types@1102.1.0
|
package/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015-2016 Rico Sta. Cruz and other contributors
|
|
4
|
+
Copyright (c) 2016-2026 Zoltan Kochan and other contributors
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
package/README.md
ADDED
package/lib/index.d.ts
ADDED
package/lib/index.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import type { PackageScripts, ProjectRootDir, ProjectsGraph, WorkspaceTasks } from '@pnpm/types';
|
|
2
|
+
/**
|
|
3
|
+
* A task is a `(project, task name)` pair; its key is the stable identifier
|
|
4
|
+
* used by the scheduler, summaries, and dry-run output.
|
|
5
|
+
*/
|
|
6
|
+
export type TaskKey = string;
|
|
7
|
+
export interface TaskNode {
|
|
8
|
+
project: ProjectRootDir;
|
|
9
|
+
taskName: string;
|
|
10
|
+
concurrency?: number;
|
|
11
|
+
/**
|
|
12
|
+
* The scripts of the project that the task name selected — several when the
|
|
13
|
+
* task name is a RegExp selector. Empty when the project has no such
|
|
14
|
+
* script: the task is then a pass-through that runs nothing, completes as
|
|
15
|
+
* soon as its dependencies have, and is reported as skipped, so that a
|
|
16
|
+
* scriptless project does not sever a dependency chain.
|
|
17
|
+
*/
|
|
18
|
+
scripts: string[];
|
|
19
|
+
/** Whether the invocation named this task, as opposed to `dependsOn` pulling it in. */
|
|
20
|
+
requested: boolean;
|
|
21
|
+
dependencies: TaskKey[];
|
|
22
|
+
}
|
|
23
|
+
export type TaskGraph = Map<TaskKey, TaskNode>;
|
|
24
|
+
export interface BuildTaskGraphOptions {
|
|
25
|
+
/**
|
|
26
|
+
* The dependency edges among the selected projects, already resolved
|
|
27
|
+
* through the full workspace graph (`filteredProjectsDependencies`). Tasks
|
|
28
|
+
* are created only for these projects: `dependsOn` never runs anything in a
|
|
29
|
+
* project the filter did not select.
|
|
30
|
+
*/
|
|
31
|
+
projectDependencies: Map<ProjectRootDir, ProjectRootDir[]>;
|
|
32
|
+
scriptsByProject: (project: ProjectRootDir) => PackageScripts;
|
|
33
|
+
selectScripts: (scripts: PackageScripts, taskName: string) => string[];
|
|
34
|
+
/** The script the invocation runs; every selected project gets a task named this. */
|
|
35
|
+
taskName: string;
|
|
36
|
+
tasks?: WorkspaceTasks;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Builds the graph of tasks the invocation runs: a task named `taskName` in
|
|
40
|
+
* every selected project, plus every task those transitively pull in through
|
|
41
|
+
* `dependsOn`. A task with no `tasks` entry behaves as
|
|
42
|
+
* `dependsOn: ['^<its own name>']`: plain topological order over the
|
|
43
|
+
* project graph.
|
|
44
|
+
*/
|
|
45
|
+
export declare function buildTaskGraph(opts: BuildTaskGraphOptions): TaskGraph;
|
|
46
|
+
export declare function taskKey(project: ProjectRootDir, taskName: string): TaskKey;
|
|
47
|
+
export interface SequenceTasksOptions {
|
|
48
|
+
workspaceDir: string;
|
|
49
|
+
/**
|
|
50
|
+
* The `ignoreWorkspaceCycles` setting: the workspace has declared its
|
|
51
|
+
* cycles deliberate, so a cyclic task graph is downgraded from an error
|
|
52
|
+
* to a warning, backward edges are dropped, and the members run in the
|
|
53
|
+
* graph sequencer's deterministic order.
|
|
54
|
+
*/
|
|
55
|
+
ignoreCycles?: boolean;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Topologically orders the task graph, throwing when the tasks form a cycle
|
|
59
|
+
* unless `ignoreCycles` tolerates it and mutates the graph's edges acyclic.
|
|
60
|
+
* Detection is scoped to this graph: a cycle among tasks the filter did not
|
|
61
|
+
* select cannot fail the run.
|
|
62
|
+
*/
|
|
63
|
+
export declare function sequenceTasks(graph: TaskGraph, opts: SequenceTasksOptions): TaskKey[];
|
|
64
|
+
export declare function formatTask(node: TaskNode, workspaceDir: string): string;
|
|
65
|
+
/** The same graph with every edge turned around: dependents run before dependencies. */
|
|
66
|
+
export declare function reverseTaskGraph(graph: TaskGraph): TaskGraph;
|
|
67
|
+
export interface ResumeTaskGraphOptions {
|
|
68
|
+
resumeFrom: string;
|
|
69
|
+
selectedProjectsGraph: ProjectsGraph;
|
|
70
|
+
/** The task of the anchor project the invocation resolves to. */
|
|
71
|
+
taskName: string;
|
|
72
|
+
/** Tasks durably completed by the matching previous invocation. */
|
|
73
|
+
completedTasks?: ReadonlySet<TaskKey>;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* When durable state is available, the graph without exactly those completed
|
|
77
|
+
* tasks. Otherwise, the graph without the anchor's transitive dependencies —
|
|
78
|
+
* the tasks inferred to have finished before a run would reach the anchor.
|
|
79
|
+
* The anchor itself and unfinished work stay, and edges into the dropped set
|
|
80
|
+
* are treated as satisfied.
|
|
81
|
+
*/
|
|
82
|
+
export declare function resumeTaskGraphFrom(graph: TaskGraph, opts: ResumeTaskGraphOptions): TaskGraph;
|
|
83
|
+
/**
|
|
84
|
+
* Whether at most one script can ever be in flight, which is when output may
|
|
85
|
+
* stay inherited rather than piped: no task runs several scripts at once, and
|
|
86
|
+
* every script-running task lies on one dependency chain, so the graph forces
|
|
87
|
+
* them to run one after another.
|
|
88
|
+
*
|
|
89
|
+
* `sequencedTasks` is {@link sequenceTasks}'s result — the proof the graph is
|
|
90
|
+
* acyclic, and the evaluation order for the longest-chain scan.
|
|
91
|
+
*/
|
|
92
|
+
export declare function isSerialTaskGraph(graph: TaskGraph, sequencedTasks: TaskKey[]): boolean;
|
|
93
|
+
export interface DryRunTaskDependency {
|
|
94
|
+
project: string;
|
|
95
|
+
script: string;
|
|
96
|
+
}
|
|
97
|
+
export interface DryRunTask extends DryRunTaskDependency {
|
|
98
|
+
missingScript: boolean;
|
|
99
|
+
dependsOn: DryRunTaskDependency[];
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* What `--dry-run --json` emits: nodes and edges rather than an order, since
|
|
103
|
+
* independent tasks have no required sequence. Identifiers are the
|
|
104
|
+
* workspace-relative project directory and the script name.
|
|
105
|
+
*/
|
|
106
|
+
export declare function taskGraphToJson(graph: TaskGraph, workspaceDir: string): {
|
|
107
|
+
tasks: DryRunTask[];
|
|
108
|
+
};
|
|
109
|
+
/**
|
|
110
|
+
* What plain `--dry-run` prints: one valid linearization of the graph — not
|
|
111
|
+
* the order the scheduler will follow. Ties among simultaneously runnable
|
|
112
|
+
* tasks are broken by project directory, so two dry runs of one workspace
|
|
113
|
+
* print the same thing and their diff is meaningful.
|
|
114
|
+
*/
|
|
115
|
+
export declare function renderTaskGraphDryRun(graph: TaskGraph, sequencedTasks: TaskKey[], workspaceDir: string): string;
|
package/lib/taskGraph.js
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { graphSequencer } from '@pnpm/deps.graph-sequencer';
|
|
3
|
+
import { PnpmError } from '@pnpm/error';
|
|
4
|
+
import { globalWarn } from '@pnpm/logger';
|
|
5
|
+
import { lexCompare } from '@pnpm/text.ordinal-comparator';
|
|
6
|
+
/**
|
|
7
|
+
* Builds the graph of tasks the invocation runs: a task named `taskName` in
|
|
8
|
+
* every selected project, plus every task those transitively pull in through
|
|
9
|
+
* `dependsOn`. A task with no `tasks` entry behaves as
|
|
10
|
+
* `dependsOn: ['^<its own name>']`: plain topological order over the
|
|
11
|
+
* project graph.
|
|
12
|
+
*/
|
|
13
|
+
export function buildTaskGraph(opts) {
|
|
14
|
+
const graph = new Map();
|
|
15
|
+
const queue = [];
|
|
16
|
+
for (const project of opts.projectDependencies.keys()) {
|
|
17
|
+
queue.push({ project, taskName: opts.taskName, requested: true });
|
|
18
|
+
}
|
|
19
|
+
// Drained by index: shift() moves every remaining element, which is
|
|
20
|
+
// quadratic over a workspace-sized queue.
|
|
21
|
+
let head = 0;
|
|
22
|
+
while (head < queue.length) {
|
|
23
|
+
const { project, taskName, requested } = queue[head++];
|
|
24
|
+
const key = taskKey(project, taskName);
|
|
25
|
+
const existing = graph.get(key);
|
|
26
|
+
if (existing != null) {
|
|
27
|
+
existing.requested ||= requested;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
const dependencies = new Set();
|
|
31
|
+
for (const entry of taskDependsOn(opts.tasks, taskName)) {
|
|
32
|
+
if (entry.startsWith('^')) {
|
|
33
|
+
const dependencyTaskName = entry.slice(1);
|
|
34
|
+
for (const dependencyProject of opts.projectDependencies.get(project) ?? []) {
|
|
35
|
+
dependencies.add(taskKey(dependencyProject, dependencyTaskName));
|
|
36
|
+
queue.push({ project: dependencyProject, taskName: dependencyTaskName, requested: false });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
dependencies.add(taskKey(project, entry));
|
|
41
|
+
queue.push({ project, taskName: entry, requested: false });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
graph.set(key, {
|
|
45
|
+
project,
|
|
46
|
+
taskName,
|
|
47
|
+
concurrency: taskConcurrency(opts.tasks, taskName),
|
|
48
|
+
scripts: opts.selectScripts(opts.scriptsByProject(project), taskName),
|
|
49
|
+
requested,
|
|
50
|
+
dependencies: [...dependencies],
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return graph;
|
|
54
|
+
}
|
|
55
|
+
function taskConcurrency(tasks, taskName) {
|
|
56
|
+
return tasks != null && Object.hasOwn(tasks, taskName)
|
|
57
|
+
? tasks[taskName].concurrency
|
|
58
|
+
: undefined;
|
|
59
|
+
}
|
|
60
|
+
export function taskKey(project, taskName) {
|
|
61
|
+
return `${project}\0${taskName}`;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The `dependsOn` entries of `taskName`. An own-property check, not a plain
|
|
65
|
+
* lookup: a script named like an `Object.prototype` member (`constructor`,
|
|
66
|
+
* `toString`, ...) must get the default rather than resolve an inherited
|
|
67
|
+
* value.
|
|
68
|
+
*/
|
|
69
|
+
function taskDependsOn(tasks, taskName) {
|
|
70
|
+
if (tasks != null && Object.hasOwn(tasks, taskName)) {
|
|
71
|
+
return tasks[taskName].dependsOn ?? [];
|
|
72
|
+
}
|
|
73
|
+
return [`^${taskName}`];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Topologically orders the task graph, throwing when the tasks form a cycle
|
|
77
|
+
* unless `ignoreCycles` tolerates it and mutates the graph's edges acyclic.
|
|
78
|
+
* Detection is scoped to this graph: a cycle among tasks the filter did not
|
|
79
|
+
* select cannot fail the run.
|
|
80
|
+
*/
|
|
81
|
+
export function sequenceTasks(graph, opts) {
|
|
82
|
+
const edges = new Map();
|
|
83
|
+
for (const [key, node] of graph) {
|
|
84
|
+
edges.set(key, node.dependencies);
|
|
85
|
+
}
|
|
86
|
+
const result = graphSequencer(edges, [...graph.keys()]);
|
|
87
|
+
if (result.cycles.length > 0) {
|
|
88
|
+
const cycles = result.cycles.map((cycle) => [...cycle, cycle[0]].map((key) => formatTask(graph.get(key), opts.workspaceDir)).join(' → ')).join('; ');
|
|
89
|
+
if (!opts.ignoreCycles) {
|
|
90
|
+
throw new PnpmError('TASK_CYCLE', `The tasks form a dependency cycle: ${cycles}`, {
|
|
91
|
+
hint: 'If the cycles are deliberate, set ignoreWorkspaceCycles to true to run their tasks in an arbitrary order.',
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
globalWarn(`The tasks form a dependency cycle and run in an arbitrary order relative to each other because ignoreWorkspaceCycles is set: ${cycles}`);
|
|
95
|
+
dropCyclicDependencies(graph, result.order);
|
|
96
|
+
}
|
|
97
|
+
return result.order;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Keeps only dependencies that point backward in the sequencer's order,
|
|
101
|
+
* making an ignored cyclic graph deterministic and runnable.
|
|
102
|
+
*/
|
|
103
|
+
function dropCyclicDependencies(graph, order) {
|
|
104
|
+
const orderIndex = new Map(order.map((key, index) => [key, index]));
|
|
105
|
+
for (const [key, node] of graph) {
|
|
106
|
+
node.dependencies = node.dependencies.filter((dependency) => orderIndex.get(dependency) < orderIndex.get(key));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
export function formatTask(node, workspaceDir) {
|
|
110
|
+
return `${relativeProjectDir(node.project, workspaceDir)}#${node.taskName}`;
|
|
111
|
+
}
|
|
112
|
+
function relativeProjectDir(project, workspaceDir) {
|
|
113
|
+
const relative = path.relative(workspaceDir, project);
|
|
114
|
+
return relative === '' ? '.' : relative.replaceAll(path.sep, '/');
|
|
115
|
+
}
|
|
116
|
+
/** The same graph with every edge turned around: dependents run before dependencies. */
|
|
117
|
+
export function reverseTaskGraph(graph) {
|
|
118
|
+
const reversed = new Map();
|
|
119
|
+
for (const [key, node] of graph) {
|
|
120
|
+
reversed.set(key, { ...node, dependencies: [] });
|
|
121
|
+
}
|
|
122
|
+
for (const [key, node] of graph) {
|
|
123
|
+
for (const dependency of node.dependencies) {
|
|
124
|
+
reversed.get(dependency).dependencies.push(key);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return reversed;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* When durable state is available, the graph without exactly those completed
|
|
131
|
+
* tasks. Otherwise, the graph without the anchor's transitive dependencies —
|
|
132
|
+
* the tasks inferred to have finished before a run would reach the anchor.
|
|
133
|
+
* The anchor itself and unfinished work stay, and edges into the dropped set
|
|
134
|
+
* are treated as satisfied.
|
|
135
|
+
*/
|
|
136
|
+
export function resumeTaskGraphFrom(graph, opts) {
|
|
137
|
+
const anchorProject = Object.keys(opts.selectedProjectsGraph)
|
|
138
|
+
.find((project) => opts.selectedProjectsGraph[project]?.package.manifest.name === opts.resumeFrom);
|
|
139
|
+
if (!anchorProject) {
|
|
140
|
+
throw new PnpmError('RESUME_FROM_NOT_FOUND', `Cannot find package ${opts.resumeFrom}. Could not determine where to resume from.`);
|
|
141
|
+
}
|
|
142
|
+
const anchor = graph.get(taskKey(anchorProject, opts.taskName));
|
|
143
|
+
if (anchor == null) {
|
|
144
|
+
// The anchor exists but its task is not in this graph (e.g. a
|
|
145
|
+
// non-recursive invocation): there is nothing to skip.
|
|
146
|
+
return graph;
|
|
147
|
+
}
|
|
148
|
+
const anchorKey = taskKey(anchorProject, opts.taskName);
|
|
149
|
+
const dropped = opts.completedTasks == null
|
|
150
|
+
? transitiveDependencies(graph, anchor)
|
|
151
|
+
: new Set([...opts.completedTasks].filter((key) => key !== anchorKey && graph.has(key)));
|
|
152
|
+
const resumed = new Map();
|
|
153
|
+
for (const [key, node] of graph) {
|
|
154
|
+
if (dropped.has(key))
|
|
155
|
+
continue;
|
|
156
|
+
resumed.set(key, { ...node, dependencies: node.dependencies.filter((dependency) => !dropped.has(dependency)) });
|
|
157
|
+
}
|
|
158
|
+
return resumed;
|
|
159
|
+
}
|
|
160
|
+
function transitiveDependencies(graph, anchor) {
|
|
161
|
+
const dependencies = new Set();
|
|
162
|
+
const stack = [...anchor.dependencies];
|
|
163
|
+
while (stack.length > 0) {
|
|
164
|
+
const key = stack.pop();
|
|
165
|
+
if (dependencies.has(key))
|
|
166
|
+
continue;
|
|
167
|
+
dependencies.add(key);
|
|
168
|
+
stack.push(...graph.get(key).dependencies);
|
|
169
|
+
}
|
|
170
|
+
return dependencies;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Whether at most one script can ever be in flight, which is when output may
|
|
174
|
+
* stay inherited rather than piped: no task runs several scripts at once, and
|
|
175
|
+
* every script-running task lies on one dependency chain, so the graph forces
|
|
176
|
+
* them to run one after another.
|
|
177
|
+
*
|
|
178
|
+
* `sequencedTasks` is {@link sequenceTasks}'s result — the proof the graph is
|
|
179
|
+
* acyclic, and the evaluation order for the longest-chain scan.
|
|
180
|
+
*/
|
|
181
|
+
export function isSerialTaskGraph(graph, sequencedTasks) {
|
|
182
|
+
let scriptTaskCount = 0;
|
|
183
|
+
for (const node of graph.values()) {
|
|
184
|
+
if (node.scripts.length > 1)
|
|
185
|
+
return false;
|
|
186
|
+
scriptTaskCount += node.scripts.length;
|
|
187
|
+
}
|
|
188
|
+
if (scriptTaskCount <= 1)
|
|
189
|
+
return true;
|
|
190
|
+
const chainLength = new Map();
|
|
191
|
+
let longestChain = 0;
|
|
192
|
+
for (const key of sequencedTasks) {
|
|
193
|
+
const node = graph.get(key);
|
|
194
|
+
let viaDependencies = 0;
|
|
195
|
+
for (const dependency of node.dependencies) {
|
|
196
|
+
viaDependencies = Math.max(viaDependencies, chainLength.get(dependency) ?? 0);
|
|
197
|
+
}
|
|
198
|
+
const length = viaDependencies + node.scripts.length;
|
|
199
|
+
chainLength.set(key, length);
|
|
200
|
+
longestChain = Math.max(longestChain, length);
|
|
201
|
+
}
|
|
202
|
+
return longestChain === scriptTaskCount;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* What `--dry-run --json` emits: nodes and edges rather than an order, since
|
|
206
|
+
* independent tasks have no required sequence. Identifiers are the
|
|
207
|
+
* workspace-relative project directory and the script name.
|
|
208
|
+
*/
|
|
209
|
+
export function taskGraphToJson(graph, workspaceDir) {
|
|
210
|
+
const tasks = [...graph.values()]
|
|
211
|
+
.map((node) => ({
|
|
212
|
+
project: relativeProjectDir(node.project, workspaceDir),
|
|
213
|
+
script: node.taskName,
|
|
214
|
+
missingScript: node.scripts.length === 0,
|
|
215
|
+
dependsOn: node.dependencies
|
|
216
|
+
.map((dependency) => {
|
|
217
|
+
const dependencyNode = graph.get(dependency);
|
|
218
|
+
return {
|
|
219
|
+
project: relativeProjectDir(dependencyNode.project, workspaceDir),
|
|
220
|
+
script: dependencyNode.taskName,
|
|
221
|
+
};
|
|
222
|
+
})
|
|
223
|
+
.sort(compareTaskIds),
|
|
224
|
+
}))
|
|
225
|
+
.sort(compareTaskIds);
|
|
226
|
+
return { tasks };
|
|
227
|
+
}
|
|
228
|
+
function compareTaskIds(left, right) {
|
|
229
|
+
return lexCompare(left.project, right.project) || lexCompare(left.script, right.script);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* What plain `--dry-run` prints: one valid linearization of the graph — not
|
|
233
|
+
* the order the scheduler will follow. Ties among simultaneously runnable
|
|
234
|
+
* tasks are broken by project directory, so two dry runs of one workspace
|
|
235
|
+
* print the same thing and their diff is meaningful.
|
|
236
|
+
*/
|
|
237
|
+
export function renderTaskGraphDryRun(graph, sequencedTasks, workspaceDir) {
|
|
238
|
+
return sequencedTasks.map((key) => {
|
|
239
|
+
const node = graph.get(key);
|
|
240
|
+
return node.scripts.length === 0
|
|
241
|
+
? `${formatTask(node, workspaceDir)} (skipped: no such script)`
|
|
242
|
+
: formatTask(node, workspaceDir);
|
|
243
|
+
}).join('\n');
|
|
244
|
+
}
|
|
245
|
+
//# sourceMappingURL=taskGraph.js.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { TaskGraph, TaskKey, TaskNode } from './taskGraph.js';
|
|
2
|
+
export type DependencyGraph<Node> = Map<Node, Node[]>;
|
|
3
|
+
export type TaskCompletion = 'passed' | 'failed'
|
|
4
|
+
/**
|
|
5
|
+
* The task's work errored before it could run — an infrastructure
|
|
6
|
+
* failure, not a script failure. Stops dispatch like a bail; the caller
|
|
7
|
+
* holds the error and rethrows it after the scheduler settles.
|
|
8
|
+
*/
|
|
9
|
+
| 'aborted';
|
|
10
|
+
export interface ScheduleTasksOptions {
|
|
11
|
+
/** When `true`, the first failure stops the run: nothing further is dispatched and the scheduler settles at once. */
|
|
12
|
+
bail: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Runs one task's work and resolves with how it ended. Never rejects:
|
|
15
|
+
* the caller records its own failure details. Not called for
|
|
16
|
+
* pass-through tasks (no scripts to run).
|
|
17
|
+
*/
|
|
18
|
+
runTask: (node: TaskNode, key: TaskKey) => Promise<TaskCompletion>;
|
|
19
|
+
/**
|
|
20
|
+
* A task that runs nothing: a pass-through with no such script, or —
|
|
21
|
+
* without `--bail` — a task some dependency of which did not pass. Both are
|
|
22
|
+
* reported as skipped.
|
|
23
|
+
*/
|
|
24
|
+
onTaskSkipped: (node: TaskNode, key: TaskKey) => void;
|
|
25
|
+
}
|
|
26
|
+
export interface ScheduleGraphOptions<Node> {
|
|
27
|
+
/** When `true`, the first failure stops the run: nothing further is dispatched and the scheduler settles at once. */
|
|
28
|
+
bail: boolean;
|
|
29
|
+
/** Maximum number of graph nodes whose work may be in flight. */
|
|
30
|
+
concurrency?: number;
|
|
31
|
+
/** Let dependents run after a failed node. Used by legacy `--no-bail` command loops. */
|
|
32
|
+
continueOnFailure?: boolean;
|
|
33
|
+
/** Wait for already-dispatched nodes after dispatch stops. Defaults to `true`. */
|
|
34
|
+
finishInFlight?: boolean;
|
|
35
|
+
runNode: (node: Node) => Promise<TaskCompletion>;
|
|
36
|
+
onNodeSkipped: (node: Node) => void;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Dispatches every task whose dependencies have all completed successfully,
|
|
40
|
+
* in dependency order and nothing else, with concurrency among ready tasks
|
|
41
|
+
* limited by the scheduler. Resolves once all tasks settled, or as
|
|
42
|
+
* soon as a bailed failure or an abort stops the run: in-flight work is then
|
|
43
|
+
* abandoned to the caller, whose exit path terminates the running commands.
|
|
44
|
+
* Tasks never dispatched are left untouched, so their caller-side status
|
|
45
|
+
* stays whatever "queued" is.
|
|
46
|
+
*
|
|
47
|
+
* The graph must be acyclic ({@link sequenceTasks} proves it); a cycle would
|
|
48
|
+
* hang this scheduler.
|
|
49
|
+
*/
|
|
50
|
+
export declare function scheduleTasks(graph: TaskGraph, opts: ScheduleTasksOptions): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Dispatches graph nodes as soon as all of their dependencies settle under the
|
|
53
|
+
* configured failure policy. Independent branches do not wait for a shared
|
|
54
|
+
* topological-group barrier. Backward edges in a cycle are dropped according
|
|
55
|
+
* to the graph sequencer's deterministic order.
|
|
56
|
+
*/
|
|
57
|
+
export declare function scheduleGraph<Node>(graph: DependencyGraph<Node>, opts: ScheduleGraphOptions<Node>): Promise<void>;
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { graphSequencer } from '@pnpm/deps.graph-sequencer';
|
|
2
|
+
/**
|
|
3
|
+
* Dispatches every task whose dependencies have all completed successfully,
|
|
4
|
+
* in dependency order and nothing else, with concurrency among ready tasks
|
|
5
|
+
* limited by the scheduler. Resolves once all tasks settled, or as
|
|
6
|
+
* soon as a bailed failure or an abort stops the run: in-flight work is then
|
|
7
|
+
* abandoned to the caller, whose exit path terminates the running commands.
|
|
8
|
+
* Tasks never dispatched are left untouched, so their caller-side status
|
|
9
|
+
* stays whatever "queued" is.
|
|
10
|
+
*
|
|
11
|
+
* The graph must be acyclic ({@link sequenceTasks} proves it); a cycle would
|
|
12
|
+
* hang this scheduler.
|
|
13
|
+
*/
|
|
14
|
+
export async function scheduleTasks(graph, opts) {
|
|
15
|
+
const dependencies = new Map();
|
|
16
|
+
for (const [key, node] of graph) {
|
|
17
|
+
dependencies.set(key, node.dependencies);
|
|
18
|
+
}
|
|
19
|
+
await scheduleGraphWithConcurrencyLimits(dependencies, {
|
|
20
|
+
bail: opts.bail,
|
|
21
|
+
finishInFlight: false,
|
|
22
|
+
runNode: async (key) => {
|
|
23
|
+
const node = graph.get(key);
|
|
24
|
+
if (node.scripts.length === 0) {
|
|
25
|
+
opts.onTaskSkipped(node, key);
|
|
26
|
+
return 'passed';
|
|
27
|
+
}
|
|
28
|
+
return opts.runTask(node, key);
|
|
29
|
+
},
|
|
30
|
+
onNodeSkipped: (key) => opts.onTaskSkipped(graph.get(key), key),
|
|
31
|
+
}, (key) => {
|
|
32
|
+
const node = graph.get(key);
|
|
33
|
+
return node.concurrency == null || node.scripts.length === 0
|
|
34
|
+
? undefined
|
|
35
|
+
: { group: node.taskName, limit: normalizeConcurrency(node.concurrency) };
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Dispatches graph nodes as soon as all of their dependencies settle under the
|
|
40
|
+
* configured failure policy. Independent branches do not wait for a shared
|
|
41
|
+
* topological-group barrier. Backward edges in a cycle are dropped according
|
|
42
|
+
* to the graph sequencer's deterministic order.
|
|
43
|
+
*/
|
|
44
|
+
export async function scheduleGraph(graph, opts) {
|
|
45
|
+
await scheduleGraphWithConcurrencyLimits(graph, opts);
|
|
46
|
+
}
|
|
47
|
+
async function scheduleGraphWithConcurrencyLimits(graph, opts, concurrencyLimit) {
|
|
48
|
+
// A rejection violates runTask's contract; held here so the run still
|
|
49
|
+
// fails with it rather than silently resolving. First error wins: a
|
|
50
|
+
// rejection landing only after something else already stopped the run is
|
|
51
|
+
// abandoned along with the rest of the in-flight work, exactly as a
|
|
52
|
+
// second script failure after a bail is.
|
|
53
|
+
let contractViolation;
|
|
54
|
+
let rejected = false;
|
|
55
|
+
const concurrency = normalizeConcurrency(opts.concurrency);
|
|
56
|
+
const pendingDependencyCount = new Map();
|
|
57
|
+
const dependents = new Map();
|
|
58
|
+
const ready = [];
|
|
59
|
+
const nodeConcurrencyGroups = new Map();
|
|
60
|
+
const concurrencyGroups = new Map();
|
|
61
|
+
const order = graphSequencer(graph).order;
|
|
62
|
+
const orderIndex = new Map(order.map((node, index) => [node, index]));
|
|
63
|
+
for (const [node, dependencies] of graph) {
|
|
64
|
+
const orderedDependencies = dependencies.filter((dependency) => orderIndex.get(dependency) < orderIndex.get(node));
|
|
65
|
+
pendingDependencyCount.set(node, orderedDependencies.length);
|
|
66
|
+
for (const dependency of orderedDependencies) {
|
|
67
|
+
let list = dependents.get(dependency);
|
|
68
|
+
if (list == null) {
|
|
69
|
+
dependents.set(dependency, list = []);
|
|
70
|
+
}
|
|
71
|
+
list.push(node);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const blocked = new Set();
|
|
75
|
+
let stopDispatch = false;
|
|
76
|
+
let unsettled = graph.size;
|
|
77
|
+
const makeReady = (node) => {
|
|
78
|
+
const concurrency = concurrencyLimit?.(node);
|
|
79
|
+
if (concurrency == null) {
|
|
80
|
+
ready.push(node);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
nodeConcurrencyGroups.set(node, concurrency.group);
|
|
84
|
+
let group = concurrencyGroups.get(concurrency.group);
|
|
85
|
+
if (group == null) {
|
|
86
|
+
concurrencyGroups.set(concurrency.group, group = {
|
|
87
|
+
limit: concurrency.limit,
|
|
88
|
+
reserved: 0,
|
|
89
|
+
waiting: [],
|
|
90
|
+
waitingHead: 0,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
if (group.reserved < group.limit) {
|
|
94
|
+
group.reserved++;
|
|
95
|
+
ready.push(node);
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
group.waiting.push(node);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
const releaseConcurrency = (node) => {
|
|
102
|
+
const groupName = nodeConcurrencyGroups.get(node);
|
|
103
|
+
if (groupName == null)
|
|
104
|
+
return;
|
|
105
|
+
const group = concurrencyGroups.get(groupName);
|
|
106
|
+
group.reserved--;
|
|
107
|
+
if (group.waitingHead < group.waiting.length) {
|
|
108
|
+
group.reserved++;
|
|
109
|
+
ready.push(group.waiting[group.waitingHead++]);
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
for (const [node, count] of pendingDependencyCount) {
|
|
113
|
+
if (count === 0)
|
|
114
|
+
makeReady(node);
|
|
115
|
+
}
|
|
116
|
+
await new Promise((resolve) => {
|
|
117
|
+
const settleIfDone = () => {
|
|
118
|
+
// Task runs may opt out because a watch-style script never finishes.
|
|
119
|
+
// Command pipelines retain their prior Promise.all behavior by waiting
|
|
120
|
+
// for work that was already dispatched.
|
|
121
|
+
if (unsettled === 0 || (stopDispatch && (opts.finishInFlight === false || active === 0))) {
|
|
122
|
+
resolve();
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
const complete = (node) => {
|
|
126
|
+
unsettled--;
|
|
127
|
+
for (const dependent of dependents.get(node) ?? []) {
|
|
128
|
+
const remaining = pendingDependencyCount.get(dependent) - 1;
|
|
129
|
+
pendingDependencyCount.set(dependent, remaining);
|
|
130
|
+
if (remaining === 0 && !blocked.has(dependent)) {
|
|
131
|
+
makeReady(dependent);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
// A failed task's transitive dependents can never become ready (their
|
|
136
|
+
// dependency count never reaches zero), so they are settled here as
|
|
137
|
+
// skipped instead.
|
|
138
|
+
const block = (node) => {
|
|
139
|
+
const stack = [node];
|
|
140
|
+
while (stack.length > 0) {
|
|
141
|
+
for (const dependent of dependents.get(stack.pop()) ?? []) {
|
|
142
|
+
if (blocked.has(dependent))
|
|
143
|
+
continue;
|
|
144
|
+
blocked.add(dependent);
|
|
145
|
+
unsettled--;
|
|
146
|
+
opts.onNodeSkipped(dependent);
|
|
147
|
+
stack.push(dependent);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
const settle = (node, completion) => {
|
|
152
|
+
switch (completion) {
|
|
153
|
+
case 'passed':
|
|
154
|
+
complete(node);
|
|
155
|
+
break;
|
|
156
|
+
case 'failed':
|
|
157
|
+
if (opts.bail) {
|
|
158
|
+
unsettled--;
|
|
159
|
+
stopDispatch = true;
|
|
160
|
+
}
|
|
161
|
+
else if (opts.continueOnFailure === true) {
|
|
162
|
+
complete(node);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
unsettled--;
|
|
166
|
+
block(node);
|
|
167
|
+
}
|
|
168
|
+
break;
|
|
169
|
+
case 'aborted':
|
|
170
|
+
unsettled--;
|
|
171
|
+
stopDispatch = true;
|
|
172
|
+
break;
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
// An explicit queue rather than recursion: a workspace-long chain of
|
|
176
|
+
// pass-through tasks completes synchronously, and call depth must not
|
|
177
|
+
// grow with chain length. Drained by index: shift() moves every
|
|
178
|
+
// remaining element, which is quadratic over a workspace-sized queue.
|
|
179
|
+
let head = 0;
|
|
180
|
+
let active = 0;
|
|
181
|
+
let pumping = false;
|
|
182
|
+
const pump = () => {
|
|
183
|
+
if (pumping)
|
|
184
|
+
return;
|
|
185
|
+
pumping = true;
|
|
186
|
+
while (!stopDispatch && active < concurrency && head < ready.length) {
|
|
187
|
+
const node = ready[head++];
|
|
188
|
+
active++;
|
|
189
|
+
opts.runNode(node).then((completion) => {
|
|
190
|
+
active--;
|
|
191
|
+
releaseConcurrency(node);
|
|
192
|
+
settle(node, completion);
|
|
193
|
+
pump();
|
|
194
|
+
}, (error) => {
|
|
195
|
+
active--;
|
|
196
|
+
releaseConcurrency(node);
|
|
197
|
+
// runTask's contract is to never reject; treated as an abort, and
|
|
198
|
+
// the error resurfaces once the scheduler settles.
|
|
199
|
+
if (!rejected) {
|
|
200
|
+
rejected = true;
|
|
201
|
+
contractViolation = error;
|
|
202
|
+
}
|
|
203
|
+
settle(node, 'aborted');
|
|
204
|
+
pump();
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
pumping = false;
|
|
208
|
+
settleIfDone();
|
|
209
|
+
};
|
|
210
|
+
pump();
|
|
211
|
+
});
|
|
212
|
+
if (rejected) {
|
|
213
|
+
throw contractViolation;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
function normalizeConcurrency(concurrency) {
|
|
217
|
+
if (concurrency === Infinity || concurrency == null)
|
|
218
|
+
return Infinity;
|
|
219
|
+
return Number.isInteger(concurrency) && concurrency > 0 ? concurrency : 1;
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=taskScheduler.js.map
|
package/package.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pnpm/workspace.task-scheduler",
|
|
3
|
+
"version": "1100.0.0",
|
|
4
|
+
"description": "Builds and schedules workspace task graphs",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pnpm",
|
|
7
|
+
"pnpm11",
|
|
8
|
+
"tasks",
|
|
9
|
+
"workspace"
|
|
10
|
+
],
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"funding": "https://opencollective.com/pnpm",
|
|
13
|
+
"repository": {
|
|
14
|
+
"type": "git",
|
|
15
|
+
"url": "https://github.com/pnpm/pnpm/tree/main/pnpm11/workspace/task-scheduler"
|
|
16
|
+
},
|
|
17
|
+
"homepage": "https://github.com/pnpm/pnpm/tree/main/pnpm11/workspace/task-scheduler#readme",
|
|
18
|
+
"bugs": {
|
|
19
|
+
"url": "https://github.com/pnpm/pnpm/issues"
|
|
20
|
+
},
|
|
21
|
+
"type": "module",
|
|
22
|
+
"main": "lib/index.js",
|
|
23
|
+
"types": "lib/index.d.ts",
|
|
24
|
+
"exports": {
|
|
25
|
+
".": "./lib/index.js"
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"lib",
|
|
29
|
+
"!*.map"
|
|
30
|
+
],
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@pnpm/deps.graph-sequencer": "1101.0.0",
|
|
33
|
+
"@pnpm/error": "1100.1.3",
|
|
34
|
+
"@pnpm/text.ordinal-comparator": "1100.0.0",
|
|
35
|
+
"@pnpm/types": "1102.1.0"
|
|
36
|
+
},
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"@pnpm/logger": "^1100.0.0"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"@jest/globals": "30.4.1",
|
|
42
|
+
"@pnpm/logger": "1100.0.0",
|
|
43
|
+
"@pnpm/workspace.task-scheduler": "1100.0.0"
|
|
44
|
+
},
|
|
45
|
+
"engines": {
|
|
46
|
+
"node": ">=22.13"
|
|
47
|
+
},
|
|
48
|
+
"jest": {
|
|
49
|
+
"preset": "@pnpm/jest-config"
|
|
50
|
+
},
|
|
51
|
+
"scripts": {
|
|
52
|
+
"lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
|
|
53
|
+
"test": "pn compile && pn .test",
|
|
54
|
+
"compile": "tsgo --build && pn lint --fix",
|
|
55
|
+
".test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest"
|
|
56
|
+
}
|
|
57
|
+
}
|