@procedures/dag 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE +21 -0
- package/README.md +498 -2
- package/README_NEW.md +736 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/dag/builder.d.ts +25 -0
- package/dist/dag/builder.d.ts.map +1 -0
- package/dist/dag/builder.js +95 -0
- package/dist/dag/builder.js.map +1 -0
- package/dist/dag/executor.d.ts +28 -0
- package/dist/dag/executor.d.ts.map +1 -0
- package/dist/dag/executor.js +147 -0
- package/dist/dag/executor.js.map +1 -0
- package/dist/dag/index.d.ts +7 -0
- package/dist/dag/index.d.ts.map +1 -0
- package/dist/dag/index.js +7 -0
- package/dist/dag/index.js.map +1 -0
- package/dist/dag/traversal.d.ts +28 -0
- package/dist/dag/traversal.d.ts.map +1 -0
- package/dist/dag/traversal.js +103 -0
- package/dist/dag/traversal.js.map +1 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +42 -0
- package/dist/index.js.map +1 -0
- package/dist/types.d.ts +80 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +8 -0
- package/dist/types.js.map +1 -0
- package/package.json +62 -3
- package/src/dag/builder.ts +124 -0
- package/src/dag/executor.ts +190 -0
- package/src/dag/index.ts +23 -0
- package/src/dag/traversal.ts +129 -0
- package/src/index.ts +64 -0
- package/src/types.ts +87 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DAG Traversal using Kahn's Algorithm
|
|
3
|
+
*
|
|
4
|
+
* Implements topological sort with level assignment for parallel execution.
|
|
5
|
+
* Level 0 = leaves (no dependencies), higher levels depend on lower levels.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Build a leveled DAG from nodes using Kahn's algorithm
|
|
9
|
+
*
|
|
10
|
+
* This assigns levels to each node where:
|
|
11
|
+
* - Level 0: Nodes with no dependencies (leaves)
|
|
12
|
+
* - Level N: Nodes whose dependencies are all at level < N
|
|
13
|
+
*
|
|
14
|
+
* Nodes at the same level can be processed in parallel.
|
|
15
|
+
*
|
|
16
|
+
* @throws Error if a cycle is detected
|
|
17
|
+
*/
|
|
18
|
+
export function buildLeveledDAG(nodes) {
|
|
19
|
+
const inDegree = new Map();
|
|
20
|
+
const dependents = new Map();
|
|
21
|
+
for (const [name, node] of nodes) {
|
|
22
|
+
// A set: a dependency that a node lists twice counts once. (`dependents` is a set, so it
|
|
23
|
+
// takes the in-degree down once: a repeated dependency was "Circular": deep dive DATA-21.)
|
|
24
|
+
const depsInDag = new Set(node.dependencies.filter((dep) => nodes.has(dep)));
|
|
25
|
+
inDegree.set(name, depsInDag.size);
|
|
26
|
+
dependents.set(name, new Set());
|
|
27
|
+
}
|
|
28
|
+
for (const [name, node] of nodes) {
|
|
29
|
+
for (const dep of node.dependencies) {
|
|
30
|
+
if (dependents.has(dep)) {
|
|
31
|
+
dependents.get(dep).add(name);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
let queue = [];
|
|
36
|
+
for (const [name, degree] of inDegree) {
|
|
37
|
+
if (degree === 0) {
|
|
38
|
+
queue.push(name);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const levels = [];
|
|
42
|
+
const nodeLevel = new Map();
|
|
43
|
+
let processedCount = 0;
|
|
44
|
+
while (queue.length > 0) {
|
|
45
|
+
const currentLevel = levels.length;
|
|
46
|
+
const levelNodes = [];
|
|
47
|
+
const nextQueue = [];
|
|
48
|
+
for (const name of queue) {
|
|
49
|
+
const node = nodes.get(name);
|
|
50
|
+
node.level = currentLevel;
|
|
51
|
+
nodeLevel.set(name, currentLevel);
|
|
52
|
+
levelNodes.push(node);
|
|
53
|
+
processedCount++;
|
|
54
|
+
for (const dependent of dependents.get(name) ?? []) {
|
|
55
|
+
const newDegree = inDegree.get(dependent) - 1;
|
|
56
|
+
inDegree.set(dependent, newDegree);
|
|
57
|
+
if (newDegree === 0) {
|
|
58
|
+
nextQueue.push(dependent);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
levels.push(levelNodes);
|
|
63
|
+
queue = nextQueue;
|
|
64
|
+
}
|
|
65
|
+
if (processedCount !== nodes.size) {
|
|
66
|
+
const cycleNodes = [...nodes.keys()].filter((n) => !nodeLevel.has(n));
|
|
67
|
+
throw new Error("Circular dependency detected involving: " + cycleNodes.join(", "));
|
|
68
|
+
}
|
|
69
|
+
const roots = [...nodes.keys()].filter((name) => {
|
|
70
|
+
const deps = dependents.get(name);
|
|
71
|
+
return !deps || deps.size === 0;
|
|
72
|
+
});
|
|
73
|
+
const leaves = [...nodes.keys()].filter((name) => {
|
|
74
|
+
const node = nodes.get(name);
|
|
75
|
+
const depsInDag = node.dependencies.filter((dep) => nodes.has(dep));
|
|
76
|
+
return depsInDag.length === 0;
|
|
77
|
+
});
|
|
78
|
+
return { nodes, levels, roots, leaves };
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Get the topological order (flat list, leaves first)
|
|
82
|
+
*/
|
|
83
|
+
export function getTopologicalOrder(dag) {
|
|
84
|
+
return dag.levels.flat();
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Visualize the DAG structure (for debugging)
|
|
88
|
+
*/
|
|
89
|
+
export function visualizeDAG(dag) {
|
|
90
|
+
const lines = ["DAG Structure:"];
|
|
91
|
+
for (let i = 0; i < dag.levels.length; i++) {
|
|
92
|
+
const level = dag.levels[i];
|
|
93
|
+
lines.push(" Level " + i + ":");
|
|
94
|
+
for (const node of level) {
|
|
95
|
+
const deps = node.dependencies.length > 0
|
|
96
|
+
? " -> [" + node.dependencies.join(", ") + "]"
|
|
97
|
+
: " (leaf)";
|
|
98
|
+
lines.push(" - " + node.id + deps);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return lines.join("\n");
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=traversal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"traversal.js","sourceRoot":"","sources":["../../src/dag/traversal.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAyB;IAEzB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAElD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;QACjC,yFAAyF;QACzF,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACnC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QACnC,MAAM,UAAU,GAAY,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;YAC1B,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,cAAc,EAAE,CAAC;YAEjB,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAE,GAAG,CAAC,CAAC;gBAC/C,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBAEnC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;oBACpB,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,IAAI,cAAc,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,OAAO,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAAyB;IAEzB,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,GAAyB;IAEzB,MAAM,KAAK,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;gBAC1B,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG;gBAC9C,CAAC,CAAC,SAAS,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @procedures/dag
|
|
3
|
+
*
|
|
4
|
+
* Generic DAG (Directed Acyclic Graph) algorithms for dependency management.
|
|
5
|
+
*
|
|
6
|
+
* Features:
|
|
7
|
+
* - Kahn's algorithm for topological sorting with level assignment
|
|
8
|
+
* - Parallel execution with concurrency control
|
|
9
|
+
* - Cycle detection
|
|
10
|
+
* - Graph filtering and traversal utilities
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import {
|
|
15
|
+
* createNode,
|
|
16
|
+
* buildNodeMap,
|
|
17
|
+
* buildLeveledDAG,
|
|
18
|
+
* executeDAG,
|
|
19
|
+
* createProcessor
|
|
20
|
+
* } from "@procedures/dag";
|
|
21
|
+
*
|
|
22
|
+
* // Create nodes
|
|
23
|
+
* const nodes = buildNodeMap([
|
|
24
|
+
* createNode("a", []),
|
|
25
|
+
* createNode("b", ["a"]),
|
|
26
|
+
* createNode("c", ["a", "b"]),
|
|
27
|
+
* ]);
|
|
28
|
+
*
|
|
29
|
+
* // Build leveled DAG
|
|
30
|
+
* const dag = buildLeveledDAG(nodes);
|
|
31
|
+
*
|
|
32
|
+
* // Execute in dependency order
|
|
33
|
+
* const processor = createProcessor(async (node) => {
|
|
34
|
+
* console.log(`Processing ${node.id}`);
|
|
35
|
+
* });
|
|
36
|
+
*
|
|
37
|
+
* const result = await executeDAG(dag, processor, { concurrency: 2 });
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export type { DAGNode, DependencyDAG, DAGExecutionOptions, NodeResult, DAGResult, NodeProcessor, } from "./types.js";
|
|
41
|
+
export { buildLeveledDAG, getTopologicalOrder, visualizeDAG, executeDAG, executeDAGSequential, createProcessor, filterDAGFromRoot, getAncestors, getDescendants, createNode, buildNodeMap, } from "./dag/index.js";
|
|
42
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAGH,YAAY,EACV,OAAO,EACP,aAAa,EACb,mBAAmB,EACnB,UAAU,EACV,SAAS,EACT,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,eAAe,EACf,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,UAAU,EACV,YAAY,GACb,MAAM,gBAAgB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @procedures/dag
|
|
3
|
+
*
|
|
4
|
+
* Generic DAG (Directed Acyclic Graph) algorithms for dependency management.
|
|
5
|
+
*
|
|
6
|
+
* Features:
|
|
7
|
+
* - Kahn's algorithm for topological sorting with level assignment
|
|
8
|
+
* - Parallel execution with concurrency control
|
|
9
|
+
* - Cycle detection
|
|
10
|
+
* - Graph filtering and traversal utilities
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import {
|
|
15
|
+
* createNode,
|
|
16
|
+
* buildNodeMap,
|
|
17
|
+
* buildLeveledDAG,
|
|
18
|
+
* executeDAG,
|
|
19
|
+
* createProcessor
|
|
20
|
+
* } from "@procedures/dag";
|
|
21
|
+
*
|
|
22
|
+
* // Create nodes
|
|
23
|
+
* const nodes = buildNodeMap([
|
|
24
|
+
* createNode("a", []),
|
|
25
|
+
* createNode("b", ["a"]),
|
|
26
|
+
* createNode("c", ["a", "b"]),
|
|
27
|
+
* ]);
|
|
28
|
+
*
|
|
29
|
+
* // Build leveled DAG
|
|
30
|
+
* const dag = buildLeveledDAG(nodes);
|
|
31
|
+
*
|
|
32
|
+
* // Execute in dependency order
|
|
33
|
+
* const processor = createProcessor(async (node) => {
|
|
34
|
+
* console.log(`Processing ${node.id}`);
|
|
35
|
+
* });
|
|
36
|
+
*
|
|
37
|
+
* const result = await executeDAG(dag, processor, { concurrency: 2 });
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
// DAG utilities
|
|
41
|
+
export { buildLeveledDAG, getTopologicalOrder, visualizeDAG, executeDAG, executeDAGSequential, createProcessor, filterDAGFromRoot, getAncestors, getDescendants, createNode, buildNodeMap, } from "./dag/index.js";
|
|
42
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAYH,gBAAgB;AAChB,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,eAAe,EACf,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,UAAU,EACV,YAAY,GACb,MAAM,gBAAgB,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic DAG Types
|
|
3
|
+
*
|
|
4
|
+
* These types are designed to be reusable across different use cases.
|
|
5
|
+
* For ecosystem-specific DAG building, see client-lib.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* A node in a dependency DAG
|
|
9
|
+
*/
|
|
10
|
+
export interface DAGNode<TData = unknown> {
|
|
11
|
+
/** Unique identifier for this node */
|
|
12
|
+
id: string;
|
|
13
|
+
/** Names/IDs of nodes this node depends on */
|
|
14
|
+
dependencies: string[];
|
|
15
|
+
/** Topological level (0 = leaves, computed by Kahn's algorithm) */
|
|
16
|
+
level?: number;
|
|
17
|
+
/** Optional user data associated with this node */
|
|
18
|
+
data?: TData | undefined;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* A dependency DAG with level information for parallel execution
|
|
22
|
+
*/
|
|
23
|
+
export interface DependencyDAG<TNode extends DAGNode = DAGNode> {
|
|
24
|
+
/** All nodes in the DAG */
|
|
25
|
+
nodes: Map<string, TNode>;
|
|
26
|
+
/** Nodes grouped by level for parallel execution */
|
|
27
|
+
levels: TNode[][];
|
|
28
|
+
/** Root nodes (no dependents in the DAG) */
|
|
29
|
+
roots: string[];
|
|
30
|
+
/** Leaf nodes (no dependencies in the DAG) */
|
|
31
|
+
leaves: string[];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Options for DAG execution
|
|
35
|
+
*/
|
|
36
|
+
export interface DAGExecutionOptions<TNode extends DAGNode = DAGNode> {
|
|
37
|
+
/** Max parallel operations per level (default: 4) */
|
|
38
|
+
concurrency?: number;
|
|
39
|
+
/** Stop on first error vs continue (default: true) */
|
|
40
|
+
failFast?: boolean;
|
|
41
|
+
/** Callback for progress reporting */
|
|
42
|
+
onNodeStart?: (node: TNode) => void;
|
|
43
|
+
/** Callback when a node completes */
|
|
44
|
+
onNodeComplete?: (result: NodeResult<TNode>) => void;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Result of processing a single node
|
|
48
|
+
*/
|
|
49
|
+
export interface NodeResult<TNode extends DAGNode = DAGNode> {
|
|
50
|
+
/** The node that was processed */
|
|
51
|
+
node: TNode;
|
|
52
|
+
/** Whether processing succeeded */
|
|
53
|
+
success: boolean;
|
|
54
|
+
/** Error if failed */
|
|
55
|
+
error?: Error | undefined;
|
|
56
|
+
/** Duration in milliseconds */
|
|
57
|
+
duration: number;
|
|
58
|
+
/** Logs from processing */
|
|
59
|
+
logs: string[];
|
|
60
|
+
/** Optional output from the processor */
|
|
61
|
+
output?: unknown | undefined;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Result of executing an entire DAG
|
|
65
|
+
*/
|
|
66
|
+
export interface DAGResult<TNode extends DAGNode = DAGNode> {
|
|
67
|
+
/** Overall success */
|
|
68
|
+
success: boolean;
|
|
69
|
+
/** Results for each node */
|
|
70
|
+
results: Map<string, NodeResult<TNode>>;
|
|
71
|
+
/** Names of failed nodes */
|
|
72
|
+
failedNodes: string[];
|
|
73
|
+
/** Total duration in milliseconds */
|
|
74
|
+
totalDuration: number;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Processor function type
|
|
78
|
+
*/
|
|
79
|
+
export type NodeProcessor<TNode extends DAGNode = DAGNode> = (node: TNode) => Promise<NodeResult<TNode>>;
|
|
80
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,MAAM,WAAW,OAAO,CAAC,KAAK,GAAG,OAAO;IACtC,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,8CAA8C;IAC9C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IAC5D,2BAA2B;IAC3B,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC1B,oDAAoD;IACpD,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;IAClB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,8CAA8C;IAC9C,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IAClE,qDAAqD;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sCAAsC;IACtC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,KAAK,IAAI,CAAC;IACpC,qCAAqC;IACrC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC;CACtD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IACzD,kCAAkC;IAClC,IAAI,EAAE,KAAK,CAAC;IACZ,mCAAmC;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,sBAAsB;IACtB,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,yCAAyC;IACzC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,SAAS,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IACxD,sBAAsB;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,4BAA4B;IAC5B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,4BAA4B;IAC5B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO,IAAI,CAC3D,IAAI,EAAE,KAAK,KACR,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,65 @@
|
|
|
1
1
|
{
|
|
2
|
+
"$schema": "https://json.schemastore.org/package",
|
|
2
3
|
"name": "@procedures/dag",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
4
|
+
"version": "0.2.0",
|
|
5
|
+
"description": "Dependency graph algorithms for TypeScript: build a DAG, detect cycles, compute topological levels and traverse in parallel with a concurrency limit.",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"dag",
|
|
8
|
+
"graph",
|
|
9
|
+
"topological-sort",
|
|
10
|
+
"dependency-graph",
|
|
11
|
+
"cycle-detection",
|
|
12
|
+
"traversal",
|
|
13
|
+
"concurrency"
|
|
14
|
+
],
|
|
15
|
+
"homepage": "https://mark1russell7.github.io/client/",
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/mark1russell7/client/issues"
|
|
18
|
+
},
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/mark1russell7/client.git",
|
|
22
|
+
"directory": "packages/client-dag"
|
|
23
|
+
},
|
|
24
|
+
"license": "MIT",
|
|
25
|
+
"author": "Mark Russell (https://github.com/mark1russell7)",
|
|
26
|
+
"type": "module",
|
|
27
|
+
"main": "./dist/index.js",
|
|
28
|
+
"types": "./dist/index.d.ts",
|
|
29
|
+
"exports": {
|
|
30
|
+
".": {
|
|
31
|
+
"types": "./dist/index.d.ts",
|
|
32
|
+
"import": "./dist/index.js"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"files": [
|
|
36
|
+
"dist",
|
|
37
|
+
"src",
|
|
38
|
+
"!src/**/*.test.ts",
|
|
39
|
+
"!src/**/*.test.tsx",
|
|
40
|
+
"!src/**/__tests__",
|
|
41
|
+
"CHANGELOG.md"
|
|
42
|
+
],
|
|
43
|
+
"sideEffects": false,
|
|
44
|
+
"engines": {
|
|
45
|
+
"node": ">=22"
|
|
46
|
+
},
|
|
47
|
+
"publishConfig": {
|
|
48
|
+
"access": "public",
|
|
49
|
+
"provenance": true
|
|
50
|
+
},
|
|
51
|
+
"devDependencies": {
|
|
52
|
+
"typescript": "^7.0.2",
|
|
53
|
+
"@types/node": "^22.0.0",
|
|
54
|
+
"@mark1russell7/cue": "^1.0.0",
|
|
55
|
+
"vitest": "^5.0.3"
|
|
56
|
+
},
|
|
57
|
+
"scripts": {
|
|
58
|
+
"build": "tsc -b",
|
|
59
|
+
"typecheck": "tsc --noEmit",
|
|
60
|
+
"clean": "rm -rf dist .tsbuildinfo",
|
|
61
|
+
"test": "vitest run",
|
|
62
|
+
"test:watch": "vitest",
|
|
63
|
+
"test:coverage": "vitest run --coverage"
|
|
64
|
+
}
|
|
6
65
|
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DAG Builder utilities
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import type { DAGNode } from "../types.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Filter DAG to only include nodes reachable from a root
|
|
9
|
+
*/
|
|
10
|
+
export function filterDAGFromRoot<TNode extends DAGNode>(
|
|
11
|
+
nodes: Map<string, TNode>,
|
|
12
|
+
rootId: string
|
|
13
|
+
): Map<string, TNode> {
|
|
14
|
+
const filtered = new Map<string, TNode>();
|
|
15
|
+
const visited = new Set<string>();
|
|
16
|
+
|
|
17
|
+
function visit(id: string): void {
|
|
18
|
+
if (visited.has(id)) return;
|
|
19
|
+
visited.add(id);
|
|
20
|
+
|
|
21
|
+
const node = nodes.get(id);
|
|
22
|
+
if (!node) return;
|
|
23
|
+
|
|
24
|
+
filtered.set(id, node);
|
|
25
|
+
|
|
26
|
+
for (const dep of node.dependencies) {
|
|
27
|
+
visit(dep);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
visit(rootId);
|
|
32
|
+
return filtered;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Get all ancestors of a node (nodes that this node depends on, transitively)
|
|
37
|
+
*/
|
|
38
|
+
export function getAncestors<TNode extends DAGNode>(
|
|
39
|
+
nodes: Map<string, TNode>,
|
|
40
|
+
id: string
|
|
41
|
+
): Set<string> {
|
|
42
|
+
const ancestors = new Set<string>();
|
|
43
|
+
const visited = new Set<string>();
|
|
44
|
+
|
|
45
|
+
function visit(n: string): void {
|
|
46
|
+
if (visited.has(n)) return;
|
|
47
|
+
visited.add(n);
|
|
48
|
+
|
|
49
|
+
const node = nodes.get(n);
|
|
50
|
+
if (!node) return;
|
|
51
|
+
|
|
52
|
+
for (const dep of node.dependencies) {
|
|
53
|
+
if (nodes.has(dep)) {
|
|
54
|
+
ancestors.add(dep);
|
|
55
|
+
visit(dep);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
visit(id);
|
|
61
|
+
return ancestors;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Get all descendants of a node (nodes that depend on this node, transitively)
|
|
66
|
+
*/
|
|
67
|
+
export function getDescendants<TNode extends DAGNode>(
|
|
68
|
+
nodes: Map<string, TNode>,
|
|
69
|
+
id: string
|
|
70
|
+
): Set<string> {
|
|
71
|
+
const descendants = new Set<string>();
|
|
72
|
+
const visited = new Set<string>();
|
|
73
|
+
|
|
74
|
+
// Build reverse dependency map
|
|
75
|
+
const dependents = new Map<string, Set<string>>();
|
|
76
|
+
for (const [nodeId, node] of nodes) {
|
|
77
|
+
for (const dep of node.dependencies) {
|
|
78
|
+
if (!dependents.has(dep)) {
|
|
79
|
+
dependents.set(dep, new Set());
|
|
80
|
+
}
|
|
81
|
+
dependents.get(dep)!.add(nodeId);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function visit(n: string): void {
|
|
86
|
+
if (visited.has(n)) return;
|
|
87
|
+
visited.add(n);
|
|
88
|
+
|
|
89
|
+
const deps = dependents.get(n);
|
|
90
|
+
if (!deps) return;
|
|
91
|
+
|
|
92
|
+
for (const dep of deps) {
|
|
93
|
+
descendants.add(dep);
|
|
94
|
+
visit(dep);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
visit(id);
|
|
99
|
+
return descendants;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Create a DAG node from minimal input
|
|
104
|
+
*/
|
|
105
|
+
export function createNode<TData = unknown>(
|
|
106
|
+
id: string,
|
|
107
|
+
dependencies: string[] = [],
|
|
108
|
+
data?: TData | undefined
|
|
109
|
+
): DAGNode<TData> {
|
|
110
|
+
const node: DAGNode<TData> = { id, dependencies };
|
|
111
|
+
if (data !== undefined) {
|
|
112
|
+
node.data = data;
|
|
113
|
+
}
|
|
114
|
+
return node;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Build a Map of nodes from an array
|
|
119
|
+
*/
|
|
120
|
+
export function buildNodeMap<TNode extends DAGNode>(
|
|
121
|
+
nodes: TNode[]
|
|
122
|
+
): Map<string, TNode> {
|
|
123
|
+
return new Map(nodes.map((n) => [n.id, n]));
|
|
124
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DAG Executor with parallel level-based execution
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
DependencyDAG,
|
|
7
|
+
DAGNode,
|
|
8
|
+
DAGExecutionOptions,
|
|
9
|
+
NodeResult,
|
|
10
|
+
DAGResult,
|
|
11
|
+
} from "../types.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Execute a batch of items with concurrency limit
|
|
15
|
+
*/
|
|
16
|
+
async function executeWithConcurrency<T, R>(
|
|
17
|
+
items: T[],
|
|
18
|
+
fn: (item: T) => Promise<R>,
|
|
19
|
+
limit: number
|
|
20
|
+
): Promise<R[]> {
|
|
21
|
+
const results: R[] = [];
|
|
22
|
+
const executing = new Set<Promise<void>>();
|
|
23
|
+
|
|
24
|
+
for (const item of items) {
|
|
25
|
+
const promise = fn(item)
|
|
26
|
+
.then((result) => {
|
|
27
|
+
results.push(result);
|
|
28
|
+
})
|
|
29
|
+
.finally(() => {
|
|
30
|
+
// Rejection-safe cleanup: always remove the settled promise from the
|
|
31
|
+
// executing set, even if `fn` rejected, so the set never leaks a
|
|
32
|
+
// settled (possibly rejected) promise into later `race`/`all` waits.
|
|
33
|
+
executing.delete(promise);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
executing.add(promise);
|
|
37
|
+
|
|
38
|
+
if (executing.size >= limit) {
|
|
39
|
+
await Promise.race(executing);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
await Promise.all(executing);
|
|
44
|
+
return results;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Call a listener. A listener that throws does not stop the run: the function returns a log
|
|
49
|
+
* line for its error (deep dive DATA-21).
|
|
50
|
+
*/
|
|
51
|
+
function notify<T>(name: string, listener: ((arg: T) => void) | undefined, arg: T): string | undefined {
|
|
52
|
+
if (!listener) return undefined;
|
|
53
|
+
try {
|
|
54
|
+
listener(arg);
|
|
55
|
+
return undefined;
|
|
56
|
+
} catch (error) {
|
|
57
|
+
return name + " threw: " + (error instanceof Error ? error.message : String(error));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Execute DAG with level-based parallelization
|
|
63
|
+
*
|
|
64
|
+
* - Levels are processed sequentially (level 0 first, then 1, etc.)
|
|
65
|
+
* - Within each level, nodes are processed in parallel up to concurrency limit
|
|
66
|
+
* - Supports fail-fast or continue-on-error modes
|
|
67
|
+
*
|
|
68
|
+
* A processor that throws does NOT reject the whole run: the throw is captured
|
|
69
|
+
* and recorded as a failed NodeResult, so partial results are always returned.
|
|
70
|
+
*
|
|
71
|
+
* `failFast` operates at level granularity. When a node in a level fails, every
|
|
72
|
+
* node already in flight in that same level still runs to completion, but no
|
|
73
|
+
* later level is started. (There is no mid-level cancellation.)
|
|
74
|
+
*/
|
|
75
|
+
export async function executeDAG<TNode extends DAGNode>(
|
|
76
|
+
dag: DependencyDAG<TNode>,
|
|
77
|
+
processor: (node: TNode) => Promise<NodeResult<TNode>>,
|
|
78
|
+
options: DAGExecutionOptions<TNode> = {}
|
|
79
|
+
): Promise<DAGResult<TNode>> {
|
|
80
|
+
const {
|
|
81
|
+
concurrency = 4,
|
|
82
|
+
failFast = true,
|
|
83
|
+
onNodeStart,
|
|
84
|
+
onNodeComplete,
|
|
85
|
+
} = options;
|
|
86
|
+
|
|
87
|
+
const results = new Map<string, NodeResult<TNode>>();
|
|
88
|
+
const failedNodes: string[] = [];
|
|
89
|
+
const startTime = Date.now();
|
|
90
|
+
let shouldStop = false;
|
|
91
|
+
|
|
92
|
+
for (const level of dag.levels) {
|
|
93
|
+
if (shouldStop) break;
|
|
94
|
+
|
|
95
|
+
const levelResults = await executeWithConcurrency(
|
|
96
|
+
level,
|
|
97
|
+
async (node) => {
|
|
98
|
+
const startError = notify("onNodeStart", onNodeStart, node);
|
|
99
|
+
const nodeStart = Date.now();
|
|
100
|
+
let result: NodeResult<TNode>;
|
|
101
|
+
try {
|
|
102
|
+
result = await processor(node);
|
|
103
|
+
} catch (error) {
|
|
104
|
+
// Convert a thrown processor error into a failed NodeResult instead
|
|
105
|
+
// of rejecting the entire run and discarding partial results.
|
|
106
|
+
const err =
|
|
107
|
+
error instanceof Error ? error : new Error(String(error));
|
|
108
|
+
result = {
|
|
109
|
+
node,
|
|
110
|
+
success: false,
|
|
111
|
+
error: err,
|
|
112
|
+
duration: Date.now() - nodeStart,
|
|
113
|
+
logs: ["Processor threw: " + err.message],
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
const completeError = notify("onNodeComplete", onNodeComplete, result);
|
|
117
|
+
const listenerErrors = [startError, completeError].filter((line): line is string => line !== undefined);
|
|
118
|
+
if (listenerErrors.length > 0) {
|
|
119
|
+
result = { ...result, logs: [...(result.logs ?? []), ...listenerErrors] };
|
|
120
|
+
}
|
|
121
|
+
return result;
|
|
122
|
+
},
|
|
123
|
+
concurrency
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
for (const result of levelResults) {
|
|
127
|
+
results.set(result.node.id, result);
|
|
128
|
+
if (!result.success) {
|
|
129
|
+
failedNodes.push(result.node.id);
|
|
130
|
+
if (failFast) {
|
|
131
|
+
shouldStop = true;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return {
|
|
138
|
+
success: failedNodes.length === 0,
|
|
139
|
+
results,
|
|
140
|
+
failedNodes,
|
|
141
|
+
totalDuration: Date.now() - startTime,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Execute DAG sequentially (no parallelization)
|
|
147
|
+
*/
|
|
148
|
+
export async function executeDAGSequential<TNode extends DAGNode>(
|
|
149
|
+
dag: DependencyDAG<TNode>,
|
|
150
|
+
processor: (node: TNode) => Promise<NodeResult<TNode>>,
|
|
151
|
+
options: Omit<DAGExecutionOptions<TNode>, "concurrency"> = {}
|
|
152
|
+
): Promise<DAGResult<TNode>> {
|
|
153
|
+
return executeDAG(dag, processor, { ...options, concurrency: 1 });
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Create a simple processor that wraps an async function
|
|
158
|
+
*/
|
|
159
|
+
export function createProcessor<TNode extends DAGNode>(
|
|
160
|
+
fn: (node: TNode) => Promise<void>
|
|
161
|
+
): (node: TNode) => Promise<NodeResult<TNode>> {
|
|
162
|
+
return async (node: TNode): Promise<NodeResult<TNode>> => {
|
|
163
|
+
const startTime = Date.now();
|
|
164
|
+
const logs: string[] = [];
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
logs.push("Starting " + node.id);
|
|
168
|
+
await fn(node);
|
|
169
|
+
logs.push("Completed " + node.id);
|
|
170
|
+
|
|
171
|
+
return {
|
|
172
|
+
node,
|
|
173
|
+
success: true,
|
|
174
|
+
duration: Date.now() - startTime,
|
|
175
|
+
logs,
|
|
176
|
+
};
|
|
177
|
+
} catch (error) {
|
|
178
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
179
|
+
logs.push("Failed " + node.id + ": " + err.message);
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
node,
|
|
183
|
+
success: false,
|
|
184
|
+
error: err,
|
|
185
|
+
duration: Date.now() - startTime,
|
|
186
|
+
logs,
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|