@workglow/task-graph 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +299 -1383
- package/dist/browser.js.map +11 -19
- package/dist/bun.js +299 -1383
- package/dist/bun.js.map +11 -19
- package/dist/node.js +299 -1383
- package/dist/node.js.map +11 -19
- package/dist/task/GraphAsTaskRunner.d.ts +3 -4
- package/dist/task/GraphAsTaskRunner.d.ts.map +1 -1
- package/dist/task/index.d.ts +1 -13
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +1 -1
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +1 -1
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +2 -1
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +8 -94
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +433 -0
- package/dist/task/BatchTask.d.ts +0 -154
- package/dist/task/BatchTask.d.ts.map +0 -1
- package/dist/task/ForEachTask.d.ts +0 -120
- package/dist/task/ForEachTask.d.ts.map +0 -1
- package/dist/task/IteratorTask.d.ts +0 -197
- package/dist/task/IteratorTask.d.ts.map +0 -1
- package/dist/task/IteratorTaskRunner.d.ts +0 -63
- package/dist/task/IteratorTaskRunner.d.ts.map +0 -1
- package/dist/task/MapTask.d.ts +0 -134
- package/dist/task/MapTask.d.ts.map +0 -1
- package/dist/task/ReduceTask.d.ts +0 -155
- package/dist/task/ReduceTask.d.ts.map +0 -1
- package/dist/task/WhileTask.d.ts +0 -176
- package/dist/task/WhileTask.d.ts.map +0 -1
package/dist/bun.js
CHANGED
|
@@ -1,55 +1,20 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __moduleCache = /* @__PURE__ */ new WeakMap;
|
|
7
|
-
var __toCommonJS = (from) => {
|
|
8
|
-
var entry = __moduleCache.get(from), desc;
|
|
9
|
-
if (entry)
|
|
10
|
-
return entry;
|
|
11
|
-
entry = __defProp({}, "__esModule", { value: true });
|
|
12
|
-
if (from && typeof from === "object" || typeof from === "function")
|
|
13
|
-
__getOwnPropNames(from).map((key) => !__hasOwnProp.call(entry, key) && __defProp(entry, key, {
|
|
14
|
-
get: () => from[key],
|
|
15
|
-
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
16
|
-
}));
|
|
17
|
-
__moduleCache.set(from, entry);
|
|
18
|
-
return entry;
|
|
19
|
-
};
|
|
20
|
-
var __export = (target, all) => {
|
|
21
|
-
for (var name in all)
|
|
22
|
-
__defProp(target, name, {
|
|
23
|
-
get: all[name],
|
|
24
|
-
enumerable: true,
|
|
25
|
-
configurable: true,
|
|
26
|
-
set: (newValue) => all[name] = () => newValue
|
|
27
|
-
});
|
|
28
|
-
};
|
|
29
|
-
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
2
|
+
// src/task-graph/Dataflow.ts
|
|
3
|
+
import { areSemanticallyCompatible, EventEmitter } from "@workglow/util";
|
|
30
4
|
|
|
31
5
|
// src/task/TaskTypes.ts
|
|
32
|
-
var TaskStatus
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
FAILED: "FAILED"
|
|
41
|
-
};
|
|
42
|
-
});
|
|
6
|
+
var TaskStatus = {
|
|
7
|
+
PENDING: "PENDING",
|
|
8
|
+
DISABLED: "DISABLED",
|
|
9
|
+
PROCESSING: "PROCESSING",
|
|
10
|
+
COMPLETED: "COMPLETED",
|
|
11
|
+
ABORTING: "ABORTING",
|
|
12
|
+
FAILED: "FAILED"
|
|
13
|
+
};
|
|
43
14
|
|
|
44
15
|
// src/task-graph/Dataflow.ts
|
|
45
|
-
var
|
|
46
|
-
|
|
47
|
-
DataflowArrow: () => DataflowArrow,
|
|
48
|
-
Dataflow: () => Dataflow,
|
|
49
|
-
DATAFLOW_ERROR_PORT: () => DATAFLOW_ERROR_PORT,
|
|
50
|
-
DATAFLOW_ALL_PORTS: () => DATAFLOW_ALL_PORTS
|
|
51
|
-
});
|
|
52
|
-
import { areSemanticallyCompatible, EventEmitter } from "@workglow/util";
|
|
16
|
+
var DATAFLOW_ALL_PORTS = "*";
|
|
17
|
+
var DATAFLOW_ERROR_PORT = "[error]";
|
|
53
18
|
|
|
54
19
|
class Dataflow {
|
|
55
20
|
sourceTaskId;
|
|
@@ -184,30 +149,22 @@ class Dataflow {
|
|
|
184
149
|
this._events?.emit(name, ...args);
|
|
185
150
|
}
|
|
186
151
|
}
|
|
187
|
-
var DATAFLOW_ALL_PORTS = "*", DATAFLOW_ERROR_PORT = "[error]", DataflowArrow;
|
|
188
|
-
var init_Dataflow = __esm(() => {
|
|
189
|
-
init_TaskTypes();
|
|
190
|
-
DataflowArrow = class DataflowArrow extends Dataflow {
|
|
191
|
-
constructor(dataflow) {
|
|
192
|
-
const pattern = /^([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\] ==> ([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\]$/;
|
|
193
|
-
const match = dataflow.match(pattern);
|
|
194
|
-
if (!match) {
|
|
195
|
-
throw new Error(`Invalid dataflow format: ${dataflow}`);
|
|
196
|
-
}
|
|
197
|
-
const [, sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId] = match;
|
|
198
|
-
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
199
|
-
}
|
|
200
|
-
};
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
// src/common.ts
|
|
204
|
-
init_Dataflow();
|
|
205
152
|
|
|
153
|
+
class DataflowArrow extends Dataflow {
|
|
154
|
+
constructor(dataflow) {
|
|
155
|
+
const pattern = /^([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\] ==> ([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\]$/;
|
|
156
|
+
const match = dataflow.match(pattern);
|
|
157
|
+
if (!match) {
|
|
158
|
+
throw new Error(`Invalid dataflow format: ${dataflow}`);
|
|
159
|
+
}
|
|
160
|
+
const [, sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId] = match;
|
|
161
|
+
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
206
164
|
// src/task-graph/TaskGraph.ts
|
|
207
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
165
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid43 } from "@workglow/util";
|
|
208
166
|
|
|
209
167
|
// src/task/GraphAsTask.ts
|
|
210
|
-
init_Dataflow();
|
|
211
168
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
212
169
|
|
|
213
170
|
// src/task-graph/TaskGraphRunner.ts
|
|
@@ -249,7 +206,6 @@ class TaskOutputRepository {
|
|
|
249
206
|
}
|
|
250
207
|
|
|
251
208
|
// src/task/Task.ts
|
|
252
|
-
init_Dataflow();
|
|
253
209
|
import {
|
|
254
210
|
compileSchema,
|
|
255
211
|
deepEqual,
|
|
@@ -376,8 +332,6 @@ async function resolveSchemaInputs(input, schema, config) {
|
|
|
376
332
|
}
|
|
377
333
|
|
|
378
334
|
// src/task/TaskRunner.ts
|
|
379
|
-
init_TaskTypes();
|
|
380
|
-
|
|
381
335
|
class TaskRunner {
|
|
382
336
|
running = false;
|
|
383
337
|
reactiveRunning = false;
|
|
@@ -570,8 +524,6 @@ class TaskRunner {
|
|
|
570
524
|
}
|
|
571
525
|
|
|
572
526
|
// src/task/Task.ts
|
|
573
|
-
init_TaskTypes();
|
|
574
|
-
|
|
575
527
|
class Task {
|
|
576
528
|
static type = "Task";
|
|
577
529
|
static category = "Hidden";
|
|
@@ -1070,13 +1022,7 @@ class ConditionalTask extends Task {
|
|
|
1070
1022
|
}
|
|
1071
1023
|
}
|
|
1072
1024
|
|
|
1073
|
-
// src/task-graph/TaskGraphRunner.ts
|
|
1074
|
-
init_TaskTypes();
|
|
1075
|
-
init_Dataflow();
|
|
1076
|
-
|
|
1077
1025
|
// src/task-graph/TaskGraphScheduler.ts
|
|
1078
|
-
init_TaskTypes();
|
|
1079
|
-
|
|
1080
1026
|
class TopologicalScheduler {
|
|
1081
1027
|
dag;
|
|
1082
1028
|
sortedNodes;
|
|
@@ -1256,16 +1202,18 @@ class TaskGraphRunner {
|
|
|
1256
1202
|
await this.handleComplete();
|
|
1257
1203
|
return results;
|
|
1258
1204
|
}
|
|
1259
|
-
async runGraphReactive() {
|
|
1205
|
+
async runGraphReactive(input = {}) {
|
|
1260
1206
|
await this.handleStartReactive();
|
|
1261
1207
|
const results = [];
|
|
1262
1208
|
try {
|
|
1263
1209
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1210
|
+
const isRootTask = this.graph.getSourceDataflows(task.config.id).length === 0;
|
|
1264
1211
|
if (task.status === TaskStatus.PENDING) {
|
|
1265
1212
|
task.resetInputData();
|
|
1266
1213
|
this.copyInputFromEdgesToNode(task);
|
|
1267
1214
|
}
|
|
1268
|
-
const
|
|
1215
|
+
const taskInput = isRootTask ? input : {};
|
|
1216
|
+
const taskResult = await task.runReactive(taskInput);
|
|
1269
1217
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1270
1218
|
if (this.graph.getTargetDataflows(task.config.id).length === 0) {
|
|
1271
1219
|
results.push({
|
|
@@ -1566,7 +1514,7 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
1566
1514
|
return results;
|
|
1567
1515
|
}
|
|
1568
1516
|
async executeTaskChildrenReactive() {
|
|
1569
|
-
return this.task.subGraph.runReactive();
|
|
1517
|
+
return this.task.subGraph.runReactive(this.task.runInputData);
|
|
1570
1518
|
}
|
|
1571
1519
|
async handleDisable() {
|
|
1572
1520
|
if (this.task.hasChildren()) {
|
|
@@ -1780,70 +1728,35 @@ class GraphAsTask extends Task {
|
|
|
1780
1728
|
}
|
|
1781
1729
|
}
|
|
1782
1730
|
|
|
1783
|
-
// src/task-graph/Conversions.ts
|
|
1784
|
-
init_Dataflow();
|
|
1785
|
-
|
|
1786
1731
|
// src/task-graph/Workflow.ts
|
|
1787
|
-
import { EventEmitter as EventEmitter4
|
|
1788
|
-
init_Dataflow();
|
|
1789
|
-
function CreateWorkflow(taskClass) {
|
|
1790
|
-
return Workflow.createWorkflow(taskClass);
|
|
1791
|
-
}
|
|
1792
|
-
function CreateLoopWorkflow(taskClass) {
|
|
1793
|
-
return function(config = {}) {
|
|
1794
|
-
const task = new taskClass({}, config);
|
|
1795
|
-
this.graph.addTask(task);
|
|
1796
|
-
const previousTask = getLastTask(this);
|
|
1797
|
-
if (previousTask && previousTask !== task) {
|
|
1798
|
-
this.graph.addDataflow(new Dataflow(previousTask.config.id, "*", task.config.id, "*"));
|
|
1799
|
-
}
|
|
1800
|
-
return new Workflow(this.outputCache(), this, task);
|
|
1801
|
-
};
|
|
1802
|
-
}
|
|
1803
|
-
function CreateEndLoopWorkflow(methodName) {
|
|
1804
|
-
return function() {
|
|
1805
|
-
if (!this.isLoopBuilder) {
|
|
1806
|
-
throw new Error(`${methodName}() can only be called on loop workflows`);
|
|
1807
|
-
}
|
|
1808
|
-
return this.finalizeAndReturn();
|
|
1809
|
-
};
|
|
1810
|
-
}
|
|
1811
|
-
|
|
1732
|
+
import { EventEmitter as EventEmitter4 } from "@workglow/util";
|
|
1812
1733
|
class WorkflowTask extends GraphAsTask {
|
|
1813
1734
|
static type = "Workflow";
|
|
1814
1735
|
static compoundMerge = PROPERTY_ARRAY;
|
|
1815
1736
|
}
|
|
1737
|
+
var taskIdCounter = 0;
|
|
1816
1738
|
|
|
1817
1739
|
class Workflow {
|
|
1818
|
-
constructor(
|
|
1819
|
-
this.
|
|
1820
|
-
this.
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
this.setupEvents();
|
|
1826
|
-
}
|
|
1740
|
+
constructor(repository) {
|
|
1741
|
+
this._repository = repository;
|
|
1742
|
+
this._graph = new TaskGraph({
|
|
1743
|
+
outputCache: this._repository
|
|
1744
|
+
});
|
|
1745
|
+
this._onChanged = this._onChanged.bind(this);
|
|
1746
|
+
this.setupEvents();
|
|
1827
1747
|
}
|
|
1828
1748
|
_graph;
|
|
1829
1749
|
_dataFlows = [];
|
|
1830
1750
|
_error = "";
|
|
1831
|
-
|
|
1751
|
+
_repository;
|
|
1832
1752
|
_abortController;
|
|
1833
|
-
_parentWorkflow;
|
|
1834
|
-
_iteratorTask;
|
|
1835
|
-
outputCache() {
|
|
1836
|
-
return this._outputCache;
|
|
1837
|
-
}
|
|
1838
|
-
get isLoopBuilder() {
|
|
1839
|
-
return this._parentWorkflow !== undefined;
|
|
1840
|
-
}
|
|
1841
1753
|
events = new EventEmitter4;
|
|
1842
1754
|
static createWorkflow(taskClass) {
|
|
1843
1755
|
const helper = function(input = {}, config = {}) {
|
|
1844
1756
|
this._error = "";
|
|
1845
1757
|
const parent = getLastTask(this);
|
|
1846
|
-
|
|
1758
|
+
taskIdCounter++;
|
|
1759
|
+
const task = this.addTask(taskClass, input, { id: String(taskIdCounter), ...config });
|
|
1847
1760
|
if (this._dataFlows.length > 0) {
|
|
1848
1761
|
this._dataFlows.forEach((dataflow) => {
|
|
1849
1762
|
const taskSchema = task.inputSchema();
|
|
@@ -1858,23 +1771,158 @@ class Workflow {
|
|
|
1858
1771
|
this._dataFlows = [];
|
|
1859
1772
|
}
|
|
1860
1773
|
if (parent && this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
1861
|
-
const
|
|
1862
|
-
const
|
|
1863
|
-
const
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1774
|
+
const matches = new Map;
|
|
1775
|
+
const sourceSchema = parent.outputSchema();
|
|
1776
|
+
const targetSchema = task.inputSchema();
|
|
1777
|
+
const makeMatch = (comparator) => {
|
|
1778
|
+
if (typeof sourceSchema === "object") {
|
|
1779
|
+
if (targetSchema === true || typeof targetSchema === "object" && targetSchema.additionalProperties === true) {
|
|
1780
|
+
for (const fromOutputPortId of Object.keys(sourceSchema.properties || {})) {
|
|
1781
|
+
matches.set(fromOutputPortId, fromOutputPortId);
|
|
1782
|
+
this.connect(parent.config.id, fromOutputPortId, task.config.id, fromOutputPortId);
|
|
1783
|
+
}
|
|
1784
|
+
return matches;
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
if (typeof sourceSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1788
|
+
return matches;
|
|
1789
|
+
}
|
|
1790
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(sourceSchema.properties || {})) {
|
|
1791
|
+
for (const [toInputPortId, toPortInputSchema] of Object.entries(targetSchema.properties || {})) {
|
|
1792
|
+
if (!matches.has(toInputPortId) && comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
1793
|
+
matches.set(toInputPortId, fromOutputPortId);
|
|
1794
|
+
this.connect(parent.config.id, fromOutputPortId, task.config.id, toInputPortId);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
return matches;
|
|
1799
|
+
};
|
|
1800
|
+
const getSpecificTypeIdentifiers = (schema) => {
|
|
1801
|
+
const formats = new Set;
|
|
1802
|
+
const ids = new Set;
|
|
1803
|
+
if (typeof schema === "boolean") {
|
|
1804
|
+
return { formats, ids };
|
|
1805
|
+
}
|
|
1806
|
+
const extractFromSchema = (s) => {
|
|
1807
|
+
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
1808
|
+
return;
|
|
1809
|
+
if (s.format)
|
|
1810
|
+
formats.add(s.format);
|
|
1811
|
+
if (s.$id)
|
|
1812
|
+
ids.add(s.$id);
|
|
1813
|
+
};
|
|
1814
|
+
extractFromSchema(schema);
|
|
1815
|
+
const checkUnion = (schemas) => {
|
|
1816
|
+
if (!schemas)
|
|
1817
|
+
return;
|
|
1818
|
+
for (const s of schemas) {
|
|
1819
|
+
if (typeof s === "boolean")
|
|
1820
|
+
continue;
|
|
1821
|
+
extractFromSchema(s);
|
|
1822
|
+
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
1823
|
+
extractFromSchema(s.items);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
};
|
|
1827
|
+
checkUnion(schema.oneOf);
|
|
1828
|
+
checkUnion(schema.anyOf);
|
|
1829
|
+
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
1830
|
+
extractFromSchema(schema.items);
|
|
1831
|
+
}
|
|
1832
|
+
return { formats, ids };
|
|
1833
|
+
};
|
|
1834
|
+
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
1835
|
+
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
1836
|
+
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
1837
|
+
}
|
|
1838
|
+
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
1839
|
+
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
1840
|
+
for (const format of outputIds.formats) {
|
|
1841
|
+
if (inputIds.formats.has(format)) {
|
|
1842
|
+
return true;
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
for (const id of outputIds.ids) {
|
|
1846
|
+
if (inputIds.ids.has(id)) {
|
|
1847
|
+
return true;
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
if (requireSpecificType) {
|
|
1851
|
+
return false;
|
|
1852
|
+
}
|
|
1853
|
+
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
1854
|
+
if (!idTypeBlank)
|
|
1855
|
+
return false;
|
|
1856
|
+
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
1857
|
+
return true;
|
|
1858
|
+
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
1859
|
+
if (typeof schema === "boolean")
|
|
1860
|
+
return schema;
|
|
1861
|
+
return schema.type === fromPortOutputSchema.type;
|
|
1862
|
+
}) ?? false;
|
|
1863
|
+
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
1864
|
+
if (typeof schema === "boolean")
|
|
1865
|
+
return schema;
|
|
1866
|
+
return schema.type === fromPortOutputSchema.type;
|
|
1867
|
+
}) ?? false;
|
|
1868
|
+
return matchesOneOf || matchesAnyOf;
|
|
1869
|
+
};
|
|
1870
|
+
makeMatch(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1871
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1872
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1873
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1874
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1871
1875
|
});
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1876
|
+
makeMatch(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1877
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1878
|
+
});
|
|
1879
|
+
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
1880
|
+
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
1881
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
1882
|
+
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
1883
|
+
if (unmatchedRequired.length > 0) {
|
|
1884
|
+
const nodes = this._graph.getTasks();
|
|
1885
|
+
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
1886
|
+
for (let i = parentIndex - 1;i >= 0 && unmatchedRequired.length > 0; i--) {
|
|
1887
|
+
const earlierTask = nodes[i];
|
|
1888
|
+
const earlierOutputSchema = earlierTask.outputSchema();
|
|
1889
|
+
const makeMatchFromEarlier = (comparator) => {
|
|
1890
|
+
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1891
|
+
return;
|
|
1892
|
+
}
|
|
1893
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
1894
|
+
for (const requiredInputId of unmatchedRequired) {
|
|
1895
|
+
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
1896
|
+
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
1897
|
+
matches.set(requiredInputId, fromOutputPortId);
|
|
1898
|
+
this.connect(earlierTask.config.id, fromOutputPortId, task.config.id, requiredInputId);
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
};
|
|
1903
|
+
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1904
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1905
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1906
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1907
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1908
|
+
});
|
|
1909
|
+
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1910
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1911
|
+
});
|
|
1912
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
const stillUnmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
1916
|
+
if (stillUnmatchedRequired.length > 0) {
|
|
1917
|
+
this._error = `Could not find matches for required inputs [${stillUnmatchedRequired.join(", ")}] of ${task.type}. ` + `Attempted to match from ${parent.type} and earlier tasks. Task not added.`;
|
|
1918
|
+
console.error(this._error);
|
|
1919
|
+
this.graph.removeTask(task.config.id);
|
|
1920
|
+
} else if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
1921
|
+
const hasRequiredInputs = requiredInputs.size > 0;
|
|
1922
|
+
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
1923
|
+
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
1924
|
+
if (!allRequiredInputsProvided && !hasInputsWithDefaults) {
|
|
1925
|
+
this._error = `Could not find a match between the outputs of ${parent.type} and the inputs of ${task.type}. ` + `You now need to connect the outputs to the inputs via connect() manually before adding this task. Task not added.`;
|
|
1878
1926
|
console.error(this._error);
|
|
1879
1927
|
this.graph.removeTask(task.config.id);
|
|
1880
1928
|
}
|
|
@@ -1917,16 +1965,12 @@ class Workflow {
|
|
|
1917
1965
|
return this.events.waitOn(name);
|
|
1918
1966
|
}
|
|
1919
1967
|
async run(input = {}) {
|
|
1920
|
-
if (this.isLoopBuilder) {
|
|
1921
|
-
this.finalizeTemplate();
|
|
1922
|
-
return this._parentWorkflow.run(input);
|
|
1923
|
-
}
|
|
1924
1968
|
this.events.emit("start");
|
|
1925
1969
|
this._abortController = new AbortController;
|
|
1926
1970
|
try {
|
|
1927
1971
|
const output = await this.graph.run(input, {
|
|
1928
1972
|
parentSignal: this._abortController.signal,
|
|
1929
|
-
outputCache: this.
|
|
1973
|
+
outputCache: this._repository
|
|
1930
1974
|
});
|
|
1931
1975
|
const results = this.graph.mergeExecuteOutputsToRunOutput(output, PROPERTY_ARRAY);
|
|
1932
1976
|
this.events.emit("complete");
|
|
@@ -1939,9 +1983,6 @@ class Workflow {
|
|
|
1939
1983
|
}
|
|
1940
1984
|
}
|
|
1941
1985
|
async abort() {
|
|
1942
|
-
if (this._parentWorkflow) {
|
|
1943
|
-
return this._parentWorkflow.abort();
|
|
1944
|
-
}
|
|
1945
1986
|
this._abortController?.abort();
|
|
1946
1987
|
}
|
|
1947
1988
|
pop() {
|
|
@@ -2010,12 +2051,10 @@ class Workflow {
|
|
|
2010
2051
|
return task;
|
|
2011
2052
|
}
|
|
2012
2053
|
reset() {
|
|
2013
|
-
|
|
2014
|
-
throw new WorkflowError("Cannot reset a loop workflow. Call reset() on the parent workflow.");
|
|
2015
|
-
}
|
|
2054
|
+
taskIdCounter = 0;
|
|
2016
2055
|
this.clearEvents();
|
|
2017
2056
|
this._graph = new TaskGraph({
|
|
2018
|
-
outputCache: this.
|
|
2057
|
+
outputCache: this._repository
|
|
2019
2058
|
});
|
|
2020
2059
|
this._dataFlows = [];
|
|
2021
2060
|
this._error = "";
|
|
@@ -2070,194 +2109,15 @@ class Workflow {
|
|
|
2070
2109
|
this.graph.addDataflow(dataflow);
|
|
2071
2110
|
return this;
|
|
2072
2111
|
}
|
|
2073
|
-
|
|
2112
|
+
addTask(taskClass, input, config) {
|
|
2074
2113
|
const task = new taskClass(input, config);
|
|
2075
2114
|
const id = this.graph.addTask(task);
|
|
2076
2115
|
this.events.emit("changed", id);
|
|
2077
2116
|
return task;
|
|
2078
2117
|
}
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
}
|
|
2083
|
-
static AutoConnectOptions = Symbol("AutoConnectOptions");
|
|
2084
|
-
static autoConnect(graph, sourceTask, targetTask, options) {
|
|
2085
|
-
const matches = new Map;
|
|
2086
|
-
const sourceSchema = sourceTask.outputSchema();
|
|
2087
|
-
const targetSchema = targetTask.inputSchema();
|
|
2088
|
-
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
2089
|
-
const earlierTasks = options?.earlierTasks ?? [];
|
|
2090
|
-
const getSpecificTypeIdentifiers = (schema) => {
|
|
2091
|
-
const formats = new Set;
|
|
2092
|
-
const ids = new Set;
|
|
2093
|
-
if (typeof schema === "boolean") {
|
|
2094
|
-
return { formats, ids };
|
|
2095
|
-
}
|
|
2096
|
-
const extractFromSchema = (s) => {
|
|
2097
|
-
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
2098
|
-
return;
|
|
2099
|
-
if (s.format)
|
|
2100
|
-
formats.add(s.format);
|
|
2101
|
-
if (s.$id)
|
|
2102
|
-
ids.add(s.$id);
|
|
2103
|
-
};
|
|
2104
|
-
extractFromSchema(schema);
|
|
2105
|
-
const checkUnion = (schemas) => {
|
|
2106
|
-
if (!schemas)
|
|
2107
|
-
return;
|
|
2108
|
-
for (const s of schemas) {
|
|
2109
|
-
if (typeof s === "boolean")
|
|
2110
|
-
continue;
|
|
2111
|
-
extractFromSchema(s);
|
|
2112
|
-
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
2113
|
-
extractFromSchema(s.items);
|
|
2114
|
-
}
|
|
2115
|
-
}
|
|
2116
|
-
};
|
|
2117
|
-
checkUnion(schema.oneOf);
|
|
2118
|
-
checkUnion(schema.anyOf);
|
|
2119
|
-
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
2120
|
-
extractFromSchema(schema.items);
|
|
2121
|
-
}
|
|
2122
|
-
return { formats, ids };
|
|
2123
|
-
};
|
|
2124
|
-
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
2125
|
-
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
2126
|
-
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
2127
|
-
}
|
|
2128
|
-
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
2129
|
-
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
2130
|
-
for (const format of outputIds.formats) {
|
|
2131
|
-
if (inputIds.formats.has(format)) {
|
|
2132
|
-
return true;
|
|
2133
|
-
}
|
|
2134
|
-
}
|
|
2135
|
-
for (const id of outputIds.ids) {
|
|
2136
|
-
if (inputIds.ids.has(id)) {
|
|
2137
|
-
return true;
|
|
2138
|
-
}
|
|
2139
|
-
}
|
|
2140
|
-
if (requireSpecificType) {
|
|
2141
|
-
return false;
|
|
2142
|
-
}
|
|
2143
|
-
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
2144
|
-
if (!idTypeBlank)
|
|
2145
|
-
return false;
|
|
2146
|
-
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
2147
|
-
return true;
|
|
2148
|
-
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
2149
|
-
if (typeof schema === "boolean")
|
|
2150
|
-
return schema;
|
|
2151
|
-
return schema.type === fromPortOutputSchema.type;
|
|
2152
|
-
}) ?? false;
|
|
2153
|
-
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
2154
|
-
if (typeof schema === "boolean")
|
|
2155
|
-
return schema;
|
|
2156
|
-
return schema.type === fromPortOutputSchema.type;
|
|
2157
|
-
}) ?? false;
|
|
2158
|
-
return matchesOneOf || matchesAnyOf;
|
|
2159
|
-
};
|
|
2160
|
-
const makeMatch = (fromSchema, toSchema, fromTaskId, toTaskId, comparator) => {
|
|
2161
|
-
if (typeof fromSchema === "object") {
|
|
2162
|
-
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2163
|
-
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
2164
|
-
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2165
|
-
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2166
|
-
}
|
|
2167
|
-
return;
|
|
2168
|
-
}
|
|
2169
|
-
}
|
|
2170
|
-
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2171
|
-
return;
|
|
2172
|
-
}
|
|
2173
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2174
|
-
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2175
|
-
if (!matches.has(toInputPortId) && comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
2176
|
-
matches.set(toInputPortId, fromOutputPortId);
|
|
2177
|
-
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, toInputPortId));
|
|
2178
|
-
}
|
|
2179
|
-
}
|
|
2180
|
-
}
|
|
2181
|
-
};
|
|
2182
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2183
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2184
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2185
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2186
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2187
|
-
});
|
|
2188
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2189
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2190
|
-
});
|
|
2191
|
-
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2192
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
2193
|
-
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2194
|
-
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2195
|
-
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2196
|
-
const earlierTask = earlierTasks[i];
|
|
2197
|
-
const earlierOutputSchema = earlierTask.outputSchema();
|
|
2198
|
-
const makeMatchFromEarlier = (comparator) => {
|
|
2199
|
-
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2200
|
-
return;
|
|
2201
|
-
}
|
|
2202
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
2203
|
-
for (const requiredInputId of unmatchedRequired) {
|
|
2204
|
-
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2205
|
-
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2206
|
-
matches.set(requiredInputId, fromOutputPortId);
|
|
2207
|
-
graph.addDataflow(new Dataflow(earlierTask.config.id, fromOutputPortId, targetTask.config.id, requiredInputId));
|
|
2208
|
-
}
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
|
-
};
|
|
2212
|
-
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2213
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2214
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2215
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2216
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2217
|
-
});
|
|
2218
|
-
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2219
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2220
|
-
});
|
|
2221
|
-
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
2222
|
-
}
|
|
2223
|
-
}
|
|
2224
|
-
const stillUnmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2225
|
-
if (stillUnmatchedRequired.length > 0) {
|
|
2226
|
-
return {
|
|
2227
|
-
matches,
|
|
2228
|
-
error: `Could not find matches for required inputs [${stillUnmatchedRequired.join(", ")}] of ${targetTask.type}. ` + `Attempted to match from ${sourceTask.type} and earlier tasks.`,
|
|
2229
|
-
unmatchedRequired: stillUnmatchedRequired
|
|
2230
|
-
};
|
|
2231
|
-
}
|
|
2232
|
-
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
2233
|
-
const hasRequiredInputs = requiredInputs.size > 0;
|
|
2234
|
-
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
2235
|
-
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
2236
|
-
if (!allRequiredInputsProvided && !hasInputsWithDefaults) {
|
|
2237
|
-
return {
|
|
2238
|
-
matches,
|
|
2239
|
-
error: `Could not find a match between the outputs of ${sourceTask.type} and the inputs of ${targetTask.type}. ` + `You may need to connect the outputs to the inputs via connect() manually.`,
|
|
2240
|
-
unmatchedRequired: []
|
|
2241
|
-
};
|
|
2242
|
-
}
|
|
2243
|
-
}
|
|
2244
|
-
return {
|
|
2245
|
-
matches,
|
|
2246
|
-
unmatchedRequired: []
|
|
2247
|
-
};
|
|
2248
|
-
}
|
|
2249
|
-
finalizeTemplate() {
|
|
2250
|
-
if (this._iteratorTask && this.graph.getTasks().length > 0) {
|
|
2251
|
-
this._iteratorTask.setTemplateGraph(this.graph);
|
|
2252
|
-
}
|
|
2253
|
-
}
|
|
2254
|
-
finalizeAndReturn() {
|
|
2255
|
-
if (!this._parentWorkflow) {
|
|
2256
|
-
throw new WorkflowError("finalizeAndReturn() can only be called on loop workflows");
|
|
2257
|
-
}
|
|
2258
|
-
this.finalizeTemplate();
|
|
2259
|
-
return this._parentWorkflow;
|
|
2260
|
-
}
|
|
2118
|
+
}
|
|
2119
|
+
function CreateWorkflow(taskClass) {
|
|
2120
|
+
return Workflow.createWorkflow(taskClass);
|
|
2261
2121
|
}
|
|
2262
2122
|
|
|
2263
2123
|
// src/task-graph/Conversions.ts
|
|
@@ -2380,9 +2240,6 @@ function parallel(args, mergeFn = PROPERTY_ARRAY, workflow = new Workflow) {
|
|
|
2380
2240
|
return workflow;
|
|
2381
2241
|
}
|
|
2382
2242
|
|
|
2383
|
-
// src/task-graph/TaskGraph.ts
|
|
2384
|
-
init_Dataflow();
|
|
2385
|
-
|
|
2386
2243
|
// src/task-graph/TaskGraphEvents.ts
|
|
2387
2244
|
var EventDagToTaskGraphMapping = {
|
|
2388
2245
|
"node-added": "task_added",
|
|
@@ -2428,8 +2285,8 @@ class TaskGraph {
|
|
|
2428
2285
|
parentSignal: config?.parentSignal || undefined
|
|
2429
2286
|
});
|
|
2430
2287
|
}
|
|
2431
|
-
runReactive() {
|
|
2432
|
-
return this.runner.runGraphReactive();
|
|
2288
|
+
runReactive(input = {}) {
|
|
2289
|
+
return this.runner.runGraphReactive(input);
|
|
2433
2290
|
}
|
|
2434
2291
|
mergeExecuteOutputsToRunOutput(results, compoundMerge) {
|
|
2435
2292
|
return this.runner.mergeExecuteOutputsToRunOutput(results, compoundMerge);
|
|
@@ -2500,7 +2357,7 @@ class TaskGraph {
|
|
|
2500
2357
|
return this._dag.removeNode(taskId);
|
|
2501
2358
|
}
|
|
2502
2359
|
resetGraph() {
|
|
2503
|
-
this.runner.resetGraph(this,
|
|
2360
|
+
this.runner.resetGraph(this, uuid43());
|
|
2504
2361
|
}
|
|
2505
2362
|
toJSON() {
|
|
2506
2363
|
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
@@ -2665,10 +2522,84 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
2665
2522
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
2666
2523
|
return graph;
|
|
2667
2524
|
}
|
|
2668
|
-
// src/task/
|
|
2669
|
-
import {
|
|
2525
|
+
// src/task/JobQueueFactory.ts
|
|
2526
|
+
import {
|
|
2527
|
+
JobQueueClient,
|
|
2528
|
+
JobQueueServer
|
|
2529
|
+
} from "@workglow/job-queue";
|
|
2670
2530
|
import { InMemoryQueueStorage } from "@workglow/storage";
|
|
2671
|
-
import {
|
|
2531
|
+
import { createServiceToken as createServiceToken2, globalServiceRegistry as globalServiceRegistry3 } from "@workglow/util";
|
|
2532
|
+
var JOB_QUEUE_FACTORY = createServiceToken2("taskgraph.jobQueueFactory");
|
|
2533
|
+
var defaultJobQueueFactory = async ({
|
|
2534
|
+
queueName,
|
|
2535
|
+
jobClass,
|
|
2536
|
+
options
|
|
2537
|
+
}) => {
|
|
2538
|
+
const storage = options?.storage ?? new InMemoryQueueStorage(queueName);
|
|
2539
|
+
await storage.setupDatabase();
|
|
2540
|
+
const server = new JobQueueServer(jobClass, {
|
|
2541
|
+
storage,
|
|
2542
|
+
queueName,
|
|
2543
|
+
limiter: options?.limiter,
|
|
2544
|
+
workerCount: options?.workerCount,
|
|
2545
|
+
pollIntervalMs: options?.pollIntervalMs,
|
|
2546
|
+
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
2547
|
+
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
2548
|
+
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
2549
|
+
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
2550
|
+
});
|
|
2551
|
+
const client = new JobQueueClient({
|
|
2552
|
+
storage,
|
|
2553
|
+
queueName
|
|
2554
|
+
});
|
|
2555
|
+
client.attach(server);
|
|
2556
|
+
return { server, client, storage };
|
|
2557
|
+
};
|
|
2558
|
+
function registerJobQueueFactory(factory) {
|
|
2559
|
+
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
2560
|
+
}
|
|
2561
|
+
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
2562
|
+
return async ({
|
|
2563
|
+
queueName,
|
|
2564
|
+
jobClass,
|
|
2565
|
+
options
|
|
2566
|
+
}) => {
|
|
2567
|
+
const mergedOptions = {
|
|
2568
|
+
...defaultOptions,
|
|
2569
|
+
...options ?? {}
|
|
2570
|
+
};
|
|
2571
|
+
const storage = mergedOptions.storage ?? new InMemoryQueueStorage(queueName);
|
|
2572
|
+
await storage.setupDatabase();
|
|
2573
|
+
const server = new JobQueueServer(jobClass, {
|
|
2574
|
+
storage,
|
|
2575
|
+
queueName,
|
|
2576
|
+
limiter: mergedOptions.limiter,
|
|
2577
|
+
workerCount: mergedOptions.workerCount,
|
|
2578
|
+
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
2579
|
+
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
2580
|
+
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
2581
|
+
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
2582
|
+
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
2583
|
+
});
|
|
2584
|
+
const client = new JobQueueClient({
|
|
2585
|
+
storage,
|
|
2586
|
+
queueName
|
|
2587
|
+
});
|
|
2588
|
+
client.attach(server);
|
|
2589
|
+
return { server, client, storage };
|
|
2590
|
+
};
|
|
2591
|
+
}
|
|
2592
|
+
function getJobQueueFactory() {
|
|
2593
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2594
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2595
|
+
}
|
|
2596
|
+
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
2597
|
+
}
|
|
2598
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2599
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2600
|
+
}
|
|
2601
|
+
// src/task/JobQueueTask.ts
|
|
2602
|
+
import { Job as Job2 } from "@workglow/job-queue";
|
|
2672
2603
|
|
|
2673
2604
|
// src/task/TaskQueueRegistry.ts
|
|
2674
2605
|
var taskQueueRegistry = null;
|
|
@@ -2718,659 +2649,24 @@ function setTaskQueueRegistry(registry) {
|
|
|
2718
2649
|
taskQueueRegistry = registry;
|
|
2719
2650
|
}
|
|
2720
2651
|
|
|
2721
|
-
// src/task/
|
|
2722
|
-
class
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
const queueName = this.task.config.queueName ?? `iterator-${this.task.config.id}-${uuid45().slice(0, 8)}`;
|
|
2734
|
-
this.iteratorQueueName = queueName;
|
|
2735
|
-
const existingQueue = getTaskQueueRegistry().getQueue(queueName);
|
|
2736
|
-
if (existingQueue) {
|
|
2737
|
-
this.iteratorQueue = existingQueue;
|
|
2738
|
-
return existingQueue;
|
|
2739
|
-
}
|
|
2740
|
-
const concurrency = this.getConcurrencyForMode(executionMode);
|
|
2741
|
-
this.iteratorQueue = await this.createIteratorQueue(queueName, concurrency);
|
|
2742
|
-
return this.iteratorQueue;
|
|
2743
|
-
}
|
|
2744
|
-
getConcurrencyForMode(mode) {
|
|
2745
|
-
switch (mode) {
|
|
2746
|
-
case "sequential":
|
|
2747
|
-
return 1;
|
|
2748
|
-
case "parallel-limited":
|
|
2749
|
-
return this.task.concurrencyLimit;
|
|
2750
|
-
case "batched":
|
|
2751
|
-
return this.task.batchSize;
|
|
2752
|
-
case "parallel":
|
|
2753
|
-
default:
|
|
2754
|
-
return Infinity;
|
|
2755
|
-
}
|
|
2756
|
-
}
|
|
2757
|
-
async createIteratorQueue(queueName, concurrency) {
|
|
2758
|
-
const storage = new InMemoryQueueStorage(queueName);
|
|
2759
|
-
await storage.setupDatabase();
|
|
2760
|
-
const JobClass = class extends Job {
|
|
2761
|
-
async execute(input) {
|
|
2762
|
-
return input;
|
|
2763
|
-
}
|
|
2764
|
-
};
|
|
2765
|
-
const server = new JobQueueServer(JobClass, {
|
|
2766
|
-
storage,
|
|
2767
|
-
queueName,
|
|
2768
|
-
workerCount: Math.min(concurrency, 10)
|
|
2769
|
-
});
|
|
2770
|
-
const client = new JobQueueClient({
|
|
2771
|
-
storage,
|
|
2772
|
-
queueName
|
|
2773
|
-
});
|
|
2774
|
-
client.attach(server);
|
|
2775
|
-
const queue = {
|
|
2776
|
-
server,
|
|
2777
|
-
client,
|
|
2778
|
-
storage
|
|
2779
|
-
};
|
|
2780
|
-
try {
|
|
2781
|
-
getTaskQueueRegistry().registerQueue(queue);
|
|
2782
|
-
} catch (err) {
|
|
2783
|
-
const existing = getTaskQueueRegistry().getQueue(queueName);
|
|
2784
|
-
if (existing) {
|
|
2785
|
-
return existing;
|
|
2786
|
-
}
|
|
2787
|
-
throw err;
|
|
2788
|
-
}
|
|
2789
|
-
await server.start();
|
|
2790
|
-
return queue;
|
|
2652
|
+
// src/task/JobQueueTask.ts
|
|
2653
|
+
class JobQueueTask extends GraphAsTask {
|
|
2654
|
+
static type = "JobQueueTask";
|
|
2655
|
+
static canRunDirectly = true;
|
|
2656
|
+
currentQueueName;
|
|
2657
|
+
currentJobId;
|
|
2658
|
+
currentRunnerId;
|
|
2659
|
+
jobClass;
|
|
2660
|
+
constructor(input = {}, config = {}) {
|
|
2661
|
+
config.queue ??= true;
|
|
2662
|
+
super(input, config);
|
|
2663
|
+
this.jobClass = Job2;
|
|
2791
2664
|
}
|
|
2792
|
-
async
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
case "parallel-limited":
|
|
2798
|
-
return this.executeParallelLimited(input);
|
|
2799
|
-
case "batched":
|
|
2800
|
-
return this.executeBatched(input);
|
|
2801
|
-
case "parallel":
|
|
2802
|
-
default:
|
|
2803
|
-
return super.executeTaskChildren(input);
|
|
2804
|
-
}
|
|
2805
|
-
}
|
|
2806
|
-
async executeSequential(input) {
|
|
2807
|
-
const tasks = this.task.subGraph.getTasks();
|
|
2808
|
-
const results = [];
|
|
2809
|
-
for (const task of tasks) {
|
|
2810
|
-
if (this.abortController?.signal.aborted) {
|
|
2811
|
-
break;
|
|
2812
|
-
}
|
|
2813
|
-
const taskResult = await task.run(input);
|
|
2814
|
-
results.push({
|
|
2815
|
-
id: task.config.id,
|
|
2816
|
-
type: task.type,
|
|
2817
|
-
data: taskResult
|
|
2818
|
-
});
|
|
2819
|
-
}
|
|
2820
|
-
return results;
|
|
2821
|
-
}
|
|
2822
|
-
async executeParallelLimited(input) {
|
|
2823
|
-
const tasks = this.task.subGraph.getTasks();
|
|
2824
|
-
const results = [];
|
|
2825
|
-
const limit = this.task.concurrencyLimit;
|
|
2826
|
-
for (let i = 0;i < tasks.length; i += limit) {
|
|
2827
|
-
if (this.abortController?.signal.aborted) {
|
|
2828
|
-
break;
|
|
2829
|
-
}
|
|
2830
|
-
const chunk = tasks.slice(i, i + limit);
|
|
2831
|
-
const chunkPromises = chunk.map(async (task) => {
|
|
2832
|
-
const taskResult = await task.run(input);
|
|
2833
|
-
return {
|
|
2834
|
-
id: task.config.id,
|
|
2835
|
-
type: task.type,
|
|
2836
|
-
data: taskResult
|
|
2837
|
-
};
|
|
2838
|
-
});
|
|
2839
|
-
const chunkResults = await Promise.all(chunkPromises);
|
|
2840
|
-
results.push(...chunkResults);
|
|
2841
|
-
}
|
|
2842
|
-
return results;
|
|
2843
|
-
}
|
|
2844
|
-
async executeBatched(input) {
|
|
2845
|
-
const tasks = this.task.subGraph.getTasks();
|
|
2846
|
-
const results = [];
|
|
2847
|
-
const batchSize = this.task.batchSize;
|
|
2848
|
-
for (let i = 0;i < tasks.length; i += batchSize) {
|
|
2849
|
-
if (this.abortController?.signal.aborted) {
|
|
2850
|
-
break;
|
|
2851
|
-
}
|
|
2852
|
-
const batch = tasks.slice(i, i + batchSize);
|
|
2853
|
-
const batchPromises = batch.map(async (task) => {
|
|
2854
|
-
const taskResult = await task.run(input);
|
|
2855
|
-
return {
|
|
2856
|
-
id: task.config.id,
|
|
2857
|
-
type: task.type,
|
|
2858
|
-
data: taskResult
|
|
2859
|
-
};
|
|
2860
|
-
});
|
|
2861
|
-
const batchResults = await Promise.all(batchPromises);
|
|
2862
|
-
results.push(...batchResults);
|
|
2863
|
-
const progress = Math.round((i + batch.length) / tasks.length * 100);
|
|
2864
|
-
this.task.emit("progress", progress, `Completed batch ${Math.ceil((i + 1) / batchSize)}`);
|
|
2865
|
-
}
|
|
2866
|
-
return results;
|
|
2867
|
-
}
|
|
2868
|
-
async cleanup() {
|
|
2869
|
-
if (this.iteratorQueue && this.iteratorQueueName) {
|
|
2870
|
-
try {
|
|
2871
|
-
this.iteratorQueue.server.stop();
|
|
2872
|
-
} catch (err) {}
|
|
2873
|
-
}
|
|
2874
|
-
}
|
|
2875
|
-
}
|
|
2876
|
-
|
|
2877
|
-
// src/task/IteratorTask.ts
|
|
2878
|
-
class IteratorTask extends GraphAsTask {
|
|
2879
|
-
static type = "IteratorTask";
|
|
2880
|
-
static category = "Flow Control";
|
|
2881
|
-
static title = "Iterator";
|
|
2882
|
-
static description = "Base class for loop-type tasks";
|
|
2883
|
-
static hasDynamicSchemas = true;
|
|
2884
|
-
_templateGraph;
|
|
2885
|
-
_iteratorPortInfo;
|
|
2886
|
-
constructor(input = {}, config = {}) {
|
|
2887
|
-
super(input, config);
|
|
2888
|
-
}
|
|
2889
|
-
get runner() {
|
|
2890
|
-
if (!this._runner) {
|
|
2891
|
-
this._runner = new IteratorTaskRunner(this);
|
|
2892
|
-
}
|
|
2893
|
-
return this._runner;
|
|
2894
|
-
}
|
|
2895
|
-
get executionMode() {
|
|
2896
|
-
return this.config.executionMode ?? "parallel";
|
|
2897
|
-
}
|
|
2898
|
-
get concurrencyLimit() {
|
|
2899
|
-
return this.config.concurrencyLimit ?? 5;
|
|
2900
|
-
}
|
|
2901
|
-
get batchSize() {
|
|
2902
|
-
return this.config.batchSize ?? 10;
|
|
2903
|
-
}
|
|
2904
|
-
detectIteratorPort() {
|
|
2905
|
-
if (this._iteratorPortInfo) {
|
|
2906
|
-
return this._iteratorPortInfo;
|
|
2907
|
-
}
|
|
2908
|
-
if (this.config.iteratorPort) {
|
|
2909
|
-
const schema2 = this.inputSchema();
|
|
2910
|
-
if (typeof schema2 === "boolean")
|
|
2911
|
-
return;
|
|
2912
|
-
const portSchema = schema2.properties?.[this.config.iteratorPort];
|
|
2913
|
-
if (portSchema && typeof portSchema === "object") {
|
|
2914
|
-
const itemSchema = portSchema.items ?? { type: "object" };
|
|
2915
|
-
this._iteratorPortInfo = {
|
|
2916
|
-
portName: this.config.iteratorPort,
|
|
2917
|
-
itemSchema
|
|
2918
|
-
};
|
|
2919
|
-
return this._iteratorPortInfo;
|
|
2920
|
-
}
|
|
2921
|
-
}
|
|
2922
|
-
const schema = this.inputSchema();
|
|
2923
|
-
if (typeof schema === "boolean")
|
|
2924
|
-
return;
|
|
2925
|
-
const properties = schema.properties || {};
|
|
2926
|
-
for (const [portName, portSchema] of Object.entries(properties)) {
|
|
2927
|
-
if (typeof portSchema !== "object" || portSchema === null)
|
|
2928
|
-
continue;
|
|
2929
|
-
const ps = portSchema;
|
|
2930
|
-
if (ps.type === "array" || ps.items !== undefined) {
|
|
2931
|
-
const itemSchema = ps.items ?? {
|
|
2932
|
-
type: "object",
|
|
2933
|
-
properties: {},
|
|
2934
|
-
additionalProperties: true
|
|
2935
|
-
};
|
|
2936
|
-
this._iteratorPortInfo = { portName, itemSchema };
|
|
2937
|
-
return this._iteratorPortInfo;
|
|
2938
|
-
}
|
|
2939
|
-
const variants = ps.oneOf ?? ps.anyOf;
|
|
2940
|
-
if (Array.isArray(variants)) {
|
|
2941
|
-
for (const variant of variants) {
|
|
2942
|
-
if (typeof variant === "object" && variant !== null) {
|
|
2943
|
-
const v = variant;
|
|
2944
|
-
if (v.type === "array" || v.items !== undefined) {
|
|
2945
|
-
const itemSchema = v.items ?? {
|
|
2946
|
-
type: "object",
|
|
2947
|
-
properties: {},
|
|
2948
|
-
additionalProperties: true
|
|
2949
|
-
};
|
|
2950
|
-
this._iteratorPortInfo = { portName, itemSchema };
|
|
2951
|
-
return this._iteratorPortInfo;
|
|
2952
|
-
}
|
|
2953
|
-
}
|
|
2954
|
-
}
|
|
2955
|
-
}
|
|
2956
|
-
}
|
|
2957
|
-
return;
|
|
2958
|
-
}
|
|
2959
|
-
getIteratorPortName() {
|
|
2960
|
-
return this.detectIteratorPort()?.portName;
|
|
2961
|
-
}
|
|
2962
|
-
getItemSchema() {
|
|
2963
|
-
return this.detectIteratorPort()?.itemSchema ?? {
|
|
2964
|
-
type: "object",
|
|
2965
|
-
properties: {},
|
|
2966
|
-
additionalProperties: true
|
|
2967
|
-
};
|
|
2968
|
-
}
|
|
2969
|
-
getIterableItems(input) {
|
|
2970
|
-
const portName = this.getIteratorPortName();
|
|
2971
|
-
if (!portName) {
|
|
2972
|
-
throw new TaskConfigurationError(`${this.type}: No array port found in input schema. ` + `Specify 'iteratorPort' in config or ensure input has an array-typed property.`);
|
|
2973
|
-
}
|
|
2974
|
-
const items = input[portName];
|
|
2975
|
-
if (items === undefined || items === null) {
|
|
2976
|
-
return [];
|
|
2977
|
-
}
|
|
2978
|
-
if (Array.isArray(items)) {
|
|
2979
|
-
return items;
|
|
2980
|
-
}
|
|
2981
|
-
return [items];
|
|
2982
|
-
}
|
|
2983
|
-
setTemplateGraph(graph) {
|
|
2984
|
-
this._templateGraph = graph;
|
|
2985
|
-
this.events.emit("regenerate");
|
|
2986
|
-
}
|
|
2987
|
-
getTemplateGraph() {
|
|
2988
|
-
return this._templateGraph;
|
|
2989
|
-
}
|
|
2990
|
-
regenerateGraph() {
|
|
2991
|
-
this.subGraph = new TaskGraph;
|
|
2992
|
-
if (!this._templateGraph || !this._templateGraph.getTasks().length) {
|
|
2993
|
-
super.regenerateGraph();
|
|
2994
|
-
return;
|
|
2995
|
-
}
|
|
2996
|
-
const items = this.getIterableItems(this.runInputData);
|
|
2997
|
-
if (items.length === 0) {
|
|
2998
|
-
super.regenerateGraph();
|
|
2999
|
-
return;
|
|
3000
|
-
}
|
|
3001
|
-
this.createIterationTasks(items);
|
|
3002
|
-
super.regenerateGraph();
|
|
3003
|
-
}
|
|
3004
|
-
createIterationTasks(items) {
|
|
3005
|
-
const portName = this.getIteratorPortName();
|
|
3006
|
-
if (!portName)
|
|
3007
|
-
return;
|
|
3008
|
-
const baseInput = {};
|
|
3009
|
-
for (const [key, value] of Object.entries(this.runInputData)) {
|
|
3010
|
-
if (key !== portName) {
|
|
3011
|
-
baseInput[key] = value;
|
|
3012
|
-
}
|
|
3013
|
-
}
|
|
3014
|
-
for (let i = 0;i < items.length; i++) {
|
|
3015
|
-
const item = items[i];
|
|
3016
|
-
const iterationInput = {
|
|
3017
|
-
...baseInput,
|
|
3018
|
-
[portName]: item,
|
|
3019
|
-
_iterationIndex: i,
|
|
3020
|
-
_iterationItem: item
|
|
3021
|
-
};
|
|
3022
|
-
this.cloneTemplateForIteration(iterationInput, i);
|
|
3023
|
-
}
|
|
3024
|
-
}
|
|
3025
|
-
cloneTemplateForIteration(iterationInput, index) {
|
|
3026
|
-
if (!this._templateGraph)
|
|
3027
|
-
return;
|
|
3028
|
-
const templateTasks = this._templateGraph.getTasks();
|
|
3029
|
-
const templateDataflows = this._templateGraph.getDataflows();
|
|
3030
|
-
const idMap = new Map;
|
|
3031
|
-
for (const templateTask of templateTasks) {
|
|
3032
|
-
const TaskClass = templateTask.constructor;
|
|
3033
|
-
const clonedTask = new TaskClass({ ...templateTask.defaults, ...iterationInput }, {
|
|
3034
|
-
...templateTask.config,
|
|
3035
|
-
id: `${templateTask.config.id}_iter${index}`,
|
|
3036
|
-
name: `${templateTask.config.name || templateTask.type} [${index}]`
|
|
3037
|
-
});
|
|
3038
|
-
this.subGraph.addTask(clonedTask);
|
|
3039
|
-
idMap.set(templateTask.config.id, clonedTask.config.id);
|
|
3040
|
-
}
|
|
3041
|
-
for (const templateDataflow of templateDataflows) {
|
|
3042
|
-
const sourceId = idMap.get(templateDataflow.sourceTaskId);
|
|
3043
|
-
const targetId = idMap.get(templateDataflow.targetTaskId);
|
|
3044
|
-
if (sourceId !== undefined && targetId !== undefined) {
|
|
3045
|
-
const { Dataflow: Dataflow2 } = (init_Dataflow(), __toCommonJS(exports_Dataflow));
|
|
3046
|
-
const clonedDataflow = new Dataflow2(sourceId, templateDataflow.sourceTaskPortId, targetId, templateDataflow.targetTaskPortId);
|
|
3047
|
-
this.subGraph.addDataflow(clonedDataflow);
|
|
3048
|
-
}
|
|
3049
|
-
}
|
|
3050
|
-
}
|
|
3051
|
-
async execute(input, context) {
|
|
3052
|
-
const items = this.getIterableItems(input);
|
|
3053
|
-
if (items.length === 0) {
|
|
3054
|
-
return this.getEmptyResult();
|
|
3055
|
-
}
|
|
3056
|
-
this.runInputData = { ...this.defaults, ...input };
|
|
3057
|
-
this.regenerateGraph();
|
|
3058
|
-
return super.execute(input, context);
|
|
3059
|
-
}
|
|
3060
|
-
getEmptyResult() {
|
|
3061
|
-
return {};
|
|
3062
|
-
}
|
|
3063
|
-
collectResults(results) {
|
|
3064
|
-
return results;
|
|
3065
|
-
}
|
|
3066
|
-
inputSchema() {
|
|
3067
|
-
if (this.hasChildren() || this._templateGraph) {
|
|
3068
|
-
return super.inputSchema();
|
|
3069
|
-
}
|
|
3070
|
-
return this.constructor.inputSchema();
|
|
3071
|
-
}
|
|
3072
|
-
outputSchema() {
|
|
3073
|
-
if (!this.hasChildren() && !this._templateGraph) {
|
|
3074
|
-
return this.constructor.outputSchema();
|
|
3075
|
-
}
|
|
3076
|
-
return this.getWrappedOutputSchema();
|
|
3077
|
-
}
|
|
3078
|
-
getWrappedOutputSchema() {
|
|
3079
|
-
const templateGraph = this._templateGraph ?? this.subGraph;
|
|
3080
|
-
if (!templateGraph) {
|
|
3081
|
-
return { type: "object", properties: {}, additionalProperties: false };
|
|
3082
|
-
}
|
|
3083
|
-
const tasks = templateGraph.getTasks();
|
|
3084
|
-
const endingNodes = tasks.filter((task) => templateGraph.getTargetDataflows(task.config.id).length === 0);
|
|
3085
|
-
if (endingNodes.length === 0) {
|
|
3086
|
-
return { type: "object", properties: {}, additionalProperties: false };
|
|
3087
|
-
}
|
|
3088
|
-
const properties = {};
|
|
3089
|
-
for (const task of endingNodes) {
|
|
3090
|
-
const taskOutputSchema = task.outputSchema();
|
|
3091
|
-
if (typeof taskOutputSchema === "boolean")
|
|
3092
|
-
continue;
|
|
3093
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
3094
|
-
for (const [key, schema] of Object.entries(taskProperties)) {
|
|
3095
|
-
properties[key] = {
|
|
3096
|
-
type: "array",
|
|
3097
|
-
items: schema
|
|
3098
|
-
};
|
|
3099
|
-
}
|
|
3100
|
-
}
|
|
3101
|
-
return {
|
|
3102
|
-
type: "object",
|
|
3103
|
-
properties,
|
|
3104
|
-
additionalProperties: false
|
|
3105
|
-
};
|
|
3106
|
-
}
|
|
3107
|
-
}
|
|
3108
|
-
|
|
3109
|
-
// src/task/BatchTask.ts
|
|
3110
|
-
class BatchTask extends IteratorTask {
|
|
3111
|
-
static type = "BatchTask";
|
|
3112
|
-
static category = "Flow Control";
|
|
3113
|
-
static title = "Batch";
|
|
3114
|
-
static description = "Processes an array in configurable batches";
|
|
3115
|
-
static compoundMerge = PROPERTY_ARRAY;
|
|
3116
|
-
static inputSchema() {
|
|
3117
|
-
return {
|
|
3118
|
-
type: "object",
|
|
3119
|
-
properties: {},
|
|
3120
|
-
additionalProperties: true
|
|
3121
|
-
};
|
|
3122
|
-
}
|
|
3123
|
-
static outputSchema() {
|
|
3124
|
-
return {
|
|
3125
|
-
type: "object",
|
|
3126
|
-
properties: {},
|
|
3127
|
-
additionalProperties: true
|
|
3128
|
-
};
|
|
3129
|
-
}
|
|
3130
|
-
get batchSize() {
|
|
3131
|
-
return this.config.batchSize ?? 10;
|
|
3132
|
-
}
|
|
3133
|
-
get flattenResults() {
|
|
3134
|
-
return this.config.flattenResults ?? true;
|
|
3135
|
-
}
|
|
3136
|
-
get batchExecutionMode() {
|
|
3137
|
-
return this.config.batchExecutionMode ?? "sequential";
|
|
3138
|
-
}
|
|
3139
|
-
getIterableItems(input) {
|
|
3140
|
-
const items = super.getIterableItems(input);
|
|
3141
|
-
return this.groupIntoBatches(items);
|
|
3142
|
-
}
|
|
3143
|
-
groupIntoBatches(items) {
|
|
3144
|
-
const batches = [];
|
|
3145
|
-
const size = this.batchSize;
|
|
3146
|
-
for (let i = 0;i < items.length; i += size) {
|
|
3147
|
-
batches.push(items.slice(i, i + size));
|
|
3148
|
-
}
|
|
3149
|
-
return batches;
|
|
3150
|
-
}
|
|
3151
|
-
createIterationTasks(batches) {
|
|
3152
|
-
const portName = this.getIteratorPortName();
|
|
3153
|
-
if (!portName)
|
|
3154
|
-
return;
|
|
3155
|
-
const baseInput = {};
|
|
3156
|
-
for (const [key, value] of Object.entries(this.runInputData)) {
|
|
3157
|
-
if (key !== portName) {
|
|
3158
|
-
baseInput[key] = value;
|
|
3159
|
-
}
|
|
3160
|
-
}
|
|
3161
|
-
for (let i = 0;i < batches.length; i++) {
|
|
3162
|
-
const batch = batches[i];
|
|
3163
|
-
const batchInput = {
|
|
3164
|
-
...baseInput,
|
|
3165
|
-
[portName]: batch,
|
|
3166
|
-
_batchIndex: i,
|
|
3167
|
-
_batchItems: batch
|
|
3168
|
-
};
|
|
3169
|
-
this.cloneTemplateForIteration(batchInput, i);
|
|
3170
|
-
}
|
|
3171
|
-
}
|
|
3172
|
-
getEmptyResult() {
|
|
3173
|
-
const schema = this.outputSchema();
|
|
3174
|
-
if (typeof schema === "boolean") {
|
|
3175
|
-
return {};
|
|
3176
|
-
}
|
|
3177
|
-
const result = {};
|
|
3178
|
-
for (const key of Object.keys(schema.properties || {})) {
|
|
3179
|
-
result[key] = [];
|
|
3180
|
-
}
|
|
3181
|
-
return result;
|
|
3182
|
-
}
|
|
3183
|
-
outputSchema() {
|
|
3184
|
-
if (!this.hasChildren() && !this._templateGraph) {
|
|
3185
|
-
return this.constructor.outputSchema();
|
|
3186
|
-
}
|
|
3187
|
-
return this.getWrappedOutputSchema();
|
|
3188
|
-
}
|
|
3189
|
-
collectResults(results) {
|
|
3190
|
-
const collected = super.collectResults(results);
|
|
3191
|
-
if (!this.flattenResults || typeof collected !== "object" || collected === null) {
|
|
3192
|
-
return collected;
|
|
3193
|
-
}
|
|
3194
|
-
const flattened = {};
|
|
3195
|
-
for (const [key, value] of Object.entries(collected)) {
|
|
3196
|
-
if (Array.isArray(value)) {
|
|
3197
|
-
flattened[key] = value.flat(2);
|
|
3198
|
-
} else {
|
|
3199
|
-
flattened[key] = value;
|
|
3200
|
-
}
|
|
3201
|
-
}
|
|
3202
|
-
return flattened;
|
|
3203
|
-
}
|
|
3204
|
-
regenerateGraph() {
|
|
3205
|
-
this.subGraph = new TaskGraph;
|
|
3206
|
-
if (!this._templateGraph || !this._templateGraph.getTasks().length) {
|
|
3207
|
-
super.regenerateGraph();
|
|
3208
|
-
return;
|
|
3209
|
-
}
|
|
3210
|
-
const batches = this.getIterableItems(this.runInputData);
|
|
3211
|
-
if (batches.length === 0) {
|
|
3212
|
-
super.regenerateGraph();
|
|
3213
|
-
return;
|
|
3214
|
-
}
|
|
3215
|
-
this.createIterationTasks(batches);
|
|
3216
|
-
this.events.emit("regenerate");
|
|
3217
|
-
}
|
|
3218
|
-
}
|
|
3219
|
-
Workflow.prototype.batch = CreateLoopWorkflow(BatchTask);
|
|
3220
|
-
Workflow.prototype.endBatch = CreateEndLoopWorkflow("endBatch");
|
|
3221
|
-
// src/task/ForEachTask.ts
|
|
3222
|
-
class ForEachTask extends IteratorTask {
|
|
3223
|
-
static type = "ForEachTask";
|
|
3224
|
-
static category = "Flow Control";
|
|
3225
|
-
static title = "For Each";
|
|
3226
|
-
static description = "Iterates over an array and runs a workflow for each element";
|
|
3227
|
-
static inputSchema() {
|
|
3228
|
-
return {
|
|
3229
|
-
type: "object",
|
|
3230
|
-
properties: {},
|
|
3231
|
-
additionalProperties: true
|
|
3232
|
-
};
|
|
3233
|
-
}
|
|
3234
|
-
static outputSchema() {
|
|
3235
|
-
return {
|
|
3236
|
-
type: "object",
|
|
3237
|
-
properties: {
|
|
3238
|
-
completed: {
|
|
3239
|
-
type: "boolean",
|
|
3240
|
-
title: "Completed",
|
|
3241
|
-
description: "Whether all iterations completed successfully"
|
|
3242
|
-
},
|
|
3243
|
-
count: {
|
|
3244
|
-
type: "number",
|
|
3245
|
-
title: "Count",
|
|
3246
|
-
description: "Number of items processed"
|
|
3247
|
-
}
|
|
3248
|
-
},
|
|
3249
|
-
additionalProperties: false
|
|
3250
|
-
};
|
|
3251
|
-
}
|
|
3252
|
-
get shouldCollectResults() {
|
|
3253
|
-
return this.config.shouldCollectResults ?? false;
|
|
3254
|
-
}
|
|
3255
|
-
getEmptyResult() {
|
|
3256
|
-
return {
|
|
3257
|
-
completed: true,
|
|
3258
|
-
count: 0
|
|
3259
|
-
};
|
|
3260
|
-
}
|
|
3261
|
-
outputSchema() {
|
|
3262
|
-
if (this.shouldCollectResults && (this.hasChildren() || this._templateGraph)) {
|
|
3263
|
-
return this.getWrappedOutputSchema();
|
|
3264
|
-
}
|
|
3265
|
-
return this.constructor.outputSchema();
|
|
3266
|
-
}
|
|
3267
|
-
collectResults(results) {
|
|
3268
|
-
if (this.config.shouldCollectResults) {
|
|
3269
|
-
return super.collectResults(results);
|
|
3270
|
-
}
|
|
3271
|
-
return {
|
|
3272
|
-
completed: true,
|
|
3273
|
-
count: results.length
|
|
3274
|
-
};
|
|
3275
|
-
}
|
|
3276
|
-
}
|
|
3277
|
-
Workflow.prototype.forEach = CreateLoopWorkflow(ForEachTask);
|
|
3278
|
-
Workflow.prototype.endForEach = CreateEndLoopWorkflow("endForEach");
|
|
3279
|
-
// src/task/JobQueueFactory.ts
|
|
3280
|
-
import {
|
|
3281
|
-
JobQueueClient as JobQueueClient2,
|
|
3282
|
-
JobQueueServer as JobQueueServer2
|
|
3283
|
-
} from "@workglow/job-queue";
|
|
3284
|
-
import { InMemoryQueueStorage as InMemoryQueueStorage2 } from "@workglow/storage";
|
|
3285
|
-
import { createServiceToken as createServiceToken2, globalServiceRegistry as globalServiceRegistry3 } from "@workglow/util";
|
|
3286
|
-
var JOB_QUEUE_FACTORY = createServiceToken2("taskgraph.jobQueueFactory");
|
|
3287
|
-
var defaultJobQueueFactory = async ({
|
|
3288
|
-
queueName,
|
|
3289
|
-
jobClass,
|
|
3290
|
-
options
|
|
3291
|
-
}) => {
|
|
3292
|
-
const storage = options?.storage ?? new InMemoryQueueStorage2(queueName);
|
|
3293
|
-
await storage.setupDatabase();
|
|
3294
|
-
const server = new JobQueueServer2(jobClass, {
|
|
3295
|
-
storage,
|
|
3296
|
-
queueName,
|
|
3297
|
-
limiter: options?.limiter,
|
|
3298
|
-
workerCount: options?.workerCount,
|
|
3299
|
-
pollIntervalMs: options?.pollIntervalMs,
|
|
3300
|
-
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
3301
|
-
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
3302
|
-
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
3303
|
-
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
3304
|
-
});
|
|
3305
|
-
const client = new JobQueueClient2({
|
|
3306
|
-
storage,
|
|
3307
|
-
queueName
|
|
3308
|
-
});
|
|
3309
|
-
client.attach(server);
|
|
3310
|
-
return { server, client, storage };
|
|
3311
|
-
};
|
|
3312
|
-
function registerJobQueueFactory(factory) {
|
|
3313
|
-
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
3314
|
-
}
|
|
3315
|
-
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
3316
|
-
return async ({
|
|
3317
|
-
queueName,
|
|
3318
|
-
jobClass,
|
|
3319
|
-
options
|
|
3320
|
-
}) => {
|
|
3321
|
-
const mergedOptions = {
|
|
3322
|
-
...defaultOptions,
|
|
3323
|
-
...options ?? {}
|
|
3324
|
-
};
|
|
3325
|
-
const storage = mergedOptions.storage ?? new InMemoryQueueStorage2(queueName);
|
|
3326
|
-
await storage.setupDatabase();
|
|
3327
|
-
const server = new JobQueueServer2(jobClass, {
|
|
3328
|
-
storage,
|
|
3329
|
-
queueName,
|
|
3330
|
-
limiter: mergedOptions.limiter,
|
|
3331
|
-
workerCount: mergedOptions.workerCount,
|
|
3332
|
-
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
3333
|
-
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
3334
|
-
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
3335
|
-
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
3336
|
-
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
3337
|
-
});
|
|
3338
|
-
const client = new JobQueueClient2({
|
|
3339
|
-
storage,
|
|
3340
|
-
queueName
|
|
3341
|
-
});
|
|
3342
|
-
client.attach(server);
|
|
3343
|
-
return { server, client, storage };
|
|
3344
|
-
};
|
|
3345
|
-
}
|
|
3346
|
-
function getJobQueueFactory() {
|
|
3347
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
3348
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
3349
|
-
}
|
|
3350
|
-
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
3351
|
-
}
|
|
3352
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
3353
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
3354
|
-
}
|
|
3355
|
-
// src/task/JobQueueTask.ts
|
|
3356
|
-
import { Job as Job3 } from "@workglow/job-queue";
|
|
3357
|
-
class JobQueueTask extends GraphAsTask {
|
|
3358
|
-
static type = "JobQueueTask";
|
|
3359
|
-
static canRunDirectly = true;
|
|
3360
|
-
currentQueueName;
|
|
3361
|
-
currentJobId;
|
|
3362
|
-
currentRunnerId;
|
|
3363
|
-
jobClass;
|
|
3364
|
-
constructor(input = {}, config = {}) {
|
|
3365
|
-
config.queue ??= true;
|
|
3366
|
-
super(input, config);
|
|
3367
|
-
this.jobClass = Job3;
|
|
3368
|
-
}
|
|
3369
|
-
async execute(input, executeContext) {
|
|
3370
|
-
let cleanup = () => {};
|
|
3371
|
-
try {
|
|
3372
|
-
if (this.config.queue === false && !this.constructor.canRunDirectly) {
|
|
3373
|
-
throw new TaskConfigurationError(`${this.type} cannot run directly without a queue`);
|
|
2665
|
+
async execute(input, executeContext) {
|
|
2666
|
+
let cleanup = () => {};
|
|
2667
|
+
try {
|
|
2668
|
+
if (this.config.queue === false && !this.constructor.canRunDirectly) {
|
|
2669
|
+
throw new TaskConfigurationError(`${this.type} cannot run directly without a queue`);
|
|
3374
2670
|
}
|
|
3375
2671
|
const registeredQueue = await this.resolveQueue(input);
|
|
3376
2672
|
if (!registeredQueue) {
|
|
@@ -3486,219 +2782,6 @@ class JobQueueTask extends GraphAsTask {
|
|
|
3486
2782
|
super.abort();
|
|
3487
2783
|
}
|
|
3488
2784
|
}
|
|
3489
|
-
// src/task/MapTask.ts
|
|
3490
|
-
class MapTask extends IteratorTask {
|
|
3491
|
-
static type = "MapTask";
|
|
3492
|
-
static category = "Flow Control";
|
|
3493
|
-
static title = "Map";
|
|
3494
|
-
static description = "Transforms an array by running a workflow for each element";
|
|
3495
|
-
static compoundMerge = PROPERTY_ARRAY;
|
|
3496
|
-
static inputSchema() {
|
|
3497
|
-
return {
|
|
3498
|
-
type: "object",
|
|
3499
|
-
properties: {},
|
|
3500
|
-
additionalProperties: true
|
|
3501
|
-
};
|
|
3502
|
-
}
|
|
3503
|
-
static outputSchema() {
|
|
3504
|
-
return {
|
|
3505
|
-
type: "object",
|
|
3506
|
-
properties: {},
|
|
3507
|
-
additionalProperties: true
|
|
3508
|
-
};
|
|
3509
|
-
}
|
|
3510
|
-
get preserveOrder() {
|
|
3511
|
-
return this.config.preserveOrder ?? true;
|
|
3512
|
-
}
|
|
3513
|
-
get flatten() {
|
|
3514
|
-
return this.config.flatten ?? false;
|
|
3515
|
-
}
|
|
3516
|
-
getEmptyResult() {
|
|
3517
|
-
const schema = this.outputSchema();
|
|
3518
|
-
if (typeof schema === "boolean") {
|
|
3519
|
-
return {};
|
|
3520
|
-
}
|
|
3521
|
-
const result = {};
|
|
3522
|
-
for (const key of Object.keys(schema.properties || {})) {
|
|
3523
|
-
result[key] = [];
|
|
3524
|
-
}
|
|
3525
|
-
return result;
|
|
3526
|
-
}
|
|
3527
|
-
outputSchema() {
|
|
3528
|
-
if (!this.hasChildren() && !this._templateGraph) {
|
|
3529
|
-
return this.constructor.outputSchema();
|
|
3530
|
-
}
|
|
3531
|
-
return this.getWrappedOutputSchema();
|
|
3532
|
-
}
|
|
3533
|
-
collectResults(results) {
|
|
3534
|
-
const collected = super.collectResults(results);
|
|
3535
|
-
if (!this.flatten || typeof collected !== "object" || collected === null) {
|
|
3536
|
-
return collected;
|
|
3537
|
-
}
|
|
3538
|
-
const flattened = {};
|
|
3539
|
-
for (const [key, value] of Object.entries(collected)) {
|
|
3540
|
-
if (Array.isArray(value)) {
|
|
3541
|
-
flattened[key] = value.flat();
|
|
3542
|
-
} else {
|
|
3543
|
-
flattened[key] = value;
|
|
3544
|
-
}
|
|
3545
|
-
}
|
|
3546
|
-
return flattened;
|
|
3547
|
-
}
|
|
3548
|
-
}
|
|
3549
|
-
Workflow.prototype.map = CreateLoopWorkflow(MapTask);
|
|
3550
|
-
Workflow.prototype.endMap = CreateEndLoopWorkflow("endMap");
|
|
3551
|
-
// src/task/ReduceTask.ts
|
|
3552
|
-
class ReduceTask extends IteratorTask {
|
|
3553
|
-
static type = "ReduceTask";
|
|
3554
|
-
static category = "Flow Control";
|
|
3555
|
-
static title = "Reduce";
|
|
3556
|
-
static description = "Processes array elements sequentially with an accumulator (fold)";
|
|
3557
|
-
constructor(input = {}, config = {}) {
|
|
3558
|
-
const reduceConfig = {
|
|
3559
|
-
...config,
|
|
3560
|
-
executionMode: "sequential"
|
|
3561
|
-
};
|
|
3562
|
-
super(input, reduceConfig);
|
|
3563
|
-
}
|
|
3564
|
-
get initialValue() {
|
|
3565
|
-
return this.config.initialValue ?? {};
|
|
3566
|
-
}
|
|
3567
|
-
get accumulatorPort() {
|
|
3568
|
-
return this.config.accumulatorPort ?? "accumulator";
|
|
3569
|
-
}
|
|
3570
|
-
get currentItemPort() {
|
|
3571
|
-
return this.config.currentItemPort ?? "currentItem";
|
|
3572
|
-
}
|
|
3573
|
-
get indexPort() {
|
|
3574
|
-
return this.config.indexPort ?? "index";
|
|
3575
|
-
}
|
|
3576
|
-
async execute(input, context) {
|
|
3577
|
-
if (!this._templateGraph || this._templateGraph.getTasks().length === 0) {
|
|
3578
|
-
return this.initialValue;
|
|
3579
|
-
}
|
|
3580
|
-
const items = this.getIterableItems(input);
|
|
3581
|
-
if (items.length === 0) {
|
|
3582
|
-
return this.initialValue;
|
|
3583
|
-
}
|
|
3584
|
-
let accumulator = { ...this.initialValue };
|
|
3585
|
-
for (let index = 0;index < items.length; index++) {
|
|
3586
|
-
if (context.signal?.aborted) {
|
|
3587
|
-
break;
|
|
3588
|
-
}
|
|
3589
|
-
const currentItem = items[index];
|
|
3590
|
-
const stepInput = {
|
|
3591
|
-
...input,
|
|
3592
|
-
[this.accumulatorPort]: accumulator,
|
|
3593
|
-
[this.currentItemPort]: currentItem,
|
|
3594
|
-
[this.indexPort]: index
|
|
3595
|
-
};
|
|
3596
|
-
this.subGraph = this.cloneTemplateForStep(index);
|
|
3597
|
-
const results = await this.subGraph.run(stepInput, {
|
|
3598
|
-
parentSignal: context.signal
|
|
3599
|
-
});
|
|
3600
|
-
accumulator = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
3601
|
-
const progress = Math.round((index + 1) / items.length * 100);
|
|
3602
|
-
await context.updateProgress(progress, `Processing item ${index + 1}/${items.length}`);
|
|
3603
|
-
}
|
|
3604
|
-
return accumulator;
|
|
3605
|
-
}
|
|
3606
|
-
getEmptyResult() {
|
|
3607
|
-
return this.initialValue;
|
|
3608
|
-
}
|
|
3609
|
-
cloneTemplateForStep(stepIndex) {
|
|
3610
|
-
const clonedGraph = new TaskGraph;
|
|
3611
|
-
if (!this._templateGraph) {
|
|
3612
|
-
return clonedGraph;
|
|
3613
|
-
}
|
|
3614
|
-
const templateTasks = this._templateGraph.getTasks();
|
|
3615
|
-
const templateDataflows = this._templateGraph.getDataflows();
|
|
3616
|
-
const idMap = new Map;
|
|
3617
|
-
for (const templateTask of templateTasks) {
|
|
3618
|
-
const TaskClass = templateTask.constructor;
|
|
3619
|
-
const clonedTask = new TaskClass({ ...templateTask.defaults }, {
|
|
3620
|
-
...templateTask.config,
|
|
3621
|
-
id: `${templateTask.config.id}_step${stepIndex}`,
|
|
3622
|
-
name: `${templateTask.config.name || templateTask.type} [${stepIndex}]`
|
|
3623
|
-
});
|
|
3624
|
-
clonedGraph.addTask(clonedTask);
|
|
3625
|
-
idMap.set(templateTask.config.id, clonedTask.config.id);
|
|
3626
|
-
}
|
|
3627
|
-
for (const templateDataflow of templateDataflows) {
|
|
3628
|
-
const sourceId = idMap.get(templateDataflow.sourceTaskId);
|
|
3629
|
-
const targetId = idMap.get(templateDataflow.targetTaskId);
|
|
3630
|
-
if (sourceId !== undefined && targetId !== undefined) {
|
|
3631
|
-
const { Dataflow: Dataflow2 } = (init_Dataflow(), __toCommonJS(exports_Dataflow));
|
|
3632
|
-
const clonedDataflow = new Dataflow2(sourceId, templateDataflow.sourceTaskPortId, targetId, templateDataflow.targetTaskPortId);
|
|
3633
|
-
clonedGraph.addDataflow(clonedDataflow);
|
|
3634
|
-
}
|
|
3635
|
-
}
|
|
3636
|
-
return clonedGraph;
|
|
3637
|
-
}
|
|
3638
|
-
static inputSchema() {
|
|
3639
|
-
return {
|
|
3640
|
-
type: "object",
|
|
3641
|
-
properties: {
|
|
3642
|
-
accumulator: {
|
|
3643
|
-
title: "Accumulator",
|
|
3644
|
-
description: "The current accumulator value"
|
|
3645
|
-
},
|
|
3646
|
-
currentItem: {
|
|
3647
|
-
title: "Current Item",
|
|
3648
|
-
description: "The current item being processed"
|
|
3649
|
-
},
|
|
3650
|
-
index: {
|
|
3651
|
-
type: "number",
|
|
3652
|
-
title: "Index",
|
|
3653
|
-
description: "The current item index"
|
|
3654
|
-
}
|
|
3655
|
-
},
|
|
3656
|
-
additionalProperties: true
|
|
3657
|
-
};
|
|
3658
|
-
}
|
|
3659
|
-
static outputSchema() {
|
|
3660
|
-
return {
|
|
3661
|
-
type: "object",
|
|
3662
|
-
properties: {},
|
|
3663
|
-
additionalProperties: true
|
|
3664
|
-
};
|
|
3665
|
-
}
|
|
3666
|
-
outputSchema() {
|
|
3667
|
-
if (!this._templateGraph) {
|
|
3668
|
-
return this.constructor.outputSchema();
|
|
3669
|
-
}
|
|
3670
|
-
const tasks = this._templateGraph.getTasks();
|
|
3671
|
-
const endingNodes = tasks.filter((task) => this._templateGraph.getTargetDataflows(task.config.id).length === 0);
|
|
3672
|
-
if (endingNodes.length === 0) {
|
|
3673
|
-
return this.constructor.outputSchema();
|
|
3674
|
-
}
|
|
3675
|
-
const properties = {};
|
|
3676
|
-
for (const task of endingNodes) {
|
|
3677
|
-
const taskOutputSchema = task.outputSchema();
|
|
3678
|
-
if (typeof taskOutputSchema === "boolean")
|
|
3679
|
-
continue;
|
|
3680
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
3681
|
-
for (const [key, schema] of Object.entries(taskProperties)) {
|
|
3682
|
-
if (!properties[key]) {
|
|
3683
|
-
properties[key] = schema;
|
|
3684
|
-
}
|
|
3685
|
-
}
|
|
3686
|
-
}
|
|
3687
|
-
return {
|
|
3688
|
-
type: "object",
|
|
3689
|
-
properties,
|
|
3690
|
-
additionalProperties: false
|
|
3691
|
-
};
|
|
3692
|
-
}
|
|
3693
|
-
regenerateGraph() {
|
|
3694
|
-
this.events.emit("regenerate");
|
|
3695
|
-
}
|
|
3696
|
-
}
|
|
3697
|
-
Workflow.prototype.reduce = CreateLoopWorkflow(ReduceTask);
|
|
3698
|
-
Workflow.prototype.endReduce = CreateEndLoopWorkflow("endReduce");
|
|
3699
|
-
// src/task/TaskJSON.ts
|
|
3700
|
-
init_Dataflow();
|
|
3701
|
-
|
|
3702
2785
|
// src/task/TaskRegistry.ts
|
|
3703
2786
|
var taskConstructors = new Map;
|
|
3704
2787
|
function registerTask(baseClass) {
|
|
@@ -3766,167 +2849,9 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
3766
2849
|
}
|
|
3767
2850
|
return subGraph;
|
|
3768
2851
|
};
|
|
3769
|
-
|
|
3770
|
-
// src/task/index.ts
|
|
3771
|
-
init_TaskTypes();
|
|
3772
|
-
|
|
3773
|
-
// src/task/WhileTask.ts
|
|
3774
|
-
class WhileTask extends GraphAsTask {
|
|
3775
|
-
static type = "WhileTask";
|
|
3776
|
-
static category = "Flow Control";
|
|
3777
|
-
static title = "While Loop";
|
|
3778
|
-
static description = "Loops until a condition function returns false";
|
|
3779
|
-
static hasDynamicSchemas = true;
|
|
3780
|
-
_templateGraph;
|
|
3781
|
-
_currentIteration = 0;
|
|
3782
|
-
constructor(input = {}, config = {}) {
|
|
3783
|
-
super(input, config);
|
|
3784
|
-
}
|
|
3785
|
-
get condition() {
|
|
3786
|
-
return this.config.condition;
|
|
3787
|
-
}
|
|
3788
|
-
get maxIterations() {
|
|
3789
|
-
return this.config.maxIterations ?? 100;
|
|
3790
|
-
}
|
|
3791
|
-
get chainIterations() {
|
|
3792
|
-
return this.config.chainIterations ?? true;
|
|
3793
|
-
}
|
|
3794
|
-
get currentIteration() {
|
|
3795
|
-
return this._currentIteration;
|
|
3796
|
-
}
|
|
3797
|
-
setTemplateGraph(graph) {
|
|
3798
|
-
this._templateGraph = graph;
|
|
3799
|
-
}
|
|
3800
|
-
getTemplateGraph() {
|
|
3801
|
-
return this._templateGraph;
|
|
3802
|
-
}
|
|
3803
|
-
async execute(input, context) {
|
|
3804
|
-
if (!this._templateGraph || this._templateGraph.getTasks().length === 0) {
|
|
3805
|
-
throw new TaskConfigurationError(`${this.type}: No template graph set for while loop`);
|
|
3806
|
-
}
|
|
3807
|
-
if (!this.condition) {
|
|
3808
|
-
throw new TaskConfigurationError(`${this.type}: No condition function provided`);
|
|
3809
|
-
}
|
|
3810
|
-
this._currentIteration = 0;
|
|
3811
|
-
let currentInput = { ...input };
|
|
3812
|
-
let currentOutput = {};
|
|
3813
|
-
while (this._currentIteration < this.maxIterations) {
|
|
3814
|
-
if (context.signal?.aborted) {
|
|
3815
|
-
break;
|
|
3816
|
-
}
|
|
3817
|
-
this.subGraph = this.cloneTemplateGraph(this._currentIteration);
|
|
3818
|
-
const results = await this.subGraph.run(currentInput, {
|
|
3819
|
-
parentSignal: context.signal
|
|
3820
|
-
});
|
|
3821
|
-
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
3822
|
-
if (!this.condition(currentOutput, this._currentIteration)) {
|
|
3823
|
-
break;
|
|
3824
|
-
}
|
|
3825
|
-
if (this.chainIterations) {
|
|
3826
|
-
currentInput = { ...currentInput, ...currentOutput };
|
|
3827
|
-
}
|
|
3828
|
-
this._currentIteration++;
|
|
3829
|
-
const progress = Math.min(this._currentIteration / this.maxIterations * 100, 99);
|
|
3830
|
-
await context.updateProgress(progress, `Iteration ${this._currentIteration}`);
|
|
3831
|
-
}
|
|
3832
|
-
return currentOutput;
|
|
3833
|
-
}
|
|
3834
|
-
cloneTemplateGraph(iteration) {
|
|
3835
|
-
const clonedGraph = new TaskGraph;
|
|
3836
|
-
if (!this._templateGraph) {
|
|
3837
|
-
return clonedGraph;
|
|
3838
|
-
}
|
|
3839
|
-
const templateTasks = this._templateGraph.getTasks();
|
|
3840
|
-
const templateDataflows = this._templateGraph.getDataflows();
|
|
3841
|
-
const idMap = new Map;
|
|
3842
|
-
for (const templateTask of templateTasks) {
|
|
3843
|
-
const TaskClass = templateTask.constructor;
|
|
3844
|
-
const clonedTask = new TaskClass({ ...templateTask.defaults }, {
|
|
3845
|
-
...templateTask.config,
|
|
3846
|
-
id: `${templateTask.config.id}_iter${iteration}`,
|
|
3847
|
-
name: `${templateTask.config.name || templateTask.type} [${iteration}]`
|
|
3848
|
-
});
|
|
3849
|
-
clonedGraph.addTask(clonedTask);
|
|
3850
|
-
idMap.set(templateTask.config.id, clonedTask.config.id);
|
|
3851
|
-
}
|
|
3852
|
-
for (const templateDataflow of templateDataflows) {
|
|
3853
|
-
const sourceId = idMap.get(templateDataflow.sourceTaskId);
|
|
3854
|
-
const targetId = idMap.get(templateDataflow.targetTaskId);
|
|
3855
|
-
if (sourceId !== undefined && targetId !== undefined) {
|
|
3856
|
-
const { Dataflow: Dataflow2 } = (init_Dataflow(), __toCommonJS(exports_Dataflow));
|
|
3857
|
-
const clonedDataflow = new Dataflow2(sourceId, templateDataflow.sourceTaskPortId, targetId, templateDataflow.targetTaskPortId);
|
|
3858
|
-
clonedGraph.addDataflow(clonedDataflow);
|
|
3859
|
-
}
|
|
3860
|
-
}
|
|
3861
|
-
return clonedGraph;
|
|
3862
|
-
}
|
|
3863
|
-
static inputSchema() {
|
|
3864
|
-
return {
|
|
3865
|
-
type: "object",
|
|
3866
|
-
properties: {},
|
|
3867
|
-
additionalProperties: true
|
|
3868
|
-
};
|
|
3869
|
-
}
|
|
3870
|
-
static outputSchema() {
|
|
3871
|
-
return {
|
|
3872
|
-
type: "object",
|
|
3873
|
-
properties: {
|
|
3874
|
-
_iterations: {
|
|
3875
|
-
type: "number",
|
|
3876
|
-
title: "Iterations",
|
|
3877
|
-
description: "Number of iterations executed"
|
|
3878
|
-
}
|
|
3879
|
-
},
|
|
3880
|
-
additionalProperties: true
|
|
3881
|
-
};
|
|
3882
|
-
}
|
|
3883
|
-
outputSchema() {
|
|
3884
|
-
if (!this._templateGraph) {
|
|
3885
|
-
return this.constructor.outputSchema();
|
|
3886
|
-
}
|
|
3887
|
-
const tasks = this._templateGraph.getTasks();
|
|
3888
|
-
const endingNodes = tasks.filter((task) => this._templateGraph.getTargetDataflows(task.config.id).length === 0);
|
|
3889
|
-
if (endingNodes.length === 0) {
|
|
3890
|
-
return this.constructor.outputSchema();
|
|
3891
|
-
}
|
|
3892
|
-
const properties = {
|
|
3893
|
-
_iterations: {
|
|
3894
|
-
type: "number",
|
|
3895
|
-
title: "Iterations",
|
|
3896
|
-
description: "Number of iterations executed"
|
|
3897
|
-
}
|
|
3898
|
-
};
|
|
3899
|
-
for (const task of endingNodes) {
|
|
3900
|
-
const taskOutputSchema = task.outputSchema();
|
|
3901
|
-
if (typeof taskOutputSchema === "boolean")
|
|
3902
|
-
continue;
|
|
3903
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
3904
|
-
for (const [key, schema] of Object.entries(taskProperties)) {
|
|
3905
|
-
if (!properties[key]) {
|
|
3906
|
-
properties[key] = schema;
|
|
3907
|
-
}
|
|
3908
|
-
}
|
|
3909
|
-
}
|
|
3910
|
-
return {
|
|
3911
|
-
type: "object",
|
|
3912
|
-
properties,
|
|
3913
|
-
additionalProperties: false
|
|
3914
|
-
};
|
|
3915
|
-
}
|
|
3916
|
-
}
|
|
3917
|
-
Workflow.prototype.while = CreateLoopWorkflow(WhileTask);
|
|
3918
|
-
Workflow.prototype.endWhile = CreateEndLoopWorkflow("endWhile");
|
|
3919
2852
|
// src/task/index.ts
|
|
3920
2853
|
var registerBaseTasks = () => {
|
|
3921
|
-
const tasks = [
|
|
3922
|
-
ConditionalTask,
|
|
3923
|
-
GraphAsTask,
|
|
3924
|
-
ForEachTask,
|
|
3925
|
-
MapTask,
|
|
3926
|
-
BatchTask,
|
|
3927
|
-
WhileTask,
|
|
3928
|
-
ReduceTask
|
|
3929
|
-
];
|
|
2854
|
+
const tasks = [ConditionalTask, GraphAsTask];
|
|
3930
2855
|
tasks.map(TaskRegistry.registerTask);
|
|
3931
2856
|
return tasks;
|
|
3932
2857
|
};
|
|
@@ -4106,7 +3031,6 @@ export {
|
|
|
4106
3031
|
connect,
|
|
4107
3032
|
WorkflowError,
|
|
4108
3033
|
Workflow,
|
|
4109
|
-
WhileTask,
|
|
4110
3034
|
TaskStatus,
|
|
4111
3035
|
TaskRegistry,
|
|
4112
3036
|
TaskQueueRegistry,
|
|
@@ -4129,18 +3053,13 @@ export {
|
|
|
4129
3053
|
Task,
|
|
4130
3054
|
TASK_OUTPUT_REPOSITORY,
|
|
4131
3055
|
TASK_GRAPH_REPOSITORY,
|
|
4132
|
-
ReduceTask,
|
|
4133
3056
|
PROPERTY_ARRAY,
|
|
4134
|
-
MapTask,
|
|
4135
3057
|
JobTaskFailedError,
|
|
4136
3058
|
JobQueueTask,
|
|
4137
3059
|
JOB_QUEUE_FACTORY,
|
|
4138
|
-
IteratorTaskRunner,
|
|
4139
|
-
IteratorTask,
|
|
4140
3060
|
GraphAsTaskRunner,
|
|
4141
3061
|
GraphAsTask,
|
|
4142
3062
|
GRAPH_RESULT_ARRAY,
|
|
4143
|
-
ForEachTask,
|
|
4144
3063
|
EventTaskGraphToDagMapping,
|
|
4145
3064
|
EventDagToTaskGraphMapping,
|
|
4146
3065
|
DataflowArrow,
|
|
@@ -4148,10 +3067,7 @@ export {
|
|
|
4148
3067
|
DATAFLOW_ERROR_PORT,
|
|
4149
3068
|
DATAFLOW_ALL_PORTS,
|
|
4150
3069
|
CreateWorkflow,
|
|
4151
|
-
|
|
4152
|
-
CreateEndLoopWorkflow,
|
|
4153
|
-
ConditionalTask,
|
|
4154
|
-
BatchTask
|
|
3070
|
+
ConditionalTask
|
|
4155
3071
|
};
|
|
4156
3072
|
|
|
4157
|
-
//# debugId=
|
|
3073
|
+
//# debugId=B92DF56229C4FC4064756E2164756E21
|