@workglow/task-graph 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +1280 -0
- package/dist/browser.d.ts +7 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +2842 -0
- package/dist/browser.js.map +33 -0
- package/dist/bun.d.ts +7 -0
- package/dist/bun.d.ts.map +1 -0
- package/dist/bun.js +2843 -0
- package/dist/bun.js.map +33 -0
- package/dist/common.d.ts +33 -0
- package/dist/common.d.ts.map +1 -0
- package/dist/node.d.ts +7 -0
- package/dist/node.d.ts.map +1 -0
- package/dist/node.js +2842 -0
- package/dist/node.js.map +33 -0
- package/dist/storage/TaskGraphRepository.d.ts +92 -0
- package/dist/storage/TaskGraphRepository.d.ts.map +1 -0
- package/dist/storage/TaskGraphTabularRepository.d.ts +73 -0
- package/dist/storage/TaskGraphTabularRepository.d.ts.map +1 -0
- package/dist/storage/TaskOutputRepository.d.ts +93 -0
- package/dist/storage/TaskOutputRepository.d.ts.map +1 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts +84 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts.map +1 -0
- package/dist/task/ArrayTask.d.ts +72 -0
- package/dist/task/ArrayTask.d.ts.map +1 -0
- package/dist/task/ConditionalTask.d.ts +278 -0
- package/dist/task/ConditionalTask.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +79 -0
- package/dist/task/GraphAsTask.d.ts.map +1 -0
- package/dist/task/GraphAsTaskRunner.d.ts +36 -0
- package/dist/task/GraphAsTaskRunner.d.ts.map +1 -0
- package/dist/task/ITask.d.ts +144 -0
- package/dist/task/ITask.d.ts.map +1 -0
- package/dist/task/ITaskRunner.d.ts +36 -0
- package/dist/task/ITaskRunner.d.ts.map +1 -0
- package/dist/task/JobQueueFactory.d.ts +23 -0
- package/dist/task/JobQueueFactory.d.ts.map +1 -0
- package/dist/task/JobQueueTask.d.ts +65 -0
- package/dist/task/JobQueueTask.d.ts.map +1 -0
- package/dist/task/Task.d.ts +334 -0
- package/dist/task/Task.d.ts.map +1 -0
- package/dist/task/TaskError.d.ts +66 -0
- package/dist/task/TaskError.d.ts.map +1 -0
- package/dist/task/TaskEvents.d.ts +40 -0
- package/dist/task/TaskEvents.d.ts.map +1 -0
- package/dist/task/TaskJSON.d.ts +82 -0
- package/dist/task/TaskJSON.d.ts.map +1 -0
- package/dist/task/TaskQueueRegistry.d.ts +69 -0
- package/dist/task/TaskQueueRegistry.d.ts.map +1 -0
- package/dist/task/TaskRegistry.d.ts +31 -0
- package/dist/task/TaskRegistry.d.ts.map +1 -0
- package/dist/task/TaskRunner.d.ts +99 -0
- package/dist/task/TaskRunner.d.ts.map +1 -0
- package/dist/task/TaskTypes.d.ts +68 -0
- package/dist/task/TaskTypes.d.ts.map +1 -0
- package/dist/task-graph/Conversions.d.ts +28 -0
- package/dist/task-graph/Conversions.d.ts.map +1 -0
- package/dist/task-graph/Dataflow.d.ts +73 -0
- package/dist/task-graph/Dataflow.d.ts.map +1 -0
- package/dist/task-graph/DataflowEvents.d.ts +34 -0
- package/dist/task-graph/DataflowEvents.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +38 -0
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -0
- package/dist/task-graph/IWorkflow.d.ts +13 -0
- package/dist/task-graph/IWorkflow.d.ts.map +1 -0
- package/dist/task-graph/TaskGraph.d.ts +230 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -0
- package/dist/task-graph/TaskGraphEvents.d.ts +54 -0
- package/dist/task-graph/TaskGraphEvents.d.ts.map +1 -0
- package/dist/task-graph/TaskGraphRunner.d.ts +202 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -0
- package/dist/task-graph/TaskGraphScheduler.d.ts +56 -0
- package/dist/task-graph/TaskGraphScheduler.d.ts.map +1 -0
- package/dist/task-graph/Workflow.d.ts +155 -0
- package/dist/task-graph/Workflow.d.ts.map +1 -0
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +59 -0
- package/src/storage/README.md +61 -0
- package/src/task/ConditionalTask.README.md +268 -0
- package/src/task/README.md +251 -0
- package/src/task-graph/README.md +142 -0
package/dist/browser.js
ADDED
|
@@ -0,0 +1,2842 @@
|
|
|
1
|
+
// src/task/ArrayTask.ts
|
|
2
|
+
import { uuid4 as uuid44 } from "@workglow/util";
|
|
3
|
+
|
|
4
|
+
// src/task-graph/TaskGraph.ts
|
|
5
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid43 } from "@workglow/util";
|
|
6
|
+
|
|
7
|
+
// src/task/GraphAsTask.ts
|
|
8
|
+
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
9
|
+
|
|
10
|
+
// src/task-graph/Dataflow.ts
|
|
11
|
+
import { areSemanticallyCompatible, EventEmitter } from "@workglow/util";
|
|
12
|
+
|
|
13
|
+
// src/task/TaskTypes.ts
|
|
14
|
+
var TaskStatus;
|
|
15
|
+
((TaskStatus2) => {
|
|
16
|
+
TaskStatus2["PENDING"] = "PENDING";
|
|
17
|
+
TaskStatus2["DISABLED"] = "DISABLED";
|
|
18
|
+
TaskStatus2["PROCESSING"] = "PROCESSING";
|
|
19
|
+
TaskStatus2["COMPLETED"] = "COMPLETED";
|
|
20
|
+
TaskStatus2["ABORTING"] = "ABORTING";
|
|
21
|
+
TaskStatus2["FAILED"] = "FAILED";
|
|
22
|
+
})(TaskStatus ||= {});
|
|
23
|
+
|
|
24
|
+
// src/task-graph/Dataflow.ts
|
|
25
|
+
var DATAFLOW_ALL_PORTS = "*";
|
|
26
|
+
var DATAFLOW_ERROR_PORT = "[error]";
|
|
27
|
+
|
|
28
|
+
class Dataflow {
|
|
29
|
+
sourceTaskId;
|
|
30
|
+
sourceTaskPortId;
|
|
31
|
+
targetTaskId;
|
|
32
|
+
targetTaskPortId;
|
|
33
|
+
constructor(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId) {
|
|
34
|
+
this.sourceTaskId = sourceTaskId;
|
|
35
|
+
this.sourceTaskPortId = sourceTaskPortId;
|
|
36
|
+
this.targetTaskId = targetTaskId;
|
|
37
|
+
this.targetTaskPortId = targetTaskPortId;
|
|
38
|
+
}
|
|
39
|
+
static createId(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId) {
|
|
40
|
+
return `${sourceTaskId}[${sourceTaskPortId}] ==> ${targetTaskId}[${targetTaskPortId}]`;
|
|
41
|
+
}
|
|
42
|
+
get id() {
|
|
43
|
+
return Dataflow.createId(this.sourceTaskId, this.sourceTaskPortId, this.targetTaskId, this.targetTaskPortId);
|
|
44
|
+
}
|
|
45
|
+
value = undefined;
|
|
46
|
+
provenance = {};
|
|
47
|
+
status = "PENDING" /* PENDING */;
|
|
48
|
+
error;
|
|
49
|
+
reset() {
|
|
50
|
+
this.status = "PENDING" /* PENDING */;
|
|
51
|
+
this.error = undefined;
|
|
52
|
+
this.value = undefined;
|
|
53
|
+
this.provenance = {};
|
|
54
|
+
this.emit("reset");
|
|
55
|
+
this.emit("status", this.status);
|
|
56
|
+
}
|
|
57
|
+
setStatus(status) {
|
|
58
|
+
if (status === this.status)
|
|
59
|
+
return;
|
|
60
|
+
this.status = status;
|
|
61
|
+
switch (status) {
|
|
62
|
+
case "PROCESSING" /* PROCESSING */:
|
|
63
|
+
this.emit("start");
|
|
64
|
+
break;
|
|
65
|
+
case "COMPLETED" /* COMPLETED */:
|
|
66
|
+
this.emit("complete");
|
|
67
|
+
break;
|
|
68
|
+
case "ABORTING" /* ABORTING */:
|
|
69
|
+
this.emit("abort");
|
|
70
|
+
break;
|
|
71
|
+
case "PENDING" /* PENDING */:
|
|
72
|
+
this.emit("reset");
|
|
73
|
+
break;
|
|
74
|
+
case "FAILED" /* FAILED */:
|
|
75
|
+
this.emit("error", this.error);
|
|
76
|
+
break;
|
|
77
|
+
case "DISABLED" /* DISABLED */:
|
|
78
|
+
this.emit("disabled");
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
this.emit("status", this.status);
|
|
82
|
+
}
|
|
83
|
+
setPortData(entireDataBlock, nodeProvenance) {
|
|
84
|
+
if (this.sourceTaskPortId === DATAFLOW_ALL_PORTS) {
|
|
85
|
+
this.value = entireDataBlock;
|
|
86
|
+
} else if (this.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
87
|
+
this.error = entireDataBlock;
|
|
88
|
+
} else {
|
|
89
|
+
this.value = entireDataBlock[this.sourceTaskPortId];
|
|
90
|
+
}
|
|
91
|
+
if (nodeProvenance)
|
|
92
|
+
this.provenance = nodeProvenance;
|
|
93
|
+
}
|
|
94
|
+
getPortData() {
|
|
95
|
+
if (this.targetTaskPortId === DATAFLOW_ALL_PORTS) {
|
|
96
|
+
return this.value;
|
|
97
|
+
} else if (this.targetTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
98
|
+
return { [DATAFLOW_ERROR_PORT]: this.error };
|
|
99
|
+
} else {
|
|
100
|
+
return { [this.targetTaskPortId]: this.value };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
toJSON() {
|
|
104
|
+
return {
|
|
105
|
+
sourceTaskId: this.sourceTaskId,
|
|
106
|
+
sourceTaskPortId: this.sourceTaskPortId,
|
|
107
|
+
targetTaskId: this.targetTaskId,
|
|
108
|
+
targetTaskPortId: this.targetTaskPortId
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
semanticallyCompatible(graph, dataflow) {
|
|
112
|
+
const targetSchema = graph.getTask(dataflow.targetTaskId).inputSchema();
|
|
113
|
+
const sourceSchema = graph.getTask(dataflow.sourceTaskId).outputSchema();
|
|
114
|
+
if (typeof targetSchema === "boolean") {
|
|
115
|
+
if (targetSchema === false) {
|
|
116
|
+
return "incompatible";
|
|
117
|
+
}
|
|
118
|
+
return "static";
|
|
119
|
+
}
|
|
120
|
+
if (typeof sourceSchema === "boolean") {
|
|
121
|
+
if (sourceSchema === false) {
|
|
122
|
+
return "incompatible";
|
|
123
|
+
}
|
|
124
|
+
return "runtime";
|
|
125
|
+
}
|
|
126
|
+
let targetSchemaProperty = DATAFLOW_ALL_PORTS === dataflow.targetTaskPortId ? true : targetSchema.properties?.[dataflow.targetTaskPortId];
|
|
127
|
+
if (targetSchemaProperty === undefined && targetSchema.additionalProperties === true) {
|
|
128
|
+
targetSchemaProperty = true;
|
|
129
|
+
}
|
|
130
|
+
let sourceSchemaProperty = DATAFLOW_ALL_PORTS === dataflow.sourceTaskPortId ? true : sourceSchema.properties?.[dataflow.sourceTaskPortId];
|
|
131
|
+
if (sourceSchemaProperty === undefined && sourceSchema.additionalProperties === true) {
|
|
132
|
+
sourceSchemaProperty = true;
|
|
133
|
+
}
|
|
134
|
+
const semanticallyCompatible = areSemanticallyCompatible(sourceSchemaProperty, targetSchemaProperty);
|
|
135
|
+
return semanticallyCompatible;
|
|
136
|
+
}
|
|
137
|
+
get events() {
|
|
138
|
+
if (!this._events) {
|
|
139
|
+
this._events = new EventEmitter;
|
|
140
|
+
}
|
|
141
|
+
return this._events;
|
|
142
|
+
}
|
|
143
|
+
_events;
|
|
144
|
+
subscribe(name, fn) {
|
|
145
|
+
return this.events.subscribe(name, fn);
|
|
146
|
+
}
|
|
147
|
+
on(name, fn) {
|
|
148
|
+
this.events.on(name, fn);
|
|
149
|
+
}
|
|
150
|
+
off(name, fn) {
|
|
151
|
+
this.events.off(name, fn);
|
|
152
|
+
}
|
|
153
|
+
once(name, fn) {
|
|
154
|
+
this.events.once(name, fn);
|
|
155
|
+
}
|
|
156
|
+
waitOn(name) {
|
|
157
|
+
return this.events.waitOn(name);
|
|
158
|
+
}
|
|
159
|
+
emit(name, ...args) {
|
|
160
|
+
this._events?.emit(name, ...args);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
class DataflowArrow extends Dataflow {
|
|
165
|
+
constructor(dataflow) {
|
|
166
|
+
const pattern = /^([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\] ==> ([a-zA-Z0-9-]+?)\[([a-zA-Z0-9-]+?)\]$/;
|
|
167
|
+
const match = dataflow.match(pattern);
|
|
168
|
+
if (!match) {
|
|
169
|
+
throw new Error(`Invalid dataflow format: ${dataflow}`);
|
|
170
|
+
}
|
|
171
|
+
const [, sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId] = match;
|
|
172
|
+
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// src/task-graph/TaskGraphRunner.ts
|
|
177
|
+
import {
|
|
178
|
+
collectPropertyValues,
|
|
179
|
+
globalServiceRegistry as globalServiceRegistry2,
|
|
180
|
+
uuid4 as uuid42
|
|
181
|
+
} from "@workglow/util";
|
|
182
|
+
|
|
183
|
+
// src/storage/TaskOutputRepository.ts
|
|
184
|
+
import { createServiceToken, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
185
|
+
var TASK_OUTPUT_REPOSITORY = createServiceToken("taskgraph.taskOutputRepository");
|
|
186
|
+
|
|
187
|
+
class TaskOutputRepository {
|
|
188
|
+
outputCompression;
|
|
189
|
+
constructor({ outputCompression = true }) {
|
|
190
|
+
this.outputCompression = outputCompression;
|
|
191
|
+
}
|
|
192
|
+
get events() {
|
|
193
|
+
if (!this._events) {
|
|
194
|
+
this._events = new EventEmitter2;
|
|
195
|
+
}
|
|
196
|
+
return this._events;
|
|
197
|
+
}
|
|
198
|
+
_events;
|
|
199
|
+
on(name, fn) {
|
|
200
|
+
this.events.on(name, fn);
|
|
201
|
+
}
|
|
202
|
+
off(name, fn) {
|
|
203
|
+
this.events.off(name, fn);
|
|
204
|
+
}
|
|
205
|
+
waitOn(name) {
|
|
206
|
+
return this.events.waitOn(name);
|
|
207
|
+
}
|
|
208
|
+
emit(name, ...args) {
|
|
209
|
+
this._events?.emit(name, ...args);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// src/task/Task.ts
|
|
214
|
+
import {
|
|
215
|
+
compileSchema,
|
|
216
|
+
deepEqual,
|
|
217
|
+
EventEmitter as EventEmitter3,
|
|
218
|
+
uuid4
|
|
219
|
+
} from "@workglow/util";
|
|
220
|
+
|
|
221
|
+
// src/task/TaskError.ts
|
|
222
|
+
import { BaseError } from "@workglow/util";
|
|
223
|
+
|
|
224
|
+
class TaskError extends BaseError {
|
|
225
|
+
static type = "TaskError";
|
|
226
|
+
constructor(message) {
|
|
227
|
+
super(message);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
class TaskConfigurationError extends TaskError {
|
|
232
|
+
static type = "TaskConfigurationError";
|
|
233
|
+
constructor(message) {
|
|
234
|
+
super(message);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
class WorkflowError extends TaskError {
|
|
239
|
+
static type = "WorkflowError";
|
|
240
|
+
constructor(message) {
|
|
241
|
+
super(message);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
class TaskAbortedError extends TaskError {
|
|
246
|
+
static type = "TaskAbortedError";
|
|
247
|
+
constructor(message = "Task aborted") {
|
|
248
|
+
super(message);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
class TaskFailedError extends TaskError {
|
|
253
|
+
static type = "TaskFailedError";
|
|
254
|
+
constructor(message = "Task failed") {
|
|
255
|
+
super(message);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
class JobTaskFailedError extends TaskFailedError {
|
|
260
|
+
static type = "JobTaskFailedError";
|
|
261
|
+
jobError;
|
|
262
|
+
constructor(err) {
|
|
263
|
+
super(String(err));
|
|
264
|
+
this.jobError = err;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
class TaskJSONError extends TaskError {
|
|
269
|
+
static type = "TaskJSONError";
|
|
270
|
+
constructor(message = "Error converting JSON to a Task") {
|
|
271
|
+
super(message);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
class TaskInvalidInputError extends TaskError {
|
|
276
|
+
static type = "TaskInvalidInputError";
|
|
277
|
+
constructor(message = "Invalid input data") {
|
|
278
|
+
super(message);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// src/task/TaskRunner.ts
|
|
283
|
+
import { globalServiceRegistry } from "@workglow/util";
|
|
284
|
+
class TaskRunner {
|
|
285
|
+
running = false;
|
|
286
|
+
reactiveRunning = false;
|
|
287
|
+
nodeProvenance = {};
|
|
288
|
+
task;
|
|
289
|
+
abortController;
|
|
290
|
+
outputCache;
|
|
291
|
+
constructor(task) {
|
|
292
|
+
this.task = task;
|
|
293
|
+
this.own = this.own.bind(this);
|
|
294
|
+
this.handleProgress = this.handleProgress.bind(this);
|
|
295
|
+
}
|
|
296
|
+
async run(overrides = {}, config = {}) {
|
|
297
|
+
await this.handleStart(config);
|
|
298
|
+
try {
|
|
299
|
+
this.task.setInput(overrides);
|
|
300
|
+
const isValid = await this.task.validateInput(this.task.runInputData);
|
|
301
|
+
if (!isValid) {
|
|
302
|
+
throw new TaskInvalidInputError("Invalid input data");
|
|
303
|
+
}
|
|
304
|
+
if (this.abortController?.signal.aborted) {
|
|
305
|
+
await this.handleAbort();
|
|
306
|
+
throw new TaskAbortedError("Promise for task created and aborted before run");
|
|
307
|
+
}
|
|
308
|
+
const inputs = this.task.runInputData;
|
|
309
|
+
let outputs;
|
|
310
|
+
if (this.task.cacheable) {
|
|
311
|
+
outputs = await this.outputCache?.getOutput(this.task.type, inputs);
|
|
312
|
+
if (outputs) {
|
|
313
|
+
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
if (!outputs) {
|
|
317
|
+
outputs = await this.executeTask(inputs);
|
|
318
|
+
if (this.task.cacheable && outputs !== undefined) {
|
|
319
|
+
await this.outputCache?.saveOutput(this.task.type, inputs, outputs);
|
|
320
|
+
}
|
|
321
|
+
this.task.runOutputData = outputs ?? {};
|
|
322
|
+
}
|
|
323
|
+
await this.handleComplete();
|
|
324
|
+
return this.task.runOutputData;
|
|
325
|
+
} catch (err) {
|
|
326
|
+
await this.handleError(err);
|
|
327
|
+
throw err;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
async runReactive(overrides = {}) {
|
|
331
|
+
if (this.task.status === "PROCESSING" /* PROCESSING */) {
|
|
332
|
+
return this.task.runOutputData;
|
|
333
|
+
}
|
|
334
|
+
this.task.setInput(overrides);
|
|
335
|
+
await this.handleStartReactive();
|
|
336
|
+
try {
|
|
337
|
+
const isValid = await this.task.validateInput(this.task.runInputData);
|
|
338
|
+
if (!isValid) {
|
|
339
|
+
throw new TaskInvalidInputError("Invalid input data");
|
|
340
|
+
}
|
|
341
|
+
const resultReactive = await this.executeTaskReactive(this.task.runInputData, this.task.runOutputData);
|
|
342
|
+
this.task.runOutputData = resultReactive;
|
|
343
|
+
await this.handleCompleteReactive();
|
|
344
|
+
} catch (err) {
|
|
345
|
+
await this.handleErrorReactive();
|
|
346
|
+
} finally {
|
|
347
|
+
return this.task.runOutputData;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
abort() {
|
|
351
|
+
if (this.task.hasChildren()) {
|
|
352
|
+
this.task.subGraph.abort();
|
|
353
|
+
}
|
|
354
|
+
this.abortController?.abort();
|
|
355
|
+
}
|
|
356
|
+
own(i) {
|
|
357
|
+
const task = ensureTask(i, { isOwned: true });
|
|
358
|
+
this.task.subGraph.addTask(task);
|
|
359
|
+
return i;
|
|
360
|
+
}
|
|
361
|
+
async executeTask(input) {
|
|
362
|
+
const result = await this.task.execute(input, {
|
|
363
|
+
signal: this.abortController.signal,
|
|
364
|
+
updateProgress: this.handleProgress.bind(this),
|
|
365
|
+
nodeProvenance: this.nodeProvenance,
|
|
366
|
+
own: this.own
|
|
367
|
+
});
|
|
368
|
+
return await this.executeTaskReactive(input, result || {});
|
|
369
|
+
}
|
|
370
|
+
async executeTaskReactive(input, output) {
|
|
371
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
372
|
+
return Object.keys(reactiveResult || {}) >= Object.keys(output || {}) ? reactiveResult : output ?? {};
|
|
373
|
+
}
|
|
374
|
+
async handleStart(config = {}) {
|
|
375
|
+
if (this.task.status === "PROCESSING" /* PROCESSING */)
|
|
376
|
+
return;
|
|
377
|
+
this.nodeProvenance = {};
|
|
378
|
+
this.running = true;
|
|
379
|
+
this.task.startedAt = new Date;
|
|
380
|
+
this.task.progress = 0;
|
|
381
|
+
this.task.status = "PROCESSING" /* PROCESSING */;
|
|
382
|
+
this.abortController = new AbortController;
|
|
383
|
+
this.abortController.signal.addEventListener("abort", () => {
|
|
384
|
+
this.handleAbort();
|
|
385
|
+
});
|
|
386
|
+
this.nodeProvenance = config.nodeProvenance ?? {};
|
|
387
|
+
const cache = this.task.config.outputCache ?? config.outputCache;
|
|
388
|
+
if (cache === true) {
|
|
389
|
+
let instance = globalServiceRegistry.get(TASK_OUTPUT_REPOSITORY);
|
|
390
|
+
this.outputCache = instance;
|
|
391
|
+
} else if (cache === false) {
|
|
392
|
+
this.outputCache = undefined;
|
|
393
|
+
} else if (cache instanceof TaskOutputRepository) {
|
|
394
|
+
this.outputCache = cache;
|
|
395
|
+
}
|
|
396
|
+
if (config.updateProgress) {
|
|
397
|
+
this.updateProgress = config.updateProgress;
|
|
398
|
+
}
|
|
399
|
+
this.task.emit("start");
|
|
400
|
+
this.task.emit("status", this.task.status);
|
|
401
|
+
}
|
|
402
|
+
updateProgress = async (task, progress, message, ...args) => {};
|
|
403
|
+
async handleStartReactive() {
|
|
404
|
+
this.reactiveRunning = true;
|
|
405
|
+
}
|
|
406
|
+
async handleAbort() {
|
|
407
|
+
if (this.task.status === "ABORTING" /* ABORTING */)
|
|
408
|
+
return;
|
|
409
|
+
this.task.status = "ABORTING" /* ABORTING */;
|
|
410
|
+
this.task.progress = 100;
|
|
411
|
+
this.task.error = new TaskAbortedError;
|
|
412
|
+
this.task.emit("abort", this.task.error);
|
|
413
|
+
this.task.emit("status", this.task.status);
|
|
414
|
+
}
|
|
415
|
+
async handleAbortReactive() {
|
|
416
|
+
this.reactiveRunning = false;
|
|
417
|
+
}
|
|
418
|
+
async handleComplete() {
|
|
419
|
+
if (this.task.status === "COMPLETED" /* COMPLETED */)
|
|
420
|
+
return;
|
|
421
|
+
this.task.completedAt = new Date;
|
|
422
|
+
this.task.progress = 100;
|
|
423
|
+
this.task.status = "COMPLETED" /* COMPLETED */;
|
|
424
|
+
this.abortController = undefined;
|
|
425
|
+
this.nodeProvenance = {};
|
|
426
|
+
this.task.emit("complete");
|
|
427
|
+
this.task.emit("status", this.task.status);
|
|
428
|
+
}
|
|
429
|
+
async handleCompleteReactive() {
|
|
430
|
+
this.reactiveRunning = false;
|
|
431
|
+
}
|
|
432
|
+
async handleDisable() {
|
|
433
|
+
if (this.task.status === "DISABLED" /* DISABLED */)
|
|
434
|
+
return;
|
|
435
|
+
this.task.status = "DISABLED" /* DISABLED */;
|
|
436
|
+
this.task.progress = 100;
|
|
437
|
+
this.task.completedAt = new Date;
|
|
438
|
+
this.abortController = undefined;
|
|
439
|
+
this.nodeProvenance = {};
|
|
440
|
+
this.task.emit("disabled");
|
|
441
|
+
this.task.emit("status", this.task.status);
|
|
442
|
+
}
|
|
443
|
+
async disable() {
|
|
444
|
+
await this.handleDisable();
|
|
445
|
+
}
|
|
446
|
+
async handleError(err) {
|
|
447
|
+
if (err instanceof TaskAbortedError)
|
|
448
|
+
return this.handleAbort();
|
|
449
|
+
if (this.task.status === "FAILED" /* FAILED */)
|
|
450
|
+
return;
|
|
451
|
+
if (this.task.hasChildren()) {
|
|
452
|
+
this.task.subGraph.abort();
|
|
453
|
+
}
|
|
454
|
+
this.task.completedAt = new Date;
|
|
455
|
+
this.task.progress = 100;
|
|
456
|
+
this.task.status = "FAILED" /* FAILED */;
|
|
457
|
+
this.task.error = err instanceof TaskError ? err : new TaskFailedError(err?.message || "Task failed");
|
|
458
|
+
this.abortController = undefined;
|
|
459
|
+
this.nodeProvenance = {};
|
|
460
|
+
this.task.emit("error", this.task.error);
|
|
461
|
+
this.task.emit("status", this.task.status);
|
|
462
|
+
}
|
|
463
|
+
async handleErrorReactive() {
|
|
464
|
+
this.reactiveRunning = false;
|
|
465
|
+
}
|
|
466
|
+
async handleProgress(progress, message, ...args) {
|
|
467
|
+
this.task.progress = progress;
|
|
468
|
+
await this.updateProgress(this.task, progress, message, ...args);
|
|
469
|
+
this.task.emit("progress", progress, message, ...args);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// src/task/Task.ts
|
|
474
|
+
class Task {
|
|
475
|
+
static type = "Task";
|
|
476
|
+
static category = "Hidden";
|
|
477
|
+
static title = "";
|
|
478
|
+
static cacheable = true;
|
|
479
|
+
static hasDynamicSchemas = false;
|
|
480
|
+
static inputSchema() {
|
|
481
|
+
return {
|
|
482
|
+
type: "object",
|
|
483
|
+
properties: {},
|
|
484
|
+
additionalProperties: false
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
static outputSchema() {
|
|
488
|
+
return {
|
|
489
|
+
type: "object",
|
|
490
|
+
properties: {},
|
|
491
|
+
additionalProperties: false
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
async execute(input, context) {
|
|
495
|
+
if (context.signal?.aborted) {
|
|
496
|
+
throw new TaskAbortedError("Task aborted");
|
|
497
|
+
}
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
async executeReactive(input, output, context) {
|
|
501
|
+
return output;
|
|
502
|
+
}
|
|
503
|
+
_runner;
|
|
504
|
+
get runner() {
|
|
505
|
+
if (!this._runner) {
|
|
506
|
+
this._runner = new TaskRunner(this);
|
|
507
|
+
}
|
|
508
|
+
return this._runner;
|
|
509
|
+
}
|
|
510
|
+
async run(overrides = {}) {
|
|
511
|
+
return this.runner.run(overrides);
|
|
512
|
+
}
|
|
513
|
+
async runReactive(overrides = {}) {
|
|
514
|
+
return this.runner.runReactive(overrides);
|
|
515
|
+
}
|
|
516
|
+
abort() {
|
|
517
|
+
this.runner.abort();
|
|
518
|
+
}
|
|
519
|
+
async disable() {
|
|
520
|
+
await this.runner.disable();
|
|
521
|
+
}
|
|
522
|
+
inputSchema() {
|
|
523
|
+
return this.constructor.inputSchema();
|
|
524
|
+
}
|
|
525
|
+
outputSchema() {
|
|
526
|
+
return this.constructor.outputSchema();
|
|
527
|
+
}
|
|
528
|
+
get type() {
|
|
529
|
+
return this.constructor.type;
|
|
530
|
+
}
|
|
531
|
+
get category() {
|
|
532
|
+
return this.constructor.category;
|
|
533
|
+
}
|
|
534
|
+
get title() {
|
|
535
|
+
return this.constructor.title;
|
|
536
|
+
}
|
|
537
|
+
get cacheable() {
|
|
538
|
+
return this.config?.cacheable ?? this.constructor.cacheable;
|
|
539
|
+
}
|
|
540
|
+
defaults;
|
|
541
|
+
runInputData = {};
|
|
542
|
+
runOutputData = {};
|
|
543
|
+
config;
|
|
544
|
+
status = "PENDING" /* PENDING */;
|
|
545
|
+
progress = 0;
|
|
546
|
+
createdAt = new Date;
|
|
547
|
+
startedAt;
|
|
548
|
+
completedAt;
|
|
549
|
+
error;
|
|
550
|
+
get events() {
|
|
551
|
+
if (!this._events) {
|
|
552
|
+
this._events = new EventEmitter3;
|
|
553
|
+
}
|
|
554
|
+
return this._events;
|
|
555
|
+
}
|
|
556
|
+
_events;
|
|
557
|
+
nodeProvenance = {};
|
|
558
|
+
constructor(callerDefaultInputs = {}, config = {}) {
|
|
559
|
+
const inputDefaults = this.getDefaultInputsFromStaticInputDefinitions();
|
|
560
|
+
const mergedDefaults = Object.assign(inputDefaults, callerDefaultInputs);
|
|
561
|
+
this.defaults = this.stripSymbols(mergedDefaults);
|
|
562
|
+
this.resetInputData();
|
|
563
|
+
const name = this.title || new.target.title || new.target.name;
|
|
564
|
+
this.config = Object.assign({
|
|
565
|
+
id: uuid4(),
|
|
566
|
+
name
|
|
567
|
+
}, config);
|
|
568
|
+
}
|
|
569
|
+
getDefaultInputsFromStaticInputDefinitions() {
|
|
570
|
+
const schema = this.inputSchema();
|
|
571
|
+
if (typeof schema === "boolean") {
|
|
572
|
+
return {};
|
|
573
|
+
}
|
|
574
|
+
try {
|
|
575
|
+
const compiledSchema = this.getInputSchemaNode(this.type);
|
|
576
|
+
const defaultData = compiledSchema.getData(undefined, {
|
|
577
|
+
addOptionalProps: true,
|
|
578
|
+
removeInvalidData: false,
|
|
579
|
+
useTypeDefaults: false
|
|
580
|
+
});
|
|
581
|
+
return defaultData || {};
|
|
582
|
+
} catch (error) {
|
|
583
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to manual extraction:`, error);
|
|
584
|
+
return Object.entries(schema.properties || {}).reduce((acc, [id, prop]) => {
|
|
585
|
+
const defaultValue = prop.default;
|
|
586
|
+
if (defaultValue !== undefined) {
|
|
587
|
+
acc[id] = defaultValue;
|
|
588
|
+
}
|
|
589
|
+
return acc;
|
|
590
|
+
}, {});
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
resetInputData() {
|
|
594
|
+
try {
|
|
595
|
+
this.runInputData = structuredClone(this.defaults);
|
|
596
|
+
} catch (err) {
|
|
597
|
+
this.runInputData = JSON.parse(JSON.stringify(this.defaults));
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
setDefaults(defaults) {
|
|
601
|
+
this.defaults = this.stripSymbols(defaults);
|
|
602
|
+
}
|
|
603
|
+
setInput(input) {
|
|
604
|
+
const schema = this.inputSchema();
|
|
605
|
+
if (typeof schema === "boolean") {
|
|
606
|
+
if (schema === true) {
|
|
607
|
+
for (const [inputId, value] of Object.entries(input)) {
|
|
608
|
+
if (value !== undefined) {
|
|
609
|
+
this.runInputData[inputId] = value;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
const properties = schema.properties || {};
|
|
616
|
+
for (const [inputId, prop] of Object.entries(properties)) {
|
|
617
|
+
if (input[inputId] !== undefined) {
|
|
618
|
+
this.runInputData[inputId] = input[inputId];
|
|
619
|
+
} else if (this.runInputData[inputId] === undefined && prop.default !== undefined) {
|
|
620
|
+
this.runInputData[inputId] = prop.default;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
if (schema.additionalProperties === true) {
|
|
624
|
+
for (const [inputId, value] of Object.entries(input)) {
|
|
625
|
+
if (value !== undefined && !(inputId in properties)) {
|
|
626
|
+
this.runInputData[inputId] = value;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
addInput(overrides) {
|
|
632
|
+
if (!overrides)
|
|
633
|
+
return false;
|
|
634
|
+
let changed = false;
|
|
635
|
+
const inputSchema = this.inputSchema();
|
|
636
|
+
if (typeof inputSchema === "boolean") {
|
|
637
|
+
if (inputSchema === false) {
|
|
638
|
+
return false;
|
|
639
|
+
}
|
|
640
|
+
for (const [key, value] of Object.entries(overrides)) {
|
|
641
|
+
if (!deepEqual(this.runInputData[key], value)) {
|
|
642
|
+
this.runInputData[key] = value;
|
|
643
|
+
changed = true;
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
return changed;
|
|
647
|
+
}
|
|
648
|
+
const properties = inputSchema.properties || {};
|
|
649
|
+
for (const [inputId, prop] of Object.entries(properties)) {
|
|
650
|
+
if (inputId === DATAFLOW_ALL_PORTS) {
|
|
651
|
+
this.runInputData = { ...this.runInputData, ...overrides };
|
|
652
|
+
changed = true;
|
|
653
|
+
} else {
|
|
654
|
+
if (overrides[inputId] === undefined)
|
|
655
|
+
continue;
|
|
656
|
+
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
657
|
+
if (isArray) {
|
|
658
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
659
|
+
const newitems = [...existingItems];
|
|
660
|
+
const overrideItem = overrides[inputId];
|
|
661
|
+
if (Array.isArray(overrideItem)) {
|
|
662
|
+
newitems.push(...overrideItem);
|
|
663
|
+
} else {
|
|
664
|
+
newitems.push(overrideItem);
|
|
665
|
+
}
|
|
666
|
+
this.runInputData[inputId] = newitems;
|
|
667
|
+
changed = true;
|
|
668
|
+
} else {
|
|
669
|
+
if (!deepEqual(this.runInputData[inputId], overrides[inputId])) {
|
|
670
|
+
this.runInputData[inputId] = overrides[inputId];
|
|
671
|
+
changed = true;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
if (inputSchema.additionalProperties === true) {
|
|
677
|
+
for (const [inputId, value] of Object.entries(overrides)) {
|
|
678
|
+
if (value !== undefined && !(inputId in properties)) {
|
|
679
|
+
if (!deepEqual(this.runInputData[inputId], value)) {
|
|
680
|
+
this.runInputData[inputId] = value;
|
|
681
|
+
changed = true;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
return changed;
|
|
687
|
+
}
|
|
688
|
+
async narrowInput(input) {
|
|
689
|
+
return input;
|
|
690
|
+
}
|
|
691
|
+
subscribe(name, fn) {
|
|
692
|
+
return this.events.subscribe(name, fn);
|
|
693
|
+
}
|
|
694
|
+
on(name, fn) {
|
|
695
|
+
this.events.on(name, fn);
|
|
696
|
+
}
|
|
697
|
+
off(name, fn) {
|
|
698
|
+
this.events.off(name, fn);
|
|
699
|
+
}
|
|
700
|
+
once(name, fn) {
|
|
701
|
+
this.events.once(name, fn);
|
|
702
|
+
}
|
|
703
|
+
waitOn(name) {
|
|
704
|
+
return this.events.waitOn(name);
|
|
705
|
+
}
|
|
706
|
+
emit(name, ...args) {
|
|
707
|
+
this._events?.emit(name, ...args);
|
|
708
|
+
}
|
|
709
|
+
emitSchemaChange(inputSchema, outputSchema) {
|
|
710
|
+
const finalInputSchema = inputSchema ?? this.inputSchema();
|
|
711
|
+
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
712
|
+
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
713
|
+
}
|
|
714
|
+
static _inputSchemaNode = new Map;
|
|
715
|
+
static generateInputSchemaNode(schema) {
|
|
716
|
+
if (typeof schema === "boolean") {
|
|
717
|
+
if (schema === false) {
|
|
718
|
+
return compileSchema({ not: {} });
|
|
719
|
+
}
|
|
720
|
+
return compileSchema({});
|
|
721
|
+
}
|
|
722
|
+
return compileSchema(schema);
|
|
723
|
+
}
|
|
724
|
+
static getInputSchemaNode(type) {
|
|
725
|
+
if (!this._inputSchemaNode.has(type)) {
|
|
726
|
+
const dataPortSchema = this.inputSchema();
|
|
727
|
+
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
728
|
+
try {
|
|
729
|
+
this._inputSchemaNode.set(type, schemaNode);
|
|
730
|
+
} catch (error) {
|
|
731
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
732
|
+
this._inputSchemaNode.set(type, compileSchema({}));
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return this._inputSchemaNode.get(type);
|
|
736
|
+
}
|
|
737
|
+
getInputSchemaNode(type) {
|
|
738
|
+
return this.constructor.getInputSchemaNode(type);
|
|
739
|
+
}
|
|
740
|
+
async validateInput(input) {
|
|
741
|
+
const schemaNode = this.getInputSchemaNode(this.type);
|
|
742
|
+
const result = schemaNode.validate(input);
|
|
743
|
+
if (!result.valid) {
|
|
744
|
+
const errorMessages = result.errors.map((e) => {
|
|
745
|
+
const path = e.data.pointer || "";
|
|
746
|
+
return `${e.message}${path ? ` (${path})` : ""}`;
|
|
747
|
+
});
|
|
748
|
+
throw new TaskInvalidInputError(`Input ${JSON.stringify(input)} does not match schema: ${errorMessages.join(", ")}`);
|
|
749
|
+
}
|
|
750
|
+
return true;
|
|
751
|
+
}
|
|
752
|
+
id() {
|
|
753
|
+
return this.config.id;
|
|
754
|
+
}
|
|
755
|
+
getProvenance() {
|
|
756
|
+
return this.config.provenance ?? {};
|
|
757
|
+
}
|
|
758
|
+
stripSymbols(obj) {
|
|
759
|
+
if (obj === null || obj === undefined) {
|
|
760
|
+
return obj;
|
|
761
|
+
}
|
|
762
|
+
if (Array.isArray(obj)) {
|
|
763
|
+
return obj.map((item) => this.stripSymbols(item));
|
|
764
|
+
}
|
|
765
|
+
if (typeof obj === "object") {
|
|
766
|
+
const result = {};
|
|
767
|
+
for (const key in obj) {
|
|
768
|
+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
769
|
+
result[key] = this.stripSymbols(obj[key]);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
return result;
|
|
773
|
+
}
|
|
774
|
+
return obj;
|
|
775
|
+
}
|
|
776
|
+
toJSON() {
|
|
777
|
+
const provenance = this.getProvenance();
|
|
778
|
+
const extras = this.config.extras;
|
|
779
|
+
let json = this.stripSymbols({
|
|
780
|
+
id: this.config.id,
|
|
781
|
+
type: this.type,
|
|
782
|
+
input: this.defaults,
|
|
783
|
+
...Object.keys(provenance).length ? { provenance } : {},
|
|
784
|
+
...extras && Object.keys(extras).length ? { extras } : {}
|
|
785
|
+
});
|
|
786
|
+
return json;
|
|
787
|
+
}
|
|
788
|
+
toDependencyJSON() {
|
|
789
|
+
const json = this.toJSON();
|
|
790
|
+
return json;
|
|
791
|
+
}
|
|
792
|
+
hasChildren() {
|
|
793
|
+
return this._subGraph !== undefined && this._subGraph !== null && this._subGraph.getTasks().length > 0;
|
|
794
|
+
}
|
|
795
|
+
_taskAddedListener = () => {
|
|
796
|
+
this.emit("regenerate");
|
|
797
|
+
};
|
|
798
|
+
_subGraph = undefined;
|
|
799
|
+
set subGraph(subGraph) {
|
|
800
|
+
if (this._subGraph) {
|
|
801
|
+
this._subGraph.off("task_added", this._taskAddedListener);
|
|
802
|
+
}
|
|
803
|
+
this._subGraph = subGraph;
|
|
804
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
805
|
+
}
|
|
806
|
+
get subGraph() {
|
|
807
|
+
if (!this._subGraph) {
|
|
808
|
+
this._subGraph = new TaskGraph;
|
|
809
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
810
|
+
}
|
|
811
|
+
return this._subGraph;
|
|
812
|
+
}
|
|
813
|
+
regenerateGraph() {
|
|
814
|
+
if (this.hasChildren()) {
|
|
815
|
+
for (const dataflow of this.subGraph.getDataflows()) {
|
|
816
|
+
this.subGraph.removeDataflow(dataflow);
|
|
817
|
+
}
|
|
818
|
+
for (const child of this.subGraph.getTasks()) {
|
|
819
|
+
this.subGraph.removeTask(child.config.id);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
this.events.emit("regenerate");
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// src/task/ConditionalTask.ts
|
|
827
|
+
class ConditionalTask extends Task {
|
|
828
|
+
static type = "ConditionalTask";
|
|
829
|
+
static category = "Flow Control";
|
|
830
|
+
static title = "Conditional Task";
|
|
831
|
+
static hasDynamicSchemas = true;
|
|
832
|
+
activeBranches = new Set;
|
|
833
|
+
async execute(input, context) {
|
|
834
|
+
if (context.signal?.aborted) {
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
this.activeBranches.clear();
|
|
838
|
+
const branches = this.config.branches ?? [];
|
|
839
|
+
const isExclusive = this.config.exclusive ?? true;
|
|
840
|
+
for (const branch of branches) {
|
|
841
|
+
try {
|
|
842
|
+
const isActive = branch.condition(input);
|
|
843
|
+
if (isActive) {
|
|
844
|
+
this.activeBranches.add(branch.id);
|
|
845
|
+
if (isExclusive) {
|
|
846
|
+
break;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
} catch (error) {
|
|
850
|
+
console.warn(`Condition evaluation failed for branch "${branch.id}":`, error);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
if (this.activeBranches.size === 0 && this.config.defaultBranch) {
|
|
854
|
+
const defaultBranchExists = branches.some((b) => b.id === this.config.defaultBranch);
|
|
855
|
+
if (defaultBranchExists) {
|
|
856
|
+
this.activeBranches.add(this.config.defaultBranch);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
return this.buildOutput(input);
|
|
860
|
+
}
|
|
861
|
+
buildOutput(input) {
|
|
862
|
+
const output = {
|
|
863
|
+
_activeBranches: Array.from(this.activeBranches)
|
|
864
|
+
};
|
|
865
|
+
const branches = this.config.branches ?? [];
|
|
866
|
+
for (const branch of branches) {
|
|
867
|
+
if (this.activeBranches.has(branch.id)) {
|
|
868
|
+
output[branch.outputPort] = { ...input };
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
return output;
|
|
872
|
+
}
|
|
873
|
+
isBranchActive(branchId) {
|
|
874
|
+
return this.activeBranches.has(branchId);
|
|
875
|
+
}
|
|
876
|
+
getActiveBranches() {
|
|
877
|
+
return new Set(this.activeBranches);
|
|
878
|
+
}
|
|
879
|
+
getPortActiveStatus() {
|
|
880
|
+
const status = new Map;
|
|
881
|
+
const branches = this.config.branches ?? [];
|
|
882
|
+
for (const branch of branches) {
|
|
883
|
+
status.set(branch.outputPort, this.activeBranches.has(branch.id));
|
|
884
|
+
}
|
|
885
|
+
return status;
|
|
886
|
+
}
|
|
887
|
+
static outputSchema() {
|
|
888
|
+
return {
|
|
889
|
+
type: "object",
|
|
890
|
+
properties: {
|
|
891
|
+
_activeBranches: {
|
|
892
|
+
type: "array",
|
|
893
|
+
items: { type: "string" },
|
|
894
|
+
description: "List of active branch IDs after condition evaluation"
|
|
895
|
+
}
|
|
896
|
+
},
|
|
897
|
+
additionalProperties: true
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
outputSchema() {
|
|
901
|
+
const branches = this.config?.branches ?? [];
|
|
902
|
+
const properties = {
|
|
903
|
+
_activeBranches: {
|
|
904
|
+
type: "array",
|
|
905
|
+
items: { type: "string" },
|
|
906
|
+
description: "List of active branch IDs after condition evaluation"
|
|
907
|
+
}
|
|
908
|
+
};
|
|
909
|
+
for (const branch of branches) {
|
|
910
|
+
properties[branch.outputPort] = {
|
|
911
|
+
type: "object",
|
|
912
|
+
description: `Output for branch "${branch.id}" when active`,
|
|
913
|
+
additionalProperties: true
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
return {
|
|
917
|
+
type: "object",
|
|
918
|
+
properties,
|
|
919
|
+
additionalProperties: false
|
|
920
|
+
};
|
|
921
|
+
}
|
|
922
|
+
static inputSchema() {
|
|
923
|
+
return {
|
|
924
|
+
type: "object",
|
|
925
|
+
properties: {},
|
|
926
|
+
additionalProperties: true
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
inputSchema() {
|
|
930
|
+
return {
|
|
931
|
+
type: "object",
|
|
932
|
+
properties: {},
|
|
933
|
+
additionalProperties: true
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// src/task-graph/TaskGraphScheduler.ts
|
|
939
|
+
class TopologicalScheduler {
|
|
940
|
+
dag;
|
|
941
|
+
sortedNodes;
|
|
942
|
+
currentIndex;
|
|
943
|
+
constructor(dag) {
|
|
944
|
+
this.dag = dag;
|
|
945
|
+
this.sortedNodes = [];
|
|
946
|
+
this.currentIndex = 0;
|
|
947
|
+
this.reset();
|
|
948
|
+
}
|
|
949
|
+
async* tasks() {
|
|
950
|
+
while (this.currentIndex < this.sortedNodes.length) {
|
|
951
|
+
yield this.sortedNodes[this.currentIndex++];
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
onTaskCompleted(taskId) {}
|
|
955
|
+
reset() {
|
|
956
|
+
this.sortedNodes = this.dag.topologicallySortedNodes();
|
|
957
|
+
this.currentIndex = 0;
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
class DependencyBasedScheduler {
|
|
962
|
+
dag;
|
|
963
|
+
completedTasks;
|
|
964
|
+
pendingTasks;
|
|
965
|
+
nextResolver = null;
|
|
966
|
+
constructor(dag) {
|
|
967
|
+
this.dag = dag;
|
|
968
|
+
this.completedTasks = new Set;
|
|
969
|
+
this.pendingTasks = new Set;
|
|
970
|
+
this.reset();
|
|
971
|
+
}
|
|
972
|
+
isTaskReady(task) {
|
|
973
|
+
if (task.status === "DISABLED" /* DISABLED */) {
|
|
974
|
+
return false;
|
|
975
|
+
}
|
|
976
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.config.id);
|
|
977
|
+
if (sourceDataflows.length > 0) {
|
|
978
|
+
const allIncomingDisabled = sourceDataflows.every((df) => df.status === "DISABLED" /* DISABLED */);
|
|
979
|
+
if (allIncomingDisabled) {
|
|
980
|
+
return false;
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
const dependencies = sourceDataflows.filter((df) => df.status !== "DISABLED" /* DISABLED */).map((dataflow) => dataflow.sourceTaskId);
|
|
984
|
+
return dependencies.every((dep) => this.completedTasks.has(dep));
|
|
985
|
+
}
|
|
986
|
+
async waitForNextTask() {
|
|
987
|
+
if (this.pendingTasks.size === 0)
|
|
988
|
+
return null;
|
|
989
|
+
for (const task of Array.from(this.pendingTasks)) {
|
|
990
|
+
if (task.status === "DISABLED" /* DISABLED */) {
|
|
991
|
+
this.pendingTasks.delete(task);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
if (this.pendingTasks.size === 0)
|
|
995
|
+
return null;
|
|
996
|
+
const readyTask = Array.from(this.pendingTasks).find((task) => this.isTaskReady(task));
|
|
997
|
+
if (readyTask) {
|
|
998
|
+
this.pendingTasks.delete(readyTask);
|
|
999
|
+
return readyTask;
|
|
1000
|
+
}
|
|
1001
|
+
if (this.pendingTasks.size > 0) {
|
|
1002
|
+
return new Promise((resolve) => {
|
|
1003
|
+
this.nextResolver = resolve;
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
return null;
|
|
1007
|
+
}
|
|
1008
|
+
async* tasks() {
|
|
1009
|
+
while (this.pendingTasks.size > 0) {
|
|
1010
|
+
const task = await this.waitForNextTask();
|
|
1011
|
+
if (task) {
|
|
1012
|
+
yield task;
|
|
1013
|
+
} else {
|
|
1014
|
+
break;
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
onTaskCompleted(taskId) {
|
|
1019
|
+
this.completedTasks.add(taskId);
|
|
1020
|
+
for (const task of Array.from(this.pendingTasks)) {
|
|
1021
|
+
if (task.status === "DISABLED" /* DISABLED */) {
|
|
1022
|
+
this.pendingTasks.delete(task);
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
if (this.nextResolver) {
|
|
1026
|
+
const readyTask = Array.from(this.pendingTasks).find((task) => this.isTaskReady(task));
|
|
1027
|
+
if (readyTask) {
|
|
1028
|
+
this.pendingTasks.delete(readyTask);
|
|
1029
|
+
const resolver = this.nextResolver;
|
|
1030
|
+
this.nextResolver = null;
|
|
1031
|
+
resolver(readyTask);
|
|
1032
|
+
} else if (this.pendingTasks.size === 0) {
|
|
1033
|
+
const resolver = this.nextResolver;
|
|
1034
|
+
this.nextResolver = null;
|
|
1035
|
+
resolver(null);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
reset() {
|
|
1040
|
+
this.completedTasks.clear();
|
|
1041
|
+
this.pendingTasks = new Set(this.dag.topologicallySortedNodes());
|
|
1042
|
+
this.nextResolver = null;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// src/task-graph/TaskGraphRunner.ts
|
|
1047
|
+
var PROPERTY_ARRAY = "PROPERTY_ARRAY";
|
|
1048
|
+
var GRAPH_RESULT_ARRAY = "GRAPH_RESULT_ARRAY";
|
|
1049
|
+
|
|
1050
|
+
class TaskGraphRunner {
|
|
1051
|
+
processScheduler;
|
|
1052
|
+
reactiveScheduler;
|
|
1053
|
+
running = false;
|
|
1054
|
+
reactiveRunning = false;
|
|
1055
|
+
provenanceInput;
|
|
1056
|
+
graph;
|
|
1057
|
+
outputCache;
|
|
1058
|
+
abortController;
|
|
1059
|
+
inProgressTasks = new Map;
|
|
1060
|
+
inProgressFunctions = new Map;
|
|
1061
|
+
failedTaskErrors = new Map;
|
|
1062
|
+
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph), reactiveScheduler = new TopologicalScheduler(graph)) {
|
|
1063
|
+
this.processScheduler = processScheduler;
|
|
1064
|
+
this.reactiveScheduler = reactiveScheduler;
|
|
1065
|
+
this.graph = graph;
|
|
1066
|
+
this.provenanceInput = new Map;
|
|
1067
|
+
graph.outputCache = outputCache;
|
|
1068
|
+
this.handleProgress = this.handleProgress.bind(this);
|
|
1069
|
+
}
|
|
1070
|
+
async runGraph(input = {}, config) {
|
|
1071
|
+
await this.handleStart(config);
|
|
1072
|
+
const results = [];
|
|
1073
|
+
let error;
|
|
1074
|
+
try {
|
|
1075
|
+
for await (const task of this.processScheduler.tasks()) {
|
|
1076
|
+
if (this.abortController?.signal.aborted) {
|
|
1077
|
+
break;
|
|
1078
|
+
}
|
|
1079
|
+
if (this.failedTaskErrors.size > 0) {
|
|
1080
|
+
break;
|
|
1081
|
+
}
|
|
1082
|
+
const isRootTask = this.graph.getSourceDataflows(task.config.id).length === 0;
|
|
1083
|
+
const runAsync = async () => {
|
|
1084
|
+
try {
|
|
1085
|
+
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1086
|
+
const taskPromise = this.runTaskWithProvenance(task, taskInput, config?.parentProvenance || {});
|
|
1087
|
+
this.inProgressTasks.set(task.config.id, taskPromise);
|
|
1088
|
+
const taskResult = await taskPromise;
|
|
1089
|
+
if (this.graph.getTargetDataflows(task.config.id).length === 0) {
|
|
1090
|
+
results.push(taskResult);
|
|
1091
|
+
}
|
|
1092
|
+
} catch (error2) {
|
|
1093
|
+
this.failedTaskErrors.set(task.config.id, error2);
|
|
1094
|
+
} finally {
|
|
1095
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
1096
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
1097
|
+
this.processScheduler.onTaskCompleted(task.config.id);
|
|
1098
|
+
}
|
|
1099
|
+
};
|
|
1100
|
+
this.inProgressFunctions.set(Symbol(task.config.id), runAsync());
|
|
1101
|
+
}
|
|
1102
|
+
} catch (err) {
|
|
1103
|
+
error = err;
|
|
1104
|
+
}
|
|
1105
|
+
await Promise.allSettled(Array.from(this.inProgressTasks.values()));
|
|
1106
|
+
await Promise.allSettled(Array.from(this.inProgressFunctions.values()));
|
|
1107
|
+
if (this.failedTaskErrors.size > 0) {
|
|
1108
|
+
const latestError = this.failedTaskErrors.values().next().value;
|
|
1109
|
+
this.handleError(latestError);
|
|
1110
|
+
throw latestError;
|
|
1111
|
+
}
|
|
1112
|
+
if (this.abortController?.signal.aborted) {
|
|
1113
|
+
await this.handleAbort();
|
|
1114
|
+
throw new TaskAbortedError;
|
|
1115
|
+
}
|
|
1116
|
+
await this.handleComplete();
|
|
1117
|
+
return results;
|
|
1118
|
+
}
|
|
1119
|
+
async runGraphReactive() {
|
|
1120
|
+
await this.handleStartReactive();
|
|
1121
|
+
const results = [];
|
|
1122
|
+
try {
|
|
1123
|
+
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1124
|
+
if (task.status === "PENDING" /* PENDING */) {
|
|
1125
|
+
task.resetInputData();
|
|
1126
|
+
this.copyInputFromEdgesToNode(task);
|
|
1127
|
+
}
|
|
1128
|
+
const taskResult = await task.runReactive();
|
|
1129
|
+
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1130
|
+
if (this.graph.getTargetDataflows(task.config.id).length === 0) {
|
|
1131
|
+
results.push({
|
|
1132
|
+
id: task.config.id,
|
|
1133
|
+
type: task.constructor.runtype || task.constructor.type,
|
|
1134
|
+
data: taskResult
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
await this.handleCompleteReactive();
|
|
1139
|
+
return results;
|
|
1140
|
+
} catch (error) {
|
|
1141
|
+
await this.handleErrorReactive();
|
|
1142
|
+
throw error;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
abort() {
|
|
1146
|
+
this.abortController?.abort();
|
|
1147
|
+
}
|
|
1148
|
+
async disable() {
|
|
1149
|
+
await this.handleDisable();
|
|
1150
|
+
}
|
|
1151
|
+
filterInputForTask(task, input) {
|
|
1152
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.config.id);
|
|
1153
|
+
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1154
|
+
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1155
|
+
const filteredInput = {};
|
|
1156
|
+
for (const [key, value] of Object.entries(input)) {
|
|
1157
|
+
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
1158
|
+
filteredInput[key] = value;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
return filteredInput;
|
|
1162
|
+
}
|
|
1163
|
+
addInputData(task, overrides) {
|
|
1164
|
+
if (!overrides)
|
|
1165
|
+
return;
|
|
1166
|
+
const changed = task.addInput(overrides);
|
|
1167
|
+
if (changed && "regenerateGraph" in task && typeof task.regenerateGraph === "function") {
|
|
1168
|
+
task.regenerateGraph();
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
mergeExecuteOutputsToRunOutput(results, compoundMerge) {
|
|
1172
|
+
if (compoundMerge === GRAPH_RESULT_ARRAY) {
|
|
1173
|
+
return results;
|
|
1174
|
+
}
|
|
1175
|
+
if (compoundMerge === PROPERTY_ARRAY) {
|
|
1176
|
+
let fixedOutput = {};
|
|
1177
|
+
const outputs = results.map((result) => result.data);
|
|
1178
|
+
if (outputs.length === 1) {
|
|
1179
|
+
fixedOutput = outputs[0];
|
|
1180
|
+
} else if (outputs.length > 1) {
|
|
1181
|
+
const collected = collectPropertyValues(outputs);
|
|
1182
|
+
if (Object.keys(collected).length > 0) {
|
|
1183
|
+
fixedOutput = collected;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
return fixedOutput;
|
|
1187
|
+
}
|
|
1188
|
+
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1189
|
+
}
|
|
1190
|
+
copyInputFromEdgesToNode(task) {
|
|
1191
|
+
const dataflows = this.graph.getSourceDataflows(task.config.id);
|
|
1192
|
+
for (const dataflow of dataflows) {
|
|
1193
|
+
this.addInputData(task, dataflow.getPortData());
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
getInputProvenance(node) {
|
|
1197
|
+
const nodeProvenance = {};
|
|
1198
|
+
this.graph.getSourceDataflows(node.config.id).forEach((dataflow) => {
|
|
1199
|
+
Object.assign(nodeProvenance, dataflow.provenance);
|
|
1200
|
+
});
|
|
1201
|
+
return nodeProvenance;
|
|
1202
|
+
}
|
|
1203
|
+
async pushOutputFromNodeToEdges(node, results, nodeProvenance) {
|
|
1204
|
+
const dataflows = this.graph.getTargetDataflows(node.config.id);
|
|
1205
|
+
for (const dataflow of dataflows) {
|
|
1206
|
+
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1207
|
+
if (compatibility === "static") {
|
|
1208
|
+
dataflow.setPortData(results, nodeProvenance);
|
|
1209
|
+
} else if (compatibility === "runtime") {
|
|
1210
|
+
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
1211
|
+
const narrowed = await task.narrowInput({ ...results });
|
|
1212
|
+
dataflow.setPortData(narrowed, nodeProvenance);
|
|
1213
|
+
} else {}
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1217
|
+
if (!node?.config?.id)
|
|
1218
|
+
return;
|
|
1219
|
+
const dataflows = graph.getTargetDataflows(node.config.id);
|
|
1220
|
+
const effectiveStatus = status ?? node.status;
|
|
1221
|
+
if (node instanceof ConditionalTask && effectiveStatus === "COMPLETED" /* COMPLETED */) {
|
|
1222
|
+
const branches = node.config.branches ?? [];
|
|
1223
|
+
const portToBranch = new Map;
|
|
1224
|
+
for (const branch of branches) {
|
|
1225
|
+
portToBranch.set(branch.outputPort, branch.id);
|
|
1226
|
+
}
|
|
1227
|
+
const activeBranches = node.getActiveBranches();
|
|
1228
|
+
for (const dataflow of dataflows) {
|
|
1229
|
+
const branchId = portToBranch.get(dataflow.sourceTaskPortId);
|
|
1230
|
+
if (branchId !== undefined) {
|
|
1231
|
+
if (activeBranches.has(branchId)) {
|
|
1232
|
+
dataflow.setStatus("COMPLETED" /* COMPLETED */);
|
|
1233
|
+
} else {
|
|
1234
|
+
dataflow.setStatus("DISABLED" /* DISABLED */);
|
|
1235
|
+
}
|
|
1236
|
+
} else {
|
|
1237
|
+
dataflow.setStatus(effectiveStatus);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
this.propagateDisabledStatus(graph);
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
dataflows.forEach((dataflow) => {
|
|
1244
|
+
dataflow.setStatus(effectiveStatus);
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1247
|
+
pushErrorFromNodeToEdges(graph, node) {
|
|
1248
|
+
if (!node?.config?.id)
|
|
1249
|
+
return;
|
|
1250
|
+
graph.getTargetDataflows(node.config.id).forEach((dataflow) => {
|
|
1251
|
+
dataflow.error = node.error;
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
propagateDisabledStatus(graph) {
|
|
1255
|
+
let changed = true;
|
|
1256
|
+
while (changed) {
|
|
1257
|
+
changed = false;
|
|
1258
|
+
for (const task of graph.getTasks()) {
|
|
1259
|
+
if (task.status !== "PENDING" /* PENDING */) {
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
const incomingDataflows = graph.getSourceDataflows(task.config.id);
|
|
1263
|
+
if (incomingDataflows.length === 0) {
|
|
1264
|
+
continue;
|
|
1265
|
+
}
|
|
1266
|
+
const allDisabled = incomingDataflows.every((df) => df.status === "DISABLED" /* DISABLED */);
|
|
1267
|
+
if (allDisabled) {
|
|
1268
|
+
task.status = "DISABLED" /* DISABLED */;
|
|
1269
|
+
task.progress = 100;
|
|
1270
|
+
task.completedAt = new Date;
|
|
1271
|
+
task.emit("disabled");
|
|
1272
|
+
task.emit("status", task.status);
|
|
1273
|
+
graph.getTargetDataflows(task.config.id).forEach((dataflow) => {
|
|
1274
|
+
dataflow.setStatus("DISABLED" /* DISABLED */);
|
|
1275
|
+
});
|
|
1276
|
+
this.processScheduler.onTaskCompleted(task.config.id);
|
|
1277
|
+
changed = true;
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
async runTaskWithProvenance(task, input, parentProvenance) {
|
|
1283
|
+
const nodeProvenance = {
|
|
1284
|
+
...parentProvenance,
|
|
1285
|
+
...this.getInputProvenance(task),
|
|
1286
|
+
...task.getProvenance()
|
|
1287
|
+
};
|
|
1288
|
+
this.provenanceInput.set(task.config.id, nodeProvenance);
|
|
1289
|
+
this.copyInputFromEdgesToNode(task);
|
|
1290
|
+
const results = await task.runner.run(input, {
|
|
1291
|
+
nodeProvenance,
|
|
1292
|
+
outputCache: this.outputCache,
|
|
1293
|
+
updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args)
|
|
1294
|
+
});
|
|
1295
|
+
await this.pushOutputFromNodeToEdges(task, results, nodeProvenance);
|
|
1296
|
+
return {
|
|
1297
|
+
id: task.config.id,
|
|
1298
|
+
type: task.constructor.runtype || task.constructor.type,
|
|
1299
|
+
data: results
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
resetTask(graph, task, runId) {
|
|
1303
|
+
task.status = "PENDING" /* PENDING */;
|
|
1304
|
+
task.resetInputData();
|
|
1305
|
+
task.runOutputData = {};
|
|
1306
|
+
task.error = undefined;
|
|
1307
|
+
task.progress = 0;
|
|
1308
|
+
if (task.config) {
|
|
1309
|
+
task.config.runnerId = runId;
|
|
1310
|
+
}
|
|
1311
|
+
this.pushStatusFromNodeToEdges(graph, task);
|
|
1312
|
+
this.pushErrorFromNodeToEdges(graph, task);
|
|
1313
|
+
task.emit("reset");
|
|
1314
|
+
task.emit("status", task.status);
|
|
1315
|
+
}
|
|
1316
|
+
resetGraph(graph, runnerId) {
|
|
1317
|
+
graph.getTasks().forEach((node) => {
|
|
1318
|
+
this.resetTask(graph, node, runnerId);
|
|
1319
|
+
node.regenerateGraph();
|
|
1320
|
+
if (node.hasChildren()) {
|
|
1321
|
+
this.resetGraph(node.subGraph, runnerId);
|
|
1322
|
+
}
|
|
1323
|
+
});
|
|
1324
|
+
graph.getDataflows().forEach((dataflow) => {
|
|
1325
|
+
dataflow.reset();
|
|
1326
|
+
});
|
|
1327
|
+
}
|
|
1328
|
+
async handleStart(config) {
|
|
1329
|
+
if (config?.outputCache !== undefined) {
|
|
1330
|
+
if (typeof config.outputCache === "boolean") {
|
|
1331
|
+
if (config.outputCache === true) {
|
|
1332
|
+
this.outputCache = globalServiceRegistry2.get(TASK_OUTPUT_REPOSITORY);
|
|
1333
|
+
} else {
|
|
1334
|
+
this.outputCache = undefined;
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
this.outputCache = config.outputCache;
|
|
1338
|
+
}
|
|
1339
|
+
this.graph.outputCache = this.outputCache;
|
|
1340
|
+
}
|
|
1341
|
+
if (this.running || this.reactiveRunning) {
|
|
1342
|
+
throw new TaskConfigurationError("Graph is already running");
|
|
1343
|
+
}
|
|
1344
|
+
this.running = true;
|
|
1345
|
+
this.abortController = new AbortController;
|
|
1346
|
+
this.abortController.signal.addEventListener("abort", () => {
|
|
1347
|
+
this.handleAbort();
|
|
1348
|
+
});
|
|
1349
|
+
if (config?.parentSignal?.aborted) {
|
|
1350
|
+
this.abortController.abort();
|
|
1351
|
+
return;
|
|
1352
|
+
} else {
|
|
1353
|
+
config?.parentSignal?.addEventListener("abort", () => {
|
|
1354
|
+
this.abortController?.abort();
|
|
1355
|
+
}, { once: true });
|
|
1356
|
+
}
|
|
1357
|
+
this.resetGraph(this.graph, uuid42());
|
|
1358
|
+
this.processScheduler.reset();
|
|
1359
|
+
this.inProgressTasks.clear();
|
|
1360
|
+
this.inProgressFunctions.clear();
|
|
1361
|
+
this.failedTaskErrors.clear();
|
|
1362
|
+
this.graph.emit("start");
|
|
1363
|
+
}
|
|
1364
|
+
async handleStartReactive() {
|
|
1365
|
+
if (this.reactiveRunning) {
|
|
1366
|
+
throw new TaskConfigurationError("Graph is already running reactively");
|
|
1367
|
+
}
|
|
1368
|
+
this.reactiveScheduler.reset();
|
|
1369
|
+
this.reactiveRunning = true;
|
|
1370
|
+
}
|
|
1371
|
+
async handleComplete() {
|
|
1372
|
+
this.running = false;
|
|
1373
|
+
this.graph.emit("complete");
|
|
1374
|
+
}
|
|
1375
|
+
async handleCompleteReactive() {
|
|
1376
|
+
this.reactiveRunning = false;
|
|
1377
|
+
}
|
|
1378
|
+
async handleError(error) {
|
|
1379
|
+
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
1380
|
+
if (["PROCESSING" /* PROCESSING */].includes(task.status)) {
|
|
1381
|
+
task.abort();
|
|
1382
|
+
}
|
|
1383
|
+
}));
|
|
1384
|
+
this.running = false;
|
|
1385
|
+
this.graph.emit("error", error);
|
|
1386
|
+
}
|
|
1387
|
+
async handleErrorReactive() {
|
|
1388
|
+
this.reactiveRunning = false;
|
|
1389
|
+
}
|
|
1390
|
+
async handleAbort() {
|
|
1391
|
+
this.graph.getTasks().map(async (task) => {
|
|
1392
|
+
if (["PROCESSING" /* PROCESSING */].includes(task.status)) {
|
|
1393
|
+
task.abort();
|
|
1394
|
+
}
|
|
1395
|
+
});
|
|
1396
|
+
this.running = false;
|
|
1397
|
+
this.graph.emit("abort");
|
|
1398
|
+
}
|
|
1399
|
+
async handleAbortReactive() {
|
|
1400
|
+
this.reactiveRunning = false;
|
|
1401
|
+
}
|
|
1402
|
+
async handleDisable() {
|
|
1403
|
+
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
1404
|
+
if (["PENDING" /* PENDING */].includes(task.status)) {
|
|
1405
|
+
return task.disable();
|
|
1406
|
+
}
|
|
1407
|
+
}));
|
|
1408
|
+
this.running = false;
|
|
1409
|
+
this.graph.emit("disabled");
|
|
1410
|
+
}
|
|
1411
|
+
async handleProgress(task, progress, message, ...args) {
|
|
1412
|
+
const total = this.graph.getTasks().length;
|
|
1413
|
+
if (total > 1) {
|
|
1414
|
+
const completed = this.graph.getTasks().reduce((acc, t) => acc + t.progress, 0);
|
|
1415
|
+
progress = Math.round(completed / total);
|
|
1416
|
+
}
|
|
1417
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
1418
|
+
await this.pushOutputFromNodeToEdges(task, task.runOutputData, task.getProvenance());
|
|
1419
|
+
this.graph.emit("graph_progress", progress, message, args);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
// src/task/GraphAsTaskRunner.ts
|
|
1424
|
+
class GraphAsTaskRunner extends TaskRunner {
|
|
1425
|
+
async executeTaskChildren(input) {
|
|
1426
|
+
const unsubscribe = this.task.subGraph.subscribe("graph_progress", (progress, message, ...args) => {
|
|
1427
|
+
this.task.emit("progress", progress, message, ...args);
|
|
1428
|
+
});
|
|
1429
|
+
const results = await this.task.subGraph.run(input, {
|
|
1430
|
+
parentProvenance: this.nodeProvenance || {},
|
|
1431
|
+
parentSignal: this.abortController?.signal,
|
|
1432
|
+
outputCache: this.outputCache
|
|
1433
|
+
});
|
|
1434
|
+
unsubscribe();
|
|
1435
|
+
return results;
|
|
1436
|
+
}
|
|
1437
|
+
async executeTaskChildrenReactive() {
|
|
1438
|
+
return this.task.subGraph.runReactive();
|
|
1439
|
+
}
|
|
1440
|
+
async handleDisable() {
|
|
1441
|
+
if (this.task.hasChildren()) {
|
|
1442
|
+
await this.task.subGraph.disable();
|
|
1443
|
+
}
|
|
1444
|
+
super.handleDisable();
|
|
1445
|
+
}
|
|
1446
|
+
fixInput(input) {
|
|
1447
|
+
const flattenedInput = Object.entries(input).reduce((acc, [key, value]) => {
|
|
1448
|
+
if (Array.isArray(value)) {
|
|
1449
|
+
return { ...acc, [key]: value[0] };
|
|
1450
|
+
}
|
|
1451
|
+
return { ...acc, [key]: value };
|
|
1452
|
+
}, {});
|
|
1453
|
+
return flattenedInput;
|
|
1454
|
+
}
|
|
1455
|
+
async executeTask(input) {
|
|
1456
|
+
if (this.task.hasChildren()) {
|
|
1457
|
+
const runExecuteOutputData = await this.executeTaskChildren(input);
|
|
1458
|
+
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(runExecuteOutputData, this.task.compoundMerge);
|
|
1459
|
+
} else {
|
|
1460
|
+
const result = await super.executeTask(this.fixInput(input));
|
|
1461
|
+
this.task.runOutputData = result ?? {};
|
|
1462
|
+
}
|
|
1463
|
+
return this.task.runOutputData;
|
|
1464
|
+
}
|
|
1465
|
+
async executeTaskReactive(input, output) {
|
|
1466
|
+
if (this.task.hasChildren()) {
|
|
1467
|
+
const reactiveResults = await this.executeTaskChildrenReactive();
|
|
1468
|
+
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(reactiveResults, this.task.compoundMerge);
|
|
1469
|
+
} else {
|
|
1470
|
+
const reactiveResults = await super.executeTaskReactive(this.fixInput(input), output);
|
|
1471
|
+
this.task.runOutputData = reactiveResults ?? output ?? {};
|
|
1472
|
+
}
|
|
1473
|
+
return this.task.runOutputData;
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
// src/task/GraphAsTask.ts
|
|
1478
|
+
class GraphAsTask extends Task {
|
|
1479
|
+
static type = "GraphAsTask";
|
|
1480
|
+
static category = "Hidden";
|
|
1481
|
+
static compoundMerge = PROPERTY_ARRAY;
|
|
1482
|
+
static hasDynamicSchemas = true;
|
|
1483
|
+
constructor(input = {}, config = {}) {
|
|
1484
|
+
const { subGraph, ...rest } = config;
|
|
1485
|
+
super(input, rest);
|
|
1486
|
+
if (subGraph) {
|
|
1487
|
+
this.subGraph = subGraph;
|
|
1488
|
+
}
|
|
1489
|
+
this.regenerateGraph();
|
|
1490
|
+
}
|
|
1491
|
+
get runner() {
|
|
1492
|
+
if (!this._runner) {
|
|
1493
|
+
this._runner = new GraphAsTaskRunner(this);
|
|
1494
|
+
}
|
|
1495
|
+
return this._runner;
|
|
1496
|
+
}
|
|
1497
|
+
get compoundMerge() {
|
|
1498
|
+
return this.config?.compoundMerge || this.constructor.compoundMerge;
|
|
1499
|
+
}
|
|
1500
|
+
get cacheable() {
|
|
1501
|
+
return this.config?.cacheable ?? (this.constructor.cacheable && !this.hasChildren());
|
|
1502
|
+
}
|
|
1503
|
+
inputSchema() {
|
|
1504
|
+
if (!this.hasChildren()) {
|
|
1505
|
+
return this.constructor.inputSchema();
|
|
1506
|
+
}
|
|
1507
|
+
const properties = {};
|
|
1508
|
+
const required = [];
|
|
1509
|
+
const tasks = this.subGraph.getTasks();
|
|
1510
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
1511
|
+
for (const task of startingNodes) {
|
|
1512
|
+
const taskInputSchema = task.inputSchema();
|
|
1513
|
+
if (typeof taskInputSchema === "boolean") {
|
|
1514
|
+
if (taskInputSchema === false) {
|
|
1515
|
+
continue;
|
|
1516
|
+
}
|
|
1517
|
+
if (taskInputSchema === true) {
|
|
1518
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
1523
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
1524
|
+
if (!properties[inputName]) {
|
|
1525
|
+
properties[inputName] = inputProp;
|
|
1526
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
1527
|
+
required.push(inputName);
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
return {
|
|
1533
|
+
type: "object",
|
|
1534
|
+
properties,
|
|
1535
|
+
...required.length > 0 ? { required } : {},
|
|
1536
|
+
additionalProperties: false
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
_inputSchemaNode;
|
|
1540
|
+
getInputSchemaNode(type) {
|
|
1541
|
+
if (!this._inputSchemaNode) {
|
|
1542
|
+
const dataPortSchema = this.inputSchema();
|
|
1543
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
1544
|
+
try {
|
|
1545
|
+
this._inputSchemaNode = schemaNode;
|
|
1546
|
+
} catch (error) {
|
|
1547
|
+
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
1548
|
+
this._inputSchemaNode = compileSchema2({});
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
return this._inputSchemaNode;
|
|
1552
|
+
}
|
|
1553
|
+
calculateNodeDepths() {
|
|
1554
|
+
const depths = new Map;
|
|
1555
|
+
const tasks = this.subGraph.getTasks();
|
|
1556
|
+
for (const task of tasks) {
|
|
1557
|
+
depths.set(task.config.id, 0);
|
|
1558
|
+
}
|
|
1559
|
+
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
1560
|
+
for (const task of sortedTasks) {
|
|
1561
|
+
const currentDepth = depths.get(task.config.id) || 0;
|
|
1562
|
+
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
1563
|
+
for (const targetTask of targetTasks) {
|
|
1564
|
+
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
1565
|
+
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
return depths;
|
|
1569
|
+
}
|
|
1570
|
+
outputSchema() {
|
|
1571
|
+
if (!this.hasChildren()) {
|
|
1572
|
+
return this.constructor.outputSchema();
|
|
1573
|
+
}
|
|
1574
|
+
const properties = {};
|
|
1575
|
+
const required = [];
|
|
1576
|
+
const tasks = this.subGraph.getTasks();
|
|
1577
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
1578
|
+
const depths = this.calculateNodeDepths();
|
|
1579
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
1580
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
1581
|
+
const propertyCount = {};
|
|
1582
|
+
const propertySchema = {};
|
|
1583
|
+
for (const task of lastLevelNodes) {
|
|
1584
|
+
const taskOutputSchema = task.outputSchema();
|
|
1585
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
1586
|
+
if (taskOutputSchema === false) {
|
|
1587
|
+
continue;
|
|
1588
|
+
}
|
|
1589
|
+
if (taskOutputSchema === true) {
|
|
1590
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
1591
|
+
continue;
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
1595
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
1596
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
1597
|
+
if (!propertySchema[outputName]) {
|
|
1598
|
+
propertySchema[outputName] = outputProp;
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
1603
|
+
const outputProp = propertySchema[outputName];
|
|
1604
|
+
if (lastLevelNodes.length === 1) {
|
|
1605
|
+
properties[outputName] = outputProp;
|
|
1606
|
+
} else {
|
|
1607
|
+
properties[outputName] = {
|
|
1608
|
+
type: "array",
|
|
1609
|
+
items: outputProp
|
|
1610
|
+
};
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
return {
|
|
1614
|
+
type: "object",
|
|
1615
|
+
properties,
|
|
1616
|
+
...required.length > 0 ? { required } : {},
|
|
1617
|
+
additionalProperties: false
|
|
1618
|
+
};
|
|
1619
|
+
}
|
|
1620
|
+
resetInputData() {
|
|
1621
|
+
super.resetInputData();
|
|
1622
|
+
if (this.hasChildren()) {
|
|
1623
|
+
this.subGraph.getTasks().forEach((node) => {
|
|
1624
|
+
node.resetInputData();
|
|
1625
|
+
});
|
|
1626
|
+
this.subGraph.getDataflows().forEach((dataflow) => {
|
|
1627
|
+
dataflow.reset();
|
|
1628
|
+
});
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
regenerateGraph() {
|
|
1632
|
+
this._inputSchemaNode = undefined;
|
|
1633
|
+
this.events.emit("regenerate");
|
|
1634
|
+
}
|
|
1635
|
+
toJSON() {
|
|
1636
|
+
let json = super.toJSON();
|
|
1637
|
+
const hasChildren = this.hasChildren();
|
|
1638
|
+
if (hasChildren) {
|
|
1639
|
+
json = {
|
|
1640
|
+
...json,
|
|
1641
|
+
merge: this.compoundMerge,
|
|
1642
|
+
subgraph: this.subGraph.toJSON()
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
return json;
|
|
1646
|
+
}
|
|
1647
|
+
toDependencyJSON() {
|
|
1648
|
+
const json = this.toJSON();
|
|
1649
|
+
if (this.hasChildren()) {
|
|
1650
|
+
if ("subgraph" in json) {
|
|
1651
|
+
delete json.subgraph;
|
|
1652
|
+
}
|
|
1653
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
1654
|
+
}
|
|
1655
|
+
return json;
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
// src/task-graph/Workflow.ts
|
|
1660
|
+
import { EventEmitter as EventEmitter4 } from "@workglow/util";
|
|
1661
|
+
class WorkflowTask extends GraphAsTask {
|
|
1662
|
+
static type = "Workflow";
|
|
1663
|
+
static compoundMerge = PROPERTY_ARRAY;
|
|
1664
|
+
}
|
|
1665
|
+
var taskIdCounter = 0;
|
|
1666
|
+
|
|
1667
|
+
class Workflow {
|
|
1668
|
+
constructor(repository) {
|
|
1669
|
+
this._repository = repository;
|
|
1670
|
+
this._graph = new TaskGraph({
|
|
1671
|
+
outputCache: this._repository
|
|
1672
|
+
});
|
|
1673
|
+
this._onChanged = this._onChanged.bind(this);
|
|
1674
|
+
this.setupEvents();
|
|
1675
|
+
}
|
|
1676
|
+
_graph;
|
|
1677
|
+
_dataFlows = [];
|
|
1678
|
+
_error = "";
|
|
1679
|
+
_repository;
|
|
1680
|
+
_abortController;
|
|
1681
|
+
events = new EventEmitter4;
|
|
1682
|
+
static createWorkflow(taskClass) {
|
|
1683
|
+
const helper = function(input = {}, config = {}) {
|
|
1684
|
+
this._error = "";
|
|
1685
|
+
const parent = getLastTask(this);
|
|
1686
|
+
taskIdCounter++;
|
|
1687
|
+
const task = this.addTask(taskClass, input, { id: String(taskIdCounter), ...config });
|
|
1688
|
+
if (this._dataFlows.length > 0) {
|
|
1689
|
+
this._dataFlows.forEach((dataflow) => {
|
|
1690
|
+
const taskSchema = task.inputSchema();
|
|
1691
|
+
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
1692
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.config.id}`;
|
|
1693
|
+
console.error(this._error);
|
|
1694
|
+
return;
|
|
1695
|
+
}
|
|
1696
|
+
dataflow.targetTaskId = task.config.id;
|
|
1697
|
+
this.graph.addDataflow(dataflow);
|
|
1698
|
+
});
|
|
1699
|
+
this._dataFlows = [];
|
|
1700
|
+
}
|
|
1701
|
+
if (parent && this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
1702
|
+
const matches = new Map;
|
|
1703
|
+
const sourceSchema = parent.outputSchema();
|
|
1704
|
+
const targetSchema = task.inputSchema();
|
|
1705
|
+
const makeMatch = (comparator) => {
|
|
1706
|
+
if (typeof sourceSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1707
|
+
return matches;
|
|
1708
|
+
}
|
|
1709
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(sourceSchema.properties || {})) {
|
|
1710
|
+
for (const [toInputPortId, toPortInputSchema] of Object.entries(targetSchema.properties || {})) {
|
|
1711
|
+
if (!matches.has(toInputPortId) && comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
1712
|
+
matches.set(toInputPortId, fromOutputPortId);
|
|
1713
|
+
this.connect(parent.config.id, fromOutputPortId, task.config.id, toInputPortId);
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
return matches;
|
|
1718
|
+
};
|
|
1719
|
+
makeMatch(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1720
|
+
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
1721
|
+
if (fromPortOutputSchema === true && toPortInputSchema === true) {
|
|
1722
|
+
return true;
|
|
1723
|
+
}
|
|
1724
|
+
return false;
|
|
1725
|
+
}
|
|
1726
|
+
const idTypeMatch = fromPortOutputSchema.$id !== undefined && fromPortOutputSchema.$id === toPortInputSchema.$id;
|
|
1727
|
+
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
1728
|
+
const typeMatch = idTypeBlank && (fromPortOutputSchema.type === toPortInputSchema.type || (toPortInputSchema.oneOf?.some((i) => i.type == fromPortOutputSchema.type) ?? false));
|
|
1729
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1730
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1731
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1732
|
+
return (idTypeMatch || typeMatch) && portIdsCompatible;
|
|
1733
|
+
});
|
|
1734
|
+
if (matches.size === 0) {
|
|
1735
|
+
this._error = `Could not find a match between the outputs of ${parent.type} and the inputs of ${task.type}. ` + `You now need to connect the outputs to the inputs via connect() manually before adding this task. Task not added.`;
|
|
1736
|
+
console.error(this._error);
|
|
1737
|
+
this.graph.removeTask(task.config.id);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
return this;
|
|
1741
|
+
};
|
|
1742
|
+
helper.type = taskClass.runtype ?? taskClass.type;
|
|
1743
|
+
helper.category = taskClass.category;
|
|
1744
|
+
helper.inputSchema = taskClass.inputSchema;
|
|
1745
|
+
helper.outputSchema = taskClass.outputSchema;
|
|
1746
|
+
helper.cacheable = taskClass.cacheable;
|
|
1747
|
+
helper.workflowCreate = true;
|
|
1748
|
+
return helper;
|
|
1749
|
+
}
|
|
1750
|
+
get graph() {
|
|
1751
|
+
return this._graph;
|
|
1752
|
+
}
|
|
1753
|
+
set graph(value) {
|
|
1754
|
+
this._dataFlows = [];
|
|
1755
|
+
this._error = "";
|
|
1756
|
+
this.clearEvents();
|
|
1757
|
+
this._graph = value;
|
|
1758
|
+
this.setupEvents();
|
|
1759
|
+
this.events.emit("reset");
|
|
1760
|
+
}
|
|
1761
|
+
get error() {
|
|
1762
|
+
return this._error;
|
|
1763
|
+
}
|
|
1764
|
+
on(name, fn) {
|
|
1765
|
+
this.events.on(name, fn);
|
|
1766
|
+
}
|
|
1767
|
+
off(name, fn) {
|
|
1768
|
+
this.events.off(name, fn);
|
|
1769
|
+
}
|
|
1770
|
+
once(name, fn) {
|
|
1771
|
+
this.events.once(name, fn);
|
|
1772
|
+
}
|
|
1773
|
+
waitOn(name) {
|
|
1774
|
+
return this.events.waitOn(name);
|
|
1775
|
+
}
|
|
1776
|
+
async run(input = {}) {
|
|
1777
|
+
this.events.emit("start");
|
|
1778
|
+
this._abortController = new AbortController;
|
|
1779
|
+
try {
|
|
1780
|
+
const output = await this.graph.run(input, {
|
|
1781
|
+
parentSignal: this._abortController.signal,
|
|
1782
|
+
parentProvenance: {},
|
|
1783
|
+
outputCache: this._repository
|
|
1784
|
+
});
|
|
1785
|
+
const results = this.graph.mergeExecuteOutputsToRunOutput(output, PROPERTY_ARRAY);
|
|
1786
|
+
this.events.emit("complete");
|
|
1787
|
+
return results;
|
|
1788
|
+
} catch (error) {
|
|
1789
|
+
this.events.emit("error", String(error));
|
|
1790
|
+
throw error;
|
|
1791
|
+
} finally {
|
|
1792
|
+
this._abortController = undefined;
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
async abort() {
|
|
1796
|
+
this._abortController?.abort();
|
|
1797
|
+
}
|
|
1798
|
+
pop() {
|
|
1799
|
+
this._error = "";
|
|
1800
|
+
const nodes = this._graph.getTasks();
|
|
1801
|
+
if (nodes.length === 0) {
|
|
1802
|
+
this._error = "No tasks to remove";
|
|
1803
|
+
console.error(this._error);
|
|
1804
|
+
return this;
|
|
1805
|
+
}
|
|
1806
|
+
const lastNode = nodes[nodes.length - 1];
|
|
1807
|
+
this._graph.removeTask(lastNode.config.id);
|
|
1808
|
+
return this;
|
|
1809
|
+
}
|
|
1810
|
+
toJSON() {
|
|
1811
|
+
return this._graph.toJSON();
|
|
1812
|
+
}
|
|
1813
|
+
toDependencyJSON() {
|
|
1814
|
+
return this._graph.toDependencyJSON();
|
|
1815
|
+
}
|
|
1816
|
+
pipe(...args) {
|
|
1817
|
+
return pipe(args, this);
|
|
1818
|
+
}
|
|
1819
|
+
static pipe(...args) {
|
|
1820
|
+
return pipe(args, new Workflow);
|
|
1821
|
+
}
|
|
1822
|
+
parallel(args, mergeFn) {
|
|
1823
|
+
return parallel(args, mergeFn ?? PROPERTY_ARRAY, this);
|
|
1824
|
+
}
|
|
1825
|
+
static parallel(args, mergeFn) {
|
|
1826
|
+
return parallel(args, mergeFn ?? PROPERTY_ARRAY, new Workflow);
|
|
1827
|
+
}
|
|
1828
|
+
rename(source, target, index = -1) {
|
|
1829
|
+
this._error = "";
|
|
1830
|
+
const nodes = this._graph.getTasks();
|
|
1831
|
+
if (-index > nodes.length) {
|
|
1832
|
+
const errorMsg = `Back index greater than number of tasks`;
|
|
1833
|
+
this._error = errorMsg;
|
|
1834
|
+
console.error(this._error);
|
|
1835
|
+
throw new WorkflowError(errorMsg);
|
|
1836
|
+
}
|
|
1837
|
+
const lastNode = nodes[nodes.length + index];
|
|
1838
|
+
const outputSchema = lastNode.outputSchema();
|
|
1839
|
+
if (typeof outputSchema === "boolean") {
|
|
1840
|
+
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
1841
|
+
const errorMsg = `Task ${lastNode.config.id} has schema 'false' and outputs nothing`;
|
|
1842
|
+
this._error = errorMsg;
|
|
1843
|
+
console.error(this._error);
|
|
1844
|
+
throw new WorkflowError(errorMsg);
|
|
1845
|
+
}
|
|
1846
|
+
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
1847
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.config.id}`;
|
|
1848
|
+
this._error = errorMsg;
|
|
1849
|
+
console.error(this._error);
|
|
1850
|
+
throw new WorkflowError(errorMsg);
|
|
1851
|
+
}
|
|
1852
|
+
this._dataFlows.push(new Dataflow(lastNode.config.id, source, undefined, target));
|
|
1853
|
+
return this;
|
|
1854
|
+
}
|
|
1855
|
+
toTaskGraph() {
|
|
1856
|
+
return this._graph;
|
|
1857
|
+
}
|
|
1858
|
+
toTask() {
|
|
1859
|
+
const task = new WorkflowTask;
|
|
1860
|
+
task.subGraph = this.toTaskGraph();
|
|
1861
|
+
return task;
|
|
1862
|
+
}
|
|
1863
|
+
reset() {
|
|
1864
|
+
taskIdCounter = 0;
|
|
1865
|
+
this.clearEvents();
|
|
1866
|
+
this._graph = new TaskGraph({
|
|
1867
|
+
outputCache: this._repository
|
|
1868
|
+
});
|
|
1869
|
+
this._dataFlows = [];
|
|
1870
|
+
this._error = "";
|
|
1871
|
+
this.setupEvents();
|
|
1872
|
+
this.events.emit("changed", undefined);
|
|
1873
|
+
this.events.emit("reset");
|
|
1874
|
+
return this;
|
|
1875
|
+
}
|
|
1876
|
+
setupEvents() {
|
|
1877
|
+
this._graph.on("task_added", this._onChanged);
|
|
1878
|
+
this._graph.on("task_replaced", this._onChanged);
|
|
1879
|
+
this._graph.on("task_removed", this._onChanged);
|
|
1880
|
+
this._graph.on("dataflow_added", this._onChanged);
|
|
1881
|
+
this._graph.on("dataflow_replaced", this._onChanged);
|
|
1882
|
+
this._graph.on("dataflow_removed", this._onChanged);
|
|
1883
|
+
}
|
|
1884
|
+
clearEvents() {
|
|
1885
|
+
this._graph.off("task_added", this._onChanged);
|
|
1886
|
+
this._graph.off("task_replaced", this._onChanged);
|
|
1887
|
+
this._graph.off("task_removed", this._onChanged);
|
|
1888
|
+
this._graph.off("dataflow_added", this._onChanged);
|
|
1889
|
+
this._graph.off("dataflow_replaced", this._onChanged);
|
|
1890
|
+
this._graph.off("dataflow_removed", this._onChanged);
|
|
1891
|
+
}
|
|
1892
|
+
_onChanged(id) {
|
|
1893
|
+
this.events.emit("changed", id);
|
|
1894
|
+
}
|
|
1895
|
+
connect(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId) {
|
|
1896
|
+
const sourceTask = this.graph.getTask(sourceTaskId);
|
|
1897
|
+
const targetTask = this.graph.getTask(targetTaskId);
|
|
1898
|
+
if (!sourceTask || !targetTask) {
|
|
1899
|
+
throw new WorkflowError("Source or target task not found");
|
|
1900
|
+
}
|
|
1901
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
1902
|
+
const targetSchema = targetTask.inputSchema();
|
|
1903
|
+
if (typeof sourceSchema === "boolean") {
|
|
1904
|
+
if (sourceSchema === false) {
|
|
1905
|
+
throw new WorkflowError(`Source task has schema 'false' and outputs nothing`);
|
|
1906
|
+
}
|
|
1907
|
+
} else if (!sourceSchema.properties?.[sourceTaskPortId]) {
|
|
1908
|
+
throw new WorkflowError(`Output ${sourceTaskPortId} not found on source task`);
|
|
1909
|
+
}
|
|
1910
|
+
if (typeof targetSchema === "boolean") {
|
|
1911
|
+
if (targetSchema === false) {
|
|
1912
|
+
throw new WorkflowError(`Target task has schema 'false' and accepts no inputs`);
|
|
1913
|
+
}
|
|
1914
|
+
} else if (!targetSchema.properties?.[targetTaskPortId]) {
|
|
1915
|
+
throw new WorkflowError(`Input ${targetTaskPortId} not found on target task`);
|
|
1916
|
+
}
|
|
1917
|
+
const dataflow = new Dataflow(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
1918
|
+
this.graph.addDataflow(dataflow);
|
|
1919
|
+
return this;
|
|
1920
|
+
}
|
|
1921
|
+
addTask(taskClass, input, config) {
|
|
1922
|
+
const task = new taskClass(input, config);
|
|
1923
|
+
const id = this.graph.addTask(task);
|
|
1924
|
+
this.events.emit("changed", id);
|
|
1925
|
+
return task;
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
function CreateWorkflow(taskClass) {
|
|
1929
|
+
return Workflow.createWorkflow(taskClass);
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
// src/task-graph/Conversions.ts
|
|
1933
|
+
class ListeningGraphAsTask extends GraphAsTask {
|
|
1934
|
+
constructor(input, config) {
|
|
1935
|
+
super(input, config);
|
|
1936
|
+
this.subGraph.on("start", () => {
|
|
1937
|
+
this.emit("start");
|
|
1938
|
+
});
|
|
1939
|
+
this.subGraph.on("complete", () => {
|
|
1940
|
+
this.emit("complete");
|
|
1941
|
+
});
|
|
1942
|
+
this.subGraph.on("error", (e) => {
|
|
1943
|
+
this.emit("error", e);
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
class OwnGraphTask extends ListeningGraphAsTask {
|
|
1949
|
+
static type = "Own[Graph]";
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
class OwnWorkflowTask extends ListeningGraphAsTask {
|
|
1953
|
+
static type = "Own[Workflow]";
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
class GraphTask extends GraphAsTask {
|
|
1957
|
+
static type = "Graph";
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1960
|
+
class WorkflowTask2 extends GraphAsTask {
|
|
1961
|
+
static type = "Workflow";
|
|
1962
|
+
}
|
|
1963
|
+
function convertPipeFunctionToTask(fn, config) {
|
|
1964
|
+
|
|
1965
|
+
class QuickTask extends Task {
|
|
1966
|
+
static type = fn.name ? `\uD835\uDC53 ${fn.name}` : "\uD835\uDC53";
|
|
1967
|
+
static inputSchema = () => {
|
|
1968
|
+
return {
|
|
1969
|
+
type: "object",
|
|
1970
|
+
properties: {
|
|
1971
|
+
[DATAFLOW_ALL_PORTS]: {}
|
|
1972
|
+
},
|
|
1973
|
+
additionalProperties: false
|
|
1974
|
+
};
|
|
1975
|
+
};
|
|
1976
|
+
static outputSchema = () => {
|
|
1977
|
+
return {
|
|
1978
|
+
type: "object",
|
|
1979
|
+
properties: {
|
|
1980
|
+
[DATAFLOW_ALL_PORTS]: {}
|
|
1981
|
+
},
|
|
1982
|
+
additionalProperties: false
|
|
1983
|
+
};
|
|
1984
|
+
};
|
|
1985
|
+
static cacheable = false;
|
|
1986
|
+
async execute(input, context) {
|
|
1987
|
+
return fn(input, context);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
return new QuickTask({}, config);
|
|
1991
|
+
}
|
|
1992
|
+
function ensureTask(arg, config = {}) {
|
|
1993
|
+
if (arg instanceof Task) {
|
|
1994
|
+
return arg;
|
|
1995
|
+
}
|
|
1996
|
+
if (arg instanceof TaskGraph) {
|
|
1997
|
+
if (config.isOwned) {
|
|
1998
|
+
return new OwnGraphTask({}, { ...config, subGraph: arg });
|
|
1999
|
+
} else {
|
|
2000
|
+
return new GraphTask({}, { ...config, subGraph: arg });
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
if (arg instanceof Workflow) {
|
|
2004
|
+
if (config.isOwned) {
|
|
2005
|
+
return new OwnWorkflowTask({}, { ...config, subGraph: arg.graph });
|
|
2006
|
+
} else {
|
|
2007
|
+
return new WorkflowTask2({}, { ...config, subGraph: arg.graph });
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
return convertPipeFunctionToTask(arg, config);
|
|
2011
|
+
}
|
|
2012
|
+
function getLastTask(workflow) {
|
|
2013
|
+
const tasks = workflow.graph.getTasks();
|
|
2014
|
+
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
2015
|
+
}
|
|
2016
|
+
function connect(source, target, workflow) {
|
|
2017
|
+
workflow.graph.addDataflow(new Dataflow(source.config.id, "*", target.config.id, "*"));
|
|
2018
|
+
}
|
|
2019
|
+
function pipe(args, workflow = new Workflow) {
|
|
2020
|
+
let previousTask = getLastTask(workflow);
|
|
2021
|
+
const tasks = args.map((arg) => ensureTask(arg));
|
|
2022
|
+
tasks.forEach((task) => {
|
|
2023
|
+
workflow.graph.addTask(task);
|
|
2024
|
+
if (previousTask) {
|
|
2025
|
+
connect(previousTask, task, workflow);
|
|
2026
|
+
}
|
|
2027
|
+
previousTask = task;
|
|
2028
|
+
});
|
|
2029
|
+
return workflow;
|
|
2030
|
+
}
|
|
2031
|
+
function parallel(args, mergeFn = PROPERTY_ARRAY, workflow = new Workflow) {
|
|
2032
|
+
let previousTask = getLastTask(workflow);
|
|
2033
|
+
const tasks = args.map((arg) => ensureTask(arg));
|
|
2034
|
+
const input = {};
|
|
2035
|
+
const config = {
|
|
2036
|
+
compoundMerge: mergeFn
|
|
2037
|
+
};
|
|
2038
|
+
const name = `‖${args.map((arg) => "\uD835\uDC53").join("‖")}‖`;
|
|
2039
|
+
|
|
2040
|
+
class ParallelTask extends GraphAsTask {
|
|
2041
|
+
static type = name;
|
|
2042
|
+
}
|
|
2043
|
+
const mergeTask = new ParallelTask(input, config);
|
|
2044
|
+
mergeTask.subGraph.addTasks(tasks);
|
|
2045
|
+
workflow.graph.addTask(mergeTask);
|
|
2046
|
+
if (previousTask) {
|
|
2047
|
+
connect(previousTask, mergeTask, workflow);
|
|
2048
|
+
}
|
|
2049
|
+
return workflow;
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
// src/task-graph/TaskGraphEvents.ts
|
|
2053
|
+
var EventDagToTaskGraphMapping = {
|
|
2054
|
+
"node-added": "task_added",
|
|
2055
|
+
"node-removed": "task_removed",
|
|
2056
|
+
"node-replaced": "task_replaced",
|
|
2057
|
+
"edge-added": "dataflow_added",
|
|
2058
|
+
"edge-removed": "dataflow_removed",
|
|
2059
|
+
"edge-replaced": "dataflow_replaced"
|
|
2060
|
+
};
|
|
2061
|
+
var EventTaskGraphToDagMapping = {
|
|
2062
|
+
task_added: "node-added",
|
|
2063
|
+
task_removed: "node-removed",
|
|
2064
|
+
task_replaced: "node-replaced",
|
|
2065
|
+
dataflow_added: "edge-added",
|
|
2066
|
+
dataflow_removed: "edge-removed",
|
|
2067
|
+
dataflow_replaced: "edge-replaced"
|
|
2068
|
+
};
|
|
2069
|
+
|
|
2070
|
+
// src/task-graph/TaskGraph.ts
|
|
2071
|
+
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
2072
|
+
constructor() {
|
|
2073
|
+
super((task) => task.config.id, (dataflow) => dataflow.id);
|
|
2074
|
+
}
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
class TaskGraph {
|
|
2078
|
+
outputCache;
|
|
2079
|
+
constructor({ outputCache, dag } = {}) {
|
|
2080
|
+
this.outputCache = outputCache;
|
|
2081
|
+
this._dag = dag || new TaskGraphDAG;
|
|
2082
|
+
}
|
|
2083
|
+
_dag;
|
|
2084
|
+
_runner;
|
|
2085
|
+
get runner() {
|
|
2086
|
+
if (!this._runner) {
|
|
2087
|
+
this._runner = new TaskGraphRunner(this, this.outputCache);
|
|
2088
|
+
}
|
|
2089
|
+
return this._runner;
|
|
2090
|
+
}
|
|
2091
|
+
run(input = {}, config = {}) {
|
|
2092
|
+
return this.runner.runGraph(input, {
|
|
2093
|
+
outputCache: config?.outputCache || this.outputCache,
|
|
2094
|
+
parentProvenance: config?.parentProvenance || {},
|
|
2095
|
+
parentSignal: config?.parentSignal || undefined
|
|
2096
|
+
});
|
|
2097
|
+
}
|
|
2098
|
+
runReactive() {
|
|
2099
|
+
return this.runner.runGraphReactive();
|
|
2100
|
+
}
|
|
2101
|
+
mergeExecuteOutputsToRunOutput(results, compoundMerge) {
|
|
2102
|
+
return this.runner.mergeExecuteOutputsToRunOutput(results, compoundMerge);
|
|
2103
|
+
}
|
|
2104
|
+
abort() {
|
|
2105
|
+
this.runner.abort();
|
|
2106
|
+
}
|
|
2107
|
+
async disable() {
|
|
2108
|
+
await this.runner.disable();
|
|
2109
|
+
}
|
|
2110
|
+
getTask(id) {
|
|
2111
|
+
return this._dag.getNode(id);
|
|
2112
|
+
}
|
|
2113
|
+
getTasks() {
|
|
2114
|
+
return this._dag.getNodes();
|
|
2115
|
+
}
|
|
2116
|
+
topologicallySortedNodes() {
|
|
2117
|
+
return this._dag.topologicallySortedNodes();
|
|
2118
|
+
}
|
|
2119
|
+
addTask(task, config) {
|
|
2120
|
+
return this._dag.addNode(ensureTask(task, config));
|
|
2121
|
+
}
|
|
2122
|
+
addTasks(tasks) {
|
|
2123
|
+
return this._dag.addNodes(tasks.map(ensureTask));
|
|
2124
|
+
}
|
|
2125
|
+
addDataflow(dataflow) {
|
|
2126
|
+
return this._dag.addEdge(dataflow.sourceTaskId, dataflow.targetTaskId, dataflow);
|
|
2127
|
+
}
|
|
2128
|
+
addDataflows(dataflows) {
|
|
2129
|
+
const addedEdges = dataflows.map((edge) => {
|
|
2130
|
+
return [edge.sourceTaskId, edge.targetTaskId, edge];
|
|
2131
|
+
});
|
|
2132
|
+
return this._dag.addEdges(addedEdges);
|
|
2133
|
+
}
|
|
2134
|
+
getDataflow(id) {
|
|
2135
|
+
for (const i in this._dag.adjacency) {
|
|
2136
|
+
for (const j in this._dag.adjacency[i]) {
|
|
2137
|
+
const maybeEdges = this._dag.adjacency[i][j];
|
|
2138
|
+
if (maybeEdges !== null) {
|
|
2139
|
+
for (const edge of maybeEdges) {
|
|
2140
|
+
if (this._dag.edgeIdentity(edge, "", "") == id) {
|
|
2141
|
+
return edge;
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
}
|
|
2148
|
+
getDataflows() {
|
|
2149
|
+
return this._dag.getEdges().map((edge) => edge[2]);
|
|
2150
|
+
}
|
|
2151
|
+
removeDataflow(dataflow) {
|
|
2152
|
+
return this._dag.removeEdge(dataflow.sourceTaskId, dataflow.targetTaskId, dataflow.id);
|
|
2153
|
+
}
|
|
2154
|
+
getSourceDataflows(taskId) {
|
|
2155
|
+
return this._dag.inEdges(taskId).map(([, , dataflow]) => dataflow);
|
|
2156
|
+
}
|
|
2157
|
+
getTargetDataflows(taskId) {
|
|
2158
|
+
return this._dag.outEdges(taskId).map(([, , dataflow]) => dataflow);
|
|
2159
|
+
}
|
|
2160
|
+
getSourceTasks(taskId) {
|
|
2161
|
+
return this.getSourceDataflows(taskId).map((dataflow) => this.getTask(dataflow.sourceTaskId));
|
|
2162
|
+
}
|
|
2163
|
+
getTargetTasks(taskId) {
|
|
2164
|
+
return this.getTargetDataflows(taskId).map((dataflow) => this.getTask(dataflow.targetTaskId));
|
|
2165
|
+
}
|
|
2166
|
+
removeTask(taskId) {
|
|
2167
|
+
return this._dag.removeNode(taskId);
|
|
2168
|
+
}
|
|
2169
|
+
resetGraph() {
|
|
2170
|
+
this.runner.resetGraph(this, uuid43());
|
|
2171
|
+
}
|
|
2172
|
+
toJSON() {
|
|
2173
|
+
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
2174
|
+
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
2175
|
+
return {
|
|
2176
|
+
tasks,
|
|
2177
|
+
dataflows
|
|
2178
|
+
};
|
|
2179
|
+
}
|
|
2180
|
+
toDependencyJSON() {
|
|
2181
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
2182
|
+
this.getDataflows().forEach((df) => {
|
|
2183
|
+
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
2184
|
+
if (!target.dependencies) {
|
|
2185
|
+
target.dependencies = {};
|
|
2186
|
+
}
|
|
2187
|
+
const targetDeps = target.dependencies[df.targetTaskPortId];
|
|
2188
|
+
if (!targetDeps) {
|
|
2189
|
+
target.dependencies[df.targetTaskPortId] = {
|
|
2190
|
+
id: df.sourceTaskId,
|
|
2191
|
+
output: df.sourceTaskPortId
|
|
2192
|
+
};
|
|
2193
|
+
} else {
|
|
2194
|
+
if (Array.isArray(targetDeps)) {
|
|
2195
|
+
targetDeps.push({
|
|
2196
|
+
id: df.sourceTaskId,
|
|
2197
|
+
output: df.sourceTaskPortId
|
|
2198
|
+
});
|
|
2199
|
+
} else {
|
|
2200
|
+
target.dependencies[df.targetTaskPortId] = [
|
|
2201
|
+
targetDeps,
|
|
2202
|
+
{ id: df.sourceTaskId, output: df.sourceTaskPortId }
|
|
2203
|
+
];
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
});
|
|
2207
|
+
return tasks;
|
|
2208
|
+
}
|
|
2209
|
+
get events() {
|
|
2210
|
+
if (!this._events) {
|
|
2211
|
+
this._events = new EventEmitter5;
|
|
2212
|
+
}
|
|
2213
|
+
return this._events;
|
|
2214
|
+
}
|
|
2215
|
+
_events;
|
|
2216
|
+
subscribe(name, fn) {
|
|
2217
|
+
this.on(name, fn);
|
|
2218
|
+
return () => this.off(name, fn);
|
|
2219
|
+
}
|
|
2220
|
+
on(name, fn) {
|
|
2221
|
+
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2222
|
+
if (dagEvent) {
|
|
2223
|
+
return this._dag.on(dagEvent, fn);
|
|
2224
|
+
}
|
|
2225
|
+
return this.events.on(name, fn);
|
|
2226
|
+
}
|
|
2227
|
+
off(name, fn) {
|
|
2228
|
+
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2229
|
+
if (dagEvent) {
|
|
2230
|
+
return this._dag.off(dagEvent, fn);
|
|
2231
|
+
}
|
|
2232
|
+
return this.events.off(name, fn);
|
|
2233
|
+
}
|
|
2234
|
+
emit(name, ...args) {
|
|
2235
|
+
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2236
|
+
if (dagEvent) {
|
|
2237
|
+
return this.emit_dag(name, ...args);
|
|
2238
|
+
} else {
|
|
2239
|
+
return this.emit_local(name, ...args);
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
emit_local(name, ...args) {
|
|
2243
|
+
return this.events?.emit(name, ...args);
|
|
2244
|
+
}
|
|
2245
|
+
emit_dag(name, ...args) {
|
|
2246
|
+
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2247
|
+
return this._dag.emit(dagEvent, ...args);
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
2251
|
+
const edges = [];
|
|
2252
|
+
for (let i = 0;i < tasks.length - 1; i++) {
|
|
2253
|
+
edges.push(new Dataflow(tasks[i].config.id, inputHandle, tasks[i + 1].config.id, outputHandle));
|
|
2254
|
+
}
|
|
2255
|
+
return edges;
|
|
2256
|
+
}
|
|
2257
|
+
function serialGraph(tasks, inputHandle, outputHandle) {
|
|
2258
|
+
const graph = new TaskGraph;
|
|
2259
|
+
graph.addTasks(tasks);
|
|
2260
|
+
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
2261
|
+
return graph;
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
// src/task/ArrayTask.ts
|
|
2265
|
+
class ArrayTask extends GraphAsTask {
|
|
2266
|
+
static type = "ArrayTask";
|
|
2267
|
+
static compoundMerge = PROPERTY_ARRAY;
|
|
2268
|
+
inputSchema() {
|
|
2269
|
+
return this.constructor.inputSchema();
|
|
2270
|
+
}
|
|
2271
|
+
outputSchema() {
|
|
2272
|
+
return this.constructor.outputSchema();
|
|
2273
|
+
}
|
|
2274
|
+
regenerateGraph() {
|
|
2275
|
+
const arrayInputs = new Map;
|
|
2276
|
+
let hasArrayInputs = false;
|
|
2277
|
+
const inputSchema = this.inputSchema();
|
|
2278
|
+
if (typeof inputSchema !== "boolean") {
|
|
2279
|
+
const keys = Object.keys(inputSchema.properties || {});
|
|
2280
|
+
for (const inputId of keys) {
|
|
2281
|
+
const inputValue = this.runInputData[inputId];
|
|
2282
|
+
const inputDef = inputSchema.properties?.[inputId];
|
|
2283
|
+
if (typeof inputDef === "object" && inputDef !== null && "x-replicate" in inputDef && inputDef["x-replicate"] === true && Array.isArray(inputValue) && inputValue.length > 1) {
|
|
2284
|
+
arrayInputs.set(inputId, inputValue);
|
|
2285
|
+
hasArrayInputs = true;
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
}
|
|
2289
|
+
this.subGraph = new TaskGraph;
|
|
2290
|
+
if (!hasArrayInputs) {
|
|
2291
|
+
super.regenerateGraph();
|
|
2292
|
+
return;
|
|
2293
|
+
}
|
|
2294
|
+
this.subGraph = new TaskGraph;
|
|
2295
|
+
const inputIds = Array.from(arrayInputs.keys());
|
|
2296
|
+
const inputObject = Object.fromEntries(arrayInputs);
|
|
2297
|
+
const combinations = this.generateCombinations(inputObject, inputIds);
|
|
2298
|
+
const tasks = combinations.map((combination) => {
|
|
2299
|
+
const { id, name, ...rest } = this.config;
|
|
2300
|
+
const task = new this.constructor({ ...this.defaults, ...this.runInputData, ...combination }, { ...rest, id: `${id}_${uuid44()}` });
|
|
2301
|
+
return task;
|
|
2302
|
+
});
|
|
2303
|
+
this.subGraph.addTasks(tasks);
|
|
2304
|
+
super.regenerateGraph();
|
|
2305
|
+
}
|
|
2306
|
+
generateCombinations(input, inputMakeArray) {
|
|
2307
|
+
const arraysToCombine = inputMakeArray.map((key) => Array.isArray(input[key]) ? input[key] : []);
|
|
2308
|
+
const indices = arraysToCombine.map(() => 0);
|
|
2309
|
+
const combinations = [];
|
|
2310
|
+
let done = false;
|
|
2311
|
+
while (!done) {
|
|
2312
|
+
combinations.push([...indices]);
|
|
2313
|
+
for (let i = indices.length - 1;i >= 0; i--) {
|
|
2314
|
+
if (++indices[i] < arraysToCombine[i].length)
|
|
2315
|
+
break;
|
|
2316
|
+
if (i === 0)
|
|
2317
|
+
done = true;
|
|
2318
|
+
else
|
|
2319
|
+
indices[i] = 0;
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
const combos = combinations.map((combination) => {
|
|
2323
|
+
const result = { ...input };
|
|
2324
|
+
combination.forEach((valueIndex, arrayIndex) => {
|
|
2325
|
+
const key = inputMakeArray[arrayIndex];
|
|
2326
|
+
if (Array.isArray(input[key]))
|
|
2327
|
+
result[key] = input[key][valueIndex];
|
|
2328
|
+
});
|
|
2329
|
+
return result;
|
|
2330
|
+
});
|
|
2331
|
+
return combos;
|
|
2332
|
+
}
|
|
2333
|
+
toJSON() {
|
|
2334
|
+
const { subgraph, ...result } = super.toJSON();
|
|
2335
|
+
return result;
|
|
2336
|
+
}
|
|
2337
|
+
toDependencyJSON() {
|
|
2338
|
+
const { subtasks, ...result } = super.toDependencyJSON();
|
|
2339
|
+
return result;
|
|
2340
|
+
}
|
|
2341
|
+
get runner() {
|
|
2342
|
+
if (!this._runner) {
|
|
2343
|
+
this._runner = new ArrayTaskRunner(this);
|
|
2344
|
+
}
|
|
2345
|
+
return this._runner;
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
class ArrayTaskRunner extends GraphAsTaskRunner {
|
|
2350
|
+
async executeTaskChildren(_input) {
|
|
2351
|
+
return super.executeTaskChildren({});
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
// src/task/JobQueueFactory.ts
|
|
2355
|
+
import { JobQueue } from "@workglow/job-queue";
|
|
2356
|
+
import { InMemoryQueueStorage } from "@workglow/storage";
|
|
2357
|
+
import { createServiceToken as createServiceToken2, globalServiceRegistry as globalServiceRegistry3 } from "@workglow/util";
|
|
2358
|
+
var JOB_QUEUE_FACTORY = createServiceToken2("taskgraph.jobQueueFactory");
|
|
2359
|
+
var defaultJobQueueFactory = async ({ queueName, jobClass, options }) => {
|
|
2360
|
+
const queueOptions = {
|
|
2361
|
+
...options ?? {}
|
|
2362
|
+
};
|
|
2363
|
+
queueOptions.storage ??= new InMemoryQueueStorage(queueName);
|
|
2364
|
+
return new JobQueue(queueName, jobClass, queueOptions);
|
|
2365
|
+
};
|
|
2366
|
+
function registerJobQueueFactory(factory) {
|
|
2367
|
+
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
2368
|
+
}
|
|
2369
|
+
function createJobQueueFactoryFromClass(QueueCtor, defaultOptions = {}) {
|
|
2370
|
+
return ({ queueName, jobClass, options }) => {
|
|
2371
|
+
const mergedOptions = {
|
|
2372
|
+
...defaultOptions,
|
|
2373
|
+
...options ?? {}
|
|
2374
|
+
};
|
|
2375
|
+
return new QueueCtor(queueName, jobClass, mergedOptions);
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2378
|
+
function getJobQueueFactory() {
|
|
2379
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2380
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2381
|
+
}
|
|
2382
|
+
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
2383
|
+
}
|
|
2384
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2385
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2386
|
+
}
|
|
2387
|
+
// src/task/JobQueueTask.ts
|
|
2388
|
+
import { Job as Job2 } from "@workglow/job-queue";
|
|
2389
|
+
|
|
2390
|
+
// src/task/TaskQueueRegistry.ts
|
|
2391
|
+
var taskQueueRegistry = null;
|
|
2392
|
+
|
|
2393
|
+
class TaskQueueRegistry {
|
|
2394
|
+
queues = new Map;
|
|
2395
|
+
registerQueue(jobQueue) {
|
|
2396
|
+
if (this.queues.has(jobQueue.queueName)) {
|
|
2397
|
+
throw new Error(`Queue with name ${jobQueue.queueName} already exists`);
|
|
2398
|
+
}
|
|
2399
|
+
this.queues.set(jobQueue.queueName, jobQueue);
|
|
2400
|
+
}
|
|
2401
|
+
getQueue(queue) {
|
|
2402
|
+
return this.queues.get(queue);
|
|
2403
|
+
}
|
|
2404
|
+
startQueues() {
|
|
2405
|
+
for (const queue of this.queues.values()) {
|
|
2406
|
+
queue.start();
|
|
2407
|
+
}
|
|
2408
|
+
return this;
|
|
2409
|
+
}
|
|
2410
|
+
stopQueues() {
|
|
2411
|
+
for (const queue of this.queues.values()) {
|
|
2412
|
+
queue.stop();
|
|
2413
|
+
}
|
|
2414
|
+
return this;
|
|
2415
|
+
}
|
|
2416
|
+
clearQueues() {
|
|
2417
|
+
for (const queue of this.queues.values()) {
|
|
2418
|
+
queue.clear();
|
|
2419
|
+
}
|
|
2420
|
+
return this;
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
function getTaskQueueRegistry() {
|
|
2424
|
+
if (!taskQueueRegistry) {
|
|
2425
|
+
taskQueueRegistry = new TaskQueueRegistry;
|
|
2426
|
+
}
|
|
2427
|
+
return taskQueueRegistry;
|
|
2428
|
+
}
|
|
2429
|
+
function setTaskQueueRegistry(registry) {
|
|
2430
|
+
if (taskQueueRegistry) {
|
|
2431
|
+
taskQueueRegistry.stopQueues();
|
|
2432
|
+
taskQueueRegistry.clearQueues();
|
|
2433
|
+
}
|
|
2434
|
+
taskQueueRegistry = registry;
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
// src/task/JobQueueTask.ts
|
|
2438
|
+
class JobQueueTask extends ArrayTask {
|
|
2439
|
+
static type = "JobQueueTask";
|
|
2440
|
+
static canRunDirectly = true;
|
|
2441
|
+
currentQueueName;
|
|
2442
|
+
currentJobId;
|
|
2443
|
+
currentRunnerId;
|
|
2444
|
+
jobClass;
|
|
2445
|
+
constructor(input = {}, config = {}) {
|
|
2446
|
+
config.queue ??= true;
|
|
2447
|
+
super(input, config);
|
|
2448
|
+
this.jobClass = Job2;
|
|
2449
|
+
}
|
|
2450
|
+
async execute(input, executeContext) {
|
|
2451
|
+
let cleanup = () => {};
|
|
2452
|
+
try {
|
|
2453
|
+
if (this.config.queue === false && !this.constructor.canRunDirectly) {
|
|
2454
|
+
throw new TaskConfigurationError(`${this.type} cannot run directly without a queue`);
|
|
2455
|
+
}
|
|
2456
|
+
const queue = await this.resolveQueue(input);
|
|
2457
|
+
const job = await this.createJob(input, queue);
|
|
2458
|
+
if (!queue) {
|
|
2459
|
+
if (!this.constructor.canRunDirectly) {
|
|
2460
|
+
const queueLabel = typeof this.config.queue === "string" ? this.config.queue : this.currentQueueName ?? this.type;
|
|
2461
|
+
throw new TaskConfigurationError(`Queue ${queueLabel} not found, and ${this.type} cannot run directly`);
|
|
2462
|
+
}
|
|
2463
|
+
this.currentJobId = undefined;
|
|
2464
|
+
cleanup = job.onJobProgress((progress, message, details) => {
|
|
2465
|
+
executeContext.updateProgress(progress, message, details);
|
|
2466
|
+
});
|
|
2467
|
+
const output2 = await job.execute(job.input, {
|
|
2468
|
+
signal: executeContext.signal,
|
|
2469
|
+
updateProgress: executeContext.updateProgress.bind(this)
|
|
2470
|
+
});
|
|
2471
|
+
return output2;
|
|
2472
|
+
}
|
|
2473
|
+
const jobId = await queue.add(job);
|
|
2474
|
+
this.currentJobId = jobId;
|
|
2475
|
+
this.currentQueueName = queue?.queueName;
|
|
2476
|
+
this.currentRunnerId = job.jobRunId;
|
|
2477
|
+
cleanup = queue.onJobProgress(jobId, (progress, message, details) => {
|
|
2478
|
+
executeContext.updateProgress(progress, message, details);
|
|
2479
|
+
});
|
|
2480
|
+
const output = await queue.waitFor(jobId);
|
|
2481
|
+
if (output === undefined) {
|
|
2482
|
+
throw new TaskConfigurationError("Job disabled, should not happen");
|
|
2483
|
+
}
|
|
2484
|
+
return output;
|
|
2485
|
+
} catch (err) {
|
|
2486
|
+
throw new JobTaskFailedError(err);
|
|
2487
|
+
} finally {
|
|
2488
|
+
cleanup();
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
async createJob(input, queue) {
|
|
2492
|
+
const jobCtor = queue?.jobClass ?? this.jobClass;
|
|
2493
|
+
return new jobCtor({
|
|
2494
|
+
queueName: queue?.queueName ?? this.currentQueueName,
|
|
2495
|
+
jobRunId: this.currentRunnerId,
|
|
2496
|
+
input
|
|
2497
|
+
});
|
|
2498
|
+
}
|
|
2499
|
+
async resolveQueue(input) {
|
|
2500
|
+
const preference = this.config.queue ?? true;
|
|
2501
|
+
if (preference === false) {
|
|
2502
|
+
this.currentQueueName = undefined;
|
|
2503
|
+
return;
|
|
2504
|
+
}
|
|
2505
|
+
if (typeof preference === "string") {
|
|
2506
|
+
const queue2 = getTaskQueueRegistry().getQueue(preference);
|
|
2507
|
+
if (queue2) {
|
|
2508
|
+
this.currentQueueName = queue2.queueName;
|
|
2509
|
+
return queue2;
|
|
2510
|
+
}
|
|
2511
|
+
this.currentQueueName = preference;
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
2514
|
+
const queueName = await this.getDefaultQueueName(input);
|
|
2515
|
+
if (!queueName) {
|
|
2516
|
+
this.currentQueueName = undefined;
|
|
2517
|
+
return;
|
|
2518
|
+
}
|
|
2519
|
+
this.currentQueueName = queueName;
|
|
2520
|
+
let queue = getTaskQueueRegistry().getQueue(queueName);
|
|
2521
|
+
if (!queue) {
|
|
2522
|
+
queue = await this.createAndRegisterQueue(queueName, input);
|
|
2523
|
+
await queue.start();
|
|
2524
|
+
}
|
|
2525
|
+
return queue;
|
|
2526
|
+
}
|
|
2527
|
+
async getDefaultQueueName(_input) {
|
|
2528
|
+
return this.type;
|
|
2529
|
+
}
|
|
2530
|
+
async createAndRegisterQueue(queueName, input) {
|
|
2531
|
+
const factory = getJobQueueFactory();
|
|
2532
|
+
let queue = await factory({
|
|
2533
|
+
queueName,
|
|
2534
|
+
jobClass: this.jobClass,
|
|
2535
|
+
input,
|
|
2536
|
+
config: this.config,
|
|
2537
|
+
task: this
|
|
2538
|
+
});
|
|
2539
|
+
const registry = getTaskQueueRegistry();
|
|
2540
|
+
try {
|
|
2541
|
+
registry.registerQueue(queue);
|
|
2542
|
+
} catch (err) {
|
|
2543
|
+
if (err instanceof Error && err.message.includes("already exists")) {
|
|
2544
|
+
const existing = registry.getQueue(queueName);
|
|
2545
|
+
if (existing) {
|
|
2546
|
+
queue = existing;
|
|
2547
|
+
}
|
|
2548
|
+
} else {
|
|
2549
|
+
throw err;
|
|
2550
|
+
}
|
|
2551
|
+
}
|
|
2552
|
+
return queue;
|
|
2553
|
+
}
|
|
2554
|
+
async abort() {
|
|
2555
|
+
if (this.currentQueueName) {
|
|
2556
|
+
const queue = getTaskQueueRegistry().getQueue(this.currentQueueName);
|
|
2557
|
+
if (queue) {
|
|
2558
|
+
await queue.abort(this.currentJobId);
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
super.abort();
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
// src/task/TaskRegistry.ts
|
|
2565
|
+
var taskConstructors = new Map;
|
|
2566
|
+
function registerTask(baseClass) {
|
|
2567
|
+
if (taskConstructors.has(baseClass.type)) {}
|
|
2568
|
+
taskConstructors.set(baseClass.type, baseClass);
|
|
2569
|
+
}
|
|
2570
|
+
var TaskRegistry = {
|
|
2571
|
+
all: taskConstructors,
|
|
2572
|
+
registerTask
|
|
2573
|
+
};
|
|
2574
|
+
|
|
2575
|
+
// src/task/TaskJSON.ts
|
|
2576
|
+
var createSingleTaskFromJSON = (item) => {
|
|
2577
|
+
if (!item.id)
|
|
2578
|
+
throw new TaskJSONError("Task id required");
|
|
2579
|
+
if (!item.type)
|
|
2580
|
+
throw new TaskJSONError("Task type required");
|
|
2581
|
+
if (item.input && (Array.isArray(item.input) || Array.isArray(item.provenance)))
|
|
2582
|
+
throw new TaskJSONError("Task input must be an object");
|
|
2583
|
+
if (item.provenance && (Array.isArray(item.provenance) || typeof item.provenance !== "object"))
|
|
2584
|
+
throw new TaskJSONError("Task provenance must be an object");
|
|
2585
|
+
const taskClass = TaskRegistry.all.get(item.type);
|
|
2586
|
+
if (!taskClass)
|
|
2587
|
+
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
2588
|
+
const taskConfig = {
|
|
2589
|
+
id: item.id,
|
|
2590
|
+
name: item.name,
|
|
2591
|
+
provenance: item.provenance ?? {},
|
|
2592
|
+
extras: item.extras
|
|
2593
|
+
};
|
|
2594
|
+
const task = new taskClass(item.input ?? {}, taskConfig);
|
|
2595
|
+
return task;
|
|
2596
|
+
};
|
|
2597
|
+
var createTaskFromDependencyJSON = (item) => {
|
|
2598
|
+
const task = createSingleTaskFromJSON(item);
|
|
2599
|
+
if (item.subtasks && item.subtasks.length > 0) {
|
|
2600
|
+
if (!(task instanceof GraphAsTask)) {
|
|
2601
|
+
throw new TaskConfigurationError("Subgraph is only supported for CompoundTasks");
|
|
2602
|
+
}
|
|
2603
|
+
task.subGraph = createGraphFromDependencyJSON(item.subtasks);
|
|
2604
|
+
}
|
|
2605
|
+
return task;
|
|
2606
|
+
};
|
|
2607
|
+
var createGraphFromDependencyJSON = (jsonItems) => {
|
|
2608
|
+
const subGraph = new TaskGraph;
|
|
2609
|
+
for (const subitem of jsonItems) {
|
|
2610
|
+
subGraph.addTask(createTaskFromDependencyJSON(subitem));
|
|
2611
|
+
}
|
|
2612
|
+
return subGraph;
|
|
2613
|
+
};
|
|
2614
|
+
var createTaskFromGraphJSON = (item) => {
|
|
2615
|
+
const task = createSingleTaskFromJSON(item);
|
|
2616
|
+
if (item.subgraph) {
|
|
2617
|
+
if (!(task instanceof GraphAsTask)) {
|
|
2618
|
+
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
2619
|
+
}
|
|
2620
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
2621
|
+
}
|
|
2622
|
+
return task;
|
|
2623
|
+
};
|
|
2624
|
+
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
2625
|
+
const subGraph = new TaskGraph;
|
|
2626
|
+
for (const subitem of graphJsonObj.tasks) {
|
|
2627
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
2628
|
+
}
|
|
2629
|
+
for (const subitem of graphJsonObj.dataflows) {
|
|
2630
|
+
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
2631
|
+
}
|
|
2632
|
+
return subGraph;
|
|
2633
|
+
};
|
|
2634
|
+
// src/storage/TaskGraphRepository.ts
|
|
2635
|
+
import { createServiceToken as createServiceToken3, EventEmitter as EventEmitter6 } from "@workglow/util";
|
|
2636
|
+
var TASK_GRAPH_REPOSITORY = createServiceToken3("taskgraph.taskGraphRepository");
|
|
2637
|
+
|
|
2638
|
+
class TaskGraphRepository {
|
|
2639
|
+
type = "TaskGraphRepository";
|
|
2640
|
+
get events() {
|
|
2641
|
+
if (!this._events) {
|
|
2642
|
+
this._events = new EventEmitter6;
|
|
2643
|
+
}
|
|
2644
|
+
return this._events;
|
|
2645
|
+
}
|
|
2646
|
+
_events;
|
|
2647
|
+
on(name, fn) {
|
|
2648
|
+
this.events.on(name, fn);
|
|
2649
|
+
}
|
|
2650
|
+
off(name, fn) {
|
|
2651
|
+
this.events.off(name, fn);
|
|
2652
|
+
}
|
|
2653
|
+
once(name, fn) {
|
|
2654
|
+
this.events.once(name, fn);
|
|
2655
|
+
}
|
|
2656
|
+
waitOn(name) {
|
|
2657
|
+
return this.events.waitOn(name);
|
|
2658
|
+
}
|
|
2659
|
+
emit(name, ...args) {
|
|
2660
|
+
this._events?.emit(name, ...args);
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
// src/storage/TaskGraphTabularRepository.ts
|
|
2664
|
+
var TaskGraphSchema = {
|
|
2665
|
+
type: "object",
|
|
2666
|
+
properties: {
|
|
2667
|
+
key: { type: "string" },
|
|
2668
|
+
value: { type: "string" }
|
|
2669
|
+
},
|
|
2670
|
+
additionalProperties: false
|
|
2671
|
+
};
|
|
2672
|
+
var TaskGraphPrimaryKeyNames = ["key"];
|
|
2673
|
+
|
|
2674
|
+
class TaskGraphTabularRepository extends TaskGraphRepository {
|
|
2675
|
+
type = "TaskGraphTabularRepository";
|
|
2676
|
+
tabularRepository;
|
|
2677
|
+
constructor({ tabularRepository }) {
|
|
2678
|
+
super();
|
|
2679
|
+
this.tabularRepository = tabularRepository;
|
|
2680
|
+
}
|
|
2681
|
+
async saveTaskGraph(key, output) {
|
|
2682
|
+
const value = JSON.stringify(output.toJSON());
|
|
2683
|
+
await this.tabularRepository.put({ key, value });
|
|
2684
|
+
this.emit("graph_saved", key);
|
|
2685
|
+
}
|
|
2686
|
+
async getTaskGraph(key) {
|
|
2687
|
+
const result = await this.tabularRepository.get({ key });
|
|
2688
|
+
const value = result?.value;
|
|
2689
|
+
if (!value) {
|
|
2690
|
+
return;
|
|
2691
|
+
}
|
|
2692
|
+
const jsonObj = JSON.parse(value);
|
|
2693
|
+
const graph = createGraphFromGraphJSON(jsonObj);
|
|
2694
|
+
this.emit("graph_retrieved", key);
|
|
2695
|
+
return graph;
|
|
2696
|
+
}
|
|
2697
|
+
async clear() {
|
|
2698
|
+
await this.tabularRepository.deleteAll();
|
|
2699
|
+
this.emit("graph_cleared");
|
|
2700
|
+
}
|
|
2701
|
+
async size() {
|
|
2702
|
+
return await this.tabularRepository.size();
|
|
2703
|
+
}
|
|
2704
|
+
}
|
|
2705
|
+
// src/storage/TaskOutputTabularRepository.ts
|
|
2706
|
+
import { compress, decompress, makeFingerprint } from "@workglow/util";
|
|
2707
|
+
var TaskOutputSchema = {
|
|
2708
|
+
type: "object",
|
|
2709
|
+
properties: {
|
|
2710
|
+
key: { type: "string" },
|
|
2711
|
+
taskType: { type: "string" },
|
|
2712
|
+
value: { type: "string", contentEncoding: "blob" },
|
|
2713
|
+
createdAt: { type: "string", format: "date-time" }
|
|
2714
|
+
},
|
|
2715
|
+
additionalProperties: false
|
|
2716
|
+
};
|
|
2717
|
+
var TaskOutputPrimaryKeyNames = ["key", "taskType"];
|
|
2718
|
+
|
|
2719
|
+
class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
2720
|
+
tabularRepository;
|
|
2721
|
+
constructor({ tabularRepository, outputCompression = true }) {
|
|
2722
|
+
super({ outputCompression });
|
|
2723
|
+
this.tabularRepository = tabularRepository;
|
|
2724
|
+
this.outputCompression = outputCompression;
|
|
2725
|
+
}
|
|
2726
|
+
async keyFromInputs(inputs) {
|
|
2727
|
+
return await makeFingerprint(inputs);
|
|
2728
|
+
}
|
|
2729
|
+
async saveOutput(taskType, inputs, output, createdAt = new Date) {
|
|
2730
|
+
const key = await this.keyFromInputs(inputs);
|
|
2731
|
+
const value = JSON.stringify(output);
|
|
2732
|
+
if (this.outputCompression) {
|
|
2733
|
+
const compressedValue = await compress(value);
|
|
2734
|
+
await this.tabularRepository.put({
|
|
2735
|
+
taskType,
|
|
2736
|
+
key,
|
|
2737
|
+
value: compressedValue,
|
|
2738
|
+
createdAt: createdAt.toISOString()
|
|
2739
|
+
});
|
|
2740
|
+
} else {
|
|
2741
|
+
const valueBuffer = Buffer.from(value);
|
|
2742
|
+
await this.tabularRepository.put({
|
|
2743
|
+
taskType,
|
|
2744
|
+
key,
|
|
2745
|
+
value: valueBuffer,
|
|
2746
|
+
createdAt: createdAt.toISOString()
|
|
2747
|
+
});
|
|
2748
|
+
}
|
|
2749
|
+
this.emit("output_saved", taskType);
|
|
2750
|
+
}
|
|
2751
|
+
async getOutput(taskType, inputs) {
|
|
2752
|
+
const key = await this.keyFromInputs(inputs);
|
|
2753
|
+
const output = await this.tabularRepository.get({ key, taskType });
|
|
2754
|
+
this.emit("output_retrieved", taskType);
|
|
2755
|
+
if (output?.value) {
|
|
2756
|
+
if (this.outputCompression) {
|
|
2757
|
+
const raw = output.value;
|
|
2758
|
+
const bytes = raw instanceof Uint8Array ? raw : Array.isArray(raw) ? new Uint8Array(raw) : raw && typeof raw === "object" ? new Uint8Array(Object.keys(raw).filter((k) => /^\d+$/.test(k)).sort((a, b) => Number(a) - Number(b)).map((k) => raw[k])) : new Uint8Array;
|
|
2759
|
+
const decompressedValue = await decompress(bytes);
|
|
2760
|
+
const value = JSON.parse(decompressedValue);
|
|
2761
|
+
return value;
|
|
2762
|
+
} else {
|
|
2763
|
+
const stringValue = output.value.toString();
|
|
2764
|
+
const value = JSON.parse(stringValue);
|
|
2765
|
+
return value;
|
|
2766
|
+
}
|
|
2767
|
+
} else {
|
|
2768
|
+
return;
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
async clear() {
|
|
2772
|
+
await this.tabularRepository.deleteAll();
|
|
2773
|
+
this.emit("output_cleared");
|
|
2774
|
+
}
|
|
2775
|
+
async size() {
|
|
2776
|
+
return await this.tabularRepository.size();
|
|
2777
|
+
}
|
|
2778
|
+
async clearOlderThan(olderThanInMs) {
|
|
2779
|
+
const date = new Date(Date.now() - olderThanInMs).toISOString();
|
|
2780
|
+
await this.tabularRepository.deleteSearch("createdAt", date, "<");
|
|
2781
|
+
this.emit("output_pruned");
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
export {
|
|
2785
|
+
setTaskQueueRegistry,
|
|
2786
|
+
serialGraph,
|
|
2787
|
+
registerJobQueueFactory,
|
|
2788
|
+
pipe,
|
|
2789
|
+
parallel,
|
|
2790
|
+
getTaskQueueRegistry,
|
|
2791
|
+
getLastTask,
|
|
2792
|
+
getJobQueueFactory,
|
|
2793
|
+
ensureTask,
|
|
2794
|
+
createTaskFromGraphJSON,
|
|
2795
|
+
createTaskFromDependencyJSON,
|
|
2796
|
+
createJobQueueFactoryFromClass,
|
|
2797
|
+
createGraphFromGraphJSON,
|
|
2798
|
+
createGraphFromDependencyJSON,
|
|
2799
|
+
connect,
|
|
2800
|
+
WorkflowError,
|
|
2801
|
+
Workflow,
|
|
2802
|
+
TaskStatus,
|
|
2803
|
+
TaskRegistry,
|
|
2804
|
+
TaskQueueRegistry,
|
|
2805
|
+
TaskOutputTabularRepository,
|
|
2806
|
+
TaskOutputSchema,
|
|
2807
|
+
TaskOutputRepository,
|
|
2808
|
+
TaskOutputPrimaryKeyNames,
|
|
2809
|
+
TaskJSONError,
|
|
2810
|
+
TaskInvalidInputError,
|
|
2811
|
+
TaskGraphTabularRepository,
|
|
2812
|
+
TaskGraphSchema,
|
|
2813
|
+
TaskGraphRunner,
|
|
2814
|
+
TaskGraphRepository,
|
|
2815
|
+
TaskGraphPrimaryKeyNames,
|
|
2816
|
+
TaskGraph,
|
|
2817
|
+
TaskFailedError,
|
|
2818
|
+
TaskError,
|
|
2819
|
+
TaskConfigurationError,
|
|
2820
|
+
TaskAbortedError,
|
|
2821
|
+
Task,
|
|
2822
|
+
TASK_OUTPUT_REPOSITORY,
|
|
2823
|
+
TASK_GRAPH_REPOSITORY,
|
|
2824
|
+
PROPERTY_ARRAY,
|
|
2825
|
+
JobTaskFailedError,
|
|
2826
|
+
JobQueueTask,
|
|
2827
|
+
JOB_QUEUE_FACTORY,
|
|
2828
|
+
GraphAsTaskRunner,
|
|
2829
|
+
GraphAsTask,
|
|
2830
|
+
GRAPH_RESULT_ARRAY,
|
|
2831
|
+
EventTaskGraphToDagMapping,
|
|
2832
|
+
EventDagToTaskGraphMapping,
|
|
2833
|
+
DataflowArrow,
|
|
2834
|
+
Dataflow,
|
|
2835
|
+
DATAFLOW_ERROR_PORT,
|
|
2836
|
+
DATAFLOW_ALL_PORTS,
|
|
2837
|
+
CreateWorkflow,
|
|
2838
|
+
ConditionalTask,
|
|
2839
|
+
ArrayTask
|
|
2840
|
+
};
|
|
2841
|
+
|
|
2842
|
+
//# debugId=32538247DA95577364756E2164756E21
|