@workglow/task-graph 0.0.89 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +2251 -340
- package/dist/browser.js.map +24 -15
- package/dist/bun.js +2251 -340
- package/dist/bun.js.map +24 -15
- package/dist/node.js +2251 -340
- package/dist/node.js.map +24 -15
- package/dist/task/ConditionUtils.d.ts +47 -0
- package/dist/task/ConditionUtils.d.ts.map +1 -0
- package/dist/task/ConditionalTask.d.ts +15 -0
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +7 -0
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +177 -0
- package/dist/task/IteratorTask.d.ts.map +1 -0
- package/dist/task/IteratorTaskRunner.d.ts +36 -0
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -0
- package/dist/task/MapTask.d.ts +82 -0
- package/dist/task/MapTask.d.ts.map +1 -0
- package/dist/task/ReduceTask.d.ts +61 -0
- package/dist/task/ReduceTask.d.ts.map +1 -0
- package/dist/task/StreamTypes.d.ts +96 -0
- package/dist/task/StreamTypes.d.ts.map +1 -0
- package/dist/task/Task.d.ts +2 -2
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +7 -0
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +3 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +7 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +4 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +214 -0
- package/dist/task/WhileTask.d.ts.map +1 -0
- package/dist/task/WhileTaskRunner.d.ts +29 -0
- package/dist/task/WhileTaskRunner.d.ts.map +1 -0
- package/dist/task/index.d.ts +13 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task/iterationSchema.d.ts +70 -0
- package/dist/task/iterationSchema.d.ts.map +1 -0
- package/dist/task-graph/Dataflow.d.ts +34 -0
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/DataflowEvents.d.ts +2 -0
- package/dist/task-graph/DataflowEvents.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +6 -0
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +14 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphEvents.d.ts +7 -0
- package/dist/task-graph/TaskGraphEvents.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +25 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphScheduler.d.ts +9 -0
- package/dist/task-graph/TaskGraphScheduler.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +148 -9
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/dist/task/TaskJSON.test.d.ts +0 -7
- package/dist/task/TaskJSON.test.d.ts.map +0 -1
package/dist/bun.js
CHANGED
|
@@ -7,6 +7,7 @@ var TaskStatus = {
|
|
|
7
7
|
PENDING: "PENDING",
|
|
8
8
|
DISABLED: "DISABLED",
|
|
9
9
|
PROCESSING: "PROCESSING",
|
|
10
|
+
STREAMING: "STREAMING",
|
|
10
11
|
COMPLETED: "COMPLETED",
|
|
11
12
|
ABORTING: "ABORTING",
|
|
12
13
|
FAILED: "FAILED"
|
|
@@ -36,10 +37,62 @@ class Dataflow {
|
|
|
36
37
|
value = undefined;
|
|
37
38
|
status = TaskStatus.PENDING;
|
|
38
39
|
error;
|
|
40
|
+
stream = undefined;
|
|
41
|
+
setStream(stream) {
|
|
42
|
+
this.stream = stream;
|
|
43
|
+
}
|
|
44
|
+
getStream() {
|
|
45
|
+
return this.stream;
|
|
46
|
+
}
|
|
47
|
+
async awaitStreamValue() {
|
|
48
|
+
if (!this.stream)
|
|
49
|
+
return;
|
|
50
|
+
const reader = this.stream.getReader();
|
|
51
|
+
let accumulatedText = "";
|
|
52
|
+
let lastSnapshotData = undefined;
|
|
53
|
+
let finishData = undefined;
|
|
54
|
+
let hasTextDelta = false;
|
|
55
|
+
try {
|
|
56
|
+
while (true) {
|
|
57
|
+
const { done, value: event } = await reader.read();
|
|
58
|
+
if (done)
|
|
59
|
+
break;
|
|
60
|
+
switch (event.type) {
|
|
61
|
+
case "text-delta":
|
|
62
|
+
hasTextDelta = true;
|
|
63
|
+
accumulatedText += event.textDelta;
|
|
64
|
+
break;
|
|
65
|
+
case "snapshot":
|
|
66
|
+
lastSnapshotData = event.data;
|
|
67
|
+
break;
|
|
68
|
+
case "finish":
|
|
69
|
+
finishData = event.data;
|
|
70
|
+
break;
|
|
71
|
+
case "error":
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
} finally {
|
|
76
|
+
reader.releaseLock();
|
|
77
|
+
this.stream = undefined;
|
|
78
|
+
}
|
|
79
|
+
if (lastSnapshotData !== undefined) {
|
|
80
|
+
this.setPortData(lastSnapshotData);
|
|
81
|
+
} else if (finishData && Object.keys(finishData).length > 0) {
|
|
82
|
+
this.setPortData(finishData);
|
|
83
|
+
} else if (hasTextDelta) {
|
|
84
|
+
if (this.sourceTaskPortId === DATAFLOW_ALL_PORTS) {
|
|
85
|
+
this.value = { text: accumulatedText };
|
|
86
|
+
} else {
|
|
87
|
+
this.value = accumulatedText;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
39
91
|
reset() {
|
|
40
92
|
this.status = TaskStatus.PENDING;
|
|
41
93
|
this.error = undefined;
|
|
42
94
|
this.value = undefined;
|
|
95
|
+
this.stream = undefined;
|
|
43
96
|
this.emit("reset");
|
|
44
97
|
this.emit("status", this.status);
|
|
45
98
|
}
|
|
@@ -51,6 +104,9 @@ class Dataflow {
|
|
|
51
104
|
case TaskStatus.PROCESSING:
|
|
52
105
|
this.emit("start");
|
|
53
106
|
break;
|
|
107
|
+
case TaskStatus.STREAMING:
|
|
108
|
+
this.emit("streaming");
|
|
109
|
+
break;
|
|
54
110
|
case TaskStatus.COMPLETED:
|
|
55
111
|
this.emit("complete");
|
|
56
112
|
break;
|
|
@@ -162,7 +218,7 @@ class DataflowArrow extends Dataflow {
|
|
|
162
218
|
}
|
|
163
219
|
}
|
|
164
220
|
// src/task-graph/TaskGraph.ts
|
|
165
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
221
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid44 } from "@workglow/util";
|
|
166
222
|
|
|
167
223
|
// src/task/GraphAsTask.ts
|
|
168
224
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -205,6 +261,73 @@ class TaskOutputRepository {
|
|
|
205
261
|
}
|
|
206
262
|
}
|
|
207
263
|
|
|
264
|
+
// src/task/ConditionUtils.ts
|
|
265
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
266
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
267
|
+
switch (operator) {
|
|
268
|
+
case "is_empty":
|
|
269
|
+
return true;
|
|
270
|
+
case "is_not_empty":
|
|
271
|
+
return false;
|
|
272
|
+
case "is_true":
|
|
273
|
+
return false;
|
|
274
|
+
case "is_false":
|
|
275
|
+
return true;
|
|
276
|
+
default:
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const strValue = String(fieldValue);
|
|
281
|
+
const numValue = Number(fieldValue);
|
|
282
|
+
switch (operator) {
|
|
283
|
+
case "equals":
|
|
284
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
285
|
+
return numValue === Number(compareValue);
|
|
286
|
+
}
|
|
287
|
+
return strValue === compareValue;
|
|
288
|
+
case "not_equals":
|
|
289
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
290
|
+
return numValue !== Number(compareValue);
|
|
291
|
+
}
|
|
292
|
+
return strValue !== compareValue;
|
|
293
|
+
case "greater_than":
|
|
294
|
+
return numValue > Number(compareValue);
|
|
295
|
+
case "greater_or_equal":
|
|
296
|
+
return numValue >= Number(compareValue);
|
|
297
|
+
case "less_than":
|
|
298
|
+
return numValue < Number(compareValue);
|
|
299
|
+
case "less_or_equal":
|
|
300
|
+
return numValue <= Number(compareValue);
|
|
301
|
+
case "contains":
|
|
302
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
303
|
+
case "starts_with":
|
|
304
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
305
|
+
case "ends_with":
|
|
306
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
307
|
+
case "is_empty":
|
|
308
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
309
|
+
case "is_not_empty":
|
|
310
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
311
|
+
case "is_true":
|
|
312
|
+
return Boolean(fieldValue) === true;
|
|
313
|
+
case "is_false":
|
|
314
|
+
return Boolean(fieldValue) === false;
|
|
315
|
+
default:
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
function getNestedValue(obj, path) {
|
|
320
|
+
const parts = path.split(".");
|
|
321
|
+
let current = obj;
|
|
322
|
+
for (const part of parts) {
|
|
323
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
current = current[part];
|
|
327
|
+
}
|
|
328
|
+
return current;
|
|
329
|
+
}
|
|
330
|
+
|
|
208
331
|
// src/task/Task.ts
|
|
209
332
|
import {
|
|
210
333
|
compileSchema,
|
|
@@ -331,6 +454,49 @@ async function resolveSchemaInputs(input, schema, config) {
|
|
|
331
454
|
return resolved;
|
|
332
455
|
}
|
|
333
456
|
|
|
457
|
+
// src/task/StreamTypes.ts
|
|
458
|
+
function getPortStreamMode(schema, portId) {
|
|
459
|
+
if (typeof schema === "boolean")
|
|
460
|
+
return "none";
|
|
461
|
+
const prop = schema.properties?.[portId];
|
|
462
|
+
if (!prop || typeof prop === "boolean")
|
|
463
|
+
return "none";
|
|
464
|
+
const xStream = prop["x-stream"];
|
|
465
|
+
if (xStream === "append" || xStream === "replace")
|
|
466
|
+
return xStream;
|
|
467
|
+
return "none";
|
|
468
|
+
}
|
|
469
|
+
function getOutputStreamMode(outputSchema) {
|
|
470
|
+
if (typeof outputSchema === "boolean")
|
|
471
|
+
return "none";
|
|
472
|
+
const props = outputSchema.properties;
|
|
473
|
+
if (!props)
|
|
474
|
+
return "none";
|
|
475
|
+
let found = "none";
|
|
476
|
+
for (const prop of Object.values(props)) {
|
|
477
|
+
if (!prop || typeof prop === "boolean")
|
|
478
|
+
continue;
|
|
479
|
+
const xStream = prop["x-stream"];
|
|
480
|
+
if (xStream === "append")
|
|
481
|
+
return "append";
|
|
482
|
+
if (xStream === "replace")
|
|
483
|
+
found = "replace";
|
|
484
|
+
}
|
|
485
|
+
return found;
|
|
486
|
+
}
|
|
487
|
+
function isTaskStreamable(task) {
|
|
488
|
+
if (typeof task.executeStream !== "function")
|
|
489
|
+
return false;
|
|
490
|
+
return getOutputStreamMode(task.outputSchema()) !== "none";
|
|
491
|
+
}
|
|
492
|
+
function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPort) {
|
|
493
|
+
const sourceMode = getPortStreamMode(sourceSchema, sourcePort);
|
|
494
|
+
if (sourceMode === "none")
|
|
495
|
+
return false;
|
|
496
|
+
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
497
|
+
return sourceMode !== targetMode;
|
|
498
|
+
}
|
|
499
|
+
|
|
334
500
|
// src/task/TaskRunner.ts
|
|
335
501
|
class TaskRunner {
|
|
336
502
|
running = false;
|
|
@@ -360,14 +526,27 @@ class TaskRunner {
|
|
|
360
526
|
}
|
|
361
527
|
const inputs = this.task.runInputData;
|
|
362
528
|
let outputs;
|
|
529
|
+
const isStreamable = isTaskStreamable(this.task);
|
|
363
530
|
if (this.task.cacheable) {
|
|
364
531
|
outputs = await this.outputCache?.getOutput(this.task.type, inputs);
|
|
365
532
|
if (outputs) {
|
|
366
|
-
|
|
533
|
+
if (isStreamable) {
|
|
534
|
+
this.task.runOutputData = outputs;
|
|
535
|
+
this.task.emit("stream_start");
|
|
536
|
+
this.task.emit("stream_chunk", { type: "finish", data: outputs });
|
|
537
|
+
this.task.emit("stream_end", outputs);
|
|
538
|
+
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
539
|
+
} else {
|
|
540
|
+
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
541
|
+
}
|
|
367
542
|
}
|
|
368
543
|
}
|
|
369
544
|
if (!outputs) {
|
|
370
|
-
|
|
545
|
+
if (isStreamable) {
|
|
546
|
+
outputs = await this.executeStreamingTask(inputs);
|
|
547
|
+
} else {
|
|
548
|
+
outputs = await this.executeTask(inputs);
|
|
549
|
+
}
|
|
371
550
|
if (this.task.cacheable && outputs !== undefined) {
|
|
372
551
|
await this.outputCache?.saveOutput(this.task.type, inputs, outputs);
|
|
373
552
|
}
|
|
@@ -426,6 +605,64 @@ class TaskRunner {
|
|
|
426
605
|
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
427
606
|
return Object.assign({}, output, reactiveResult ?? {});
|
|
428
607
|
}
|
|
608
|
+
async executeStreamingTask(input) {
|
|
609
|
+
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
610
|
+
let accumulated = "";
|
|
611
|
+
let chunkCount = 0;
|
|
612
|
+
let finalOutput;
|
|
613
|
+
this.task.emit("stream_start");
|
|
614
|
+
const stream = this.task.executeStream(input, {
|
|
615
|
+
signal: this.abortController.signal,
|
|
616
|
+
updateProgress: this.handleProgress.bind(this),
|
|
617
|
+
own: this.own,
|
|
618
|
+
registry: this.registry
|
|
619
|
+
});
|
|
620
|
+
for await (const event of stream) {
|
|
621
|
+
chunkCount++;
|
|
622
|
+
if (chunkCount === 1) {
|
|
623
|
+
this.task.status = TaskStatus.STREAMING;
|
|
624
|
+
this.task.emit("status", this.task.status);
|
|
625
|
+
}
|
|
626
|
+
if (event.type === "snapshot") {
|
|
627
|
+
this.task.runOutputData = event.data;
|
|
628
|
+
}
|
|
629
|
+
this.task.emit("stream_chunk", event);
|
|
630
|
+
switch (event.type) {
|
|
631
|
+
case "text-delta": {
|
|
632
|
+
accumulated += event.textDelta;
|
|
633
|
+
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.02 * chunkCount))));
|
|
634
|
+
await this.handleProgress(progress);
|
|
635
|
+
break;
|
|
636
|
+
}
|
|
637
|
+
case "snapshot": {
|
|
638
|
+
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.02 * chunkCount))));
|
|
639
|
+
await this.handleProgress(progress);
|
|
640
|
+
break;
|
|
641
|
+
}
|
|
642
|
+
case "finish": {
|
|
643
|
+
if (streamMode === "append") {
|
|
644
|
+
const text = accumulated.length > 0 ? accumulated : event.data?.text ?? "";
|
|
645
|
+
finalOutput = { ...event.data || {}, text };
|
|
646
|
+
} else if (streamMode === "replace") {
|
|
647
|
+
finalOutput = event.data;
|
|
648
|
+
}
|
|
649
|
+
break;
|
|
650
|
+
}
|
|
651
|
+
case "error": {
|
|
652
|
+
throw event.error;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
if (this.abortController?.signal.aborted) {
|
|
657
|
+
throw new TaskAbortedError("Task aborted during streaming");
|
|
658
|
+
}
|
|
659
|
+
if (finalOutput !== undefined) {
|
|
660
|
+
this.task.runOutputData = finalOutput;
|
|
661
|
+
}
|
|
662
|
+
this.task.emit("stream_end", this.task.runOutputData);
|
|
663
|
+
const reactiveResult = await this.executeTaskReactive(input, this.task.runOutputData || {});
|
|
664
|
+
return reactiveResult;
|
|
665
|
+
}
|
|
429
666
|
async handleStart(config = {}) {
|
|
430
667
|
if (this.task.status === TaskStatus.PROCESSING)
|
|
431
668
|
return;
|
|
@@ -544,13 +781,13 @@ class Task {
|
|
|
544
781
|
additionalProperties: false
|
|
545
782
|
};
|
|
546
783
|
}
|
|
547
|
-
async execute(
|
|
784
|
+
async execute(_input, context) {
|
|
548
785
|
if (context.signal?.aborted) {
|
|
549
786
|
throw new TaskAbortedError("Task aborted");
|
|
550
787
|
}
|
|
551
788
|
return;
|
|
552
789
|
}
|
|
553
|
-
async executeReactive(
|
|
790
|
+
async executeReactive(_input, output, _context) {
|
|
554
791
|
return output;
|
|
555
792
|
}
|
|
556
793
|
_runner;
|
|
@@ -912,18 +1149,62 @@ class Task {
|
|
|
912
1149
|
// src/task/ConditionalTask.ts
|
|
913
1150
|
class ConditionalTask extends Task {
|
|
914
1151
|
static type = "ConditionalTask";
|
|
915
|
-
static category = "
|
|
916
|
-
static title = "
|
|
917
|
-
static description = "
|
|
1152
|
+
static category = "Flow Control";
|
|
1153
|
+
static title = "Condition";
|
|
1154
|
+
static description = "Route data based on conditions";
|
|
918
1155
|
static hasDynamicSchemas = true;
|
|
919
1156
|
activeBranches = new Set;
|
|
1157
|
+
buildBranchesFromConditionConfig(conditionConfig) {
|
|
1158
|
+
if (!conditionConfig?.branches || conditionConfig.branches.length === 0) {
|
|
1159
|
+
return [
|
|
1160
|
+
{
|
|
1161
|
+
id: "default",
|
|
1162
|
+
condition: () => true,
|
|
1163
|
+
outputPort: "1"
|
|
1164
|
+
}
|
|
1165
|
+
];
|
|
1166
|
+
}
|
|
1167
|
+
return conditionConfig.branches.map((branch, index) => ({
|
|
1168
|
+
id: branch.id,
|
|
1169
|
+
outputPort: String(index + 1),
|
|
1170
|
+
condition: (inputData) => {
|
|
1171
|
+
const fieldValue = getNestedValue(inputData, branch.field);
|
|
1172
|
+
return evaluateCondition(fieldValue, branch.operator, branch.value);
|
|
1173
|
+
}
|
|
1174
|
+
}));
|
|
1175
|
+
}
|
|
1176
|
+
resolveBranches(input) {
|
|
1177
|
+
const configBranches = this.config.branches ?? [];
|
|
1178
|
+
if (configBranches.length > 0 && typeof configBranches[0].condition === "function") {
|
|
1179
|
+
return {
|
|
1180
|
+
branches: configBranches,
|
|
1181
|
+
isExclusive: this.config.exclusive ?? true,
|
|
1182
|
+
defaultBranch: this.config.defaultBranch,
|
|
1183
|
+
fromConditionConfig: false
|
|
1184
|
+
};
|
|
1185
|
+
}
|
|
1186
|
+
const conditionConfig = input.conditionConfig ?? this.config.extras?.conditionConfig;
|
|
1187
|
+
if (conditionConfig) {
|
|
1188
|
+
return {
|
|
1189
|
+
branches: this.buildBranchesFromConditionConfig(conditionConfig),
|
|
1190
|
+
isExclusive: conditionConfig.exclusive ?? true,
|
|
1191
|
+
defaultBranch: conditionConfig.defaultBranch,
|
|
1192
|
+
fromConditionConfig: true
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
return {
|
|
1196
|
+
branches: configBranches,
|
|
1197
|
+
isExclusive: this.config.exclusive ?? true,
|
|
1198
|
+
defaultBranch: this.config.defaultBranch,
|
|
1199
|
+
fromConditionConfig: false
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
920
1202
|
async execute(input, context) {
|
|
921
1203
|
if (context.signal?.aborted) {
|
|
922
1204
|
return;
|
|
923
1205
|
}
|
|
924
1206
|
this.activeBranches.clear();
|
|
925
|
-
const branches = this.
|
|
926
|
-
const isExclusive = this.config.exclusive ?? true;
|
|
1207
|
+
const { branches, isExclusive, defaultBranch, fromConditionConfig } = this.resolveBranches(input);
|
|
927
1208
|
for (const branch of branches) {
|
|
928
1209
|
try {
|
|
929
1210
|
const isActive = branch.condition(input);
|
|
@@ -937,14 +1218,50 @@ class ConditionalTask extends Task {
|
|
|
937
1218
|
console.warn(`Condition evaluation failed for branch "${branch.id}":`, error);
|
|
938
1219
|
}
|
|
939
1220
|
}
|
|
940
|
-
if (this.activeBranches.size === 0 &&
|
|
941
|
-
const defaultBranchExists = branches.some((b) => b.id ===
|
|
1221
|
+
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
1222
|
+
const defaultBranchExists = branches.some((b) => b.id === defaultBranch);
|
|
942
1223
|
if (defaultBranchExists) {
|
|
943
|
-
this.activeBranches.add(
|
|
1224
|
+
this.activeBranches.add(defaultBranch);
|
|
944
1225
|
}
|
|
945
1226
|
}
|
|
1227
|
+
if (fromConditionConfig) {
|
|
1228
|
+
return this.buildConditionConfigOutput(input, branches, isExclusive);
|
|
1229
|
+
}
|
|
946
1230
|
return this.buildOutput(input);
|
|
947
1231
|
}
|
|
1232
|
+
buildConditionConfigOutput(input, branches, isExclusive) {
|
|
1233
|
+
const output = {};
|
|
1234
|
+
const { conditionConfig, ...passThrough } = input;
|
|
1235
|
+
const inputKeys = Object.keys(passThrough);
|
|
1236
|
+
let matchedBranchNumber = null;
|
|
1237
|
+
for (let i = 0;i < branches.length; i++) {
|
|
1238
|
+
if (this.activeBranches.has(branches[i].id)) {
|
|
1239
|
+
if (matchedBranchNumber === null) {
|
|
1240
|
+
matchedBranchNumber = i + 1;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
if (isExclusive) {
|
|
1245
|
+
if (matchedBranchNumber !== null) {
|
|
1246
|
+
for (const key of inputKeys) {
|
|
1247
|
+
output[`${key}_${matchedBranchNumber}`] = passThrough[key];
|
|
1248
|
+
}
|
|
1249
|
+
} else {
|
|
1250
|
+
for (const key of inputKeys) {
|
|
1251
|
+
output[`${key}_else`] = passThrough[key];
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
} else {
|
|
1255
|
+
for (let i = 0;i < branches.length; i++) {
|
|
1256
|
+
if (this.activeBranches.has(branches[i].id)) {
|
|
1257
|
+
for (const key of inputKeys) {
|
|
1258
|
+
output[`${key}_${i + 1}`] = passThrough[key];
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
return output;
|
|
1264
|
+
}
|
|
948
1265
|
buildOutput(input) {
|
|
949
1266
|
const output = {
|
|
950
1267
|
_activeBranches: Array.from(this.activeBranches)
|
|
@@ -1039,6 +1356,7 @@ class TopologicalScheduler {
|
|
|
1039
1356
|
}
|
|
1040
1357
|
}
|
|
1041
1358
|
onTaskCompleted(taskId) {}
|
|
1359
|
+
onTaskStreaming(taskId) {}
|
|
1042
1360
|
reset() {
|
|
1043
1361
|
this.sortedNodes = this.dag.topologicallySortedNodes();
|
|
1044
1362
|
this.currentIndex = 0;
|
|
@@ -1048,11 +1366,13 @@ class TopologicalScheduler {
|
|
|
1048
1366
|
class DependencyBasedScheduler {
|
|
1049
1367
|
dag;
|
|
1050
1368
|
completedTasks;
|
|
1369
|
+
streamingTasks;
|
|
1051
1370
|
pendingTasks;
|
|
1052
1371
|
nextResolver = null;
|
|
1053
1372
|
constructor(dag) {
|
|
1054
1373
|
this.dag = dag;
|
|
1055
1374
|
this.completedTasks = new Set;
|
|
1375
|
+
this.streamingTasks = new Set;
|
|
1056
1376
|
this.pendingTasks = new Set;
|
|
1057
1377
|
this.reset();
|
|
1058
1378
|
}
|
|
@@ -1067,8 +1387,23 @@ class DependencyBasedScheduler {
|
|
|
1067
1387
|
return false;
|
|
1068
1388
|
}
|
|
1069
1389
|
}
|
|
1070
|
-
const
|
|
1071
|
-
return
|
|
1390
|
+
const activeDataflows = sourceDataflows.filter((df) => df.status !== TaskStatus.DISABLED);
|
|
1391
|
+
return activeDataflows.every((df) => {
|
|
1392
|
+
const depId = df.sourceTaskId;
|
|
1393
|
+
if (this.completedTasks.has(depId))
|
|
1394
|
+
return true;
|
|
1395
|
+
if (this.streamingTasks.has(depId)) {
|
|
1396
|
+
const sourceTask = this.dag.getTask(depId);
|
|
1397
|
+
if (sourceTask) {
|
|
1398
|
+
const sourceMode = getPortStreamMode(sourceTask.outputSchema(), df.sourceTaskPortId);
|
|
1399
|
+
const targetMode = getPortStreamMode(task.inputSchema(), df.targetTaskPortId);
|
|
1400
|
+
if (sourceMode !== "none" && sourceMode === targetMode) {
|
|
1401
|
+
return true;
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
return false;
|
|
1406
|
+
});
|
|
1072
1407
|
}
|
|
1073
1408
|
async waitForNextTask() {
|
|
1074
1409
|
if (this.pendingTasks.size === 0)
|
|
@@ -1123,8 +1458,26 @@ class DependencyBasedScheduler {
|
|
|
1123
1458
|
}
|
|
1124
1459
|
}
|
|
1125
1460
|
}
|
|
1461
|
+
onTaskStreaming(taskId) {
|
|
1462
|
+
this.streamingTasks.add(taskId);
|
|
1463
|
+
for (const task of Array.from(this.pendingTasks)) {
|
|
1464
|
+
if (task.status === TaskStatus.DISABLED) {
|
|
1465
|
+
this.pendingTasks.delete(task);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
if (this.nextResolver) {
|
|
1469
|
+
const readyTask = Array.from(this.pendingTasks).find((task) => this.isTaskReady(task));
|
|
1470
|
+
if (readyTask) {
|
|
1471
|
+
this.pendingTasks.delete(readyTask);
|
|
1472
|
+
const resolver = this.nextResolver;
|
|
1473
|
+
this.nextResolver = null;
|
|
1474
|
+
resolver(readyTask);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1126
1478
|
reset() {
|
|
1127
1479
|
this.completedTasks.clear();
|
|
1480
|
+
this.streamingTasks.clear();
|
|
1128
1481
|
this.pendingTasks = new Set(this.dag.topologicallySortedNodes());
|
|
1129
1482
|
this.nextResolver = null;
|
|
1130
1483
|
}
|
|
@@ -1361,7 +1714,12 @@ class TaskGraphRunner {
|
|
|
1361
1714
|
}
|
|
1362
1715
|
}
|
|
1363
1716
|
async runTask(task, input) {
|
|
1717
|
+
const isStreamable = isTaskStreamable(task);
|
|
1718
|
+
await this.awaitStreamInputs(task);
|
|
1364
1719
|
this.copyInputFromEdgesToNode(task);
|
|
1720
|
+
if (isStreamable) {
|
|
1721
|
+
return this.runStreamingTask(task, input);
|
|
1722
|
+
}
|
|
1365
1723
|
const results = await task.runner.run(input, {
|
|
1366
1724
|
outputCache: this.outputCache,
|
|
1367
1725
|
updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
|
|
@@ -1374,6 +1732,93 @@ class TaskGraphRunner {
|
|
|
1374
1732
|
data: results
|
|
1375
1733
|
};
|
|
1376
1734
|
}
|
|
1735
|
+
async awaitStreamInputs(task) {
|
|
1736
|
+
const dataflows = this.graph.getSourceDataflows(task.config.id);
|
|
1737
|
+
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1738
|
+
if (streamPromises.length > 0) {
|
|
1739
|
+
await Promise.all(streamPromises);
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
async runStreamingTask(task, input) {
|
|
1743
|
+
const streamMode = getOutputStreamMode(task.outputSchema());
|
|
1744
|
+
let streamingNotified = false;
|
|
1745
|
+
const onStatus = (status) => {
|
|
1746
|
+
if (status === TaskStatus.STREAMING && !streamingNotified) {
|
|
1747
|
+
streamingNotified = true;
|
|
1748
|
+
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1749
|
+
this.pushStreamToEdges(task, streamMode);
|
|
1750
|
+
this.processScheduler.onTaskStreaming(task.config.id);
|
|
1751
|
+
}
|
|
1752
|
+
};
|
|
1753
|
+
const onStreamStart = () => {
|
|
1754
|
+
this.graph.emit("task_stream_start", task.config.id);
|
|
1755
|
+
};
|
|
1756
|
+
const onStreamChunk = (event) => {
|
|
1757
|
+
this.graph.emit("task_stream_chunk", task.config.id, event);
|
|
1758
|
+
};
|
|
1759
|
+
const onStreamEnd = (output) => {
|
|
1760
|
+
this.graph.emit("task_stream_end", task.config.id, output);
|
|
1761
|
+
};
|
|
1762
|
+
task.on("status", onStatus);
|
|
1763
|
+
task.on("stream_start", onStreamStart);
|
|
1764
|
+
task.on("stream_chunk", onStreamChunk);
|
|
1765
|
+
task.on("stream_end", onStreamEnd);
|
|
1766
|
+
try {
|
|
1767
|
+
const results = await task.runner.run(input, {
|
|
1768
|
+
outputCache: this.outputCache,
|
|
1769
|
+
updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
|
|
1770
|
+
registry: this.registry
|
|
1771
|
+
});
|
|
1772
|
+
await this.pushOutputFromNodeToEdges(task, results);
|
|
1773
|
+
return {
|
|
1774
|
+
id: task.config.id,
|
|
1775
|
+
type: task.constructor.runtype || task.constructor.type,
|
|
1776
|
+
data: results
|
|
1777
|
+
};
|
|
1778
|
+
} finally {
|
|
1779
|
+
task.off("status", onStatus);
|
|
1780
|
+
task.off("stream_start", onStreamStart);
|
|
1781
|
+
task.off("stream_chunk", onStreamChunk);
|
|
1782
|
+
task.off("stream_end", onStreamEnd);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
pushStreamToEdges(task, streamMode) {
|
|
1786
|
+
const targetDataflows = this.graph.getTargetDataflows(task.config.id);
|
|
1787
|
+
if (targetDataflows.length === 0)
|
|
1788
|
+
return;
|
|
1789
|
+
const stream = new ReadableStream({
|
|
1790
|
+
start: (controller) => {
|
|
1791
|
+
const onChunk = (event) => {
|
|
1792
|
+
try {
|
|
1793
|
+
controller.enqueue(event);
|
|
1794
|
+
} catch {}
|
|
1795
|
+
};
|
|
1796
|
+
const onEnd = () => {
|
|
1797
|
+
try {
|
|
1798
|
+
controller.close();
|
|
1799
|
+
} catch {}
|
|
1800
|
+
task.off("stream_chunk", onChunk);
|
|
1801
|
+
task.off("stream_end", onEnd);
|
|
1802
|
+
};
|
|
1803
|
+
task.on("stream_chunk", onChunk);
|
|
1804
|
+
task.on("stream_end", onEnd);
|
|
1805
|
+
}
|
|
1806
|
+
});
|
|
1807
|
+
if (targetDataflows.length === 1) {
|
|
1808
|
+
targetDataflows[0].setStream(stream);
|
|
1809
|
+
} else {
|
|
1810
|
+
let currentStream = stream;
|
|
1811
|
+
for (let i = 0;i < targetDataflows.length; i++) {
|
|
1812
|
+
if (i === targetDataflows.length - 1) {
|
|
1813
|
+
targetDataflows[i].setStream(currentStream);
|
|
1814
|
+
} else {
|
|
1815
|
+
const [s1, s2] = currentStream.tee();
|
|
1816
|
+
targetDataflows[i].setStream(s1);
|
|
1817
|
+
currentStream = s2;
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1377
1822
|
resetTask(graph, task, runId) {
|
|
1378
1823
|
task.status = TaskStatus.PENDING;
|
|
1379
1824
|
task.resetInputData();
|
|
@@ -1457,7 +1902,7 @@ class TaskGraphRunner {
|
|
|
1457
1902
|
}
|
|
1458
1903
|
async handleError(error) {
|
|
1459
1904
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
1460
|
-
if (task.status === TaskStatus.PROCESSING) {
|
|
1905
|
+
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
1461
1906
|
task.abort();
|
|
1462
1907
|
}
|
|
1463
1908
|
}));
|
|
@@ -1469,7 +1914,7 @@ class TaskGraphRunner {
|
|
|
1469
1914
|
}
|
|
1470
1915
|
async handleAbort() {
|
|
1471
1916
|
this.graph.getTasks().map(async (task) => {
|
|
1472
|
-
if (task.status === TaskStatus.PROCESSING) {
|
|
1917
|
+
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
1473
1918
|
task.abort();
|
|
1474
1919
|
}
|
|
1475
1920
|
});
|
|
@@ -1547,9 +1992,9 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
1547
1992
|
// src/task/GraphAsTask.ts
|
|
1548
1993
|
class GraphAsTask extends Task {
|
|
1549
1994
|
static type = "GraphAsTask";
|
|
1550
|
-
static title = "
|
|
1551
|
-
static description = "A
|
|
1552
|
-
static category = "
|
|
1995
|
+
static title = "Group";
|
|
1996
|
+
static description = "A group of tasks that are executed together";
|
|
1997
|
+
static category = "Flow Control";
|
|
1553
1998
|
static compoundMerge = PROPERTY_ARRAY;
|
|
1554
1999
|
static hasDynamicSchemas = true;
|
|
1555
2000
|
constructor(input = {}, config = {}) {
|
|
@@ -1729,38 +2174,112 @@ class GraphAsTask extends Task {
|
|
|
1729
2174
|
}
|
|
1730
2175
|
|
|
1731
2176
|
// src/task-graph/Workflow.ts
|
|
1732
|
-
import { EventEmitter as EventEmitter4 } from "@workglow/util";
|
|
2177
|
+
import { EventEmitter as EventEmitter4, uuid4 as uuid43 } from "@workglow/util";
|
|
2178
|
+
function CreateWorkflow(taskClass) {
|
|
2179
|
+
return Workflow.createWorkflow(taskClass);
|
|
2180
|
+
}
|
|
2181
|
+
function CreateLoopWorkflow(taskClass) {
|
|
2182
|
+
return function(config = {}) {
|
|
2183
|
+
return this.addLoopTask(taskClass, config);
|
|
2184
|
+
};
|
|
2185
|
+
}
|
|
2186
|
+
function CreateEndLoopWorkflow(methodName) {
|
|
2187
|
+
return function() {
|
|
2188
|
+
if (!this.isLoopBuilder) {
|
|
2189
|
+
throw new Error(`${methodName}() can only be called on loop workflows`);
|
|
2190
|
+
}
|
|
2191
|
+
return this.finalizeAndReturn();
|
|
2192
|
+
};
|
|
2193
|
+
}
|
|
2194
|
+
var TYPED_ARRAY_FORMAT_PREFIX = "TypedArray";
|
|
2195
|
+
function schemaHasTypedArrayFormat(schema) {
|
|
2196
|
+
if (typeof schema === "boolean")
|
|
2197
|
+
return false;
|
|
2198
|
+
if (!schema || typeof schema !== "object" || Array.isArray(schema))
|
|
2199
|
+
return false;
|
|
2200
|
+
const s = schema;
|
|
2201
|
+
if (typeof s.format === "string" && s.format.startsWith(TYPED_ARRAY_FORMAT_PREFIX)) {
|
|
2202
|
+
return true;
|
|
2203
|
+
}
|
|
2204
|
+
const checkUnion = (schemas) => {
|
|
2205
|
+
if (!Array.isArray(schemas))
|
|
2206
|
+
return false;
|
|
2207
|
+
return schemas.some((sub) => schemaHasTypedArrayFormat(sub));
|
|
2208
|
+
};
|
|
2209
|
+
if (checkUnion(s.oneOf) || checkUnion(s.anyOf))
|
|
2210
|
+
return true;
|
|
2211
|
+
const items = s.items;
|
|
2212
|
+
if (items && typeof items === "object" && !Array.isArray(items)) {
|
|
2213
|
+
if (schemaHasTypedArrayFormat(items))
|
|
2214
|
+
return true;
|
|
2215
|
+
}
|
|
2216
|
+
return false;
|
|
2217
|
+
}
|
|
2218
|
+
function hasVectorOutput(task) {
|
|
2219
|
+
const outputSchema = task.outputSchema();
|
|
2220
|
+
if (typeof outputSchema === "boolean" || !outputSchema?.properties)
|
|
2221
|
+
return false;
|
|
2222
|
+
return Object.values(outputSchema.properties).some((prop) => schemaHasTypedArrayFormat(prop));
|
|
2223
|
+
}
|
|
2224
|
+
function hasVectorLikeInput(input) {
|
|
2225
|
+
if (!input || typeof input !== "object")
|
|
2226
|
+
return false;
|
|
2227
|
+
const v = input.vectors;
|
|
2228
|
+
return Array.isArray(v) && v.length > 0 && typeof v[0] === "object" && v[0] !== null && ArrayBuffer.isView(v[0]);
|
|
2229
|
+
}
|
|
2230
|
+
function CreateAdaptiveWorkflow(scalarClass, vectorClass) {
|
|
2231
|
+
const scalarHelper = Workflow.createWorkflow(scalarClass);
|
|
2232
|
+
const vectorHelper = Workflow.createWorkflow(vectorClass);
|
|
2233
|
+
return function(input = {}, config = {}) {
|
|
2234
|
+
const parent = getLastTask(this);
|
|
2235
|
+
const useVector = parent !== undefined && hasVectorOutput(parent) || hasVectorLikeInput(input);
|
|
2236
|
+
if (useVector) {
|
|
2237
|
+
return vectorHelper.call(this, input, config);
|
|
2238
|
+
}
|
|
2239
|
+
return scalarHelper.call(this, input, config);
|
|
2240
|
+
};
|
|
2241
|
+
}
|
|
2242
|
+
|
|
1733
2243
|
class WorkflowTask extends GraphAsTask {
|
|
1734
2244
|
static type = "Workflow";
|
|
1735
2245
|
static compoundMerge = PROPERTY_ARRAY;
|
|
1736
2246
|
}
|
|
1737
|
-
var taskIdCounter = 0;
|
|
1738
2247
|
|
|
1739
2248
|
class Workflow {
|
|
1740
|
-
constructor(
|
|
1741
|
-
this.
|
|
1742
|
-
this.
|
|
1743
|
-
|
|
1744
|
-
});
|
|
1745
|
-
|
|
1746
|
-
|
|
2249
|
+
constructor(cache, parent, iteratorTask) {
|
|
2250
|
+
this._outputCache = cache;
|
|
2251
|
+
this._parentWorkflow = parent;
|
|
2252
|
+
this._iteratorTask = iteratorTask;
|
|
2253
|
+
this._graph = new TaskGraph({ outputCache: this._outputCache });
|
|
2254
|
+
if (!parent) {
|
|
2255
|
+
this._onChanged = this._onChanged.bind(this);
|
|
2256
|
+
this.setupEvents();
|
|
2257
|
+
}
|
|
1747
2258
|
}
|
|
1748
2259
|
_graph;
|
|
1749
2260
|
_dataFlows = [];
|
|
1750
2261
|
_error = "";
|
|
1751
|
-
|
|
2262
|
+
_outputCache;
|
|
1752
2263
|
_abortController;
|
|
2264
|
+
_parentWorkflow;
|
|
2265
|
+
_iteratorTask;
|
|
2266
|
+
_pendingLoopConnect;
|
|
2267
|
+
outputCache() {
|
|
2268
|
+
return this._outputCache;
|
|
2269
|
+
}
|
|
2270
|
+
get isLoopBuilder() {
|
|
2271
|
+
return this._parentWorkflow !== undefined;
|
|
2272
|
+
}
|
|
1753
2273
|
events = new EventEmitter4;
|
|
1754
2274
|
static createWorkflow(taskClass) {
|
|
1755
2275
|
const helper = function(input = {}, config = {}) {
|
|
1756
2276
|
this._error = "";
|
|
1757
2277
|
const parent = getLastTask(this);
|
|
1758
|
-
|
|
1759
|
-
const task = this.addTask(taskClass, input, { id: String(taskIdCounter), ...config });
|
|
2278
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid43(), ...config });
|
|
1760
2279
|
if (this._dataFlows.length > 0) {
|
|
1761
2280
|
this._dataFlows.forEach((dataflow) => {
|
|
1762
2281
|
const taskSchema = task.inputSchema();
|
|
1763
|
-
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2282
|
+
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
1764
2283
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.config.id}`;
|
|
1765
2284
|
console.error(this._error);
|
|
1766
2285
|
return;
|
|
@@ -1771,158 +2290,23 @@ class Workflow {
|
|
|
1771
2290
|
this._dataFlows = [];
|
|
1772
2291
|
}
|
|
1773
2292
|
if (parent && this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
1774
|
-
const
|
|
1775
|
-
const
|
|
1776
|
-
const
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
if (targetSchema === true || typeof targetSchema === "object" && targetSchema.additionalProperties === true) {
|
|
1780
|
-
for (const fromOutputPortId of Object.keys(sourceSchema.properties || {})) {
|
|
1781
|
-
matches.set(fromOutputPortId, fromOutputPortId);
|
|
1782
|
-
this.connect(parent.config.id, fromOutputPortId, task.config.id, fromOutputPortId);
|
|
1783
|
-
}
|
|
1784
|
-
return matches;
|
|
1785
|
-
}
|
|
1786
|
-
}
|
|
1787
|
-
if (typeof sourceSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1788
|
-
return matches;
|
|
1789
|
-
}
|
|
1790
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(sourceSchema.properties || {})) {
|
|
1791
|
-
for (const [toInputPortId, toPortInputSchema] of Object.entries(targetSchema.properties || {})) {
|
|
1792
|
-
if (!matches.has(toInputPortId) && comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
1793
|
-
matches.set(toInputPortId, fromOutputPortId);
|
|
1794
|
-
this.connect(parent.config.id, fromOutputPortId, task.config.id, toInputPortId);
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
}
|
|
1798
|
-
return matches;
|
|
1799
|
-
};
|
|
1800
|
-
const getSpecificTypeIdentifiers = (schema) => {
|
|
1801
|
-
const formats = new Set;
|
|
1802
|
-
const ids = new Set;
|
|
1803
|
-
if (typeof schema === "boolean") {
|
|
1804
|
-
return { formats, ids };
|
|
1805
|
-
}
|
|
1806
|
-
const extractFromSchema = (s) => {
|
|
1807
|
-
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
1808
|
-
return;
|
|
1809
|
-
if (s.format)
|
|
1810
|
-
formats.add(s.format);
|
|
1811
|
-
if (s.$id)
|
|
1812
|
-
ids.add(s.$id);
|
|
1813
|
-
};
|
|
1814
|
-
extractFromSchema(schema);
|
|
1815
|
-
const checkUnion = (schemas) => {
|
|
1816
|
-
if (!schemas)
|
|
1817
|
-
return;
|
|
1818
|
-
for (const s of schemas) {
|
|
1819
|
-
if (typeof s === "boolean")
|
|
1820
|
-
continue;
|
|
1821
|
-
extractFromSchema(s);
|
|
1822
|
-
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
1823
|
-
extractFromSchema(s.items);
|
|
1824
|
-
}
|
|
1825
|
-
}
|
|
1826
|
-
};
|
|
1827
|
-
checkUnion(schema.oneOf);
|
|
1828
|
-
checkUnion(schema.anyOf);
|
|
1829
|
-
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
1830
|
-
extractFromSchema(schema.items);
|
|
1831
|
-
}
|
|
1832
|
-
return { formats, ids };
|
|
1833
|
-
};
|
|
1834
|
-
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
1835
|
-
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
1836
|
-
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
1837
|
-
}
|
|
1838
|
-
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
1839
|
-
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
1840
|
-
for (const format of outputIds.formats) {
|
|
1841
|
-
if (inputIds.formats.has(format)) {
|
|
1842
|
-
return true;
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
for (const id of outputIds.ids) {
|
|
1846
|
-
if (inputIds.ids.has(id)) {
|
|
1847
|
-
return true;
|
|
1848
|
-
}
|
|
1849
|
-
}
|
|
1850
|
-
if (requireSpecificType) {
|
|
1851
|
-
return false;
|
|
1852
|
-
}
|
|
1853
|
-
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
1854
|
-
if (!idTypeBlank)
|
|
1855
|
-
return false;
|
|
1856
|
-
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
1857
|
-
return true;
|
|
1858
|
-
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
1859
|
-
if (typeof schema === "boolean")
|
|
1860
|
-
return schema;
|
|
1861
|
-
return schema.type === fromPortOutputSchema.type;
|
|
1862
|
-
}) ?? false;
|
|
1863
|
-
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
1864
|
-
if (typeof schema === "boolean")
|
|
1865
|
-
return schema;
|
|
1866
|
-
return schema.type === fromPortOutputSchema.type;
|
|
1867
|
-
}) ?? false;
|
|
1868
|
-
return matchesOneOf || matchesAnyOf;
|
|
1869
|
-
};
|
|
1870
|
-
makeMatch(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1871
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1872
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1873
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1874
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1875
|
-
});
|
|
1876
|
-
makeMatch(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1877
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1878
|
-
});
|
|
1879
|
-
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
1880
|
-
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
1881
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
1882
|
-
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
1883
|
-
if (unmatchedRequired.length > 0) {
|
|
1884
|
-
const nodes = this._graph.getTasks();
|
|
1885
|
-
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
1886
|
-
for (let i = parentIndex - 1;i >= 0 && unmatchedRequired.length > 0; i--) {
|
|
1887
|
-
const earlierTask = nodes[i];
|
|
1888
|
-
const earlierOutputSchema = earlierTask.outputSchema();
|
|
1889
|
-
const makeMatchFromEarlier = (comparator) => {
|
|
1890
|
-
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1891
|
-
return;
|
|
1892
|
-
}
|
|
1893
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
1894
|
-
for (const requiredInputId of unmatchedRequired) {
|
|
1895
|
-
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
1896
|
-
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
1897
|
-
matches.set(requiredInputId, fromOutputPortId);
|
|
1898
|
-
this.connect(earlierTask.config.id, fromOutputPortId, task.config.id, requiredInputId);
|
|
1899
|
-
}
|
|
1900
|
-
}
|
|
1901
|
-
}
|
|
1902
|
-
};
|
|
1903
|
-
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1904
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1905
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1906
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1907
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1908
|
-
});
|
|
1909
|
-
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1910
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1911
|
-
});
|
|
1912
|
-
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
1913
|
-
}
|
|
2293
|
+
const nodes = this._graph.getTasks();
|
|
2294
|
+
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
2295
|
+
const earlierTasks = [];
|
|
2296
|
+
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2297
|
+
earlierTasks.push(nodes[i]);
|
|
1914
2298
|
}
|
|
1915
|
-
const
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
this._error =
|
|
2299
|
+
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2300
|
+
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2301
|
+
providedInputKeys,
|
|
2302
|
+
earlierTasks
|
|
2303
|
+
});
|
|
2304
|
+
if (result.error) {
|
|
2305
|
+
if (this.isLoopBuilder) {
|
|
2306
|
+
this._error = result.error;
|
|
2307
|
+
console.warn(this._error);
|
|
2308
|
+
} else {
|
|
2309
|
+
this._error = result.error + " Task not added.";
|
|
1926
2310
|
console.error(this._error);
|
|
1927
2311
|
this.graph.removeTask(task.config.id);
|
|
1928
2312
|
}
|
|
@@ -1965,12 +2349,25 @@ class Workflow {
|
|
|
1965
2349
|
return this.events.waitOn(name);
|
|
1966
2350
|
}
|
|
1967
2351
|
async run(input = {}) {
|
|
2352
|
+
if (this.isLoopBuilder) {
|
|
2353
|
+
this.finalizeTemplate();
|
|
2354
|
+
if (this._pendingLoopConnect) {
|
|
2355
|
+
this._parentWorkflow.autoConnectLoopTask(this._pendingLoopConnect);
|
|
2356
|
+
this._pendingLoopConnect = undefined;
|
|
2357
|
+
}
|
|
2358
|
+
return this._parentWorkflow.run(input);
|
|
2359
|
+
}
|
|
1968
2360
|
this.events.emit("start");
|
|
1969
2361
|
this._abortController = new AbortController;
|
|
2362
|
+
const unsubStreaming = this.graph.subscribeToTaskStreaming({
|
|
2363
|
+
onStreamStart: (taskId) => this.events.emit("stream_start", taskId),
|
|
2364
|
+
onStreamChunk: (taskId, event) => this.events.emit("stream_chunk", taskId, event),
|
|
2365
|
+
onStreamEnd: (taskId, output) => this.events.emit("stream_end", taskId, output)
|
|
2366
|
+
});
|
|
1970
2367
|
try {
|
|
1971
2368
|
const output = await this.graph.run(input, {
|
|
1972
2369
|
parentSignal: this._abortController.signal,
|
|
1973
|
-
outputCache: this.
|
|
2370
|
+
outputCache: this._outputCache
|
|
1974
2371
|
});
|
|
1975
2372
|
const results = this.graph.mergeExecuteOutputsToRunOutput(output, PROPERTY_ARRAY);
|
|
1976
2373
|
this.events.emit("complete");
|
|
@@ -1979,10 +2376,14 @@ class Workflow {
|
|
|
1979
2376
|
this.events.emit("error", String(error));
|
|
1980
2377
|
throw error;
|
|
1981
2378
|
} finally {
|
|
2379
|
+
unsubStreaming();
|
|
1982
2380
|
this._abortController = undefined;
|
|
1983
2381
|
}
|
|
1984
2382
|
}
|
|
1985
2383
|
async abort() {
|
|
2384
|
+
if (this._parentWorkflow) {
|
|
2385
|
+
return this._parentWorkflow.abort();
|
|
2386
|
+
}
|
|
1986
2387
|
this._abortController?.abort();
|
|
1987
2388
|
}
|
|
1988
2389
|
pop() {
|
|
@@ -2051,10 +2452,12 @@ class Workflow {
|
|
|
2051
2452
|
return task;
|
|
2052
2453
|
}
|
|
2053
2454
|
reset() {
|
|
2054
|
-
|
|
2455
|
+
if (this._parentWorkflow) {
|
|
2456
|
+
throw new WorkflowError("Cannot reset a loop workflow. Call reset() on the parent workflow.");
|
|
2457
|
+
}
|
|
2055
2458
|
this.clearEvents();
|
|
2056
2459
|
this._graph = new TaskGraph({
|
|
2057
|
-
outputCache: this.
|
|
2460
|
+
outputCache: this._outputCache
|
|
2058
2461
|
});
|
|
2059
2462
|
this._dataFlows = [];
|
|
2060
2463
|
this._error = "";
|
|
@@ -2109,53 +2512,294 @@ class Workflow {
|
|
|
2109
2512
|
this.graph.addDataflow(dataflow);
|
|
2110
2513
|
return this;
|
|
2111
2514
|
}
|
|
2112
|
-
|
|
2515
|
+
addTaskToGraph(taskClass, input, config) {
|
|
2113
2516
|
const task = new taskClass(input, config);
|
|
2114
2517
|
const id = this.graph.addTask(task);
|
|
2115
2518
|
this.events.emit("changed", id);
|
|
2116
2519
|
return task;
|
|
2117
2520
|
}
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
|
-
// src/task-graph/Conversions.ts
|
|
2124
|
-
class ListeningGraphAsTask extends GraphAsTask {
|
|
2125
|
-
constructor(input, config) {
|
|
2126
|
-
super(input, config);
|
|
2127
|
-
this.subGraph.on("start", () => {
|
|
2128
|
-
this.emit("start");
|
|
2129
|
-
});
|
|
2130
|
-
this.subGraph.on("complete", () => {
|
|
2131
|
-
this.emit("complete");
|
|
2132
|
-
});
|
|
2133
|
-
this.subGraph.on("error", (e) => {
|
|
2134
|
-
this.emit("error", e);
|
|
2135
|
-
});
|
|
2521
|
+
addTask(taskClass, input, config) {
|
|
2522
|
+
const helper = Workflow.createWorkflow(taskClass);
|
|
2523
|
+
return helper.call(this, input, config);
|
|
2136
2524
|
}
|
|
2137
|
-
}
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
}
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
}
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2525
|
+
addLoopTask(taskClass, config = {}) {
|
|
2526
|
+
this._error = "";
|
|
2527
|
+
const parent = getLastTask(this);
|
|
2528
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid43(), ...config });
|
|
2529
|
+
if (this._dataFlows.length > 0) {
|
|
2530
|
+
this._dataFlows.forEach((dataflow) => {
|
|
2531
|
+
const taskSchema = task.inputSchema();
|
|
2532
|
+
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2533
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.config.id}`;
|
|
2534
|
+
console.error(this._error);
|
|
2535
|
+
return;
|
|
2536
|
+
}
|
|
2537
|
+
dataflow.targetTaskId = task.config.id;
|
|
2538
|
+
this.graph.addDataflow(dataflow);
|
|
2539
|
+
});
|
|
2540
|
+
this._dataFlows = [];
|
|
2541
|
+
}
|
|
2542
|
+
const loopBuilder = new Workflow(this.outputCache(), this, task);
|
|
2543
|
+
if (parent) {
|
|
2544
|
+
loopBuilder._pendingLoopConnect = { parent, iteratorTask: task };
|
|
2545
|
+
}
|
|
2546
|
+
return loopBuilder;
|
|
2547
|
+
}
|
|
2548
|
+
autoConnectLoopTask(pending) {
|
|
2549
|
+
if (!pending)
|
|
2550
|
+
return;
|
|
2551
|
+
const { parent, iteratorTask } = pending;
|
|
2552
|
+
if (this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
2553
|
+
const nodes = this._graph.getTasks();
|
|
2554
|
+
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
2555
|
+
const earlierTasks = [];
|
|
2556
|
+
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2557
|
+
earlierTasks.push(nodes[i]);
|
|
2558
|
+
}
|
|
2559
|
+
const result = Workflow.autoConnect(this.graph, parent, iteratorTask, {
|
|
2560
|
+
earlierTasks
|
|
2561
|
+
});
|
|
2562
|
+
if (result.error) {
|
|
2563
|
+
this._error = result.error + " Task not added.";
|
|
2564
|
+
console.error(this._error);
|
|
2565
|
+
this.graph.removeTask(iteratorTask.config.id);
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
static AutoConnectOptions = Symbol("AutoConnectOptions");
|
|
2570
|
+
static autoConnect(graph, sourceTask, targetTask, options) {
|
|
2571
|
+
const matches = new Map;
|
|
2572
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
2573
|
+
const targetSchema = targetTask.inputSchema();
|
|
2574
|
+
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
2575
|
+
const earlierTasks = options?.earlierTasks ?? [];
|
|
2576
|
+
const getSpecificTypeIdentifiers = (schema) => {
|
|
2577
|
+
const formats = new Set;
|
|
2578
|
+
const ids = new Set;
|
|
2579
|
+
if (typeof schema === "boolean") {
|
|
2580
|
+
return { formats, ids };
|
|
2581
|
+
}
|
|
2582
|
+
const extractFromSchema = (s) => {
|
|
2583
|
+
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
2584
|
+
return;
|
|
2585
|
+
if (s.format)
|
|
2586
|
+
formats.add(s.format);
|
|
2587
|
+
if (s.$id)
|
|
2588
|
+
ids.add(s.$id);
|
|
2589
|
+
};
|
|
2590
|
+
extractFromSchema(schema);
|
|
2591
|
+
const checkUnion = (schemas) => {
|
|
2592
|
+
if (!schemas)
|
|
2593
|
+
return;
|
|
2594
|
+
for (const s of schemas) {
|
|
2595
|
+
if (typeof s === "boolean")
|
|
2596
|
+
continue;
|
|
2597
|
+
extractFromSchema(s);
|
|
2598
|
+
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
2599
|
+
extractFromSchema(s.items);
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
};
|
|
2603
|
+
checkUnion(schema.oneOf);
|
|
2604
|
+
checkUnion(schema.anyOf);
|
|
2605
|
+
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
2606
|
+
extractFromSchema(schema.items);
|
|
2607
|
+
}
|
|
2608
|
+
return { formats, ids };
|
|
2609
|
+
};
|
|
2610
|
+
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
2611
|
+
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
2612
|
+
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
2613
|
+
}
|
|
2614
|
+
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
2615
|
+
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
2616
|
+
for (const format of outputIds.formats) {
|
|
2617
|
+
if (inputIds.formats.has(format)) {
|
|
2618
|
+
return true;
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
for (const id of outputIds.ids) {
|
|
2622
|
+
if (inputIds.ids.has(id)) {
|
|
2623
|
+
return true;
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
if (requireSpecificType) {
|
|
2627
|
+
return false;
|
|
2628
|
+
}
|
|
2629
|
+
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
2630
|
+
if (!idTypeBlank)
|
|
2631
|
+
return false;
|
|
2632
|
+
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
2633
|
+
return true;
|
|
2634
|
+
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
2635
|
+
if (typeof schema === "boolean")
|
|
2636
|
+
return schema;
|
|
2637
|
+
return schema.type === fromPortOutputSchema.type;
|
|
2638
|
+
}) ?? false;
|
|
2639
|
+
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
2640
|
+
if (typeof schema === "boolean")
|
|
2641
|
+
return schema;
|
|
2642
|
+
return schema.type === fromPortOutputSchema.type;
|
|
2643
|
+
}) ?? false;
|
|
2644
|
+
return matchesOneOf || matchesAnyOf;
|
|
2645
|
+
};
|
|
2646
|
+
const makeMatch = (fromSchema, toSchema, fromTaskId, toTaskId, comparator) => {
|
|
2647
|
+
if (typeof fromSchema === "object") {
|
|
2648
|
+
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2649
|
+
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
2650
|
+
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2651
|
+
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2652
|
+
}
|
|
2653
|
+
return;
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2657
|
+
return;
|
|
2658
|
+
}
|
|
2659
|
+
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2660
|
+
if (matches.has(toInputPortId))
|
|
2661
|
+
continue;
|
|
2662
|
+
const candidates = [];
|
|
2663
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2664
|
+
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
2665
|
+
candidates.push(fromOutputPortId);
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
if (candidates.length === 0)
|
|
2669
|
+
continue;
|
|
2670
|
+
let winner = candidates[0];
|
|
2671
|
+
if (candidates.length > 1) {
|
|
2672
|
+
const targetStreamMode = getPortStreamMode(toSchema, toInputPortId);
|
|
2673
|
+
const streamMatch = candidates.find((portId) => getPortStreamMode(fromSchema, portId) === targetStreamMode);
|
|
2674
|
+
if (streamMatch)
|
|
2675
|
+
winner = streamMatch;
|
|
2676
|
+
}
|
|
2677
|
+
matches.set(toInputPortId, winner);
|
|
2678
|
+
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2679
|
+
}
|
|
2680
|
+
};
|
|
2681
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2682
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2683
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2684
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2685
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2686
|
+
});
|
|
2687
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2688
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2689
|
+
});
|
|
2690
|
+
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2691
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
2692
|
+
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2693
|
+
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2694
|
+
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2695
|
+
const earlierTask = earlierTasks[i];
|
|
2696
|
+
const earlierOutputSchema = earlierTask.outputSchema();
|
|
2697
|
+
const makeMatchFromEarlier = (comparator) => {
|
|
2698
|
+
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2699
|
+
return;
|
|
2700
|
+
}
|
|
2701
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
2702
|
+
for (const requiredInputId of unmatchedRequired) {
|
|
2703
|
+
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2704
|
+
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2705
|
+
matches.set(requiredInputId, fromOutputPortId);
|
|
2706
|
+
graph.addDataflow(new Dataflow(earlierTask.config.id, fromOutputPortId, targetTask.config.id, requiredInputId));
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
}
|
|
2710
|
+
};
|
|
2711
|
+
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2712
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2713
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2714
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2715
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2716
|
+
});
|
|
2717
|
+
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2718
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2719
|
+
});
|
|
2720
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
const stillUnmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2724
|
+
if (stillUnmatchedRequired.length > 0) {
|
|
2725
|
+
return {
|
|
2726
|
+
matches,
|
|
2727
|
+
error: `Could not find matches for required inputs [${stillUnmatchedRequired.join(", ")}] of ${targetTask.type}. ` + `Attempted to match from ${sourceTask.type} and earlier tasks.`,
|
|
2728
|
+
unmatchedRequired: stillUnmatchedRequired
|
|
2729
|
+
};
|
|
2730
|
+
}
|
|
2731
|
+
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
2732
|
+
const hasRequiredInputs = requiredInputs.size > 0;
|
|
2733
|
+
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
2734
|
+
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
2735
|
+
if (!allRequiredInputsProvided && !hasInputsWithDefaults) {
|
|
2736
|
+
return {
|
|
2737
|
+
matches,
|
|
2738
|
+
error: `Could not find a match between the outputs of ${sourceTask.type} and the inputs of ${targetTask.type}. ` + `You may need to connect the outputs to the inputs via connect() manually.`,
|
|
2739
|
+
unmatchedRequired: []
|
|
2740
|
+
};
|
|
2741
|
+
}
|
|
2742
|
+
}
|
|
2743
|
+
return {
|
|
2744
|
+
matches,
|
|
2745
|
+
unmatchedRequired: []
|
|
2746
|
+
};
|
|
2747
|
+
}
|
|
2748
|
+
finalizeTemplate() {
|
|
2749
|
+
if (!this._iteratorTask || this.graph.getTasks().length === 0) {
|
|
2750
|
+
return;
|
|
2751
|
+
}
|
|
2752
|
+
this._iteratorTask.subGraph = this.graph;
|
|
2753
|
+
}
|
|
2754
|
+
finalizeAndReturn() {
|
|
2755
|
+
if (!this._parentWorkflow) {
|
|
2756
|
+
throw new WorkflowError("finalizeAndReturn() can only be called on loop workflows");
|
|
2757
|
+
}
|
|
2758
|
+
this.finalizeTemplate();
|
|
2759
|
+
if (this._pendingLoopConnect) {
|
|
2760
|
+
this._parentWorkflow.autoConnectLoopTask(this._pendingLoopConnect);
|
|
2761
|
+
this._pendingLoopConnect = undefined;
|
|
2762
|
+
}
|
|
2763
|
+
return this._parentWorkflow;
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
// src/task-graph/Conversions.ts
|
|
2768
|
+
class ListeningGraphAsTask extends GraphAsTask {
|
|
2769
|
+
constructor(input, config) {
|
|
2770
|
+
super(input, config);
|
|
2771
|
+
this.subGraph.on("start", () => {
|
|
2772
|
+
this.emit("start");
|
|
2773
|
+
});
|
|
2774
|
+
this.subGraph.on("complete", () => {
|
|
2775
|
+
this.emit("complete");
|
|
2776
|
+
});
|
|
2777
|
+
this.subGraph.on("error", (e) => {
|
|
2778
|
+
this.emit("error", e);
|
|
2779
|
+
});
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
class OwnGraphTask extends ListeningGraphAsTask {
|
|
2784
|
+
static type = "Own[Graph]";
|
|
2785
|
+
}
|
|
2786
|
+
|
|
2787
|
+
class OwnWorkflowTask extends ListeningGraphAsTask {
|
|
2788
|
+
static type = "Own[Workflow]";
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
class GraphTask extends GraphAsTask {
|
|
2792
|
+
static type = "Graph";
|
|
2793
|
+
}
|
|
2794
|
+
|
|
2795
|
+
class WorkflowTask2 extends GraphAsTask {
|
|
2796
|
+
static type = "Workflow";
|
|
2797
|
+
}
|
|
2798
|
+
function convertPipeFunctionToTask(fn, config) {
|
|
2799
|
+
|
|
2800
|
+
class QuickTask extends Task {
|
|
2801
|
+
static type = fn.name ? `\uD835\uDC53 ${fn.name}` : "\uD835\uDC53";
|
|
2802
|
+
static inputSchema = () => {
|
|
2159
2803
|
return {
|
|
2160
2804
|
type: "object",
|
|
2161
2805
|
properties: {
|
|
@@ -2357,7 +3001,7 @@ class TaskGraph {
|
|
|
2357
3001
|
return this._dag.removeNode(taskId);
|
|
2358
3002
|
}
|
|
2359
3003
|
resetGraph() {
|
|
2360
|
-
this.runner.resetGraph(this,
|
|
3004
|
+
this.runner.resetGraph(this, uuid44());
|
|
2361
3005
|
}
|
|
2362
3006
|
toJSON() {
|
|
2363
3007
|
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
@@ -2479,6 +3123,24 @@ class TaskGraph {
|
|
|
2479
3123
|
unsubscribes.forEach((unsub) => unsub());
|
|
2480
3124
|
};
|
|
2481
3125
|
}
|
|
3126
|
+
subscribeToTaskStreaming(callbacks) {
|
|
3127
|
+
const unsubscribes = [];
|
|
3128
|
+
if (callbacks.onStreamStart) {
|
|
3129
|
+
const unsub = this.subscribe("task_stream_start", callbacks.onStreamStart);
|
|
3130
|
+
unsubscribes.push(unsub);
|
|
3131
|
+
}
|
|
3132
|
+
if (callbacks.onStreamChunk) {
|
|
3133
|
+
const unsub = this.subscribe("task_stream_chunk", callbacks.onStreamChunk);
|
|
3134
|
+
unsubscribes.push(unsub);
|
|
3135
|
+
}
|
|
3136
|
+
if (callbacks.onStreamEnd) {
|
|
3137
|
+
const unsub = this.subscribe("task_stream_end", callbacks.onStreamEnd);
|
|
3138
|
+
unsubscribes.push(unsub);
|
|
3139
|
+
}
|
|
3140
|
+
return () => {
|
|
3141
|
+
unsubscribes.forEach((unsub) => unsub());
|
|
3142
|
+
};
|
|
3143
|
+
}
|
|
2482
3144
|
on(name, fn) {
|
|
2483
3145
|
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2484
3146
|
if (dagEvent) {
|
|
@@ -2522,122 +3184,1186 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
2522
3184
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
2523
3185
|
return graph;
|
|
2524
3186
|
}
|
|
2525
|
-
// src/task/
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
var defaultJobQueueFactory = async ({
|
|
2534
|
-
queueName,
|
|
2535
|
-
jobClass,
|
|
2536
|
-
options
|
|
2537
|
-
}) => {
|
|
2538
|
-
const storage = options?.storage ?? new InMemoryQueueStorage(queueName);
|
|
2539
|
-
await storage.setupDatabase();
|
|
2540
|
-
const server = new JobQueueServer(jobClass, {
|
|
2541
|
-
storage,
|
|
2542
|
-
queueName,
|
|
2543
|
-
limiter: options?.limiter,
|
|
2544
|
-
workerCount: options?.workerCount,
|
|
2545
|
-
pollIntervalMs: options?.pollIntervalMs,
|
|
2546
|
-
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
2547
|
-
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
2548
|
-
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
2549
|
-
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
2550
|
-
});
|
|
2551
|
-
const client = new JobQueueClient({
|
|
2552
|
-
storage,
|
|
2553
|
-
queueName
|
|
2554
|
-
});
|
|
2555
|
-
client.attach(server);
|
|
2556
|
-
return { server, client, storage };
|
|
2557
|
-
};
|
|
2558
|
-
function registerJobQueueFactory(factory) {
|
|
2559
|
-
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
2560
|
-
}
|
|
2561
|
-
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
2562
|
-
return async ({
|
|
2563
|
-
queueName,
|
|
2564
|
-
jobClass,
|
|
2565
|
-
options
|
|
2566
|
-
}) => {
|
|
2567
|
-
const mergedOptions = {
|
|
2568
|
-
...defaultOptions,
|
|
2569
|
-
...options ?? {}
|
|
2570
|
-
};
|
|
2571
|
-
const storage = mergedOptions.storage ?? new InMemoryQueueStorage(queueName);
|
|
2572
|
-
await storage.setupDatabase();
|
|
2573
|
-
const server = new JobQueueServer(jobClass, {
|
|
2574
|
-
storage,
|
|
2575
|
-
queueName,
|
|
2576
|
-
limiter: mergedOptions.limiter,
|
|
2577
|
-
workerCount: mergedOptions.workerCount,
|
|
2578
|
-
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
2579
|
-
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
2580
|
-
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
2581
|
-
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
2582
|
-
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
2583
|
-
});
|
|
2584
|
-
const client = new JobQueueClient({
|
|
2585
|
-
storage,
|
|
2586
|
-
queueName
|
|
2587
|
-
});
|
|
2588
|
-
client.attach(server);
|
|
2589
|
-
return { server, client, storage };
|
|
2590
|
-
};
|
|
2591
|
-
}
|
|
2592
|
-
function getJobQueueFactory() {
|
|
2593
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2594
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2595
|
-
}
|
|
2596
|
-
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
2597
|
-
}
|
|
2598
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2599
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2600
|
-
}
|
|
2601
|
-
// src/task/JobQueueTask.ts
|
|
2602
|
-
import { Job as Job2 } from "@workglow/job-queue";
|
|
2603
|
-
|
|
2604
|
-
// src/task/TaskQueueRegistry.ts
|
|
2605
|
-
var taskQueueRegistry = null;
|
|
2606
|
-
|
|
2607
|
-
class TaskQueueRegistry {
|
|
2608
|
-
queues = new Map;
|
|
2609
|
-
registerQueue(queue) {
|
|
2610
|
-
const queueName = queue.server.queueName;
|
|
2611
|
-
if (this.queues.has(queueName)) {
|
|
2612
|
-
throw new Error(`Queue with name ${queueName} already exists`);
|
|
3187
|
+
// src/task/IteratorTaskRunner.ts
|
|
3188
|
+
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3189
|
+
subGraphRunChain = Promise.resolve();
|
|
3190
|
+
async executeTask(input) {
|
|
3191
|
+
const analysis = this.task.analyzeIterationInput(input);
|
|
3192
|
+
if (analysis.iterationCount === 0) {
|
|
3193
|
+
const emptyResult = this.task.getEmptyResult();
|
|
3194
|
+
return this.executeTaskReactive(input, emptyResult);
|
|
2613
3195
|
}
|
|
2614
|
-
this.
|
|
2615
|
-
|
|
2616
|
-
getQueue(queueName) {
|
|
2617
|
-
return this.queues.get(queueName);
|
|
3196
|
+
const result = this.task.isReduceTask() ? await this.executeReduceIterations(analysis) : await this.executeCollectIterations(analysis);
|
|
3197
|
+
return this.executeTaskReactive(input, result);
|
|
2618
3198
|
}
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
}
|
|
2623
|
-
return this;
|
|
3199
|
+
async executeTaskReactive(input, output) {
|
|
3200
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
3201
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
2624
3202
|
}
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
3203
|
+
async executeCollectIterations(analysis) {
|
|
3204
|
+
const iterationCount = analysis.iterationCount;
|
|
3205
|
+
const preserveOrder = this.task.preserveIterationOrder();
|
|
3206
|
+
const batchSize = this.task.batchSize !== undefined && this.task.batchSize > 0 ? this.task.batchSize : iterationCount;
|
|
3207
|
+
const requestedConcurrency = this.task.concurrencyLimit ?? iterationCount;
|
|
3208
|
+
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3209
|
+
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3210
|
+
const completionOrderResults = [];
|
|
3211
|
+
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3212
|
+
if (this.abortController?.signal.aborted) {
|
|
3213
|
+
break;
|
|
3214
|
+
}
|
|
3215
|
+
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3216
|
+
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3217
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency);
|
|
3218
|
+
for (const { index, result } of batchResults) {
|
|
3219
|
+
if (result === undefined)
|
|
3220
|
+
continue;
|
|
3221
|
+
if (preserveOrder) {
|
|
3222
|
+
orderedResults[index] = result;
|
|
3223
|
+
} else {
|
|
3224
|
+
completionOrderResults.push(result);
|
|
3225
|
+
}
|
|
3226
|
+
}
|
|
3227
|
+
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3228
|
+
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
2628
3229
|
}
|
|
2629
|
-
|
|
3230
|
+
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3231
|
+
return this.task.collectResults(collected);
|
|
2630
3232
|
}
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
3233
|
+
async executeReduceIterations(analysis) {
|
|
3234
|
+
const iterationCount = analysis.iterationCount;
|
|
3235
|
+
let accumulator = this.task.getInitialAccumulator();
|
|
3236
|
+
for (let index = 0;index < iterationCount; index++) {
|
|
3237
|
+
if (this.abortController?.signal.aborted) {
|
|
3238
|
+
break;
|
|
3239
|
+
}
|
|
3240
|
+
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount, {
|
|
3241
|
+
accumulator
|
|
3242
|
+
});
|
|
3243
|
+
const iterationResult = await this.executeSubgraphIteration(iterationInput);
|
|
3244
|
+
accumulator = this.task.mergeIterationIntoAccumulator(accumulator, iterationResult, index);
|
|
3245
|
+
const progress = Math.round((index + 1) / iterationCount * 100);
|
|
3246
|
+
await this.handleProgress(progress, `Completed ${index + 1}/${iterationCount} iterations`);
|
|
2634
3247
|
}
|
|
2635
|
-
return
|
|
3248
|
+
return accumulator;
|
|
2636
3249
|
}
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
3250
|
+
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
3251
|
+
const results = [];
|
|
3252
|
+
let cursor = 0;
|
|
3253
|
+
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
3254
|
+
const workers = Array.from({ length: workerCount }, async () => {
|
|
3255
|
+
while (true) {
|
|
3256
|
+
if (this.abortController?.signal.aborted) {
|
|
3257
|
+
return;
|
|
3258
|
+
}
|
|
3259
|
+
const position = cursor;
|
|
3260
|
+
cursor += 1;
|
|
3261
|
+
if (position >= indices.length) {
|
|
3262
|
+
return;
|
|
3263
|
+
}
|
|
3264
|
+
const index = indices[position];
|
|
3265
|
+
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3266
|
+
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3267
|
+
results.push({ index, result });
|
|
3268
|
+
}
|
|
3269
|
+
});
|
|
3270
|
+
await Promise.all(workers);
|
|
3271
|
+
return results;
|
|
3272
|
+
}
|
|
3273
|
+
async executeSubgraphIteration(input) {
|
|
3274
|
+
let releaseTurn;
|
|
3275
|
+
const waitForPreviousRun = this.subGraphRunChain;
|
|
3276
|
+
this.subGraphRunChain = new Promise((resolve) => {
|
|
3277
|
+
releaseTurn = resolve;
|
|
3278
|
+
});
|
|
3279
|
+
await waitForPreviousRun;
|
|
3280
|
+
try {
|
|
3281
|
+
if (this.abortController?.signal.aborted) {
|
|
3282
|
+
return;
|
|
3283
|
+
}
|
|
3284
|
+
const results = await this.task.subGraph.run(input, {
|
|
3285
|
+
parentSignal: this.abortController?.signal,
|
|
3286
|
+
outputCache: this.outputCache
|
|
3287
|
+
});
|
|
3288
|
+
if (results.length === 0) {
|
|
3289
|
+
return;
|
|
3290
|
+
}
|
|
3291
|
+
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3292
|
+
} finally {
|
|
3293
|
+
releaseTurn?.();
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
// src/task/IteratorTask.ts
|
|
3299
|
+
var ITERATOR_CONTEXT_SCHEMA = {
|
|
3300
|
+
type: "object",
|
|
3301
|
+
properties: {
|
|
3302
|
+
_iterationIndex: {
|
|
3303
|
+
type: "integer",
|
|
3304
|
+
minimum: 0,
|
|
3305
|
+
title: "Iteration Index",
|
|
3306
|
+
description: "Current iteration index (0-based)",
|
|
3307
|
+
"x-ui-iteration": true
|
|
3308
|
+
},
|
|
3309
|
+
_iterationCount: {
|
|
3310
|
+
type: "integer",
|
|
3311
|
+
minimum: 0,
|
|
3312
|
+
title: "Iteration Count",
|
|
3313
|
+
description: "Total number of iterations",
|
|
3314
|
+
"x-ui-iteration": true
|
|
3315
|
+
}
|
|
3316
|
+
}
|
|
3317
|
+
};
|
|
3318
|
+
function isArrayVariant(schema) {
|
|
3319
|
+
if (!schema || typeof schema !== "object")
|
|
3320
|
+
return false;
|
|
3321
|
+
const record = schema;
|
|
3322
|
+
return record.type === "array" || record.items !== undefined;
|
|
3323
|
+
}
|
|
3324
|
+
function getExplicitIterationFlag(schema) {
|
|
3325
|
+
if (!schema || typeof schema !== "object")
|
|
3326
|
+
return;
|
|
3327
|
+
const record = schema;
|
|
3328
|
+
const flag = record["x-ui-iteration"];
|
|
3329
|
+
if (flag === true)
|
|
3330
|
+
return true;
|
|
3331
|
+
if (flag === false)
|
|
3332
|
+
return false;
|
|
3333
|
+
return;
|
|
3334
|
+
}
|
|
3335
|
+
function inferIterationFromSchema(schema) {
|
|
3336
|
+
if (!schema || typeof schema !== "object")
|
|
3337
|
+
return;
|
|
3338
|
+
const record = schema;
|
|
3339
|
+
if (record.type === "array" || record.items !== undefined) {
|
|
3340
|
+
return true;
|
|
3341
|
+
}
|
|
3342
|
+
const variants = record.oneOf ?? record.anyOf;
|
|
3343
|
+
if (!Array.isArray(variants) || variants.length === 0) {
|
|
3344
|
+
if (record.type !== undefined) {
|
|
3345
|
+
return false;
|
|
3346
|
+
}
|
|
3347
|
+
return;
|
|
3348
|
+
}
|
|
3349
|
+
let hasArrayVariant = false;
|
|
3350
|
+
let hasNonArrayVariant = false;
|
|
3351
|
+
for (const variant of variants) {
|
|
3352
|
+
if (isArrayVariant(variant)) {
|
|
3353
|
+
hasArrayVariant = true;
|
|
3354
|
+
} else {
|
|
3355
|
+
hasNonArrayVariant = true;
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
if (hasArrayVariant && hasNonArrayVariant)
|
|
3359
|
+
return;
|
|
3360
|
+
if (hasArrayVariant)
|
|
3361
|
+
return true;
|
|
3362
|
+
return false;
|
|
3363
|
+
}
|
|
3364
|
+
function createFlexibleSchema(baseSchema) {
|
|
3365
|
+
if (typeof baseSchema === "boolean")
|
|
3366
|
+
return baseSchema;
|
|
3367
|
+
return {
|
|
3368
|
+
anyOf: [baseSchema, { type: "array", items: baseSchema }]
|
|
3369
|
+
};
|
|
3370
|
+
}
|
|
3371
|
+
function createArraySchema(baseSchema) {
|
|
3372
|
+
if (typeof baseSchema === "boolean")
|
|
3373
|
+
return baseSchema;
|
|
3374
|
+
return {
|
|
3375
|
+
type: "array",
|
|
3376
|
+
items: baseSchema
|
|
3377
|
+
};
|
|
3378
|
+
}
|
|
3379
|
+
function extractBaseSchema(schema) {
|
|
3380
|
+
if (typeof schema === "boolean")
|
|
3381
|
+
return schema;
|
|
3382
|
+
const schemaType = schema.type;
|
|
3383
|
+
if (schemaType === "array" && schema.items) {
|
|
3384
|
+
return schema.items;
|
|
3385
|
+
}
|
|
3386
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
3387
|
+
if (Array.isArray(variants)) {
|
|
3388
|
+
for (const variant of variants) {
|
|
3389
|
+
if (typeof variant === "object") {
|
|
3390
|
+
const variantType = variant.type;
|
|
3391
|
+
if (variantType !== "array") {
|
|
3392
|
+
return variant;
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
3396
|
+
for (const variant of variants) {
|
|
3397
|
+
if (typeof variant === "object") {
|
|
3398
|
+
const variantType = variant.type;
|
|
3399
|
+
if (variantType === "array" && variant.items) {
|
|
3400
|
+
return variant.items;
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
return schema;
|
|
3406
|
+
}
|
|
3407
|
+
function schemaAcceptsArray(schema) {
|
|
3408
|
+
if (typeof schema === "boolean")
|
|
3409
|
+
return false;
|
|
3410
|
+
const schemaType = schema.type;
|
|
3411
|
+
if (schemaType === "array")
|
|
3412
|
+
return true;
|
|
3413
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
3414
|
+
if (Array.isArray(variants)) {
|
|
3415
|
+
return variants.some((variant) => isArrayVariant(variant));
|
|
3416
|
+
}
|
|
3417
|
+
return false;
|
|
3418
|
+
}
|
|
3419
|
+
|
|
3420
|
+
class IteratorTask extends GraphAsTask {
|
|
3421
|
+
static type = "IteratorTask";
|
|
3422
|
+
static category = "Flow Control";
|
|
3423
|
+
static title = "Iterator";
|
|
3424
|
+
static description = "Base class for loop-type tasks";
|
|
3425
|
+
static hasDynamicSchemas = true;
|
|
3426
|
+
static getIterationContextSchema() {
|
|
3427
|
+
return ITERATOR_CONTEXT_SCHEMA;
|
|
3428
|
+
}
|
|
3429
|
+
_iteratorPortInfo;
|
|
3430
|
+
_iterationInputSchema;
|
|
3431
|
+
constructor(input = {}, config = {}) {
|
|
3432
|
+
super(input, config);
|
|
3433
|
+
}
|
|
3434
|
+
get runner() {
|
|
3435
|
+
if (!this._runner) {
|
|
3436
|
+
this._runner = new IteratorTaskRunner(this);
|
|
3437
|
+
}
|
|
3438
|
+
return this._runner;
|
|
3439
|
+
}
|
|
3440
|
+
set subGraph(subGraph) {
|
|
3441
|
+
super.subGraph = subGraph;
|
|
3442
|
+
this.invalidateIterationInputSchema();
|
|
3443
|
+
this.events.emit("regenerate");
|
|
3444
|
+
}
|
|
3445
|
+
get subGraph() {
|
|
3446
|
+
return super.subGraph;
|
|
3447
|
+
}
|
|
3448
|
+
regenerateGraph() {
|
|
3449
|
+
this.invalidateIterationInputSchema();
|
|
3450
|
+
super.regenerateGraph();
|
|
3451
|
+
}
|
|
3452
|
+
preserveIterationOrder() {
|
|
3453
|
+
return true;
|
|
3454
|
+
}
|
|
3455
|
+
isReduceTask() {
|
|
3456
|
+
return false;
|
|
3457
|
+
}
|
|
3458
|
+
getInitialAccumulator() {
|
|
3459
|
+
return {};
|
|
3460
|
+
}
|
|
3461
|
+
buildIterationRunInput(analysis, index, iterationCount, extraInput = {}) {
|
|
3462
|
+
return {
|
|
3463
|
+
...analysis.getIterationInput(index),
|
|
3464
|
+
...extraInput,
|
|
3465
|
+
_iterationIndex: index,
|
|
3466
|
+
_iterationCount: iterationCount
|
|
3467
|
+
};
|
|
3468
|
+
}
|
|
3469
|
+
mergeIterationIntoAccumulator(accumulator, iterationResult, _index) {
|
|
3470
|
+
return iterationResult ?? accumulator;
|
|
3471
|
+
}
|
|
3472
|
+
getEmptyResult() {
|
|
3473
|
+
return {};
|
|
3474
|
+
}
|
|
3475
|
+
collectResults(results) {
|
|
3476
|
+
if (results.length === 0) {
|
|
3477
|
+
return {};
|
|
3478
|
+
}
|
|
3479
|
+
const merged = {};
|
|
3480
|
+
for (const result of results) {
|
|
3481
|
+
if (!result || typeof result !== "object")
|
|
3482
|
+
continue;
|
|
3483
|
+
for (const [key, value] of Object.entries(result)) {
|
|
3484
|
+
if (!merged[key]) {
|
|
3485
|
+
merged[key] = [];
|
|
3486
|
+
}
|
|
3487
|
+
merged[key].push(value);
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
return merged;
|
|
3491
|
+
}
|
|
3492
|
+
get concurrencyLimit() {
|
|
3493
|
+
return this.config.concurrencyLimit;
|
|
3494
|
+
}
|
|
3495
|
+
get batchSize() {
|
|
3496
|
+
return this.config.batchSize;
|
|
3497
|
+
}
|
|
3498
|
+
get iterationInputConfig() {
|
|
3499
|
+
return this.config.iterationInputConfig;
|
|
3500
|
+
}
|
|
3501
|
+
buildDefaultIterationInputSchema() {
|
|
3502
|
+
const innerSchema = this.getInnerInputSchema();
|
|
3503
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
3504
|
+
return { type: "object", properties: {}, additionalProperties: true };
|
|
3505
|
+
}
|
|
3506
|
+
const properties = {};
|
|
3507
|
+
const innerProps = innerSchema.properties || {};
|
|
3508
|
+
for (const [key, propSchema] of Object.entries(innerProps)) {
|
|
3509
|
+
if (typeof propSchema === "boolean")
|
|
3510
|
+
continue;
|
|
3511
|
+
if (propSchema["x-ui-iteration"]) {
|
|
3512
|
+
continue;
|
|
3513
|
+
}
|
|
3514
|
+
const baseSchema = propSchema;
|
|
3515
|
+
properties[key] = createFlexibleSchema(baseSchema);
|
|
3516
|
+
}
|
|
3517
|
+
return {
|
|
3518
|
+
type: "object",
|
|
3519
|
+
properties,
|
|
3520
|
+
additionalProperties: innerSchema.additionalProperties ?? true
|
|
3521
|
+
};
|
|
3522
|
+
}
|
|
3523
|
+
buildConfiguredIterationInputSchema() {
|
|
3524
|
+
const innerSchema = this.getInnerInputSchema();
|
|
3525
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
3526
|
+
return { type: "object", properties: {}, additionalProperties: true };
|
|
3527
|
+
}
|
|
3528
|
+
const config = this.iterationInputConfig || {};
|
|
3529
|
+
const properties = {};
|
|
3530
|
+
const innerProps = innerSchema.properties || {};
|
|
3531
|
+
for (const [key, propSchema] of Object.entries(innerProps)) {
|
|
3532
|
+
if (typeof propSchema === "boolean")
|
|
3533
|
+
continue;
|
|
3534
|
+
if (propSchema["x-ui-iteration"]) {
|
|
3535
|
+
continue;
|
|
3536
|
+
}
|
|
3537
|
+
const baseSchema = propSchema;
|
|
3538
|
+
const propConfig = config[key];
|
|
3539
|
+
if (!propConfig) {
|
|
3540
|
+
properties[key] = createFlexibleSchema(baseSchema);
|
|
3541
|
+
continue;
|
|
3542
|
+
}
|
|
3543
|
+
switch (propConfig.mode) {
|
|
3544
|
+
case "array":
|
|
3545
|
+
properties[key] = createArraySchema(propConfig.baseSchema);
|
|
3546
|
+
break;
|
|
3547
|
+
case "scalar":
|
|
3548
|
+
properties[key] = propConfig.baseSchema;
|
|
3549
|
+
break;
|
|
3550
|
+
case "flexible":
|
|
3551
|
+
default:
|
|
3552
|
+
properties[key] = createFlexibleSchema(propConfig.baseSchema);
|
|
3553
|
+
break;
|
|
3554
|
+
}
|
|
3555
|
+
}
|
|
3556
|
+
return {
|
|
3557
|
+
type: "object",
|
|
3558
|
+
properties,
|
|
3559
|
+
additionalProperties: innerSchema.additionalProperties ?? true
|
|
3560
|
+
};
|
|
3561
|
+
}
|
|
3562
|
+
getInnerInputSchema() {
|
|
3563
|
+
if (!this.hasChildren())
|
|
3564
|
+
return;
|
|
3565
|
+
const tasks = this.subGraph.getTasks();
|
|
3566
|
+
if (tasks.length === 0)
|
|
3567
|
+
return;
|
|
3568
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
3569
|
+
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3570
|
+
const properties = {};
|
|
3571
|
+
const required = [];
|
|
3572
|
+
let additionalProperties = false;
|
|
3573
|
+
for (const task of sources) {
|
|
3574
|
+
const inputSchema = task.inputSchema();
|
|
3575
|
+
if (typeof inputSchema === "boolean") {
|
|
3576
|
+
if (inputSchema === true) {
|
|
3577
|
+
additionalProperties = true;
|
|
3578
|
+
}
|
|
3579
|
+
continue;
|
|
3580
|
+
}
|
|
3581
|
+
additionalProperties = additionalProperties || inputSchema.additionalProperties === true;
|
|
3582
|
+
for (const [key, prop] of Object.entries(inputSchema.properties || {})) {
|
|
3583
|
+
if (typeof prop === "boolean")
|
|
3584
|
+
continue;
|
|
3585
|
+
if (!properties[key]) {
|
|
3586
|
+
properties[key] = prop;
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3589
|
+
for (const key of inputSchema.required || []) {
|
|
3590
|
+
if (!required.includes(key)) {
|
|
3591
|
+
required.push(key);
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
return {
|
|
3596
|
+
type: "object",
|
|
3597
|
+
properties,
|
|
3598
|
+
...required.length > 0 ? { required } : {},
|
|
3599
|
+
additionalProperties
|
|
3600
|
+
};
|
|
3601
|
+
}
|
|
3602
|
+
getIterationInputSchema() {
|
|
3603
|
+
if (this._iterationInputSchema) {
|
|
3604
|
+
return this._iterationInputSchema;
|
|
3605
|
+
}
|
|
3606
|
+
this._iterationInputSchema = this.iterationInputConfig ? this.buildConfiguredIterationInputSchema() : this.buildDefaultIterationInputSchema();
|
|
3607
|
+
return this._iterationInputSchema;
|
|
3608
|
+
}
|
|
3609
|
+
setIterationInputSchema(schema) {
|
|
3610
|
+
this._iterationInputSchema = schema;
|
|
3611
|
+
this._inputSchemaNode = undefined;
|
|
3612
|
+
this.events.emit("regenerate");
|
|
3613
|
+
}
|
|
3614
|
+
setPropertyInputMode(propertyName, mode, baseSchema) {
|
|
3615
|
+
const currentSchema = this.getIterationInputSchema();
|
|
3616
|
+
if (typeof currentSchema === "boolean")
|
|
3617
|
+
return;
|
|
3618
|
+
const currentProps = currentSchema.properties || {};
|
|
3619
|
+
const existingProp = currentProps[propertyName];
|
|
3620
|
+
const base = baseSchema ?? (existingProp ? extractBaseSchema(existingProp) : { type: "string" });
|
|
3621
|
+
let newPropSchema;
|
|
3622
|
+
switch (mode) {
|
|
3623
|
+
case "array":
|
|
3624
|
+
newPropSchema = createArraySchema(base);
|
|
3625
|
+
break;
|
|
3626
|
+
case "scalar":
|
|
3627
|
+
newPropSchema = base;
|
|
3628
|
+
break;
|
|
3629
|
+
case "flexible":
|
|
3630
|
+
default:
|
|
3631
|
+
newPropSchema = createFlexibleSchema(base);
|
|
3632
|
+
break;
|
|
3633
|
+
}
|
|
3634
|
+
this._iterationInputSchema = {
|
|
3635
|
+
...currentSchema,
|
|
3636
|
+
properties: {
|
|
3637
|
+
...currentProps,
|
|
3638
|
+
[propertyName]: newPropSchema
|
|
3639
|
+
}
|
|
3640
|
+
};
|
|
3641
|
+
this._inputSchemaNode = undefined;
|
|
3642
|
+
this.events.emit("regenerate");
|
|
3643
|
+
}
|
|
3644
|
+
invalidateIterationInputSchema() {
|
|
3645
|
+
this._iterationInputSchema = undefined;
|
|
3646
|
+
this._iteratorPortInfo = undefined;
|
|
3647
|
+
this._inputSchemaNode = undefined;
|
|
3648
|
+
}
|
|
3649
|
+
analyzeIterationInput(input) {
|
|
3650
|
+
const inputData = input;
|
|
3651
|
+
const schema = this.hasChildren() ? this.getIterationInputSchema() : this.inputSchema();
|
|
3652
|
+
const schemaProps = typeof schema === "object" && schema.properties ? schema.properties : {};
|
|
3653
|
+
const keys = new Set([...Object.keys(schemaProps), ...Object.keys(inputData)]);
|
|
3654
|
+
const arrayPorts = [];
|
|
3655
|
+
const scalarPorts = [];
|
|
3656
|
+
const iteratedValues = {};
|
|
3657
|
+
const arrayLengths = [];
|
|
3658
|
+
for (const key of keys) {
|
|
3659
|
+
if (key.startsWith("_iteration"))
|
|
3660
|
+
continue;
|
|
3661
|
+
const value = inputData[key];
|
|
3662
|
+
const portSchema = schemaProps[key];
|
|
3663
|
+
let shouldIterate;
|
|
3664
|
+
const explicitFlag = getExplicitIterationFlag(portSchema);
|
|
3665
|
+
if (explicitFlag !== undefined) {
|
|
3666
|
+
shouldIterate = explicitFlag;
|
|
3667
|
+
} else {
|
|
3668
|
+
const schemaInference = inferIterationFromSchema(portSchema);
|
|
3669
|
+
shouldIterate = schemaInference ?? Array.isArray(value);
|
|
3670
|
+
}
|
|
3671
|
+
if (!shouldIterate) {
|
|
3672
|
+
scalarPorts.push(key);
|
|
3673
|
+
continue;
|
|
3674
|
+
}
|
|
3675
|
+
if (!Array.isArray(value)) {
|
|
3676
|
+
throw new TaskConfigurationError(`${this.type}: Input '${key}' is configured for iteration but value is not an array.`);
|
|
3677
|
+
}
|
|
3678
|
+
iteratedValues[key] = value;
|
|
3679
|
+
arrayPorts.push(key);
|
|
3680
|
+
arrayLengths.push(value.length);
|
|
3681
|
+
}
|
|
3682
|
+
if (arrayPorts.length === 0) {
|
|
3683
|
+
throw new TaskConfigurationError(`${this.type}: At least one array input is required for iteration. ` + `Mark a port with x-ui-iteration=true, provide array-typed schema, or pass array values at runtime.`);
|
|
3684
|
+
}
|
|
3685
|
+
const uniqueLengths = new Set(arrayLengths);
|
|
3686
|
+
if (uniqueLengths.size > 1) {
|
|
3687
|
+
const lengthInfo = arrayPorts.map((port, index) => `${port}=${arrayLengths[index]}`).join(", ");
|
|
3688
|
+
throw new TaskConfigurationError(`${this.type}: All iterated array inputs must have the same length (zip semantics). ` + `Found different lengths: ${lengthInfo}`);
|
|
3689
|
+
}
|
|
3690
|
+
const iterationCount = arrayLengths[0] ?? 0;
|
|
3691
|
+
const getIterationInput = (index) => {
|
|
3692
|
+
const iterInput = {};
|
|
3693
|
+
for (const key of arrayPorts) {
|
|
3694
|
+
iterInput[key] = iteratedValues[key][index];
|
|
3695
|
+
}
|
|
3696
|
+
for (const key of scalarPorts) {
|
|
3697
|
+
if (key in inputData) {
|
|
3698
|
+
iterInput[key] = inputData[key];
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
return iterInput;
|
|
3702
|
+
};
|
|
3703
|
+
return {
|
|
3704
|
+
iterationCount,
|
|
3705
|
+
arrayPorts,
|
|
3706
|
+
scalarPorts,
|
|
3707
|
+
getIterationInput
|
|
3708
|
+
};
|
|
3709
|
+
}
|
|
3710
|
+
getIterationContextSchema() {
|
|
3711
|
+
return this.constructor.getIterationContextSchema();
|
|
3712
|
+
}
|
|
3713
|
+
inputSchema() {
|
|
3714
|
+
if (this.hasChildren()) {
|
|
3715
|
+
return this.getIterationInputSchema();
|
|
3716
|
+
}
|
|
3717
|
+
return this.constructor.inputSchema();
|
|
3718
|
+
}
|
|
3719
|
+
outputSchema() {
|
|
3720
|
+
if (!this.hasChildren()) {
|
|
3721
|
+
return this.constructor.outputSchema();
|
|
3722
|
+
}
|
|
3723
|
+
return this.getWrappedOutputSchema();
|
|
3724
|
+
}
|
|
3725
|
+
getWrappedOutputSchema() {
|
|
3726
|
+
if (!this.hasChildren()) {
|
|
3727
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
3728
|
+
}
|
|
3729
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
3730
|
+
if (endingNodes.length === 0) {
|
|
3731
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
3732
|
+
}
|
|
3733
|
+
const properties = {};
|
|
3734
|
+
for (const task of endingNodes) {
|
|
3735
|
+
const taskOutputSchema = task.outputSchema();
|
|
3736
|
+
if (typeof taskOutputSchema === "boolean")
|
|
3737
|
+
continue;
|
|
3738
|
+
for (const [key, schema] of Object.entries(taskOutputSchema.properties || {})) {
|
|
3739
|
+
properties[key] = {
|
|
3740
|
+
type: "array",
|
|
3741
|
+
items: schema
|
|
3742
|
+
};
|
|
3743
|
+
}
|
|
3744
|
+
}
|
|
3745
|
+
return {
|
|
3746
|
+
type: "object",
|
|
3747
|
+
properties,
|
|
3748
|
+
additionalProperties: false
|
|
3749
|
+
};
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
|
|
3753
|
+
// src/task/WhileTaskRunner.ts
|
|
3754
|
+
class WhileTaskRunner extends GraphAsTaskRunner {
|
|
3755
|
+
async executeTask(input) {
|
|
3756
|
+
const result = await this.task.execute(input, {
|
|
3757
|
+
signal: this.abortController.signal,
|
|
3758
|
+
updateProgress: this.handleProgress.bind(this),
|
|
3759
|
+
own: this.own,
|
|
3760
|
+
registry: this.registry
|
|
3761
|
+
});
|
|
3762
|
+
return result;
|
|
3763
|
+
}
|
|
3764
|
+
async executeTaskReactive(input, output) {
|
|
3765
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
3766
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
3767
|
+
}
|
|
3768
|
+
}
|
|
3769
|
+
|
|
3770
|
+
// src/task/WhileTask.ts
|
|
3771
|
+
var WHILE_CONTEXT_SCHEMA = {
|
|
3772
|
+
type: "object",
|
|
3773
|
+
properties: {
|
|
3774
|
+
_iterationIndex: {
|
|
3775
|
+
type: "integer",
|
|
3776
|
+
minimum: 0,
|
|
3777
|
+
title: "Iteration Number",
|
|
3778
|
+
description: "Current iteration number (0-based)",
|
|
3779
|
+
"x-ui-iteration": true
|
|
3780
|
+
}
|
|
3781
|
+
}
|
|
3782
|
+
};
|
|
3783
|
+
|
|
3784
|
+
class WhileTask extends GraphAsTask {
|
|
3785
|
+
static type = "WhileTask";
|
|
3786
|
+
static category = "Flow Control";
|
|
3787
|
+
static title = "While Loop";
|
|
3788
|
+
static description = "Loops until a condition function returns false";
|
|
3789
|
+
static hasDynamicSchemas = true;
|
|
3790
|
+
static getIterationContextSchema() {
|
|
3791
|
+
return WHILE_CONTEXT_SCHEMA;
|
|
3792
|
+
}
|
|
3793
|
+
_currentIteration = 0;
|
|
3794
|
+
constructor(input = {}, config = {}) {
|
|
3795
|
+
super(input, config);
|
|
3796
|
+
}
|
|
3797
|
+
get runner() {
|
|
3798
|
+
if (!this._runner) {
|
|
3799
|
+
this._runner = new WhileTaskRunner(this);
|
|
3800
|
+
}
|
|
3801
|
+
return this._runner;
|
|
3802
|
+
}
|
|
3803
|
+
get condition() {
|
|
3804
|
+
return this.config.condition;
|
|
3805
|
+
}
|
|
3806
|
+
get maxIterations() {
|
|
3807
|
+
if (this.config.maxIterations !== undefined)
|
|
3808
|
+
return this.config.maxIterations;
|
|
3809
|
+
const wc = this.config.extras?.whileConfig;
|
|
3810
|
+
return wc?.maxIterations ?? 100;
|
|
3811
|
+
}
|
|
3812
|
+
get chainIterations() {
|
|
3813
|
+
if (this.config.chainIterations !== undefined)
|
|
3814
|
+
return this.config.chainIterations;
|
|
3815
|
+
const wc = this.config.extras?.whileConfig;
|
|
3816
|
+
return wc?.chainIterations ?? true;
|
|
3817
|
+
}
|
|
3818
|
+
get currentIteration() {
|
|
3819
|
+
return this._currentIteration;
|
|
3820
|
+
}
|
|
3821
|
+
buildConditionFromExtras() {
|
|
3822
|
+
const wc = this.config.extras?.whileConfig;
|
|
3823
|
+
if (!wc?.conditionOperator) {
|
|
3824
|
+
return;
|
|
3825
|
+
}
|
|
3826
|
+
const { conditionField, conditionOperator, conditionValue } = wc;
|
|
3827
|
+
return (output) => {
|
|
3828
|
+
const fieldValue = conditionField ? getNestedValue(output, conditionField) : output;
|
|
3829
|
+
return evaluateCondition(fieldValue, conditionOperator, conditionValue ?? "");
|
|
3830
|
+
};
|
|
3831
|
+
}
|
|
3832
|
+
analyzeArrayInputs(input) {
|
|
3833
|
+
const wc = this.config.extras?.whileConfig;
|
|
3834
|
+
if (!wc?.iterationInputConfig) {
|
|
3835
|
+
return null;
|
|
3836
|
+
}
|
|
3837
|
+
const inputData = input;
|
|
3838
|
+
const config = wc.iterationInputConfig;
|
|
3839
|
+
const arrayPorts = [];
|
|
3840
|
+
const scalarPorts = [];
|
|
3841
|
+
const iteratedValues = {};
|
|
3842
|
+
const arrayLengths = [];
|
|
3843
|
+
for (const [key, propConfig] of Object.entries(config)) {
|
|
3844
|
+
const value = inputData[key];
|
|
3845
|
+
if (propConfig.mode === "array") {
|
|
3846
|
+
if (!Array.isArray(value)) {
|
|
3847
|
+
scalarPorts.push(key);
|
|
3848
|
+
continue;
|
|
3849
|
+
}
|
|
3850
|
+
iteratedValues[key] = value;
|
|
3851
|
+
arrayPorts.push(key);
|
|
3852
|
+
arrayLengths.push(value.length);
|
|
3853
|
+
} else {
|
|
3854
|
+
scalarPorts.push(key);
|
|
3855
|
+
}
|
|
3856
|
+
}
|
|
3857
|
+
for (const key of Object.keys(inputData)) {
|
|
3858
|
+
if (!config[key] && !key.startsWith("_iteration")) {
|
|
3859
|
+
scalarPorts.push(key);
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
if (arrayPorts.length === 0) {
|
|
3863
|
+
return null;
|
|
3864
|
+
}
|
|
3865
|
+
const uniqueLengths = new Set(arrayLengths);
|
|
3866
|
+
if (uniqueLengths.size > 1) {
|
|
3867
|
+
const lengthInfo = arrayPorts.map((port, index) => `${port}=${arrayLengths[index]}`).join(", ");
|
|
3868
|
+
throw new TaskConfigurationError(`${this.type}: All iterated array inputs must have the same length. ` + `Found different lengths: ${lengthInfo}`);
|
|
3869
|
+
}
|
|
3870
|
+
return {
|
|
3871
|
+
arrayPorts,
|
|
3872
|
+
scalarPorts,
|
|
3873
|
+
iteratedValues,
|
|
3874
|
+
iterationCount: arrayLengths[0] ?? 0
|
|
3875
|
+
};
|
|
3876
|
+
}
|
|
3877
|
+
buildIterationInput(input, analysis, index) {
|
|
3878
|
+
const inputData = input;
|
|
3879
|
+
const iterInput = {};
|
|
3880
|
+
for (const key of analysis.arrayPorts) {
|
|
3881
|
+
iterInput[key] = analysis.iteratedValues[key][index];
|
|
3882
|
+
}
|
|
3883
|
+
for (const key of analysis.scalarPorts) {
|
|
3884
|
+
if (key in inputData) {
|
|
3885
|
+
iterInput[key] = inputData[key];
|
|
3886
|
+
}
|
|
3887
|
+
}
|
|
3888
|
+
return iterInput;
|
|
3889
|
+
}
|
|
3890
|
+
async execute(input, context) {
|
|
3891
|
+
if (!this.hasChildren()) {
|
|
3892
|
+
throw new TaskConfigurationError(`${this.type}: No subgraph set for while loop`);
|
|
3893
|
+
}
|
|
3894
|
+
const condition = this.condition ?? this.buildConditionFromExtras();
|
|
3895
|
+
if (!condition) {
|
|
3896
|
+
throw new TaskConfigurationError(`${this.type}: No condition function provided`);
|
|
3897
|
+
}
|
|
3898
|
+
const arrayAnalysis = this.analyzeArrayInputs(input);
|
|
3899
|
+
this._currentIteration = 0;
|
|
3900
|
+
let currentInput = { ...input };
|
|
3901
|
+
let currentOutput = {};
|
|
3902
|
+
const effectiveMax = arrayAnalysis ? Math.min(this.maxIterations, arrayAnalysis.iterationCount) : this.maxIterations;
|
|
3903
|
+
while (this._currentIteration < effectiveMax) {
|
|
3904
|
+
if (context.signal?.aborted) {
|
|
3905
|
+
break;
|
|
3906
|
+
}
|
|
3907
|
+
let iterationInput;
|
|
3908
|
+
if (arrayAnalysis) {
|
|
3909
|
+
iterationInput = {
|
|
3910
|
+
...this.buildIterationInput(currentInput, arrayAnalysis, this._currentIteration),
|
|
3911
|
+
_iterationIndex: this._currentIteration
|
|
3912
|
+
};
|
|
3913
|
+
} else {
|
|
3914
|
+
iterationInput = {
|
|
3915
|
+
...currentInput,
|
|
3916
|
+
_iterationIndex: this._currentIteration
|
|
3917
|
+
};
|
|
3918
|
+
}
|
|
3919
|
+
const results = await this.subGraph.run(iterationInput, {
|
|
3920
|
+
parentSignal: context.signal
|
|
3921
|
+
});
|
|
3922
|
+
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
3923
|
+
if (!condition(currentOutput, this._currentIteration)) {
|
|
3924
|
+
break;
|
|
3925
|
+
}
|
|
3926
|
+
if (this.chainIterations) {
|
|
3927
|
+
currentInput = { ...currentInput, ...currentOutput };
|
|
3928
|
+
}
|
|
3929
|
+
this._currentIteration++;
|
|
3930
|
+
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
3931
|
+
await context.updateProgress(progress, `Iteration ${this._currentIteration}`);
|
|
3932
|
+
}
|
|
3933
|
+
return currentOutput;
|
|
3934
|
+
}
|
|
3935
|
+
getIterationContextSchema() {
|
|
3936
|
+
return this.constructor.getIterationContextSchema();
|
|
3937
|
+
}
|
|
3938
|
+
getChainedOutputSchema() {
|
|
3939
|
+
if (!this.chainIterations)
|
|
3940
|
+
return;
|
|
3941
|
+
const outputSchema = this.outputSchema();
|
|
3942
|
+
if (typeof outputSchema === "boolean")
|
|
3943
|
+
return;
|
|
3944
|
+
const properties = {};
|
|
3945
|
+
if (outputSchema.properties && typeof outputSchema.properties === "object") {
|
|
3946
|
+
for (const [key, schema] of Object.entries(outputSchema.properties)) {
|
|
3947
|
+
if (key === "_iterations")
|
|
3948
|
+
continue;
|
|
3949
|
+
if (typeof schema === "object" && schema !== null) {
|
|
3950
|
+
properties[key] = { ...schema, "x-ui-iteration": true };
|
|
3951
|
+
}
|
|
3952
|
+
}
|
|
3953
|
+
}
|
|
3954
|
+
if (Object.keys(properties).length === 0)
|
|
3955
|
+
return;
|
|
3956
|
+
return { type: "object", properties };
|
|
3957
|
+
}
|
|
3958
|
+
inputSchema() {
|
|
3959
|
+
if (!this.hasChildren()) {
|
|
3960
|
+
return this.constructor.inputSchema();
|
|
3961
|
+
}
|
|
3962
|
+
const baseSchema = super.inputSchema();
|
|
3963
|
+
if (typeof baseSchema === "boolean")
|
|
3964
|
+
return baseSchema;
|
|
3965
|
+
const wc = this.config.extras?.whileConfig;
|
|
3966
|
+
if (!wc?.iterationInputConfig) {
|
|
3967
|
+
return baseSchema;
|
|
3968
|
+
}
|
|
3969
|
+
const properties = { ...baseSchema.properties || {} };
|
|
3970
|
+
for (const [key, propConfig] of Object.entries(wc.iterationInputConfig)) {
|
|
3971
|
+
if (propConfig.mode === "array" && properties[key]) {
|
|
3972
|
+
const scalarSchema = properties[key];
|
|
3973
|
+
properties[key] = {
|
|
3974
|
+
anyOf: [scalarSchema, { type: "array", items: scalarSchema }]
|
|
3975
|
+
};
|
|
3976
|
+
}
|
|
3977
|
+
}
|
|
3978
|
+
return {
|
|
3979
|
+
...baseSchema,
|
|
3980
|
+
properties
|
|
3981
|
+
};
|
|
3982
|
+
}
|
|
3983
|
+
static inputSchema() {
|
|
3984
|
+
return {
|
|
3985
|
+
type: "object",
|
|
3986
|
+
properties: {},
|
|
3987
|
+
additionalProperties: true
|
|
3988
|
+
};
|
|
3989
|
+
}
|
|
3990
|
+
static outputSchema() {
|
|
3991
|
+
return {
|
|
3992
|
+
type: "object",
|
|
3993
|
+
properties: {
|
|
3994
|
+
_iterations: {
|
|
3995
|
+
type: "number",
|
|
3996
|
+
title: "Iterations",
|
|
3997
|
+
description: "Number of iterations executed"
|
|
3998
|
+
}
|
|
3999
|
+
},
|
|
4000
|
+
additionalProperties: true
|
|
4001
|
+
};
|
|
4002
|
+
}
|
|
4003
|
+
outputSchema() {
|
|
4004
|
+
if (!this.hasChildren()) {
|
|
4005
|
+
return this.constructor.outputSchema();
|
|
4006
|
+
}
|
|
4007
|
+
const tasks = this.subGraph.getTasks();
|
|
4008
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
4009
|
+
if (endingNodes.length === 0) {
|
|
4010
|
+
return this.constructor.outputSchema();
|
|
4011
|
+
}
|
|
4012
|
+
const properties = {
|
|
4013
|
+
_iterations: {
|
|
4014
|
+
type: "number",
|
|
4015
|
+
title: "Iterations",
|
|
4016
|
+
description: "Number of iterations executed"
|
|
4017
|
+
}
|
|
4018
|
+
};
|
|
4019
|
+
for (const task of endingNodes) {
|
|
4020
|
+
const taskOutputSchema = task.outputSchema();
|
|
4021
|
+
if (typeof taskOutputSchema === "boolean")
|
|
4022
|
+
continue;
|
|
4023
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
4024
|
+
for (const [key, schema] of Object.entries(taskProperties)) {
|
|
4025
|
+
if (!properties[key]) {
|
|
4026
|
+
properties[key] = schema;
|
|
4027
|
+
}
|
|
4028
|
+
}
|
|
4029
|
+
}
|
|
4030
|
+
return {
|
|
4031
|
+
type: "object",
|
|
4032
|
+
properties,
|
|
4033
|
+
additionalProperties: false
|
|
4034
|
+
};
|
|
4035
|
+
}
|
|
4036
|
+
}
|
|
4037
|
+
Workflow.prototype.while = CreateLoopWorkflow(WhileTask);
|
|
4038
|
+
Workflow.prototype.endWhile = CreateEndLoopWorkflow("endWhile");
|
|
4039
|
+
|
|
4040
|
+
// src/task/iterationSchema.ts
|
|
4041
|
+
function isFlexibleSchema(schema) {
|
|
4042
|
+
if (typeof schema === "boolean")
|
|
4043
|
+
return false;
|
|
4044
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
4045
|
+
const arr = Array.isArray(variants) ? variants : undefined;
|
|
4046
|
+
if (!arr || arr.length !== 2)
|
|
4047
|
+
return false;
|
|
4048
|
+
let hasScalar = false;
|
|
4049
|
+
let hasArray = false;
|
|
4050
|
+
for (const variant of arr) {
|
|
4051
|
+
if (typeof variant !== "object")
|
|
4052
|
+
continue;
|
|
4053
|
+
const v = variant;
|
|
4054
|
+
if (v.type === "array" || "items" in v) {
|
|
4055
|
+
hasArray = true;
|
|
4056
|
+
} else {
|
|
4057
|
+
hasScalar = true;
|
|
4058
|
+
}
|
|
4059
|
+
}
|
|
4060
|
+
return hasScalar && hasArray;
|
|
4061
|
+
}
|
|
4062
|
+
function isStrictArraySchema(schema) {
|
|
4063
|
+
if (typeof schema === "boolean")
|
|
4064
|
+
return false;
|
|
4065
|
+
const s = schema;
|
|
4066
|
+
return s.type === "array" && !isFlexibleSchema(schema);
|
|
4067
|
+
}
|
|
4068
|
+
function getInputModeFromSchema(schema) {
|
|
4069
|
+
if (isFlexibleSchema(schema))
|
|
4070
|
+
return "flexible";
|
|
4071
|
+
if (isStrictArraySchema(schema))
|
|
4072
|
+
return "array";
|
|
4073
|
+
return "scalar";
|
|
4074
|
+
}
|
|
4075
|
+
function getIterationContextSchemaForType(taskType) {
|
|
4076
|
+
if (taskType === "MapTask" || taskType === "ReduceTask") {
|
|
4077
|
+
return ITERATOR_CONTEXT_SCHEMA;
|
|
4078
|
+
}
|
|
4079
|
+
if (taskType === "WhileTask") {
|
|
4080
|
+
return WHILE_CONTEXT_SCHEMA;
|
|
4081
|
+
}
|
|
4082
|
+
return;
|
|
4083
|
+
}
|
|
4084
|
+
function addIterationContextToSchema(existingSchema, parentTaskType) {
|
|
4085
|
+
const contextSchema = getIterationContextSchemaForType(parentTaskType);
|
|
4086
|
+
if (!contextSchema) {
|
|
4087
|
+
return existingSchema ?? { type: "object", properties: {} };
|
|
4088
|
+
}
|
|
4089
|
+
const baseProperties = existingSchema && typeof existingSchema !== "boolean" && existingSchema.properties && typeof existingSchema.properties !== "boolean" ? existingSchema.properties : {};
|
|
4090
|
+
const contextProperties = typeof contextSchema !== "boolean" && contextSchema.properties && typeof contextSchema.properties !== "boolean" ? contextSchema.properties : {};
|
|
4091
|
+
return {
|
|
4092
|
+
type: "object",
|
|
4093
|
+
properties: {
|
|
4094
|
+
...baseProperties,
|
|
4095
|
+
...contextProperties
|
|
4096
|
+
}
|
|
4097
|
+
};
|
|
4098
|
+
}
|
|
4099
|
+
function isIterationProperty(schema) {
|
|
4100
|
+
if (!schema || typeof schema === "boolean")
|
|
4101
|
+
return false;
|
|
4102
|
+
return schema["x-ui-iteration"] === true;
|
|
4103
|
+
}
|
|
4104
|
+
function filterIterationProperties(schema) {
|
|
4105
|
+
if (!schema || typeof schema === "boolean")
|
|
4106
|
+
return schema;
|
|
4107
|
+
const props = schema.properties;
|
|
4108
|
+
if (!props || typeof props === "boolean")
|
|
4109
|
+
return schema;
|
|
4110
|
+
const filteredProps = {};
|
|
4111
|
+
for (const [key, propSchema] of Object.entries(props)) {
|
|
4112
|
+
if (!isIterationProperty(propSchema)) {
|
|
4113
|
+
filteredProps[key] = propSchema;
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
if (Object.keys(filteredProps).length === 0) {
|
|
4117
|
+
return { type: "object", properties: {} };
|
|
4118
|
+
}
|
|
4119
|
+
return { ...schema, properties: filteredProps };
|
|
4120
|
+
}
|
|
4121
|
+
function extractIterationProperties(schema) {
|
|
4122
|
+
if (!schema || typeof schema === "boolean")
|
|
4123
|
+
return;
|
|
4124
|
+
const props = schema.properties;
|
|
4125
|
+
if (!props || typeof props === "boolean")
|
|
4126
|
+
return;
|
|
4127
|
+
const iterProps = {};
|
|
4128
|
+
for (const [key, propSchema] of Object.entries(props)) {
|
|
4129
|
+
if (isIterationProperty(propSchema)) {
|
|
4130
|
+
iterProps[key] = propSchema;
|
|
4131
|
+
}
|
|
4132
|
+
}
|
|
4133
|
+
if (Object.keys(iterProps).length === 0)
|
|
4134
|
+
return;
|
|
4135
|
+
return { type: "object", properties: iterProps };
|
|
4136
|
+
}
|
|
4137
|
+
function removeIterationProperties(schema) {
|
|
4138
|
+
return filterIterationProperties(schema);
|
|
4139
|
+
}
|
|
4140
|
+
function mergeChainedOutputToInput(inputSchema, outputSchema) {
|
|
4141
|
+
const baseSchema = filterIterationProperties(inputSchema) ?? {
|
|
4142
|
+
type: "object",
|
|
4143
|
+
properties: {}
|
|
4144
|
+
};
|
|
4145
|
+
if (!outputSchema || typeof outputSchema === "boolean") {
|
|
4146
|
+
return baseSchema;
|
|
4147
|
+
}
|
|
4148
|
+
const outProps = outputSchema.properties;
|
|
4149
|
+
if (!outProps || typeof outProps === "boolean") {
|
|
4150
|
+
return baseSchema;
|
|
4151
|
+
}
|
|
4152
|
+
const baseProps = typeof baseSchema !== "boolean" && baseSchema.properties && typeof baseSchema.properties !== "boolean" ? baseSchema.properties : {};
|
|
4153
|
+
const mergedProperties = { ...baseProps };
|
|
4154
|
+
for (const [key, propSchema] of Object.entries(outProps)) {
|
|
4155
|
+
if (typeof propSchema === "object" && propSchema !== null) {
|
|
4156
|
+
mergedProperties[key] = { ...propSchema, "x-ui-iteration": true };
|
|
4157
|
+
}
|
|
4158
|
+
}
|
|
4159
|
+
return {
|
|
4160
|
+
type: "object",
|
|
4161
|
+
properties: mergedProperties
|
|
4162
|
+
};
|
|
4163
|
+
}
|
|
4164
|
+
function buildIterationInputSchema(innerSchema, config) {
|
|
4165
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
4166
|
+
return { type: "object", properties: {} };
|
|
4167
|
+
}
|
|
4168
|
+
const innerProps = innerSchema.properties;
|
|
4169
|
+
if (!innerProps || typeof innerProps === "boolean") {
|
|
4170
|
+
return { type: "object", properties: {} };
|
|
4171
|
+
}
|
|
4172
|
+
const properties = {};
|
|
4173
|
+
const propsRecord = innerProps;
|
|
4174
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4175
|
+
if (typeof propSchema === "boolean")
|
|
4176
|
+
continue;
|
|
4177
|
+
if (propSchema["x-ui-iteration"]) {
|
|
4178
|
+
continue;
|
|
4179
|
+
}
|
|
4180
|
+
const originalProps = propSchema;
|
|
4181
|
+
const metadata = {};
|
|
4182
|
+
for (const metaKey of Object.keys(originalProps)) {
|
|
4183
|
+
if (metaKey === "title" || metaKey === "description" || metaKey.startsWith("x-")) {
|
|
4184
|
+
metadata[metaKey] = originalProps[metaKey];
|
|
4185
|
+
}
|
|
4186
|
+
}
|
|
4187
|
+
const baseSchema = extractBaseSchema(propSchema);
|
|
4188
|
+
const propConfig = config?.[key];
|
|
4189
|
+
const mode = propConfig?.mode ?? "flexible";
|
|
4190
|
+
const base = propConfig?.baseSchema ?? baseSchema;
|
|
4191
|
+
let wrappedSchema;
|
|
4192
|
+
switch (mode) {
|
|
4193
|
+
case "array":
|
|
4194
|
+
wrappedSchema = createArraySchema(base);
|
|
4195
|
+
break;
|
|
4196
|
+
case "scalar":
|
|
4197
|
+
wrappedSchema = base;
|
|
4198
|
+
break;
|
|
4199
|
+
case "flexible":
|
|
4200
|
+
default:
|
|
4201
|
+
wrappedSchema = createFlexibleSchema(base);
|
|
4202
|
+
break;
|
|
4203
|
+
}
|
|
4204
|
+
if (Object.keys(metadata).length > 0 && typeof wrappedSchema === "object") {
|
|
4205
|
+
properties[key] = { ...metadata, ...wrappedSchema };
|
|
4206
|
+
} else {
|
|
4207
|
+
properties[key] = wrappedSchema;
|
|
4208
|
+
}
|
|
4209
|
+
}
|
|
4210
|
+
return {
|
|
4211
|
+
type: "object",
|
|
4212
|
+
properties
|
|
4213
|
+
};
|
|
4214
|
+
}
|
|
4215
|
+
function findArrayPorts(schema) {
|
|
4216
|
+
if (!schema || typeof schema === "boolean")
|
|
4217
|
+
return [];
|
|
4218
|
+
const props = schema.properties;
|
|
4219
|
+
if (!props || typeof props === "boolean")
|
|
4220
|
+
return [];
|
|
4221
|
+
const arrayPorts = [];
|
|
4222
|
+
const propsRecord = props;
|
|
4223
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4224
|
+
if (typeof propSchema === "boolean")
|
|
4225
|
+
continue;
|
|
4226
|
+
if (propSchema.type === "array") {
|
|
4227
|
+
arrayPorts.push(key);
|
|
4228
|
+
}
|
|
4229
|
+
}
|
|
4230
|
+
return arrayPorts;
|
|
4231
|
+
}
|
|
4232
|
+
function wrapSchemaInArray(schema) {
|
|
4233
|
+
if (!schema || typeof schema === "boolean")
|
|
4234
|
+
return schema;
|
|
4235
|
+
const props = schema.properties;
|
|
4236
|
+
if (!props || typeof props === "boolean")
|
|
4237
|
+
return schema;
|
|
4238
|
+
const propsRecord = props;
|
|
4239
|
+
const wrappedProperties = {};
|
|
4240
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4241
|
+
wrappedProperties[key] = {
|
|
4242
|
+
type: "array",
|
|
4243
|
+
items: propSchema
|
|
4244
|
+
};
|
|
4245
|
+
}
|
|
4246
|
+
return {
|
|
4247
|
+
type: "object",
|
|
4248
|
+
properties: wrappedProperties
|
|
4249
|
+
};
|
|
4250
|
+
}
|
|
4251
|
+
// src/task/JobQueueFactory.ts
|
|
4252
|
+
import {
|
|
4253
|
+
JobQueueClient,
|
|
4254
|
+
JobQueueServer
|
|
4255
|
+
} from "@workglow/job-queue";
|
|
4256
|
+
import { InMemoryQueueStorage } from "@workglow/storage";
|
|
4257
|
+
import { createServiceToken as createServiceToken2, globalServiceRegistry as globalServiceRegistry3 } from "@workglow/util";
|
|
4258
|
+
var JOB_QUEUE_FACTORY = createServiceToken2("taskgraph.jobQueueFactory");
|
|
4259
|
+
var defaultJobQueueFactory = async ({
|
|
4260
|
+
queueName,
|
|
4261
|
+
jobClass,
|
|
4262
|
+
options
|
|
4263
|
+
}) => {
|
|
4264
|
+
const storage = options?.storage ?? new InMemoryQueueStorage(queueName);
|
|
4265
|
+
await storage.setupDatabase();
|
|
4266
|
+
const server = new JobQueueServer(jobClass, {
|
|
4267
|
+
storage,
|
|
4268
|
+
queueName,
|
|
4269
|
+
limiter: options?.limiter,
|
|
4270
|
+
workerCount: options?.workerCount,
|
|
4271
|
+
pollIntervalMs: options?.pollIntervalMs,
|
|
4272
|
+
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
4273
|
+
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
4274
|
+
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
4275
|
+
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
4276
|
+
});
|
|
4277
|
+
const client = new JobQueueClient({
|
|
4278
|
+
storage,
|
|
4279
|
+
queueName
|
|
4280
|
+
});
|
|
4281
|
+
client.attach(server);
|
|
4282
|
+
return { server, client, storage };
|
|
4283
|
+
};
|
|
4284
|
+
function registerJobQueueFactory(factory) {
|
|
4285
|
+
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
4286
|
+
}
|
|
4287
|
+
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
4288
|
+
return async ({
|
|
4289
|
+
queueName,
|
|
4290
|
+
jobClass,
|
|
4291
|
+
options
|
|
4292
|
+
}) => {
|
|
4293
|
+
const mergedOptions = {
|
|
4294
|
+
...defaultOptions,
|
|
4295
|
+
...options ?? {}
|
|
4296
|
+
};
|
|
4297
|
+
const storage = mergedOptions.storage ?? new InMemoryQueueStorage(queueName);
|
|
4298
|
+
await storage.setupDatabase();
|
|
4299
|
+
const server = new JobQueueServer(jobClass, {
|
|
4300
|
+
storage,
|
|
4301
|
+
queueName,
|
|
4302
|
+
limiter: mergedOptions.limiter,
|
|
4303
|
+
workerCount: mergedOptions.workerCount,
|
|
4304
|
+
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
4305
|
+
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
4306
|
+
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
4307
|
+
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
4308
|
+
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
4309
|
+
});
|
|
4310
|
+
const client = new JobQueueClient({
|
|
4311
|
+
storage,
|
|
4312
|
+
queueName
|
|
4313
|
+
});
|
|
4314
|
+
client.attach(server);
|
|
4315
|
+
return { server, client, storage };
|
|
4316
|
+
};
|
|
4317
|
+
}
|
|
4318
|
+
function getJobQueueFactory() {
|
|
4319
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
4320
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
4321
|
+
}
|
|
4322
|
+
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
4323
|
+
}
|
|
4324
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
4325
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
4326
|
+
}
|
|
4327
|
+
// src/task/JobQueueTask.ts
|
|
4328
|
+
import { Job as Job2 } from "@workglow/job-queue";
|
|
4329
|
+
|
|
4330
|
+
// src/task/TaskQueueRegistry.ts
|
|
4331
|
+
var taskQueueRegistry = null;
|
|
4332
|
+
|
|
4333
|
+
class TaskQueueRegistry {
|
|
4334
|
+
queues = new Map;
|
|
4335
|
+
registerQueue(queue) {
|
|
4336
|
+
const queueName = queue.server.queueName;
|
|
4337
|
+
if (this.queues.has(queueName)) {
|
|
4338
|
+
throw new Error(`Queue with name ${queueName} already exists`);
|
|
4339
|
+
}
|
|
4340
|
+
this.queues.set(queueName, queue);
|
|
4341
|
+
}
|
|
4342
|
+
getQueue(queueName) {
|
|
4343
|
+
return this.queues.get(queueName);
|
|
4344
|
+
}
|
|
4345
|
+
startQueues() {
|
|
4346
|
+
for (const queue of this.queues.values()) {
|
|
4347
|
+
queue.server.start();
|
|
4348
|
+
}
|
|
4349
|
+
return this;
|
|
4350
|
+
}
|
|
4351
|
+
stopQueues() {
|
|
4352
|
+
for (const queue of this.queues.values()) {
|
|
4353
|
+
queue.server.stop();
|
|
4354
|
+
}
|
|
4355
|
+
return this;
|
|
4356
|
+
}
|
|
4357
|
+
clearQueues() {
|
|
4358
|
+
for (const queue of this.queues.values()) {
|
|
4359
|
+
queue.storage.deleteAll();
|
|
4360
|
+
}
|
|
4361
|
+
return this;
|
|
4362
|
+
}
|
|
4363
|
+
}
|
|
4364
|
+
function getTaskQueueRegistry() {
|
|
4365
|
+
if (!taskQueueRegistry) {
|
|
4366
|
+
taskQueueRegistry = new TaskQueueRegistry;
|
|
2641
4367
|
}
|
|
2642
4368
|
return taskQueueRegistry;
|
|
2643
4369
|
}
|
|
@@ -2782,6 +4508,155 @@ class JobQueueTask extends GraphAsTask {
|
|
|
2782
4508
|
super.abort();
|
|
2783
4509
|
}
|
|
2784
4510
|
}
|
|
4511
|
+
// src/task/MapTask.ts
|
|
4512
|
+
class MapTask extends IteratorTask {
|
|
4513
|
+
static type = "MapTask";
|
|
4514
|
+
static category = "Flow Control";
|
|
4515
|
+
static title = "Map";
|
|
4516
|
+
static description = "Transforms array inputs by running a workflow per item";
|
|
4517
|
+
static compoundMerge = PROPERTY_ARRAY;
|
|
4518
|
+
static inputSchema() {
|
|
4519
|
+
return {
|
|
4520
|
+
type: "object",
|
|
4521
|
+
properties: {},
|
|
4522
|
+
additionalProperties: true
|
|
4523
|
+
};
|
|
4524
|
+
}
|
|
4525
|
+
static outputSchema() {
|
|
4526
|
+
return {
|
|
4527
|
+
type: "object",
|
|
4528
|
+
properties: {},
|
|
4529
|
+
additionalProperties: true
|
|
4530
|
+
};
|
|
4531
|
+
}
|
|
4532
|
+
get preserveOrder() {
|
|
4533
|
+
return this.config.preserveOrder ?? true;
|
|
4534
|
+
}
|
|
4535
|
+
get flatten() {
|
|
4536
|
+
return this.config.flatten ?? false;
|
|
4537
|
+
}
|
|
4538
|
+
preserveIterationOrder() {
|
|
4539
|
+
return this.preserveOrder;
|
|
4540
|
+
}
|
|
4541
|
+
getEmptyResult() {
|
|
4542
|
+
const schema = this.outputSchema();
|
|
4543
|
+
if (typeof schema === "boolean") {
|
|
4544
|
+
return {};
|
|
4545
|
+
}
|
|
4546
|
+
const result = {};
|
|
4547
|
+
for (const key of Object.keys(schema.properties || {})) {
|
|
4548
|
+
result[key] = [];
|
|
4549
|
+
}
|
|
4550
|
+
return result;
|
|
4551
|
+
}
|
|
4552
|
+
outputSchema() {
|
|
4553
|
+
if (!this.hasChildren()) {
|
|
4554
|
+
return this.constructor.outputSchema();
|
|
4555
|
+
}
|
|
4556
|
+
return this.getWrappedOutputSchema();
|
|
4557
|
+
}
|
|
4558
|
+
collectResults(results) {
|
|
4559
|
+
const collected = super.collectResults(results);
|
|
4560
|
+
if (!this.flatten || typeof collected !== "object" || collected === null) {
|
|
4561
|
+
return collected;
|
|
4562
|
+
}
|
|
4563
|
+
const flattened = {};
|
|
4564
|
+
for (const [key, value] of Object.entries(collected)) {
|
|
4565
|
+
if (Array.isArray(value)) {
|
|
4566
|
+
flattened[key] = value.flat();
|
|
4567
|
+
} else {
|
|
4568
|
+
flattened[key] = value;
|
|
4569
|
+
}
|
|
4570
|
+
}
|
|
4571
|
+
return flattened;
|
|
4572
|
+
}
|
|
4573
|
+
}
|
|
4574
|
+
queueMicrotask(() => {
|
|
4575
|
+
Workflow.prototype.map = CreateLoopWorkflow(MapTask);
|
|
4576
|
+
Workflow.prototype.endMap = CreateEndLoopWorkflow("endMap");
|
|
4577
|
+
});
|
|
4578
|
+
// src/task/ReduceTask.ts
|
|
4579
|
+
class ReduceTask extends IteratorTask {
|
|
4580
|
+
static type = "ReduceTask";
|
|
4581
|
+
static category = "Flow Control";
|
|
4582
|
+
static title = "Reduce";
|
|
4583
|
+
static description = "Processes iterated inputs sequentially with an accumulator (fold)";
|
|
4584
|
+
constructor(input = {}, config = {}) {
|
|
4585
|
+
const reduceConfig = {
|
|
4586
|
+
...config,
|
|
4587
|
+
concurrencyLimit: 1,
|
|
4588
|
+
batchSize: 1
|
|
4589
|
+
};
|
|
4590
|
+
super(input, reduceConfig);
|
|
4591
|
+
}
|
|
4592
|
+
get initialValue() {
|
|
4593
|
+
return this.config.initialValue ?? {};
|
|
4594
|
+
}
|
|
4595
|
+
isReduceTask() {
|
|
4596
|
+
return true;
|
|
4597
|
+
}
|
|
4598
|
+
getInitialAccumulator() {
|
|
4599
|
+
const value = this.initialValue;
|
|
4600
|
+
if (Array.isArray(value)) {
|
|
4601
|
+
return [...value];
|
|
4602
|
+
}
|
|
4603
|
+
if (value && typeof value === "object") {
|
|
4604
|
+
return { ...value };
|
|
4605
|
+
}
|
|
4606
|
+
return value;
|
|
4607
|
+
}
|
|
4608
|
+
buildIterationRunInput(analysis, index, iterationCount, extraInput = {}) {
|
|
4609
|
+
return super.buildIterationRunInput(analysis, index, iterationCount, {
|
|
4610
|
+
accumulator: extraInput.accumulator
|
|
4611
|
+
});
|
|
4612
|
+
}
|
|
4613
|
+
getEmptyResult() {
|
|
4614
|
+
return this.getInitialAccumulator();
|
|
4615
|
+
}
|
|
4616
|
+
static inputSchema() {
|
|
4617
|
+
return {
|
|
4618
|
+
type: "object",
|
|
4619
|
+
properties: {},
|
|
4620
|
+
additionalProperties: true
|
|
4621
|
+
};
|
|
4622
|
+
}
|
|
4623
|
+
static outputSchema() {
|
|
4624
|
+
return {
|
|
4625
|
+
type: "object",
|
|
4626
|
+
properties: {},
|
|
4627
|
+
additionalProperties: true
|
|
4628
|
+
};
|
|
4629
|
+
}
|
|
4630
|
+
outputSchema() {
|
|
4631
|
+
if (!this.hasChildren()) {
|
|
4632
|
+
return this.constructor.outputSchema();
|
|
4633
|
+
}
|
|
4634
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
4635
|
+
if (endingNodes.length === 0) {
|
|
4636
|
+
return this.constructor.outputSchema();
|
|
4637
|
+
}
|
|
4638
|
+
const properties = {};
|
|
4639
|
+
for (const task of endingNodes) {
|
|
4640
|
+
const taskOutputSchema = task.outputSchema();
|
|
4641
|
+
if (typeof taskOutputSchema === "boolean")
|
|
4642
|
+
continue;
|
|
4643
|
+
for (const [key, schema] of Object.entries(taskOutputSchema.properties || {})) {
|
|
4644
|
+
if (!properties[key]) {
|
|
4645
|
+
properties[key] = schema;
|
|
4646
|
+
}
|
|
4647
|
+
}
|
|
4648
|
+
}
|
|
4649
|
+
return {
|
|
4650
|
+
type: "object",
|
|
4651
|
+
properties,
|
|
4652
|
+
additionalProperties: false
|
|
4653
|
+
};
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
queueMicrotask(() => {
|
|
4657
|
+
Workflow.prototype.reduce = CreateLoopWorkflow(ReduceTask);
|
|
4658
|
+
Workflow.prototype.endReduce = CreateEndLoopWorkflow("endReduce");
|
|
4659
|
+
});
|
|
2785
4660
|
// src/task/TaskRegistry.ts
|
|
2786
4661
|
var taskConstructors = new Map;
|
|
2787
4662
|
function registerTask(baseClass) {
|
|
@@ -2851,7 +4726,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
2851
4726
|
};
|
|
2852
4727
|
// src/task/index.ts
|
|
2853
4728
|
var registerBaseTasks = () => {
|
|
2854
|
-
const tasks = [ConditionalTask,
|
|
4729
|
+
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
2855
4730
|
tasks.map(TaskRegistry.registerTask);
|
|
2856
4731
|
return tasks;
|
|
2857
4732
|
};
|
|
@@ -3012,25 +4887,53 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
3012
4887
|
}
|
|
3013
4888
|
}
|
|
3014
4889
|
export {
|
|
4890
|
+
wrapSchemaInArray,
|
|
3015
4891
|
setTaskQueueRegistry,
|
|
3016
4892
|
serialGraph,
|
|
4893
|
+
schemaAcceptsArray,
|
|
3017
4894
|
resolveSchemaInputs,
|
|
4895
|
+
removeIterationProperties,
|
|
3018
4896
|
registerJobQueueFactory,
|
|
3019
4897
|
registerBaseTasks,
|
|
3020
4898
|
pipe,
|
|
3021
4899
|
parallel,
|
|
4900
|
+
mergeChainedOutputToInput,
|
|
4901
|
+
isTaskStreamable,
|
|
4902
|
+
isStrictArraySchema,
|
|
4903
|
+
isIterationProperty,
|
|
4904
|
+
isFlexibleSchema,
|
|
4905
|
+
hasVectorOutput,
|
|
4906
|
+
hasVectorLikeInput,
|
|
3022
4907
|
getTaskQueueRegistry,
|
|
4908
|
+
getPortStreamMode,
|
|
4909
|
+
getOutputStreamMode,
|
|
4910
|
+
getNestedValue,
|
|
3023
4911
|
getLastTask,
|
|
3024
4912
|
getJobQueueFactory,
|
|
4913
|
+
getIterationContextSchemaForType,
|
|
4914
|
+
getInputModeFromSchema,
|
|
4915
|
+
findArrayPorts,
|
|
4916
|
+
filterIterationProperties,
|
|
4917
|
+
extractIterationProperties,
|
|
4918
|
+
extractBaseSchema,
|
|
4919
|
+
evaluateCondition,
|
|
3025
4920
|
ensureTask,
|
|
4921
|
+
edgeNeedsAccumulation,
|
|
3026
4922
|
createTaskFromGraphJSON,
|
|
3027
4923
|
createTaskFromDependencyJSON,
|
|
3028
4924
|
createJobQueueFactoryWithOptions,
|
|
3029
4925
|
createGraphFromGraphJSON,
|
|
3030
4926
|
createGraphFromDependencyJSON,
|
|
4927
|
+
createFlexibleSchema,
|
|
4928
|
+
createArraySchema,
|
|
3031
4929
|
connect,
|
|
4930
|
+
buildIterationInputSchema,
|
|
4931
|
+
addIterationContextToSchema,
|
|
3032
4932
|
WorkflowError,
|
|
3033
4933
|
Workflow,
|
|
4934
|
+
WhileTaskRunner,
|
|
4935
|
+
WhileTask,
|
|
4936
|
+
WHILE_CONTEXT_SCHEMA,
|
|
3034
4937
|
TaskStatus,
|
|
3035
4938
|
TaskRegistry,
|
|
3036
4939
|
TaskQueueRegistry,
|
|
@@ -3053,10 +4956,15 @@ export {
|
|
|
3053
4956
|
Task,
|
|
3054
4957
|
TASK_OUTPUT_REPOSITORY,
|
|
3055
4958
|
TASK_GRAPH_REPOSITORY,
|
|
4959
|
+
ReduceTask,
|
|
3056
4960
|
PROPERTY_ARRAY,
|
|
4961
|
+
MapTask,
|
|
3057
4962
|
JobTaskFailedError,
|
|
3058
4963
|
JobQueueTask,
|
|
3059
4964
|
JOB_QUEUE_FACTORY,
|
|
4965
|
+
IteratorTaskRunner,
|
|
4966
|
+
IteratorTask,
|
|
4967
|
+
ITERATOR_CONTEXT_SCHEMA,
|
|
3060
4968
|
GraphAsTaskRunner,
|
|
3061
4969
|
GraphAsTask,
|
|
3062
4970
|
GRAPH_RESULT_ARRAY,
|
|
@@ -3067,7 +4975,10 @@ export {
|
|
|
3067
4975
|
DATAFLOW_ERROR_PORT,
|
|
3068
4976
|
DATAFLOW_ALL_PORTS,
|
|
3069
4977
|
CreateWorkflow,
|
|
4978
|
+
CreateLoopWorkflow,
|
|
4979
|
+
CreateEndLoopWorkflow,
|
|
4980
|
+
CreateAdaptiveWorkflow,
|
|
3070
4981
|
ConditionalTask
|
|
3071
4982
|
};
|
|
3072
4983
|
|
|
3073
|
-
//# debugId=
|
|
4984
|
+
//# debugId=65C6DA6762951D4864756E2164756E21
|