@workglow/task-graph 0.0.89 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +2251 -340
- package/dist/browser.js.map +24 -15
- package/dist/bun.js +2251 -340
- package/dist/bun.js.map +24 -15
- package/dist/node.js +2251 -340
- package/dist/node.js.map +24 -15
- package/dist/task/ConditionUtils.d.ts +47 -0
- package/dist/task/ConditionUtils.d.ts.map +1 -0
- package/dist/task/ConditionalTask.d.ts +15 -0
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +7 -0
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +177 -0
- package/dist/task/IteratorTask.d.ts.map +1 -0
- package/dist/task/IteratorTaskRunner.d.ts +36 -0
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -0
- package/dist/task/MapTask.d.ts +82 -0
- package/dist/task/MapTask.d.ts.map +1 -0
- package/dist/task/ReduceTask.d.ts +61 -0
- package/dist/task/ReduceTask.d.ts.map +1 -0
- package/dist/task/StreamTypes.d.ts +96 -0
- package/dist/task/StreamTypes.d.ts.map +1 -0
- package/dist/task/Task.d.ts +2 -2
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +7 -0
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +3 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +7 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +4 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +214 -0
- package/dist/task/WhileTask.d.ts.map +1 -0
- package/dist/task/WhileTaskRunner.d.ts +29 -0
- package/dist/task/WhileTaskRunner.d.ts.map +1 -0
- package/dist/task/index.d.ts +13 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task/iterationSchema.d.ts +70 -0
- package/dist/task/iterationSchema.d.ts.map +1 -0
- package/dist/task-graph/Dataflow.d.ts +34 -0
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/DataflowEvents.d.ts +2 -0
- package/dist/task-graph/DataflowEvents.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +6 -0
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +14 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphEvents.d.ts +7 -0
- package/dist/task-graph/TaskGraphEvents.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +25 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphScheduler.d.ts +9 -0
- package/dist/task-graph/TaskGraphScheduler.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +148 -9
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/dist/task/TaskJSON.test.d.ts +0 -7
- package/dist/task/TaskJSON.test.d.ts.map +0 -1
package/dist/browser.js
CHANGED
|
@@ -6,6 +6,7 @@ var TaskStatus = {
|
|
|
6
6
|
PENDING: "PENDING",
|
|
7
7
|
DISABLED: "DISABLED",
|
|
8
8
|
PROCESSING: "PROCESSING",
|
|
9
|
+
STREAMING: "STREAMING",
|
|
9
10
|
COMPLETED: "COMPLETED",
|
|
10
11
|
ABORTING: "ABORTING",
|
|
11
12
|
FAILED: "FAILED"
|
|
@@ -35,10 +36,62 @@ class Dataflow {
|
|
|
35
36
|
value = undefined;
|
|
36
37
|
status = TaskStatus.PENDING;
|
|
37
38
|
error;
|
|
39
|
+
stream = undefined;
|
|
40
|
+
setStream(stream) {
|
|
41
|
+
this.stream = stream;
|
|
42
|
+
}
|
|
43
|
+
getStream() {
|
|
44
|
+
return this.stream;
|
|
45
|
+
}
|
|
46
|
+
async awaitStreamValue() {
|
|
47
|
+
if (!this.stream)
|
|
48
|
+
return;
|
|
49
|
+
const reader = this.stream.getReader();
|
|
50
|
+
let accumulatedText = "";
|
|
51
|
+
let lastSnapshotData = undefined;
|
|
52
|
+
let finishData = undefined;
|
|
53
|
+
let hasTextDelta = false;
|
|
54
|
+
try {
|
|
55
|
+
while (true) {
|
|
56
|
+
const { done, value: event } = await reader.read();
|
|
57
|
+
if (done)
|
|
58
|
+
break;
|
|
59
|
+
switch (event.type) {
|
|
60
|
+
case "text-delta":
|
|
61
|
+
hasTextDelta = true;
|
|
62
|
+
accumulatedText += event.textDelta;
|
|
63
|
+
break;
|
|
64
|
+
case "snapshot":
|
|
65
|
+
lastSnapshotData = event.data;
|
|
66
|
+
break;
|
|
67
|
+
case "finish":
|
|
68
|
+
finishData = event.data;
|
|
69
|
+
break;
|
|
70
|
+
case "error":
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
} finally {
|
|
75
|
+
reader.releaseLock();
|
|
76
|
+
this.stream = undefined;
|
|
77
|
+
}
|
|
78
|
+
if (lastSnapshotData !== undefined) {
|
|
79
|
+
this.setPortData(lastSnapshotData);
|
|
80
|
+
} else if (finishData && Object.keys(finishData).length > 0) {
|
|
81
|
+
this.setPortData(finishData);
|
|
82
|
+
} else if (hasTextDelta) {
|
|
83
|
+
if (this.sourceTaskPortId === DATAFLOW_ALL_PORTS) {
|
|
84
|
+
this.value = { text: accumulatedText };
|
|
85
|
+
} else {
|
|
86
|
+
this.value = accumulatedText;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
38
90
|
reset() {
|
|
39
91
|
this.status = TaskStatus.PENDING;
|
|
40
92
|
this.error = undefined;
|
|
41
93
|
this.value = undefined;
|
|
94
|
+
this.stream = undefined;
|
|
42
95
|
this.emit("reset");
|
|
43
96
|
this.emit("status", this.status);
|
|
44
97
|
}
|
|
@@ -50,6 +103,9 @@ class Dataflow {
|
|
|
50
103
|
case TaskStatus.PROCESSING:
|
|
51
104
|
this.emit("start");
|
|
52
105
|
break;
|
|
106
|
+
case TaskStatus.STREAMING:
|
|
107
|
+
this.emit("streaming");
|
|
108
|
+
break;
|
|
53
109
|
case TaskStatus.COMPLETED:
|
|
54
110
|
this.emit("complete");
|
|
55
111
|
break;
|
|
@@ -161,7 +217,7 @@ class DataflowArrow extends Dataflow {
|
|
|
161
217
|
}
|
|
162
218
|
}
|
|
163
219
|
// src/task-graph/TaskGraph.ts
|
|
164
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
220
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid44 } from "@workglow/util";
|
|
165
221
|
|
|
166
222
|
// src/task/GraphAsTask.ts
|
|
167
223
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -204,6 +260,73 @@ class TaskOutputRepository {
|
|
|
204
260
|
}
|
|
205
261
|
}
|
|
206
262
|
|
|
263
|
+
// src/task/ConditionUtils.ts
|
|
264
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
265
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
266
|
+
switch (operator) {
|
|
267
|
+
case "is_empty":
|
|
268
|
+
return true;
|
|
269
|
+
case "is_not_empty":
|
|
270
|
+
return false;
|
|
271
|
+
case "is_true":
|
|
272
|
+
return false;
|
|
273
|
+
case "is_false":
|
|
274
|
+
return true;
|
|
275
|
+
default:
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
const strValue = String(fieldValue);
|
|
280
|
+
const numValue = Number(fieldValue);
|
|
281
|
+
switch (operator) {
|
|
282
|
+
case "equals":
|
|
283
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
284
|
+
return numValue === Number(compareValue);
|
|
285
|
+
}
|
|
286
|
+
return strValue === compareValue;
|
|
287
|
+
case "not_equals":
|
|
288
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
289
|
+
return numValue !== Number(compareValue);
|
|
290
|
+
}
|
|
291
|
+
return strValue !== compareValue;
|
|
292
|
+
case "greater_than":
|
|
293
|
+
return numValue > Number(compareValue);
|
|
294
|
+
case "greater_or_equal":
|
|
295
|
+
return numValue >= Number(compareValue);
|
|
296
|
+
case "less_than":
|
|
297
|
+
return numValue < Number(compareValue);
|
|
298
|
+
case "less_or_equal":
|
|
299
|
+
return numValue <= Number(compareValue);
|
|
300
|
+
case "contains":
|
|
301
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
302
|
+
case "starts_with":
|
|
303
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
304
|
+
case "ends_with":
|
|
305
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
306
|
+
case "is_empty":
|
|
307
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
308
|
+
case "is_not_empty":
|
|
309
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
310
|
+
case "is_true":
|
|
311
|
+
return Boolean(fieldValue) === true;
|
|
312
|
+
case "is_false":
|
|
313
|
+
return Boolean(fieldValue) === false;
|
|
314
|
+
default:
|
|
315
|
+
return false;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function getNestedValue(obj, path) {
|
|
319
|
+
const parts = path.split(".");
|
|
320
|
+
let current = obj;
|
|
321
|
+
for (const part of parts) {
|
|
322
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
current = current[part];
|
|
326
|
+
}
|
|
327
|
+
return current;
|
|
328
|
+
}
|
|
329
|
+
|
|
207
330
|
// src/task/Task.ts
|
|
208
331
|
import {
|
|
209
332
|
compileSchema,
|
|
@@ -330,6 +453,49 @@ async function resolveSchemaInputs(input, schema, config) {
|
|
|
330
453
|
return resolved;
|
|
331
454
|
}
|
|
332
455
|
|
|
456
|
+
// src/task/StreamTypes.ts
|
|
457
|
+
function getPortStreamMode(schema, portId) {
|
|
458
|
+
if (typeof schema === "boolean")
|
|
459
|
+
return "none";
|
|
460
|
+
const prop = schema.properties?.[portId];
|
|
461
|
+
if (!prop || typeof prop === "boolean")
|
|
462
|
+
return "none";
|
|
463
|
+
const xStream = prop["x-stream"];
|
|
464
|
+
if (xStream === "append" || xStream === "replace")
|
|
465
|
+
return xStream;
|
|
466
|
+
return "none";
|
|
467
|
+
}
|
|
468
|
+
function getOutputStreamMode(outputSchema) {
|
|
469
|
+
if (typeof outputSchema === "boolean")
|
|
470
|
+
return "none";
|
|
471
|
+
const props = outputSchema.properties;
|
|
472
|
+
if (!props)
|
|
473
|
+
return "none";
|
|
474
|
+
let found = "none";
|
|
475
|
+
for (const prop of Object.values(props)) {
|
|
476
|
+
if (!prop || typeof prop === "boolean")
|
|
477
|
+
continue;
|
|
478
|
+
const xStream = prop["x-stream"];
|
|
479
|
+
if (xStream === "append")
|
|
480
|
+
return "append";
|
|
481
|
+
if (xStream === "replace")
|
|
482
|
+
found = "replace";
|
|
483
|
+
}
|
|
484
|
+
return found;
|
|
485
|
+
}
|
|
486
|
+
function isTaskStreamable(task) {
|
|
487
|
+
if (typeof task.executeStream !== "function")
|
|
488
|
+
return false;
|
|
489
|
+
return getOutputStreamMode(task.outputSchema()) !== "none";
|
|
490
|
+
}
|
|
491
|
+
function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPort) {
|
|
492
|
+
const sourceMode = getPortStreamMode(sourceSchema, sourcePort);
|
|
493
|
+
if (sourceMode === "none")
|
|
494
|
+
return false;
|
|
495
|
+
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
496
|
+
return sourceMode !== targetMode;
|
|
497
|
+
}
|
|
498
|
+
|
|
333
499
|
// src/task/TaskRunner.ts
|
|
334
500
|
class TaskRunner {
|
|
335
501
|
running = false;
|
|
@@ -359,14 +525,27 @@ class TaskRunner {
|
|
|
359
525
|
}
|
|
360
526
|
const inputs = this.task.runInputData;
|
|
361
527
|
let outputs;
|
|
528
|
+
const isStreamable = isTaskStreamable(this.task);
|
|
362
529
|
if (this.task.cacheable) {
|
|
363
530
|
outputs = await this.outputCache?.getOutput(this.task.type, inputs);
|
|
364
531
|
if (outputs) {
|
|
365
|
-
|
|
532
|
+
if (isStreamable) {
|
|
533
|
+
this.task.runOutputData = outputs;
|
|
534
|
+
this.task.emit("stream_start");
|
|
535
|
+
this.task.emit("stream_chunk", { type: "finish", data: outputs });
|
|
536
|
+
this.task.emit("stream_end", outputs);
|
|
537
|
+
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
538
|
+
} else {
|
|
539
|
+
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
540
|
+
}
|
|
366
541
|
}
|
|
367
542
|
}
|
|
368
543
|
if (!outputs) {
|
|
369
|
-
|
|
544
|
+
if (isStreamable) {
|
|
545
|
+
outputs = await this.executeStreamingTask(inputs);
|
|
546
|
+
} else {
|
|
547
|
+
outputs = await this.executeTask(inputs);
|
|
548
|
+
}
|
|
370
549
|
if (this.task.cacheable && outputs !== undefined) {
|
|
371
550
|
await this.outputCache?.saveOutput(this.task.type, inputs, outputs);
|
|
372
551
|
}
|
|
@@ -425,6 +604,64 @@ class TaskRunner {
|
|
|
425
604
|
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
426
605
|
return Object.assign({}, output, reactiveResult ?? {});
|
|
427
606
|
}
|
|
607
|
+
async executeStreamingTask(input) {
|
|
608
|
+
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
609
|
+
let accumulated = "";
|
|
610
|
+
let chunkCount = 0;
|
|
611
|
+
let finalOutput;
|
|
612
|
+
this.task.emit("stream_start");
|
|
613
|
+
const stream = this.task.executeStream(input, {
|
|
614
|
+
signal: this.abortController.signal,
|
|
615
|
+
updateProgress: this.handleProgress.bind(this),
|
|
616
|
+
own: this.own,
|
|
617
|
+
registry: this.registry
|
|
618
|
+
});
|
|
619
|
+
for await (const event of stream) {
|
|
620
|
+
chunkCount++;
|
|
621
|
+
if (chunkCount === 1) {
|
|
622
|
+
this.task.status = TaskStatus.STREAMING;
|
|
623
|
+
this.task.emit("status", this.task.status);
|
|
624
|
+
}
|
|
625
|
+
if (event.type === "snapshot") {
|
|
626
|
+
this.task.runOutputData = event.data;
|
|
627
|
+
}
|
|
628
|
+
this.task.emit("stream_chunk", event);
|
|
629
|
+
switch (event.type) {
|
|
630
|
+
case "text-delta": {
|
|
631
|
+
accumulated += event.textDelta;
|
|
632
|
+
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.02 * chunkCount))));
|
|
633
|
+
await this.handleProgress(progress);
|
|
634
|
+
break;
|
|
635
|
+
}
|
|
636
|
+
case "snapshot": {
|
|
637
|
+
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.02 * chunkCount))));
|
|
638
|
+
await this.handleProgress(progress);
|
|
639
|
+
break;
|
|
640
|
+
}
|
|
641
|
+
case "finish": {
|
|
642
|
+
if (streamMode === "append") {
|
|
643
|
+
const text = accumulated.length > 0 ? accumulated : event.data?.text ?? "";
|
|
644
|
+
finalOutput = { ...event.data || {}, text };
|
|
645
|
+
} else if (streamMode === "replace") {
|
|
646
|
+
finalOutput = event.data;
|
|
647
|
+
}
|
|
648
|
+
break;
|
|
649
|
+
}
|
|
650
|
+
case "error": {
|
|
651
|
+
throw event.error;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
if (this.abortController?.signal.aborted) {
|
|
656
|
+
throw new TaskAbortedError("Task aborted during streaming");
|
|
657
|
+
}
|
|
658
|
+
if (finalOutput !== undefined) {
|
|
659
|
+
this.task.runOutputData = finalOutput;
|
|
660
|
+
}
|
|
661
|
+
this.task.emit("stream_end", this.task.runOutputData);
|
|
662
|
+
const reactiveResult = await this.executeTaskReactive(input, this.task.runOutputData || {});
|
|
663
|
+
return reactiveResult;
|
|
664
|
+
}
|
|
428
665
|
async handleStart(config = {}) {
|
|
429
666
|
if (this.task.status === TaskStatus.PROCESSING)
|
|
430
667
|
return;
|
|
@@ -543,13 +780,13 @@ class Task {
|
|
|
543
780
|
additionalProperties: false
|
|
544
781
|
};
|
|
545
782
|
}
|
|
546
|
-
async execute(
|
|
783
|
+
async execute(_input, context) {
|
|
547
784
|
if (context.signal?.aborted) {
|
|
548
785
|
throw new TaskAbortedError("Task aborted");
|
|
549
786
|
}
|
|
550
787
|
return;
|
|
551
788
|
}
|
|
552
|
-
async executeReactive(
|
|
789
|
+
async executeReactive(_input, output, _context) {
|
|
553
790
|
return output;
|
|
554
791
|
}
|
|
555
792
|
_runner;
|
|
@@ -911,18 +1148,62 @@ class Task {
|
|
|
911
1148
|
// src/task/ConditionalTask.ts
|
|
912
1149
|
class ConditionalTask extends Task {
|
|
913
1150
|
static type = "ConditionalTask";
|
|
914
|
-
static category = "
|
|
915
|
-
static title = "
|
|
916
|
-
static description = "
|
|
1151
|
+
static category = "Flow Control";
|
|
1152
|
+
static title = "Condition";
|
|
1153
|
+
static description = "Route data based on conditions";
|
|
917
1154
|
static hasDynamicSchemas = true;
|
|
918
1155
|
activeBranches = new Set;
|
|
1156
|
+
buildBranchesFromConditionConfig(conditionConfig) {
|
|
1157
|
+
if (!conditionConfig?.branches || conditionConfig.branches.length === 0) {
|
|
1158
|
+
return [
|
|
1159
|
+
{
|
|
1160
|
+
id: "default",
|
|
1161
|
+
condition: () => true,
|
|
1162
|
+
outputPort: "1"
|
|
1163
|
+
}
|
|
1164
|
+
];
|
|
1165
|
+
}
|
|
1166
|
+
return conditionConfig.branches.map((branch, index) => ({
|
|
1167
|
+
id: branch.id,
|
|
1168
|
+
outputPort: String(index + 1),
|
|
1169
|
+
condition: (inputData) => {
|
|
1170
|
+
const fieldValue = getNestedValue(inputData, branch.field);
|
|
1171
|
+
return evaluateCondition(fieldValue, branch.operator, branch.value);
|
|
1172
|
+
}
|
|
1173
|
+
}));
|
|
1174
|
+
}
|
|
1175
|
+
resolveBranches(input) {
|
|
1176
|
+
const configBranches = this.config.branches ?? [];
|
|
1177
|
+
if (configBranches.length > 0 && typeof configBranches[0].condition === "function") {
|
|
1178
|
+
return {
|
|
1179
|
+
branches: configBranches,
|
|
1180
|
+
isExclusive: this.config.exclusive ?? true,
|
|
1181
|
+
defaultBranch: this.config.defaultBranch,
|
|
1182
|
+
fromConditionConfig: false
|
|
1183
|
+
};
|
|
1184
|
+
}
|
|
1185
|
+
const conditionConfig = input.conditionConfig ?? this.config.extras?.conditionConfig;
|
|
1186
|
+
if (conditionConfig) {
|
|
1187
|
+
return {
|
|
1188
|
+
branches: this.buildBranchesFromConditionConfig(conditionConfig),
|
|
1189
|
+
isExclusive: conditionConfig.exclusive ?? true,
|
|
1190
|
+
defaultBranch: conditionConfig.defaultBranch,
|
|
1191
|
+
fromConditionConfig: true
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
return {
|
|
1195
|
+
branches: configBranches,
|
|
1196
|
+
isExclusive: this.config.exclusive ?? true,
|
|
1197
|
+
defaultBranch: this.config.defaultBranch,
|
|
1198
|
+
fromConditionConfig: false
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
919
1201
|
async execute(input, context) {
|
|
920
1202
|
if (context.signal?.aborted) {
|
|
921
1203
|
return;
|
|
922
1204
|
}
|
|
923
1205
|
this.activeBranches.clear();
|
|
924
|
-
const branches = this.
|
|
925
|
-
const isExclusive = this.config.exclusive ?? true;
|
|
1206
|
+
const { branches, isExclusive, defaultBranch, fromConditionConfig } = this.resolveBranches(input);
|
|
926
1207
|
for (const branch of branches) {
|
|
927
1208
|
try {
|
|
928
1209
|
const isActive = branch.condition(input);
|
|
@@ -936,14 +1217,50 @@ class ConditionalTask extends Task {
|
|
|
936
1217
|
console.warn(`Condition evaluation failed for branch "${branch.id}":`, error);
|
|
937
1218
|
}
|
|
938
1219
|
}
|
|
939
|
-
if (this.activeBranches.size === 0 &&
|
|
940
|
-
const defaultBranchExists = branches.some((b) => b.id ===
|
|
1220
|
+
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
1221
|
+
const defaultBranchExists = branches.some((b) => b.id === defaultBranch);
|
|
941
1222
|
if (defaultBranchExists) {
|
|
942
|
-
this.activeBranches.add(
|
|
1223
|
+
this.activeBranches.add(defaultBranch);
|
|
943
1224
|
}
|
|
944
1225
|
}
|
|
1226
|
+
if (fromConditionConfig) {
|
|
1227
|
+
return this.buildConditionConfigOutput(input, branches, isExclusive);
|
|
1228
|
+
}
|
|
945
1229
|
return this.buildOutput(input);
|
|
946
1230
|
}
|
|
1231
|
+
buildConditionConfigOutput(input, branches, isExclusive) {
|
|
1232
|
+
const output = {};
|
|
1233
|
+
const { conditionConfig, ...passThrough } = input;
|
|
1234
|
+
const inputKeys = Object.keys(passThrough);
|
|
1235
|
+
let matchedBranchNumber = null;
|
|
1236
|
+
for (let i = 0;i < branches.length; i++) {
|
|
1237
|
+
if (this.activeBranches.has(branches[i].id)) {
|
|
1238
|
+
if (matchedBranchNumber === null) {
|
|
1239
|
+
matchedBranchNumber = i + 1;
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
if (isExclusive) {
|
|
1244
|
+
if (matchedBranchNumber !== null) {
|
|
1245
|
+
for (const key of inputKeys) {
|
|
1246
|
+
output[`${key}_${matchedBranchNumber}`] = passThrough[key];
|
|
1247
|
+
}
|
|
1248
|
+
} else {
|
|
1249
|
+
for (const key of inputKeys) {
|
|
1250
|
+
output[`${key}_else`] = passThrough[key];
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
} else {
|
|
1254
|
+
for (let i = 0;i < branches.length; i++) {
|
|
1255
|
+
if (this.activeBranches.has(branches[i].id)) {
|
|
1256
|
+
for (const key of inputKeys) {
|
|
1257
|
+
output[`${key}_${i + 1}`] = passThrough[key];
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
return output;
|
|
1263
|
+
}
|
|
947
1264
|
buildOutput(input) {
|
|
948
1265
|
const output = {
|
|
949
1266
|
_activeBranches: Array.from(this.activeBranches)
|
|
@@ -1038,6 +1355,7 @@ class TopologicalScheduler {
|
|
|
1038
1355
|
}
|
|
1039
1356
|
}
|
|
1040
1357
|
onTaskCompleted(taskId) {}
|
|
1358
|
+
onTaskStreaming(taskId) {}
|
|
1041
1359
|
reset() {
|
|
1042
1360
|
this.sortedNodes = this.dag.topologicallySortedNodes();
|
|
1043
1361
|
this.currentIndex = 0;
|
|
@@ -1047,11 +1365,13 @@ class TopologicalScheduler {
|
|
|
1047
1365
|
class DependencyBasedScheduler {
|
|
1048
1366
|
dag;
|
|
1049
1367
|
completedTasks;
|
|
1368
|
+
streamingTasks;
|
|
1050
1369
|
pendingTasks;
|
|
1051
1370
|
nextResolver = null;
|
|
1052
1371
|
constructor(dag) {
|
|
1053
1372
|
this.dag = dag;
|
|
1054
1373
|
this.completedTasks = new Set;
|
|
1374
|
+
this.streamingTasks = new Set;
|
|
1055
1375
|
this.pendingTasks = new Set;
|
|
1056
1376
|
this.reset();
|
|
1057
1377
|
}
|
|
@@ -1066,8 +1386,23 @@ class DependencyBasedScheduler {
|
|
|
1066
1386
|
return false;
|
|
1067
1387
|
}
|
|
1068
1388
|
}
|
|
1069
|
-
const
|
|
1070
|
-
return
|
|
1389
|
+
const activeDataflows = sourceDataflows.filter((df) => df.status !== TaskStatus.DISABLED);
|
|
1390
|
+
return activeDataflows.every((df) => {
|
|
1391
|
+
const depId = df.sourceTaskId;
|
|
1392
|
+
if (this.completedTasks.has(depId))
|
|
1393
|
+
return true;
|
|
1394
|
+
if (this.streamingTasks.has(depId)) {
|
|
1395
|
+
const sourceTask = this.dag.getTask(depId);
|
|
1396
|
+
if (sourceTask) {
|
|
1397
|
+
const sourceMode = getPortStreamMode(sourceTask.outputSchema(), df.sourceTaskPortId);
|
|
1398
|
+
const targetMode = getPortStreamMode(task.inputSchema(), df.targetTaskPortId);
|
|
1399
|
+
if (sourceMode !== "none" && sourceMode === targetMode) {
|
|
1400
|
+
return true;
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
return false;
|
|
1405
|
+
});
|
|
1071
1406
|
}
|
|
1072
1407
|
async waitForNextTask() {
|
|
1073
1408
|
if (this.pendingTasks.size === 0)
|
|
@@ -1122,8 +1457,26 @@ class DependencyBasedScheduler {
|
|
|
1122
1457
|
}
|
|
1123
1458
|
}
|
|
1124
1459
|
}
|
|
1460
|
+
onTaskStreaming(taskId) {
|
|
1461
|
+
this.streamingTasks.add(taskId);
|
|
1462
|
+
for (const task of Array.from(this.pendingTasks)) {
|
|
1463
|
+
if (task.status === TaskStatus.DISABLED) {
|
|
1464
|
+
this.pendingTasks.delete(task);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
if (this.nextResolver) {
|
|
1468
|
+
const readyTask = Array.from(this.pendingTasks).find((task) => this.isTaskReady(task));
|
|
1469
|
+
if (readyTask) {
|
|
1470
|
+
this.pendingTasks.delete(readyTask);
|
|
1471
|
+
const resolver = this.nextResolver;
|
|
1472
|
+
this.nextResolver = null;
|
|
1473
|
+
resolver(readyTask);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1125
1477
|
reset() {
|
|
1126
1478
|
this.completedTasks.clear();
|
|
1479
|
+
this.streamingTasks.clear();
|
|
1127
1480
|
this.pendingTasks = new Set(this.dag.topologicallySortedNodes());
|
|
1128
1481
|
this.nextResolver = null;
|
|
1129
1482
|
}
|
|
@@ -1360,7 +1713,12 @@ class TaskGraphRunner {
|
|
|
1360
1713
|
}
|
|
1361
1714
|
}
|
|
1362
1715
|
async runTask(task, input) {
|
|
1716
|
+
const isStreamable = isTaskStreamable(task);
|
|
1717
|
+
await this.awaitStreamInputs(task);
|
|
1363
1718
|
this.copyInputFromEdgesToNode(task);
|
|
1719
|
+
if (isStreamable) {
|
|
1720
|
+
return this.runStreamingTask(task, input);
|
|
1721
|
+
}
|
|
1364
1722
|
const results = await task.runner.run(input, {
|
|
1365
1723
|
outputCache: this.outputCache,
|
|
1366
1724
|
updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
|
|
@@ -1373,6 +1731,93 @@ class TaskGraphRunner {
|
|
|
1373
1731
|
data: results
|
|
1374
1732
|
};
|
|
1375
1733
|
}
|
|
1734
|
+
async awaitStreamInputs(task) {
|
|
1735
|
+
const dataflows = this.graph.getSourceDataflows(task.config.id);
|
|
1736
|
+
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1737
|
+
if (streamPromises.length > 0) {
|
|
1738
|
+
await Promise.all(streamPromises);
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
async runStreamingTask(task, input) {
|
|
1742
|
+
const streamMode = getOutputStreamMode(task.outputSchema());
|
|
1743
|
+
let streamingNotified = false;
|
|
1744
|
+
const onStatus = (status) => {
|
|
1745
|
+
if (status === TaskStatus.STREAMING && !streamingNotified) {
|
|
1746
|
+
streamingNotified = true;
|
|
1747
|
+
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1748
|
+
this.pushStreamToEdges(task, streamMode);
|
|
1749
|
+
this.processScheduler.onTaskStreaming(task.config.id);
|
|
1750
|
+
}
|
|
1751
|
+
};
|
|
1752
|
+
const onStreamStart = () => {
|
|
1753
|
+
this.graph.emit("task_stream_start", task.config.id);
|
|
1754
|
+
};
|
|
1755
|
+
const onStreamChunk = (event) => {
|
|
1756
|
+
this.graph.emit("task_stream_chunk", task.config.id, event);
|
|
1757
|
+
};
|
|
1758
|
+
const onStreamEnd = (output) => {
|
|
1759
|
+
this.graph.emit("task_stream_end", task.config.id, output);
|
|
1760
|
+
};
|
|
1761
|
+
task.on("status", onStatus);
|
|
1762
|
+
task.on("stream_start", onStreamStart);
|
|
1763
|
+
task.on("stream_chunk", onStreamChunk);
|
|
1764
|
+
task.on("stream_end", onStreamEnd);
|
|
1765
|
+
try {
|
|
1766
|
+
const results = await task.runner.run(input, {
|
|
1767
|
+
outputCache: this.outputCache,
|
|
1768
|
+
updateProgress: async (task2, progress, message, ...args) => await this.handleProgress(task2, progress, message, ...args),
|
|
1769
|
+
registry: this.registry
|
|
1770
|
+
});
|
|
1771
|
+
await this.pushOutputFromNodeToEdges(task, results);
|
|
1772
|
+
return {
|
|
1773
|
+
id: task.config.id,
|
|
1774
|
+
type: task.constructor.runtype || task.constructor.type,
|
|
1775
|
+
data: results
|
|
1776
|
+
};
|
|
1777
|
+
} finally {
|
|
1778
|
+
task.off("status", onStatus);
|
|
1779
|
+
task.off("stream_start", onStreamStart);
|
|
1780
|
+
task.off("stream_chunk", onStreamChunk);
|
|
1781
|
+
task.off("stream_end", onStreamEnd);
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
pushStreamToEdges(task, streamMode) {
|
|
1785
|
+
const targetDataflows = this.graph.getTargetDataflows(task.config.id);
|
|
1786
|
+
if (targetDataflows.length === 0)
|
|
1787
|
+
return;
|
|
1788
|
+
const stream = new ReadableStream({
|
|
1789
|
+
start: (controller) => {
|
|
1790
|
+
const onChunk = (event) => {
|
|
1791
|
+
try {
|
|
1792
|
+
controller.enqueue(event);
|
|
1793
|
+
} catch {}
|
|
1794
|
+
};
|
|
1795
|
+
const onEnd = () => {
|
|
1796
|
+
try {
|
|
1797
|
+
controller.close();
|
|
1798
|
+
} catch {}
|
|
1799
|
+
task.off("stream_chunk", onChunk);
|
|
1800
|
+
task.off("stream_end", onEnd);
|
|
1801
|
+
};
|
|
1802
|
+
task.on("stream_chunk", onChunk);
|
|
1803
|
+
task.on("stream_end", onEnd);
|
|
1804
|
+
}
|
|
1805
|
+
});
|
|
1806
|
+
if (targetDataflows.length === 1) {
|
|
1807
|
+
targetDataflows[0].setStream(stream);
|
|
1808
|
+
} else {
|
|
1809
|
+
let currentStream = stream;
|
|
1810
|
+
for (let i = 0;i < targetDataflows.length; i++) {
|
|
1811
|
+
if (i === targetDataflows.length - 1) {
|
|
1812
|
+
targetDataflows[i].setStream(currentStream);
|
|
1813
|
+
} else {
|
|
1814
|
+
const [s1, s2] = currentStream.tee();
|
|
1815
|
+
targetDataflows[i].setStream(s1);
|
|
1816
|
+
currentStream = s2;
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1376
1821
|
resetTask(graph, task, runId) {
|
|
1377
1822
|
task.status = TaskStatus.PENDING;
|
|
1378
1823
|
task.resetInputData();
|
|
@@ -1456,7 +1901,7 @@ class TaskGraphRunner {
|
|
|
1456
1901
|
}
|
|
1457
1902
|
async handleError(error) {
|
|
1458
1903
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
1459
|
-
if (task.status === TaskStatus.PROCESSING) {
|
|
1904
|
+
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
1460
1905
|
task.abort();
|
|
1461
1906
|
}
|
|
1462
1907
|
}));
|
|
@@ -1468,7 +1913,7 @@ class TaskGraphRunner {
|
|
|
1468
1913
|
}
|
|
1469
1914
|
async handleAbort() {
|
|
1470
1915
|
this.graph.getTasks().map(async (task) => {
|
|
1471
|
-
if (task.status === TaskStatus.PROCESSING) {
|
|
1916
|
+
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
1472
1917
|
task.abort();
|
|
1473
1918
|
}
|
|
1474
1919
|
});
|
|
@@ -1546,9 +1991,9 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
1546
1991
|
// src/task/GraphAsTask.ts
|
|
1547
1992
|
class GraphAsTask extends Task {
|
|
1548
1993
|
static type = "GraphAsTask";
|
|
1549
|
-
static title = "
|
|
1550
|
-
static description = "A
|
|
1551
|
-
static category = "
|
|
1994
|
+
static title = "Group";
|
|
1995
|
+
static description = "A group of tasks that are executed together";
|
|
1996
|
+
static category = "Flow Control";
|
|
1552
1997
|
static compoundMerge = PROPERTY_ARRAY;
|
|
1553
1998
|
static hasDynamicSchemas = true;
|
|
1554
1999
|
constructor(input = {}, config = {}) {
|
|
@@ -1728,38 +2173,112 @@ class GraphAsTask extends Task {
|
|
|
1728
2173
|
}
|
|
1729
2174
|
|
|
1730
2175
|
// src/task-graph/Workflow.ts
|
|
1731
|
-
import { EventEmitter as EventEmitter4 } from "@workglow/util";
|
|
2176
|
+
import { EventEmitter as EventEmitter4, uuid4 as uuid43 } from "@workglow/util";
|
|
2177
|
+
function CreateWorkflow(taskClass) {
|
|
2178
|
+
return Workflow.createWorkflow(taskClass);
|
|
2179
|
+
}
|
|
2180
|
+
function CreateLoopWorkflow(taskClass) {
|
|
2181
|
+
return function(config = {}) {
|
|
2182
|
+
return this.addLoopTask(taskClass, config);
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
function CreateEndLoopWorkflow(methodName) {
|
|
2186
|
+
return function() {
|
|
2187
|
+
if (!this.isLoopBuilder) {
|
|
2188
|
+
throw new Error(`${methodName}() can only be called on loop workflows`);
|
|
2189
|
+
}
|
|
2190
|
+
return this.finalizeAndReturn();
|
|
2191
|
+
};
|
|
2192
|
+
}
|
|
2193
|
+
var TYPED_ARRAY_FORMAT_PREFIX = "TypedArray";
|
|
2194
|
+
function schemaHasTypedArrayFormat(schema) {
|
|
2195
|
+
if (typeof schema === "boolean")
|
|
2196
|
+
return false;
|
|
2197
|
+
if (!schema || typeof schema !== "object" || Array.isArray(schema))
|
|
2198
|
+
return false;
|
|
2199
|
+
const s = schema;
|
|
2200
|
+
if (typeof s.format === "string" && s.format.startsWith(TYPED_ARRAY_FORMAT_PREFIX)) {
|
|
2201
|
+
return true;
|
|
2202
|
+
}
|
|
2203
|
+
const checkUnion = (schemas) => {
|
|
2204
|
+
if (!Array.isArray(schemas))
|
|
2205
|
+
return false;
|
|
2206
|
+
return schemas.some((sub) => schemaHasTypedArrayFormat(sub));
|
|
2207
|
+
};
|
|
2208
|
+
if (checkUnion(s.oneOf) || checkUnion(s.anyOf))
|
|
2209
|
+
return true;
|
|
2210
|
+
const items = s.items;
|
|
2211
|
+
if (items && typeof items === "object" && !Array.isArray(items)) {
|
|
2212
|
+
if (schemaHasTypedArrayFormat(items))
|
|
2213
|
+
return true;
|
|
2214
|
+
}
|
|
2215
|
+
return false;
|
|
2216
|
+
}
|
|
2217
|
+
function hasVectorOutput(task) {
|
|
2218
|
+
const outputSchema = task.outputSchema();
|
|
2219
|
+
if (typeof outputSchema === "boolean" || !outputSchema?.properties)
|
|
2220
|
+
return false;
|
|
2221
|
+
return Object.values(outputSchema.properties).some((prop) => schemaHasTypedArrayFormat(prop));
|
|
2222
|
+
}
|
|
2223
|
+
function hasVectorLikeInput(input) {
|
|
2224
|
+
if (!input || typeof input !== "object")
|
|
2225
|
+
return false;
|
|
2226
|
+
const v = input.vectors;
|
|
2227
|
+
return Array.isArray(v) && v.length > 0 && typeof v[0] === "object" && v[0] !== null && ArrayBuffer.isView(v[0]);
|
|
2228
|
+
}
|
|
2229
|
+
function CreateAdaptiveWorkflow(scalarClass, vectorClass) {
|
|
2230
|
+
const scalarHelper = Workflow.createWorkflow(scalarClass);
|
|
2231
|
+
const vectorHelper = Workflow.createWorkflow(vectorClass);
|
|
2232
|
+
return function(input = {}, config = {}) {
|
|
2233
|
+
const parent = getLastTask(this);
|
|
2234
|
+
const useVector = parent !== undefined && hasVectorOutput(parent) || hasVectorLikeInput(input);
|
|
2235
|
+
if (useVector) {
|
|
2236
|
+
return vectorHelper.call(this, input, config);
|
|
2237
|
+
}
|
|
2238
|
+
return scalarHelper.call(this, input, config);
|
|
2239
|
+
};
|
|
2240
|
+
}
|
|
2241
|
+
|
|
1732
2242
|
class WorkflowTask extends GraphAsTask {
|
|
1733
2243
|
static type = "Workflow";
|
|
1734
2244
|
static compoundMerge = PROPERTY_ARRAY;
|
|
1735
2245
|
}
|
|
1736
|
-
var taskIdCounter = 0;
|
|
1737
2246
|
|
|
1738
2247
|
class Workflow {
|
|
1739
|
-
constructor(
|
|
1740
|
-
this.
|
|
1741
|
-
this.
|
|
1742
|
-
|
|
1743
|
-
});
|
|
1744
|
-
|
|
1745
|
-
|
|
2248
|
+
constructor(cache, parent, iteratorTask) {
|
|
2249
|
+
this._outputCache = cache;
|
|
2250
|
+
this._parentWorkflow = parent;
|
|
2251
|
+
this._iteratorTask = iteratorTask;
|
|
2252
|
+
this._graph = new TaskGraph({ outputCache: this._outputCache });
|
|
2253
|
+
if (!parent) {
|
|
2254
|
+
this._onChanged = this._onChanged.bind(this);
|
|
2255
|
+
this.setupEvents();
|
|
2256
|
+
}
|
|
1746
2257
|
}
|
|
1747
2258
|
_graph;
|
|
1748
2259
|
_dataFlows = [];
|
|
1749
2260
|
_error = "";
|
|
1750
|
-
|
|
2261
|
+
_outputCache;
|
|
1751
2262
|
_abortController;
|
|
2263
|
+
_parentWorkflow;
|
|
2264
|
+
_iteratorTask;
|
|
2265
|
+
_pendingLoopConnect;
|
|
2266
|
+
outputCache() {
|
|
2267
|
+
return this._outputCache;
|
|
2268
|
+
}
|
|
2269
|
+
get isLoopBuilder() {
|
|
2270
|
+
return this._parentWorkflow !== undefined;
|
|
2271
|
+
}
|
|
1752
2272
|
events = new EventEmitter4;
|
|
1753
2273
|
static createWorkflow(taskClass) {
|
|
1754
2274
|
const helper = function(input = {}, config = {}) {
|
|
1755
2275
|
this._error = "";
|
|
1756
2276
|
const parent = getLastTask(this);
|
|
1757
|
-
|
|
1758
|
-
const task = this.addTask(taskClass, input, { id: String(taskIdCounter), ...config });
|
|
2277
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid43(), ...config });
|
|
1759
2278
|
if (this._dataFlows.length > 0) {
|
|
1760
2279
|
this._dataFlows.forEach((dataflow) => {
|
|
1761
2280
|
const taskSchema = task.inputSchema();
|
|
1762
|
-
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2281
|
+
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
1763
2282
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.config.id}`;
|
|
1764
2283
|
console.error(this._error);
|
|
1765
2284
|
return;
|
|
@@ -1770,158 +2289,23 @@ class Workflow {
|
|
|
1770
2289
|
this._dataFlows = [];
|
|
1771
2290
|
}
|
|
1772
2291
|
if (parent && this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
1773
|
-
const
|
|
1774
|
-
const
|
|
1775
|
-
const
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
if (targetSchema === true || typeof targetSchema === "object" && targetSchema.additionalProperties === true) {
|
|
1779
|
-
for (const fromOutputPortId of Object.keys(sourceSchema.properties || {})) {
|
|
1780
|
-
matches.set(fromOutputPortId, fromOutputPortId);
|
|
1781
|
-
this.connect(parent.config.id, fromOutputPortId, task.config.id, fromOutputPortId);
|
|
1782
|
-
}
|
|
1783
|
-
return matches;
|
|
1784
|
-
}
|
|
1785
|
-
}
|
|
1786
|
-
if (typeof sourceSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1787
|
-
return matches;
|
|
1788
|
-
}
|
|
1789
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(sourceSchema.properties || {})) {
|
|
1790
|
-
for (const [toInputPortId, toPortInputSchema] of Object.entries(targetSchema.properties || {})) {
|
|
1791
|
-
if (!matches.has(toInputPortId) && comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
1792
|
-
matches.set(toInputPortId, fromOutputPortId);
|
|
1793
|
-
this.connect(parent.config.id, fromOutputPortId, task.config.id, toInputPortId);
|
|
1794
|
-
}
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
return matches;
|
|
1798
|
-
};
|
|
1799
|
-
const getSpecificTypeIdentifiers = (schema) => {
|
|
1800
|
-
const formats = new Set;
|
|
1801
|
-
const ids = new Set;
|
|
1802
|
-
if (typeof schema === "boolean") {
|
|
1803
|
-
return { formats, ids };
|
|
1804
|
-
}
|
|
1805
|
-
const extractFromSchema = (s) => {
|
|
1806
|
-
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
1807
|
-
return;
|
|
1808
|
-
if (s.format)
|
|
1809
|
-
formats.add(s.format);
|
|
1810
|
-
if (s.$id)
|
|
1811
|
-
ids.add(s.$id);
|
|
1812
|
-
};
|
|
1813
|
-
extractFromSchema(schema);
|
|
1814
|
-
const checkUnion = (schemas) => {
|
|
1815
|
-
if (!schemas)
|
|
1816
|
-
return;
|
|
1817
|
-
for (const s of schemas) {
|
|
1818
|
-
if (typeof s === "boolean")
|
|
1819
|
-
continue;
|
|
1820
|
-
extractFromSchema(s);
|
|
1821
|
-
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
1822
|
-
extractFromSchema(s.items);
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
};
|
|
1826
|
-
checkUnion(schema.oneOf);
|
|
1827
|
-
checkUnion(schema.anyOf);
|
|
1828
|
-
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
1829
|
-
extractFromSchema(schema.items);
|
|
1830
|
-
}
|
|
1831
|
-
return { formats, ids };
|
|
1832
|
-
};
|
|
1833
|
-
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
1834
|
-
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
1835
|
-
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
1836
|
-
}
|
|
1837
|
-
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
1838
|
-
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
1839
|
-
for (const format of outputIds.formats) {
|
|
1840
|
-
if (inputIds.formats.has(format)) {
|
|
1841
|
-
return true;
|
|
1842
|
-
}
|
|
1843
|
-
}
|
|
1844
|
-
for (const id of outputIds.ids) {
|
|
1845
|
-
if (inputIds.ids.has(id)) {
|
|
1846
|
-
return true;
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
|
-
if (requireSpecificType) {
|
|
1850
|
-
return false;
|
|
1851
|
-
}
|
|
1852
|
-
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
1853
|
-
if (!idTypeBlank)
|
|
1854
|
-
return false;
|
|
1855
|
-
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
1856
|
-
return true;
|
|
1857
|
-
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
1858
|
-
if (typeof schema === "boolean")
|
|
1859
|
-
return schema;
|
|
1860
|
-
return schema.type === fromPortOutputSchema.type;
|
|
1861
|
-
}) ?? false;
|
|
1862
|
-
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
1863
|
-
if (typeof schema === "boolean")
|
|
1864
|
-
return schema;
|
|
1865
|
-
return schema.type === fromPortOutputSchema.type;
|
|
1866
|
-
}) ?? false;
|
|
1867
|
-
return matchesOneOf || matchesAnyOf;
|
|
1868
|
-
};
|
|
1869
|
-
makeMatch(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1870
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1871
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1872
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1873
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1874
|
-
});
|
|
1875
|
-
makeMatch(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1876
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1877
|
-
});
|
|
1878
|
-
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
1879
|
-
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
1880
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
1881
|
-
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
1882
|
-
if (unmatchedRequired.length > 0) {
|
|
1883
|
-
const nodes = this._graph.getTasks();
|
|
1884
|
-
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
1885
|
-
for (let i = parentIndex - 1;i >= 0 && unmatchedRequired.length > 0; i--) {
|
|
1886
|
-
const earlierTask = nodes[i];
|
|
1887
|
-
const earlierOutputSchema = earlierTask.outputSchema();
|
|
1888
|
-
const makeMatchFromEarlier = (comparator) => {
|
|
1889
|
-
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
1890
|
-
return;
|
|
1891
|
-
}
|
|
1892
|
-
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
1893
|
-
for (const requiredInputId of unmatchedRequired) {
|
|
1894
|
-
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
1895
|
-
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
1896
|
-
matches.set(requiredInputId, fromOutputPortId);
|
|
1897
|
-
this.connect(earlierTask.config.id, fromOutputPortId, task.config.id, requiredInputId);
|
|
1898
|
-
}
|
|
1899
|
-
}
|
|
1900
|
-
}
|
|
1901
|
-
};
|
|
1902
|
-
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
1903
|
-
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
1904
|
-
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
1905
|
-
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
1906
|
-
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
1907
|
-
});
|
|
1908
|
-
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
1909
|
-
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
1910
|
-
});
|
|
1911
|
-
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
1912
|
-
}
|
|
2292
|
+
const nodes = this._graph.getTasks();
|
|
2293
|
+
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
2294
|
+
const earlierTasks = [];
|
|
2295
|
+
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2296
|
+
earlierTasks.push(nodes[i]);
|
|
1913
2297
|
}
|
|
1914
|
-
const
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
this._error =
|
|
2298
|
+
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2299
|
+
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2300
|
+
providedInputKeys,
|
|
2301
|
+
earlierTasks
|
|
2302
|
+
});
|
|
2303
|
+
if (result.error) {
|
|
2304
|
+
if (this.isLoopBuilder) {
|
|
2305
|
+
this._error = result.error;
|
|
2306
|
+
console.warn(this._error);
|
|
2307
|
+
} else {
|
|
2308
|
+
this._error = result.error + " Task not added.";
|
|
1925
2309
|
console.error(this._error);
|
|
1926
2310
|
this.graph.removeTask(task.config.id);
|
|
1927
2311
|
}
|
|
@@ -1964,12 +2348,25 @@ class Workflow {
|
|
|
1964
2348
|
return this.events.waitOn(name);
|
|
1965
2349
|
}
|
|
1966
2350
|
async run(input = {}) {
|
|
2351
|
+
if (this.isLoopBuilder) {
|
|
2352
|
+
this.finalizeTemplate();
|
|
2353
|
+
if (this._pendingLoopConnect) {
|
|
2354
|
+
this._parentWorkflow.autoConnectLoopTask(this._pendingLoopConnect);
|
|
2355
|
+
this._pendingLoopConnect = undefined;
|
|
2356
|
+
}
|
|
2357
|
+
return this._parentWorkflow.run(input);
|
|
2358
|
+
}
|
|
1967
2359
|
this.events.emit("start");
|
|
1968
2360
|
this._abortController = new AbortController;
|
|
2361
|
+
const unsubStreaming = this.graph.subscribeToTaskStreaming({
|
|
2362
|
+
onStreamStart: (taskId) => this.events.emit("stream_start", taskId),
|
|
2363
|
+
onStreamChunk: (taskId, event) => this.events.emit("stream_chunk", taskId, event),
|
|
2364
|
+
onStreamEnd: (taskId, output) => this.events.emit("stream_end", taskId, output)
|
|
2365
|
+
});
|
|
1969
2366
|
try {
|
|
1970
2367
|
const output = await this.graph.run(input, {
|
|
1971
2368
|
parentSignal: this._abortController.signal,
|
|
1972
|
-
outputCache: this.
|
|
2369
|
+
outputCache: this._outputCache
|
|
1973
2370
|
});
|
|
1974
2371
|
const results = this.graph.mergeExecuteOutputsToRunOutput(output, PROPERTY_ARRAY);
|
|
1975
2372
|
this.events.emit("complete");
|
|
@@ -1978,10 +2375,14 @@ class Workflow {
|
|
|
1978
2375
|
this.events.emit("error", String(error));
|
|
1979
2376
|
throw error;
|
|
1980
2377
|
} finally {
|
|
2378
|
+
unsubStreaming();
|
|
1981
2379
|
this._abortController = undefined;
|
|
1982
2380
|
}
|
|
1983
2381
|
}
|
|
1984
2382
|
async abort() {
|
|
2383
|
+
if (this._parentWorkflow) {
|
|
2384
|
+
return this._parentWorkflow.abort();
|
|
2385
|
+
}
|
|
1985
2386
|
this._abortController?.abort();
|
|
1986
2387
|
}
|
|
1987
2388
|
pop() {
|
|
@@ -2050,10 +2451,12 @@ class Workflow {
|
|
|
2050
2451
|
return task;
|
|
2051
2452
|
}
|
|
2052
2453
|
reset() {
|
|
2053
|
-
|
|
2454
|
+
if (this._parentWorkflow) {
|
|
2455
|
+
throw new WorkflowError("Cannot reset a loop workflow. Call reset() on the parent workflow.");
|
|
2456
|
+
}
|
|
2054
2457
|
this.clearEvents();
|
|
2055
2458
|
this._graph = new TaskGraph({
|
|
2056
|
-
outputCache: this.
|
|
2459
|
+
outputCache: this._outputCache
|
|
2057
2460
|
});
|
|
2058
2461
|
this._dataFlows = [];
|
|
2059
2462
|
this._error = "";
|
|
@@ -2108,53 +2511,294 @@ class Workflow {
|
|
|
2108
2511
|
this.graph.addDataflow(dataflow);
|
|
2109
2512
|
return this;
|
|
2110
2513
|
}
|
|
2111
|
-
|
|
2514
|
+
addTaskToGraph(taskClass, input, config) {
|
|
2112
2515
|
const task = new taskClass(input, config);
|
|
2113
2516
|
const id = this.graph.addTask(task);
|
|
2114
2517
|
this.events.emit("changed", id);
|
|
2115
2518
|
return task;
|
|
2116
2519
|
}
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
}
|
|
2121
|
-
|
|
2122
|
-
// src/task-graph/Conversions.ts
|
|
2123
|
-
class ListeningGraphAsTask extends GraphAsTask {
|
|
2124
|
-
constructor(input, config) {
|
|
2125
|
-
super(input, config);
|
|
2126
|
-
this.subGraph.on("start", () => {
|
|
2127
|
-
this.emit("start");
|
|
2128
|
-
});
|
|
2129
|
-
this.subGraph.on("complete", () => {
|
|
2130
|
-
this.emit("complete");
|
|
2131
|
-
});
|
|
2132
|
-
this.subGraph.on("error", (e) => {
|
|
2133
|
-
this.emit("error", e);
|
|
2134
|
-
});
|
|
2520
|
+
addTask(taskClass, input, config) {
|
|
2521
|
+
const helper = Workflow.createWorkflow(taskClass);
|
|
2522
|
+
return helper.call(this, input, config);
|
|
2135
2523
|
}
|
|
2136
|
-
}
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
}
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2524
|
+
addLoopTask(taskClass, config = {}) {
|
|
2525
|
+
this._error = "";
|
|
2526
|
+
const parent = getLastTask(this);
|
|
2527
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid43(), ...config });
|
|
2528
|
+
if (this._dataFlows.length > 0) {
|
|
2529
|
+
this._dataFlows.forEach((dataflow) => {
|
|
2530
|
+
const taskSchema = task.inputSchema();
|
|
2531
|
+
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2532
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.config.id}`;
|
|
2533
|
+
console.error(this._error);
|
|
2534
|
+
return;
|
|
2535
|
+
}
|
|
2536
|
+
dataflow.targetTaskId = task.config.id;
|
|
2537
|
+
this.graph.addDataflow(dataflow);
|
|
2538
|
+
});
|
|
2539
|
+
this._dataFlows = [];
|
|
2540
|
+
}
|
|
2541
|
+
const loopBuilder = new Workflow(this.outputCache(), this, task);
|
|
2542
|
+
if (parent) {
|
|
2543
|
+
loopBuilder._pendingLoopConnect = { parent, iteratorTask: task };
|
|
2544
|
+
}
|
|
2545
|
+
return loopBuilder;
|
|
2546
|
+
}
|
|
2547
|
+
autoConnectLoopTask(pending) {
|
|
2548
|
+
if (!pending)
|
|
2549
|
+
return;
|
|
2550
|
+
const { parent, iteratorTask } = pending;
|
|
2551
|
+
if (this.graph.getTargetDataflows(parent.config.id).length === 0) {
|
|
2552
|
+
const nodes = this._graph.getTasks();
|
|
2553
|
+
const parentIndex = nodes.findIndex((n) => n.config.id === parent.config.id);
|
|
2554
|
+
const earlierTasks = [];
|
|
2555
|
+
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2556
|
+
earlierTasks.push(nodes[i]);
|
|
2557
|
+
}
|
|
2558
|
+
const result = Workflow.autoConnect(this.graph, parent, iteratorTask, {
|
|
2559
|
+
earlierTasks
|
|
2560
|
+
});
|
|
2561
|
+
if (result.error) {
|
|
2562
|
+
this._error = result.error + " Task not added.";
|
|
2563
|
+
console.error(this._error);
|
|
2564
|
+
this.graph.removeTask(iteratorTask.config.id);
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
static AutoConnectOptions = Symbol("AutoConnectOptions");
|
|
2569
|
+
static autoConnect(graph, sourceTask, targetTask, options) {
|
|
2570
|
+
const matches = new Map;
|
|
2571
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
2572
|
+
const targetSchema = targetTask.inputSchema();
|
|
2573
|
+
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
2574
|
+
const earlierTasks = options?.earlierTasks ?? [];
|
|
2575
|
+
const getSpecificTypeIdentifiers = (schema) => {
|
|
2576
|
+
const formats = new Set;
|
|
2577
|
+
const ids = new Set;
|
|
2578
|
+
if (typeof schema === "boolean") {
|
|
2579
|
+
return { formats, ids };
|
|
2580
|
+
}
|
|
2581
|
+
const extractFromSchema = (s) => {
|
|
2582
|
+
if (!s || typeof s !== "object" || Array.isArray(s))
|
|
2583
|
+
return;
|
|
2584
|
+
if (s.format)
|
|
2585
|
+
formats.add(s.format);
|
|
2586
|
+
if (s.$id)
|
|
2587
|
+
ids.add(s.$id);
|
|
2588
|
+
};
|
|
2589
|
+
extractFromSchema(schema);
|
|
2590
|
+
const checkUnion = (schemas) => {
|
|
2591
|
+
if (!schemas)
|
|
2592
|
+
return;
|
|
2593
|
+
for (const s of schemas) {
|
|
2594
|
+
if (typeof s === "boolean")
|
|
2595
|
+
continue;
|
|
2596
|
+
extractFromSchema(s);
|
|
2597
|
+
if (s.items && typeof s.items === "object" && !Array.isArray(s.items)) {
|
|
2598
|
+
extractFromSchema(s.items);
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
};
|
|
2602
|
+
checkUnion(schema.oneOf);
|
|
2603
|
+
checkUnion(schema.anyOf);
|
|
2604
|
+
if (schema.items && typeof schema.items === "object" && !Array.isArray(schema.items)) {
|
|
2605
|
+
extractFromSchema(schema.items);
|
|
2606
|
+
}
|
|
2607
|
+
return { formats, ids };
|
|
2608
|
+
};
|
|
2609
|
+
const isTypeCompatible = (fromPortOutputSchema, toPortInputSchema, requireSpecificType = false) => {
|
|
2610
|
+
if (typeof fromPortOutputSchema === "boolean" || typeof toPortInputSchema === "boolean") {
|
|
2611
|
+
return fromPortOutputSchema === true && toPortInputSchema === true;
|
|
2612
|
+
}
|
|
2613
|
+
const outputIds = getSpecificTypeIdentifiers(fromPortOutputSchema);
|
|
2614
|
+
const inputIds = getSpecificTypeIdentifiers(toPortInputSchema);
|
|
2615
|
+
for (const format of outputIds.formats) {
|
|
2616
|
+
if (inputIds.formats.has(format)) {
|
|
2617
|
+
return true;
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
for (const id of outputIds.ids) {
|
|
2621
|
+
if (inputIds.ids.has(id)) {
|
|
2622
|
+
return true;
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
if (requireSpecificType) {
|
|
2626
|
+
return false;
|
|
2627
|
+
}
|
|
2628
|
+
const idTypeBlank = fromPortOutputSchema.$id === undefined && toPortInputSchema.$id === undefined;
|
|
2629
|
+
if (!idTypeBlank)
|
|
2630
|
+
return false;
|
|
2631
|
+
if (fromPortOutputSchema.type === toPortInputSchema.type)
|
|
2632
|
+
return true;
|
|
2633
|
+
const matchesOneOf = toPortInputSchema.oneOf?.some((schema) => {
|
|
2634
|
+
if (typeof schema === "boolean")
|
|
2635
|
+
return schema;
|
|
2636
|
+
return schema.type === fromPortOutputSchema.type;
|
|
2637
|
+
}) ?? false;
|
|
2638
|
+
const matchesAnyOf = toPortInputSchema.anyOf?.some((schema) => {
|
|
2639
|
+
if (typeof schema === "boolean")
|
|
2640
|
+
return schema;
|
|
2641
|
+
return schema.type === fromPortOutputSchema.type;
|
|
2642
|
+
}) ?? false;
|
|
2643
|
+
return matchesOneOf || matchesAnyOf;
|
|
2644
|
+
};
|
|
2645
|
+
const makeMatch = (fromSchema, toSchema, fromTaskId, toTaskId, comparator) => {
|
|
2646
|
+
if (typeof fromSchema === "object") {
|
|
2647
|
+
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2648
|
+
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
2649
|
+
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2650
|
+
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2651
|
+
}
|
|
2652
|
+
return;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2656
|
+
return;
|
|
2657
|
+
}
|
|
2658
|
+
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2659
|
+
if (matches.has(toInputPortId))
|
|
2660
|
+
continue;
|
|
2661
|
+
const candidates = [];
|
|
2662
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2663
|
+
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
2664
|
+
candidates.push(fromOutputPortId);
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2667
|
+
if (candidates.length === 0)
|
|
2668
|
+
continue;
|
|
2669
|
+
let winner = candidates[0];
|
|
2670
|
+
if (candidates.length > 1) {
|
|
2671
|
+
const targetStreamMode = getPortStreamMode(toSchema, toInputPortId);
|
|
2672
|
+
const streamMatch = candidates.find((portId) => getPortStreamMode(fromSchema, portId) === targetStreamMode);
|
|
2673
|
+
if (streamMatch)
|
|
2674
|
+
winner = streamMatch;
|
|
2675
|
+
}
|
|
2676
|
+
matches.set(toInputPortId, winner);
|
|
2677
|
+
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2678
|
+
}
|
|
2679
|
+
};
|
|
2680
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2681
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2682
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2683
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2684
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2685
|
+
});
|
|
2686
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.config.id, targetTask.config.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2687
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2688
|
+
});
|
|
2689
|
+
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2690
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
2691
|
+
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2692
|
+
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2693
|
+
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2694
|
+
const earlierTask = earlierTasks[i];
|
|
2695
|
+
const earlierOutputSchema = earlierTask.outputSchema();
|
|
2696
|
+
const makeMatchFromEarlier = (comparator) => {
|
|
2697
|
+
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2698
|
+
return;
|
|
2699
|
+
}
|
|
2700
|
+
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(earlierOutputSchema.properties || {})) {
|
|
2701
|
+
for (const requiredInputId of unmatchedRequired) {
|
|
2702
|
+
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2703
|
+
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2704
|
+
matches.set(requiredInputId, fromOutputPortId);
|
|
2705
|
+
graph.addDataflow(new Dataflow(earlierTask.config.id, fromOutputPortId, targetTask.config.id, requiredInputId));
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
};
|
|
2710
|
+
makeMatchFromEarlier(([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2711
|
+
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2712
|
+
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2713
|
+
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2714
|
+
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2715
|
+
});
|
|
2716
|
+
makeMatchFromEarlier(([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2717
|
+
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2718
|
+
});
|
|
2719
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
2720
|
+
}
|
|
2721
|
+
}
|
|
2722
|
+
const stillUnmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2723
|
+
if (stillUnmatchedRequired.length > 0) {
|
|
2724
|
+
return {
|
|
2725
|
+
matches,
|
|
2726
|
+
error: `Could not find matches for required inputs [${stillUnmatchedRequired.join(", ")}] of ${targetTask.type}. ` + `Attempted to match from ${sourceTask.type} and earlier tasks.`,
|
|
2727
|
+
unmatchedRequired: stillUnmatchedRequired
|
|
2728
|
+
};
|
|
2729
|
+
}
|
|
2730
|
+
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
2731
|
+
const hasRequiredInputs = requiredInputs.size > 0;
|
|
2732
|
+
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
2733
|
+
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
2734
|
+
if (!allRequiredInputsProvided && !hasInputsWithDefaults) {
|
|
2735
|
+
return {
|
|
2736
|
+
matches,
|
|
2737
|
+
error: `Could not find a match between the outputs of ${sourceTask.type} and the inputs of ${targetTask.type}. ` + `You may need to connect the outputs to the inputs via connect() manually.`,
|
|
2738
|
+
unmatchedRequired: []
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
return {
|
|
2743
|
+
matches,
|
|
2744
|
+
unmatchedRequired: []
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
finalizeTemplate() {
|
|
2748
|
+
if (!this._iteratorTask || this.graph.getTasks().length === 0) {
|
|
2749
|
+
return;
|
|
2750
|
+
}
|
|
2751
|
+
this._iteratorTask.subGraph = this.graph;
|
|
2752
|
+
}
|
|
2753
|
+
finalizeAndReturn() {
|
|
2754
|
+
if (!this._parentWorkflow) {
|
|
2755
|
+
throw new WorkflowError("finalizeAndReturn() can only be called on loop workflows");
|
|
2756
|
+
}
|
|
2757
|
+
this.finalizeTemplate();
|
|
2758
|
+
if (this._pendingLoopConnect) {
|
|
2759
|
+
this._parentWorkflow.autoConnectLoopTask(this._pendingLoopConnect);
|
|
2760
|
+
this._pendingLoopConnect = undefined;
|
|
2761
|
+
}
|
|
2762
|
+
return this._parentWorkflow;
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
|
|
2766
|
+
// src/task-graph/Conversions.ts
|
|
2767
|
+
class ListeningGraphAsTask extends GraphAsTask {
|
|
2768
|
+
constructor(input, config) {
|
|
2769
|
+
super(input, config);
|
|
2770
|
+
this.subGraph.on("start", () => {
|
|
2771
|
+
this.emit("start");
|
|
2772
|
+
});
|
|
2773
|
+
this.subGraph.on("complete", () => {
|
|
2774
|
+
this.emit("complete");
|
|
2775
|
+
});
|
|
2776
|
+
this.subGraph.on("error", (e) => {
|
|
2777
|
+
this.emit("error", e);
|
|
2778
|
+
});
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
class OwnGraphTask extends ListeningGraphAsTask {
|
|
2783
|
+
static type = "Own[Graph]";
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2786
|
+
class OwnWorkflowTask extends ListeningGraphAsTask {
|
|
2787
|
+
static type = "Own[Workflow]";
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2790
|
+
class GraphTask extends GraphAsTask {
|
|
2791
|
+
static type = "Graph";
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
class WorkflowTask2 extends GraphAsTask {
|
|
2795
|
+
static type = "Workflow";
|
|
2796
|
+
}
|
|
2797
|
+
function convertPipeFunctionToTask(fn, config) {
|
|
2798
|
+
|
|
2799
|
+
class QuickTask extends Task {
|
|
2800
|
+
static type = fn.name ? `\uD835\uDC53 ${fn.name}` : "\uD835\uDC53";
|
|
2801
|
+
static inputSchema = () => {
|
|
2158
2802
|
return {
|
|
2159
2803
|
type: "object",
|
|
2160
2804
|
properties: {
|
|
@@ -2356,7 +3000,7 @@ class TaskGraph {
|
|
|
2356
3000
|
return this._dag.removeNode(taskId);
|
|
2357
3001
|
}
|
|
2358
3002
|
resetGraph() {
|
|
2359
|
-
this.runner.resetGraph(this,
|
|
3003
|
+
this.runner.resetGraph(this, uuid44());
|
|
2360
3004
|
}
|
|
2361
3005
|
toJSON() {
|
|
2362
3006
|
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
@@ -2478,6 +3122,24 @@ class TaskGraph {
|
|
|
2478
3122
|
unsubscribes.forEach((unsub) => unsub());
|
|
2479
3123
|
};
|
|
2480
3124
|
}
|
|
3125
|
+
subscribeToTaskStreaming(callbacks) {
|
|
3126
|
+
const unsubscribes = [];
|
|
3127
|
+
if (callbacks.onStreamStart) {
|
|
3128
|
+
const unsub = this.subscribe("task_stream_start", callbacks.onStreamStart);
|
|
3129
|
+
unsubscribes.push(unsub);
|
|
3130
|
+
}
|
|
3131
|
+
if (callbacks.onStreamChunk) {
|
|
3132
|
+
const unsub = this.subscribe("task_stream_chunk", callbacks.onStreamChunk);
|
|
3133
|
+
unsubscribes.push(unsub);
|
|
3134
|
+
}
|
|
3135
|
+
if (callbacks.onStreamEnd) {
|
|
3136
|
+
const unsub = this.subscribe("task_stream_end", callbacks.onStreamEnd);
|
|
3137
|
+
unsubscribes.push(unsub);
|
|
3138
|
+
}
|
|
3139
|
+
return () => {
|
|
3140
|
+
unsubscribes.forEach((unsub) => unsub());
|
|
3141
|
+
};
|
|
3142
|
+
}
|
|
2481
3143
|
on(name, fn) {
|
|
2482
3144
|
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
2483
3145
|
if (dagEvent) {
|
|
@@ -2521,122 +3183,1186 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
2521
3183
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
2522
3184
|
return graph;
|
|
2523
3185
|
}
|
|
2524
|
-
// src/task/
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
var defaultJobQueueFactory = async ({
|
|
2533
|
-
queueName,
|
|
2534
|
-
jobClass,
|
|
2535
|
-
options
|
|
2536
|
-
}) => {
|
|
2537
|
-
const storage = options?.storage ?? new InMemoryQueueStorage(queueName);
|
|
2538
|
-
await storage.setupDatabase();
|
|
2539
|
-
const server = new JobQueueServer(jobClass, {
|
|
2540
|
-
storage,
|
|
2541
|
-
queueName,
|
|
2542
|
-
limiter: options?.limiter,
|
|
2543
|
-
workerCount: options?.workerCount,
|
|
2544
|
-
pollIntervalMs: options?.pollIntervalMs,
|
|
2545
|
-
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
2546
|
-
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
2547
|
-
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
2548
|
-
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
2549
|
-
});
|
|
2550
|
-
const client = new JobQueueClient({
|
|
2551
|
-
storage,
|
|
2552
|
-
queueName
|
|
2553
|
-
});
|
|
2554
|
-
client.attach(server);
|
|
2555
|
-
return { server, client, storage };
|
|
2556
|
-
};
|
|
2557
|
-
function registerJobQueueFactory(factory) {
|
|
2558
|
-
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
2559
|
-
}
|
|
2560
|
-
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
2561
|
-
return async ({
|
|
2562
|
-
queueName,
|
|
2563
|
-
jobClass,
|
|
2564
|
-
options
|
|
2565
|
-
}) => {
|
|
2566
|
-
const mergedOptions = {
|
|
2567
|
-
...defaultOptions,
|
|
2568
|
-
...options ?? {}
|
|
2569
|
-
};
|
|
2570
|
-
const storage = mergedOptions.storage ?? new InMemoryQueueStorage(queueName);
|
|
2571
|
-
await storage.setupDatabase();
|
|
2572
|
-
const server = new JobQueueServer(jobClass, {
|
|
2573
|
-
storage,
|
|
2574
|
-
queueName,
|
|
2575
|
-
limiter: mergedOptions.limiter,
|
|
2576
|
-
workerCount: mergedOptions.workerCount,
|
|
2577
|
-
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
2578
|
-
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
2579
|
-
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
2580
|
-
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
2581
|
-
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
2582
|
-
});
|
|
2583
|
-
const client = new JobQueueClient({
|
|
2584
|
-
storage,
|
|
2585
|
-
queueName
|
|
2586
|
-
});
|
|
2587
|
-
client.attach(server);
|
|
2588
|
-
return { server, client, storage };
|
|
2589
|
-
};
|
|
2590
|
-
}
|
|
2591
|
-
function getJobQueueFactory() {
|
|
2592
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2593
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2594
|
-
}
|
|
2595
|
-
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
2596
|
-
}
|
|
2597
|
-
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
2598
|
-
registerJobQueueFactory(defaultJobQueueFactory);
|
|
2599
|
-
}
|
|
2600
|
-
// src/task/JobQueueTask.ts
|
|
2601
|
-
import { Job as Job2 } from "@workglow/job-queue";
|
|
2602
|
-
|
|
2603
|
-
// src/task/TaskQueueRegistry.ts
|
|
2604
|
-
var taskQueueRegistry = null;
|
|
2605
|
-
|
|
2606
|
-
class TaskQueueRegistry {
|
|
2607
|
-
queues = new Map;
|
|
2608
|
-
registerQueue(queue) {
|
|
2609
|
-
const queueName = queue.server.queueName;
|
|
2610
|
-
if (this.queues.has(queueName)) {
|
|
2611
|
-
throw new Error(`Queue with name ${queueName} already exists`);
|
|
3186
|
+
// src/task/IteratorTaskRunner.ts
|
|
3187
|
+
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3188
|
+
subGraphRunChain = Promise.resolve();
|
|
3189
|
+
async executeTask(input) {
|
|
3190
|
+
const analysis = this.task.analyzeIterationInput(input);
|
|
3191
|
+
if (analysis.iterationCount === 0) {
|
|
3192
|
+
const emptyResult = this.task.getEmptyResult();
|
|
3193
|
+
return this.executeTaskReactive(input, emptyResult);
|
|
2612
3194
|
}
|
|
2613
|
-
this.
|
|
2614
|
-
|
|
2615
|
-
getQueue(queueName) {
|
|
2616
|
-
return this.queues.get(queueName);
|
|
3195
|
+
const result = this.task.isReduceTask() ? await this.executeReduceIterations(analysis) : await this.executeCollectIterations(analysis);
|
|
3196
|
+
return this.executeTaskReactive(input, result);
|
|
2617
3197
|
}
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
}
|
|
2622
|
-
return this;
|
|
3198
|
+
async executeTaskReactive(input, output) {
|
|
3199
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
3200
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
2623
3201
|
}
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
3202
|
+
async executeCollectIterations(analysis) {
|
|
3203
|
+
const iterationCount = analysis.iterationCount;
|
|
3204
|
+
const preserveOrder = this.task.preserveIterationOrder();
|
|
3205
|
+
const batchSize = this.task.batchSize !== undefined && this.task.batchSize > 0 ? this.task.batchSize : iterationCount;
|
|
3206
|
+
const requestedConcurrency = this.task.concurrencyLimit ?? iterationCount;
|
|
3207
|
+
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3208
|
+
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3209
|
+
const completionOrderResults = [];
|
|
3210
|
+
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3211
|
+
if (this.abortController?.signal.aborted) {
|
|
3212
|
+
break;
|
|
3213
|
+
}
|
|
3214
|
+
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3215
|
+
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3216
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency);
|
|
3217
|
+
for (const { index, result } of batchResults) {
|
|
3218
|
+
if (result === undefined)
|
|
3219
|
+
continue;
|
|
3220
|
+
if (preserveOrder) {
|
|
3221
|
+
orderedResults[index] = result;
|
|
3222
|
+
} else {
|
|
3223
|
+
completionOrderResults.push(result);
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
3226
|
+
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3227
|
+
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
2627
3228
|
}
|
|
2628
|
-
|
|
3229
|
+
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3230
|
+
return this.task.collectResults(collected);
|
|
2629
3231
|
}
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
3232
|
+
async executeReduceIterations(analysis) {
|
|
3233
|
+
const iterationCount = analysis.iterationCount;
|
|
3234
|
+
let accumulator = this.task.getInitialAccumulator();
|
|
3235
|
+
for (let index = 0;index < iterationCount; index++) {
|
|
3236
|
+
if (this.abortController?.signal.aborted) {
|
|
3237
|
+
break;
|
|
3238
|
+
}
|
|
3239
|
+
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount, {
|
|
3240
|
+
accumulator
|
|
3241
|
+
});
|
|
3242
|
+
const iterationResult = await this.executeSubgraphIteration(iterationInput);
|
|
3243
|
+
accumulator = this.task.mergeIterationIntoAccumulator(accumulator, iterationResult, index);
|
|
3244
|
+
const progress = Math.round((index + 1) / iterationCount * 100);
|
|
3245
|
+
await this.handleProgress(progress, `Completed ${index + 1}/${iterationCount} iterations`);
|
|
2633
3246
|
}
|
|
2634
|
-
return
|
|
3247
|
+
return accumulator;
|
|
2635
3248
|
}
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
3249
|
+
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
3250
|
+
const results = [];
|
|
3251
|
+
let cursor = 0;
|
|
3252
|
+
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
3253
|
+
const workers = Array.from({ length: workerCount }, async () => {
|
|
3254
|
+
while (true) {
|
|
3255
|
+
if (this.abortController?.signal.aborted) {
|
|
3256
|
+
return;
|
|
3257
|
+
}
|
|
3258
|
+
const position = cursor;
|
|
3259
|
+
cursor += 1;
|
|
3260
|
+
if (position >= indices.length) {
|
|
3261
|
+
return;
|
|
3262
|
+
}
|
|
3263
|
+
const index = indices[position];
|
|
3264
|
+
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3265
|
+
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3266
|
+
results.push({ index, result });
|
|
3267
|
+
}
|
|
3268
|
+
});
|
|
3269
|
+
await Promise.all(workers);
|
|
3270
|
+
return results;
|
|
3271
|
+
}
|
|
3272
|
+
async executeSubgraphIteration(input) {
|
|
3273
|
+
let releaseTurn;
|
|
3274
|
+
const waitForPreviousRun = this.subGraphRunChain;
|
|
3275
|
+
this.subGraphRunChain = new Promise((resolve) => {
|
|
3276
|
+
releaseTurn = resolve;
|
|
3277
|
+
});
|
|
3278
|
+
await waitForPreviousRun;
|
|
3279
|
+
try {
|
|
3280
|
+
if (this.abortController?.signal.aborted) {
|
|
3281
|
+
return;
|
|
3282
|
+
}
|
|
3283
|
+
const results = await this.task.subGraph.run(input, {
|
|
3284
|
+
parentSignal: this.abortController?.signal,
|
|
3285
|
+
outputCache: this.outputCache
|
|
3286
|
+
});
|
|
3287
|
+
if (results.length === 0) {
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3291
|
+
} finally {
|
|
3292
|
+
releaseTurn?.();
|
|
3293
|
+
}
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3297
|
+
// src/task/IteratorTask.ts
|
|
3298
|
+
var ITERATOR_CONTEXT_SCHEMA = {
|
|
3299
|
+
type: "object",
|
|
3300
|
+
properties: {
|
|
3301
|
+
_iterationIndex: {
|
|
3302
|
+
type: "integer",
|
|
3303
|
+
minimum: 0,
|
|
3304
|
+
title: "Iteration Index",
|
|
3305
|
+
description: "Current iteration index (0-based)",
|
|
3306
|
+
"x-ui-iteration": true
|
|
3307
|
+
},
|
|
3308
|
+
_iterationCount: {
|
|
3309
|
+
type: "integer",
|
|
3310
|
+
minimum: 0,
|
|
3311
|
+
title: "Iteration Count",
|
|
3312
|
+
description: "Total number of iterations",
|
|
3313
|
+
"x-ui-iteration": true
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
};
|
|
3317
|
+
function isArrayVariant(schema) {
|
|
3318
|
+
if (!schema || typeof schema !== "object")
|
|
3319
|
+
return false;
|
|
3320
|
+
const record = schema;
|
|
3321
|
+
return record.type === "array" || record.items !== undefined;
|
|
3322
|
+
}
|
|
3323
|
+
function getExplicitIterationFlag(schema) {
|
|
3324
|
+
if (!schema || typeof schema !== "object")
|
|
3325
|
+
return;
|
|
3326
|
+
const record = schema;
|
|
3327
|
+
const flag = record["x-ui-iteration"];
|
|
3328
|
+
if (flag === true)
|
|
3329
|
+
return true;
|
|
3330
|
+
if (flag === false)
|
|
3331
|
+
return false;
|
|
3332
|
+
return;
|
|
3333
|
+
}
|
|
3334
|
+
function inferIterationFromSchema(schema) {
|
|
3335
|
+
if (!schema || typeof schema !== "object")
|
|
3336
|
+
return;
|
|
3337
|
+
const record = schema;
|
|
3338
|
+
if (record.type === "array" || record.items !== undefined) {
|
|
3339
|
+
return true;
|
|
3340
|
+
}
|
|
3341
|
+
const variants = record.oneOf ?? record.anyOf;
|
|
3342
|
+
if (!Array.isArray(variants) || variants.length === 0) {
|
|
3343
|
+
if (record.type !== undefined) {
|
|
3344
|
+
return false;
|
|
3345
|
+
}
|
|
3346
|
+
return;
|
|
3347
|
+
}
|
|
3348
|
+
let hasArrayVariant = false;
|
|
3349
|
+
let hasNonArrayVariant = false;
|
|
3350
|
+
for (const variant of variants) {
|
|
3351
|
+
if (isArrayVariant(variant)) {
|
|
3352
|
+
hasArrayVariant = true;
|
|
3353
|
+
} else {
|
|
3354
|
+
hasNonArrayVariant = true;
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
if (hasArrayVariant && hasNonArrayVariant)
|
|
3358
|
+
return;
|
|
3359
|
+
if (hasArrayVariant)
|
|
3360
|
+
return true;
|
|
3361
|
+
return false;
|
|
3362
|
+
}
|
|
3363
|
+
function createFlexibleSchema(baseSchema) {
|
|
3364
|
+
if (typeof baseSchema === "boolean")
|
|
3365
|
+
return baseSchema;
|
|
3366
|
+
return {
|
|
3367
|
+
anyOf: [baseSchema, { type: "array", items: baseSchema }]
|
|
3368
|
+
};
|
|
3369
|
+
}
|
|
3370
|
+
function createArraySchema(baseSchema) {
|
|
3371
|
+
if (typeof baseSchema === "boolean")
|
|
3372
|
+
return baseSchema;
|
|
3373
|
+
return {
|
|
3374
|
+
type: "array",
|
|
3375
|
+
items: baseSchema
|
|
3376
|
+
};
|
|
3377
|
+
}
|
|
3378
|
+
function extractBaseSchema(schema) {
|
|
3379
|
+
if (typeof schema === "boolean")
|
|
3380
|
+
return schema;
|
|
3381
|
+
const schemaType = schema.type;
|
|
3382
|
+
if (schemaType === "array" && schema.items) {
|
|
3383
|
+
return schema.items;
|
|
3384
|
+
}
|
|
3385
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
3386
|
+
if (Array.isArray(variants)) {
|
|
3387
|
+
for (const variant of variants) {
|
|
3388
|
+
if (typeof variant === "object") {
|
|
3389
|
+
const variantType = variant.type;
|
|
3390
|
+
if (variantType !== "array") {
|
|
3391
|
+
return variant;
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
for (const variant of variants) {
|
|
3396
|
+
if (typeof variant === "object") {
|
|
3397
|
+
const variantType = variant.type;
|
|
3398
|
+
if (variantType === "array" && variant.items) {
|
|
3399
|
+
return variant.items;
|
|
3400
|
+
}
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
return schema;
|
|
3405
|
+
}
|
|
3406
|
+
function schemaAcceptsArray(schema) {
|
|
3407
|
+
if (typeof schema === "boolean")
|
|
3408
|
+
return false;
|
|
3409
|
+
const schemaType = schema.type;
|
|
3410
|
+
if (schemaType === "array")
|
|
3411
|
+
return true;
|
|
3412
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
3413
|
+
if (Array.isArray(variants)) {
|
|
3414
|
+
return variants.some((variant) => isArrayVariant(variant));
|
|
3415
|
+
}
|
|
3416
|
+
return false;
|
|
3417
|
+
}
|
|
3418
|
+
|
|
3419
|
+
class IteratorTask extends GraphAsTask {
|
|
3420
|
+
static type = "IteratorTask";
|
|
3421
|
+
static category = "Flow Control";
|
|
3422
|
+
static title = "Iterator";
|
|
3423
|
+
static description = "Base class for loop-type tasks";
|
|
3424
|
+
static hasDynamicSchemas = true;
|
|
3425
|
+
static getIterationContextSchema() {
|
|
3426
|
+
return ITERATOR_CONTEXT_SCHEMA;
|
|
3427
|
+
}
|
|
3428
|
+
_iteratorPortInfo;
|
|
3429
|
+
_iterationInputSchema;
|
|
3430
|
+
constructor(input = {}, config = {}) {
|
|
3431
|
+
super(input, config);
|
|
3432
|
+
}
|
|
3433
|
+
get runner() {
|
|
3434
|
+
if (!this._runner) {
|
|
3435
|
+
this._runner = new IteratorTaskRunner(this);
|
|
3436
|
+
}
|
|
3437
|
+
return this._runner;
|
|
3438
|
+
}
|
|
3439
|
+
set subGraph(subGraph) {
|
|
3440
|
+
super.subGraph = subGraph;
|
|
3441
|
+
this.invalidateIterationInputSchema();
|
|
3442
|
+
this.events.emit("regenerate");
|
|
3443
|
+
}
|
|
3444
|
+
get subGraph() {
|
|
3445
|
+
return super.subGraph;
|
|
3446
|
+
}
|
|
3447
|
+
regenerateGraph() {
|
|
3448
|
+
this.invalidateIterationInputSchema();
|
|
3449
|
+
super.regenerateGraph();
|
|
3450
|
+
}
|
|
3451
|
+
preserveIterationOrder() {
|
|
3452
|
+
return true;
|
|
3453
|
+
}
|
|
3454
|
+
isReduceTask() {
|
|
3455
|
+
return false;
|
|
3456
|
+
}
|
|
3457
|
+
getInitialAccumulator() {
|
|
3458
|
+
return {};
|
|
3459
|
+
}
|
|
3460
|
+
buildIterationRunInput(analysis, index, iterationCount, extraInput = {}) {
|
|
3461
|
+
return {
|
|
3462
|
+
...analysis.getIterationInput(index),
|
|
3463
|
+
...extraInput,
|
|
3464
|
+
_iterationIndex: index,
|
|
3465
|
+
_iterationCount: iterationCount
|
|
3466
|
+
};
|
|
3467
|
+
}
|
|
3468
|
+
mergeIterationIntoAccumulator(accumulator, iterationResult, _index) {
|
|
3469
|
+
return iterationResult ?? accumulator;
|
|
3470
|
+
}
|
|
3471
|
+
getEmptyResult() {
|
|
3472
|
+
return {};
|
|
3473
|
+
}
|
|
3474
|
+
collectResults(results) {
|
|
3475
|
+
if (results.length === 0) {
|
|
3476
|
+
return {};
|
|
3477
|
+
}
|
|
3478
|
+
const merged = {};
|
|
3479
|
+
for (const result of results) {
|
|
3480
|
+
if (!result || typeof result !== "object")
|
|
3481
|
+
continue;
|
|
3482
|
+
for (const [key, value] of Object.entries(result)) {
|
|
3483
|
+
if (!merged[key]) {
|
|
3484
|
+
merged[key] = [];
|
|
3485
|
+
}
|
|
3486
|
+
merged[key].push(value);
|
|
3487
|
+
}
|
|
3488
|
+
}
|
|
3489
|
+
return merged;
|
|
3490
|
+
}
|
|
3491
|
+
get concurrencyLimit() {
|
|
3492
|
+
return this.config.concurrencyLimit;
|
|
3493
|
+
}
|
|
3494
|
+
get batchSize() {
|
|
3495
|
+
return this.config.batchSize;
|
|
3496
|
+
}
|
|
3497
|
+
get iterationInputConfig() {
|
|
3498
|
+
return this.config.iterationInputConfig;
|
|
3499
|
+
}
|
|
3500
|
+
buildDefaultIterationInputSchema() {
|
|
3501
|
+
const innerSchema = this.getInnerInputSchema();
|
|
3502
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
3503
|
+
return { type: "object", properties: {}, additionalProperties: true };
|
|
3504
|
+
}
|
|
3505
|
+
const properties = {};
|
|
3506
|
+
const innerProps = innerSchema.properties || {};
|
|
3507
|
+
for (const [key, propSchema] of Object.entries(innerProps)) {
|
|
3508
|
+
if (typeof propSchema === "boolean")
|
|
3509
|
+
continue;
|
|
3510
|
+
if (propSchema["x-ui-iteration"]) {
|
|
3511
|
+
continue;
|
|
3512
|
+
}
|
|
3513
|
+
const baseSchema = propSchema;
|
|
3514
|
+
properties[key] = createFlexibleSchema(baseSchema);
|
|
3515
|
+
}
|
|
3516
|
+
return {
|
|
3517
|
+
type: "object",
|
|
3518
|
+
properties,
|
|
3519
|
+
additionalProperties: innerSchema.additionalProperties ?? true
|
|
3520
|
+
};
|
|
3521
|
+
}
|
|
3522
|
+
buildConfiguredIterationInputSchema() {
|
|
3523
|
+
const innerSchema = this.getInnerInputSchema();
|
|
3524
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
3525
|
+
return { type: "object", properties: {}, additionalProperties: true };
|
|
3526
|
+
}
|
|
3527
|
+
const config = this.iterationInputConfig || {};
|
|
3528
|
+
const properties = {};
|
|
3529
|
+
const innerProps = innerSchema.properties || {};
|
|
3530
|
+
for (const [key, propSchema] of Object.entries(innerProps)) {
|
|
3531
|
+
if (typeof propSchema === "boolean")
|
|
3532
|
+
continue;
|
|
3533
|
+
if (propSchema["x-ui-iteration"]) {
|
|
3534
|
+
continue;
|
|
3535
|
+
}
|
|
3536
|
+
const baseSchema = propSchema;
|
|
3537
|
+
const propConfig = config[key];
|
|
3538
|
+
if (!propConfig) {
|
|
3539
|
+
properties[key] = createFlexibleSchema(baseSchema);
|
|
3540
|
+
continue;
|
|
3541
|
+
}
|
|
3542
|
+
switch (propConfig.mode) {
|
|
3543
|
+
case "array":
|
|
3544
|
+
properties[key] = createArraySchema(propConfig.baseSchema);
|
|
3545
|
+
break;
|
|
3546
|
+
case "scalar":
|
|
3547
|
+
properties[key] = propConfig.baseSchema;
|
|
3548
|
+
break;
|
|
3549
|
+
case "flexible":
|
|
3550
|
+
default:
|
|
3551
|
+
properties[key] = createFlexibleSchema(propConfig.baseSchema);
|
|
3552
|
+
break;
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
return {
|
|
3556
|
+
type: "object",
|
|
3557
|
+
properties,
|
|
3558
|
+
additionalProperties: innerSchema.additionalProperties ?? true
|
|
3559
|
+
};
|
|
3560
|
+
}
|
|
3561
|
+
getInnerInputSchema() {
|
|
3562
|
+
if (!this.hasChildren())
|
|
3563
|
+
return;
|
|
3564
|
+
const tasks = this.subGraph.getTasks();
|
|
3565
|
+
if (tasks.length === 0)
|
|
3566
|
+
return;
|
|
3567
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
3568
|
+
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3569
|
+
const properties = {};
|
|
3570
|
+
const required = [];
|
|
3571
|
+
let additionalProperties = false;
|
|
3572
|
+
for (const task of sources) {
|
|
3573
|
+
const inputSchema = task.inputSchema();
|
|
3574
|
+
if (typeof inputSchema === "boolean") {
|
|
3575
|
+
if (inputSchema === true) {
|
|
3576
|
+
additionalProperties = true;
|
|
3577
|
+
}
|
|
3578
|
+
continue;
|
|
3579
|
+
}
|
|
3580
|
+
additionalProperties = additionalProperties || inputSchema.additionalProperties === true;
|
|
3581
|
+
for (const [key, prop] of Object.entries(inputSchema.properties || {})) {
|
|
3582
|
+
if (typeof prop === "boolean")
|
|
3583
|
+
continue;
|
|
3584
|
+
if (!properties[key]) {
|
|
3585
|
+
properties[key] = prop;
|
|
3586
|
+
}
|
|
3587
|
+
}
|
|
3588
|
+
for (const key of inputSchema.required || []) {
|
|
3589
|
+
if (!required.includes(key)) {
|
|
3590
|
+
required.push(key);
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3594
|
+
return {
|
|
3595
|
+
type: "object",
|
|
3596
|
+
properties,
|
|
3597
|
+
...required.length > 0 ? { required } : {},
|
|
3598
|
+
additionalProperties
|
|
3599
|
+
};
|
|
3600
|
+
}
|
|
3601
|
+
getIterationInputSchema() {
|
|
3602
|
+
if (this._iterationInputSchema) {
|
|
3603
|
+
return this._iterationInputSchema;
|
|
3604
|
+
}
|
|
3605
|
+
this._iterationInputSchema = this.iterationInputConfig ? this.buildConfiguredIterationInputSchema() : this.buildDefaultIterationInputSchema();
|
|
3606
|
+
return this._iterationInputSchema;
|
|
3607
|
+
}
|
|
3608
|
+
setIterationInputSchema(schema) {
|
|
3609
|
+
this._iterationInputSchema = schema;
|
|
3610
|
+
this._inputSchemaNode = undefined;
|
|
3611
|
+
this.events.emit("regenerate");
|
|
3612
|
+
}
|
|
3613
|
+
setPropertyInputMode(propertyName, mode, baseSchema) {
|
|
3614
|
+
const currentSchema = this.getIterationInputSchema();
|
|
3615
|
+
if (typeof currentSchema === "boolean")
|
|
3616
|
+
return;
|
|
3617
|
+
const currentProps = currentSchema.properties || {};
|
|
3618
|
+
const existingProp = currentProps[propertyName];
|
|
3619
|
+
const base = baseSchema ?? (existingProp ? extractBaseSchema(existingProp) : { type: "string" });
|
|
3620
|
+
let newPropSchema;
|
|
3621
|
+
switch (mode) {
|
|
3622
|
+
case "array":
|
|
3623
|
+
newPropSchema = createArraySchema(base);
|
|
3624
|
+
break;
|
|
3625
|
+
case "scalar":
|
|
3626
|
+
newPropSchema = base;
|
|
3627
|
+
break;
|
|
3628
|
+
case "flexible":
|
|
3629
|
+
default:
|
|
3630
|
+
newPropSchema = createFlexibleSchema(base);
|
|
3631
|
+
break;
|
|
3632
|
+
}
|
|
3633
|
+
this._iterationInputSchema = {
|
|
3634
|
+
...currentSchema,
|
|
3635
|
+
properties: {
|
|
3636
|
+
...currentProps,
|
|
3637
|
+
[propertyName]: newPropSchema
|
|
3638
|
+
}
|
|
3639
|
+
};
|
|
3640
|
+
this._inputSchemaNode = undefined;
|
|
3641
|
+
this.events.emit("regenerate");
|
|
3642
|
+
}
|
|
3643
|
+
invalidateIterationInputSchema() {
|
|
3644
|
+
this._iterationInputSchema = undefined;
|
|
3645
|
+
this._iteratorPortInfo = undefined;
|
|
3646
|
+
this._inputSchemaNode = undefined;
|
|
3647
|
+
}
|
|
3648
|
+
analyzeIterationInput(input) {
|
|
3649
|
+
const inputData = input;
|
|
3650
|
+
const schema = this.hasChildren() ? this.getIterationInputSchema() : this.inputSchema();
|
|
3651
|
+
const schemaProps = typeof schema === "object" && schema.properties ? schema.properties : {};
|
|
3652
|
+
const keys = new Set([...Object.keys(schemaProps), ...Object.keys(inputData)]);
|
|
3653
|
+
const arrayPorts = [];
|
|
3654
|
+
const scalarPorts = [];
|
|
3655
|
+
const iteratedValues = {};
|
|
3656
|
+
const arrayLengths = [];
|
|
3657
|
+
for (const key of keys) {
|
|
3658
|
+
if (key.startsWith("_iteration"))
|
|
3659
|
+
continue;
|
|
3660
|
+
const value = inputData[key];
|
|
3661
|
+
const portSchema = schemaProps[key];
|
|
3662
|
+
let shouldIterate;
|
|
3663
|
+
const explicitFlag = getExplicitIterationFlag(portSchema);
|
|
3664
|
+
if (explicitFlag !== undefined) {
|
|
3665
|
+
shouldIterate = explicitFlag;
|
|
3666
|
+
} else {
|
|
3667
|
+
const schemaInference = inferIterationFromSchema(portSchema);
|
|
3668
|
+
shouldIterate = schemaInference ?? Array.isArray(value);
|
|
3669
|
+
}
|
|
3670
|
+
if (!shouldIterate) {
|
|
3671
|
+
scalarPorts.push(key);
|
|
3672
|
+
continue;
|
|
3673
|
+
}
|
|
3674
|
+
if (!Array.isArray(value)) {
|
|
3675
|
+
throw new TaskConfigurationError(`${this.type}: Input '${key}' is configured for iteration but value is not an array.`);
|
|
3676
|
+
}
|
|
3677
|
+
iteratedValues[key] = value;
|
|
3678
|
+
arrayPorts.push(key);
|
|
3679
|
+
arrayLengths.push(value.length);
|
|
3680
|
+
}
|
|
3681
|
+
if (arrayPorts.length === 0) {
|
|
3682
|
+
throw new TaskConfigurationError(`${this.type}: At least one array input is required for iteration. ` + `Mark a port with x-ui-iteration=true, provide array-typed schema, or pass array values at runtime.`);
|
|
3683
|
+
}
|
|
3684
|
+
const uniqueLengths = new Set(arrayLengths);
|
|
3685
|
+
if (uniqueLengths.size > 1) {
|
|
3686
|
+
const lengthInfo = arrayPorts.map((port, index) => `${port}=${arrayLengths[index]}`).join(", ");
|
|
3687
|
+
throw new TaskConfigurationError(`${this.type}: All iterated array inputs must have the same length (zip semantics). ` + `Found different lengths: ${lengthInfo}`);
|
|
3688
|
+
}
|
|
3689
|
+
const iterationCount = arrayLengths[0] ?? 0;
|
|
3690
|
+
const getIterationInput = (index) => {
|
|
3691
|
+
const iterInput = {};
|
|
3692
|
+
for (const key of arrayPorts) {
|
|
3693
|
+
iterInput[key] = iteratedValues[key][index];
|
|
3694
|
+
}
|
|
3695
|
+
for (const key of scalarPorts) {
|
|
3696
|
+
if (key in inputData) {
|
|
3697
|
+
iterInput[key] = inputData[key];
|
|
3698
|
+
}
|
|
3699
|
+
}
|
|
3700
|
+
return iterInput;
|
|
3701
|
+
};
|
|
3702
|
+
return {
|
|
3703
|
+
iterationCount,
|
|
3704
|
+
arrayPorts,
|
|
3705
|
+
scalarPorts,
|
|
3706
|
+
getIterationInput
|
|
3707
|
+
};
|
|
3708
|
+
}
|
|
3709
|
+
getIterationContextSchema() {
|
|
3710
|
+
return this.constructor.getIterationContextSchema();
|
|
3711
|
+
}
|
|
3712
|
+
inputSchema() {
|
|
3713
|
+
if (this.hasChildren()) {
|
|
3714
|
+
return this.getIterationInputSchema();
|
|
3715
|
+
}
|
|
3716
|
+
return this.constructor.inputSchema();
|
|
3717
|
+
}
|
|
3718
|
+
outputSchema() {
|
|
3719
|
+
if (!this.hasChildren()) {
|
|
3720
|
+
return this.constructor.outputSchema();
|
|
3721
|
+
}
|
|
3722
|
+
return this.getWrappedOutputSchema();
|
|
3723
|
+
}
|
|
3724
|
+
getWrappedOutputSchema() {
|
|
3725
|
+
if (!this.hasChildren()) {
|
|
3726
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
3727
|
+
}
|
|
3728
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
3729
|
+
if (endingNodes.length === 0) {
|
|
3730
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
3731
|
+
}
|
|
3732
|
+
const properties = {};
|
|
3733
|
+
for (const task of endingNodes) {
|
|
3734
|
+
const taskOutputSchema = task.outputSchema();
|
|
3735
|
+
if (typeof taskOutputSchema === "boolean")
|
|
3736
|
+
continue;
|
|
3737
|
+
for (const [key, schema] of Object.entries(taskOutputSchema.properties || {})) {
|
|
3738
|
+
properties[key] = {
|
|
3739
|
+
type: "array",
|
|
3740
|
+
items: schema
|
|
3741
|
+
};
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
return {
|
|
3745
|
+
type: "object",
|
|
3746
|
+
properties,
|
|
3747
|
+
additionalProperties: false
|
|
3748
|
+
};
|
|
3749
|
+
}
|
|
3750
|
+
}
|
|
3751
|
+
|
|
3752
|
+
// src/task/WhileTaskRunner.ts
|
|
3753
|
+
class WhileTaskRunner extends GraphAsTaskRunner {
|
|
3754
|
+
async executeTask(input) {
|
|
3755
|
+
const result = await this.task.execute(input, {
|
|
3756
|
+
signal: this.abortController.signal,
|
|
3757
|
+
updateProgress: this.handleProgress.bind(this),
|
|
3758
|
+
own: this.own,
|
|
3759
|
+
registry: this.registry
|
|
3760
|
+
});
|
|
3761
|
+
return result;
|
|
3762
|
+
}
|
|
3763
|
+
async executeTaskReactive(input, output) {
|
|
3764
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
3765
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
|
|
3769
|
+
// src/task/WhileTask.ts
|
|
3770
|
+
var WHILE_CONTEXT_SCHEMA = {
|
|
3771
|
+
type: "object",
|
|
3772
|
+
properties: {
|
|
3773
|
+
_iterationIndex: {
|
|
3774
|
+
type: "integer",
|
|
3775
|
+
minimum: 0,
|
|
3776
|
+
title: "Iteration Number",
|
|
3777
|
+
description: "Current iteration number (0-based)",
|
|
3778
|
+
"x-ui-iteration": true
|
|
3779
|
+
}
|
|
3780
|
+
}
|
|
3781
|
+
};
|
|
3782
|
+
|
|
3783
|
+
class WhileTask extends GraphAsTask {
|
|
3784
|
+
static type = "WhileTask";
|
|
3785
|
+
static category = "Flow Control";
|
|
3786
|
+
static title = "While Loop";
|
|
3787
|
+
static description = "Loops until a condition function returns false";
|
|
3788
|
+
static hasDynamicSchemas = true;
|
|
3789
|
+
static getIterationContextSchema() {
|
|
3790
|
+
return WHILE_CONTEXT_SCHEMA;
|
|
3791
|
+
}
|
|
3792
|
+
_currentIteration = 0;
|
|
3793
|
+
constructor(input = {}, config = {}) {
|
|
3794
|
+
super(input, config);
|
|
3795
|
+
}
|
|
3796
|
+
get runner() {
|
|
3797
|
+
if (!this._runner) {
|
|
3798
|
+
this._runner = new WhileTaskRunner(this);
|
|
3799
|
+
}
|
|
3800
|
+
return this._runner;
|
|
3801
|
+
}
|
|
3802
|
+
get condition() {
|
|
3803
|
+
return this.config.condition;
|
|
3804
|
+
}
|
|
3805
|
+
get maxIterations() {
|
|
3806
|
+
if (this.config.maxIterations !== undefined)
|
|
3807
|
+
return this.config.maxIterations;
|
|
3808
|
+
const wc = this.config.extras?.whileConfig;
|
|
3809
|
+
return wc?.maxIterations ?? 100;
|
|
3810
|
+
}
|
|
3811
|
+
get chainIterations() {
|
|
3812
|
+
if (this.config.chainIterations !== undefined)
|
|
3813
|
+
return this.config.chainIterations;
|
|
3814
|
+
const wc = this.config.extras?.whileConfig;
|
|
3815
|
+
return wc?.chainIterations ?? true;
|
|
3816
|
+
}
|
|
3817
|
+
get currentIteration() {
|
|
3818
|
+
return this._currentIteration;
|
|
3819
|
+
}
|
|
3820
|
+
buildConditionFromExtras() {
|
|
3821
|
+
const wc = this.config.extras?.whileConfig;
|
|
3822
|
+
if (!wc?.conditionOperator) {
|
|
3823
|
+
return;
|
|
3824
|
+
}
|
|
3825
|
+
const { conditionField, conditionOperator, conditionValue } = wc;
|
|
3826
|
+
return (output) => {
|
|
3827
|
+
const fieldValue = conditionField ? getNestedValue(output, conditionField) : output;
|
|
3828
|
+
return evaluateCondition(fieldValue, conditionOperator, conditionValue ?? "");
|
|
3829
|
+
};
|
|
3830
|
+
}
|
|
3831
|
+
analyzeArrayInputs(input) {
|
|
3832
|
+
const wc = this.config.extras?.whileConfig;
|
|
3833
|
+
if (!wc?.iterationInputConfig) {
|
|
3834
|
+
return null;
|
|
3835
|
+
}
|
|
3836
|
+
const inputData = input;
|
|
3837
|
+
const config = wc.iterationInputConfig;
|
|
3838
|
+
const arrayPorts = [];
|
|
3839
|
+
const scalarPorts = [];
|
|
3840
|
+
const iteratedValues = {};
|
|
3841
|
+
const arrayLengths = [];
|
|
3842
|
+
for (const [key, propConfig] of Object.entries(config)) {
|
|
3843
|
+
const value = inputData[key];
|
|
3844
|
+
if (propConfig.mode === "array") {
|
|
3845
|
+
if (!Array.isArray(value)) {
|
|
3846
|
+
scalarPorts.push(key);
|
|
3847
|
+
continue;
|
|
3848
|
+
}
|
|
3849
|
+
iteratedValues[key] = value;
|
|
3850
|
+
arrayPorts.push(key);
|
|
3851
|
+
arrayLengths.push(value.length);
|
|
3852
|
+
} else {
|
|
3853
|
+
scalarPorts.push(key);
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
3856
|
+
for (const key of Object.keys(inputData)) {
|
|
3857
|
+
if (!config[key] && !key.startsWith("_iteration")) {
|
|
3858
|
+
scalarPorts.push(key);
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
if (arrayPorts.length === 0) {
|
|
3862
|
+
return null;
|
|
3863
|
+
}
|
|
3864
|
+
const uniqueLengths = new Set(arrayLengths);
|
|
3865
|
+
if (uniqueLengths.size > 1) {
|
|
3866
|
+
const lengthInfo = arrayPorts.map((port, index) => `${port}=${arrayLengths[index]}`).join(", ");
|
|
3867
|
+
throw new TaskConfigurationError(`${this.type}: All iterated array inputs must have the same length. ` + `Found different lengths: ${lengthInfo}`);
|
|
3868
|
+
}
|
|
3869
|
+
return {
|
|
3870
|
+
arrayPorts,
|
|
3871
|
+
scalarPorts,
|
|
3872
|
+
iteratedValues,
|
|
3873
|
+
iterationCount: arrayLengths[0] ?? 0
|
|
3874
|
+
};
|
|
3875
|
+
}
|
|
3876
|
+
buildIterationInput(input, analysis, index) {
|
|
3877
|
+
const inputData = input;
|
|
3878
|
+
const iterInput = {};
|
|
3879
|
+
for (const key of analysis.arrayPorts) {
|
|
3880
|
+
iterInput[key] = analysis.iteratedValues[key][index];
|
|
3881
|
+
}
|
|
3882
|
+
for (const key of analysis.scalarPorts) {
|
|
3883
|
+
if (key in inputData) {
|
|
3884
|
+
iterInput[key] = inputData[key];
|
|
3885
|
+
}
|
|
3886
|
+
}
|
|
3887
|
+
return iterInput;
|
|
3888
|
+
}
|
|
3889
|
+
async execute(input, context) {
|
|
3890
|
+
if (!this.hasChildren()) {
|
|
3891
|
+
throw new TaskConfigurationError(`${this.type}: No subgraph set for while loop`);
|
|
3892
|
+
}
|
|
3893
|
+
const condition = this.condition ?? this.buildConditionFromExtras();
|
|
3894
|
+
if (!condition) {
|
|
3895
|
+
throw new TaskConfigurationError(`${this.type}: No condition function provided`);
|
|
3896
|
+
}
|
|
3897
|
+
const arrayAnalysis = this.analyzeArrayInputs(input);
|
|
3898
|
+
this._currentIteration = 0;
|
|
3899
|
+
let currentInput = { ...input };
|
|
3900
|
+
let currentOutput = {};
|
|
3901
|
+
const effectiveMax = arrayAnalysis ? Math.min(this.maxIterations, arrayAnalysis.iterationCount) : this.maxIterations;
|
|
3902
|
+
while (this._currentIteration < effectiveMax) {
|
|
3903
|
+
if (context.signal?.aborted) {
|
|
3904
|
+
break;
|
|
3905
|
+
}
|
|
3906
|
+
let iterationInput;
|
|
3907
|
+
if (arrayAnalysis) {
|
|
3908
|
+
iterationInput = {
|
|
3909
|
+
...this.buildIterationInput(currentInput, arrayAnalysis, this._currentIteration),
|
|
3910
|
+
_iterationIndex: this._currentIteration
|
|
3911
|
+
};
|
|
3912
|
+
} else {
|
|
3913
|
+
iterationInput = {
|
|
3914
|
+
...currentInput,
|
|
3915
|
+
_iterationIndex: this._currentIteration
|
|
3916
|
+
};
|
|
3917
|
+
}
|
|
3918
|
+
const results = await this.subGraph.run(iterationInput, {
|
|
3919
|
+
parentSignal: context.signal
|
|
3920
|
+
});
|
|
3921
|
+
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
3922
|
+
if (!condition(currentOutput, this._currentIteration)) {
|
|
3923
|
+
break;
|
|
3924
|
+
}
|
|
3925
|
+
if (this.chainIterations) {
|
|
3926
|
+
currentInput = { ...currentInput, ...currentOutput };
|
|
3927
|
+
}
|
|
3928
|
+
this._currentIteration++;
|
|
3929
|
+
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
3930
|
+
await context.updateProgress(progress, `Iteration ${this._currentIteration}`);
|
|
3931
|
+
}
|
|
3932
|
+
return currentOutput;
|
|
3933
|
+
}
|
|
3934
|
+
getIterationContextSchema() {
|
|
3935
|
+
return this.constructor.getIterationContextSchema();
|
|
3936
|
+
}
|
|
3937
|
+
getChainedOutputSchema() {
|
|
3938
|
+
if (!this.chainIterations)
|
|
3939
|
+
return;
|
|
3940
|
+
const outputSchema = this.outputSchema();
|
|
3941
|
+
if (typeof outputSchema === "boolean")
|
|
3942
|
+
return;
|
|
3943
|
+
const properties = {};
|
|
3944
|
+
if (outputSchema.properties && typeof outputSchema.properties === "object") {
|
|
3945
|
+
for (const [key, schema] of Object.entries(outputSchema.properties)) {
|
|
3946
|
+
if (key === "_iterations")
|
|
3947
|
+
continue;
|
|
3948
|
+
if (typeof schema === "object" && schema !== null) {
|
|
3949
|
+
properties[key] = { ...schema, "x-ui-iteration": true };
|
|
3950
|
+
}
|
|
3951
|
+
}
|
|
3952
|
+
}
|
|
3953
|
+
if (Object.keys(properties).length === 0)
|
|
3954
|
+
return;
|
|
3955
|
+
return { type: "object", properties };
|
|
3956
|
+
}
|
|
3957
|
+
inputSchema() {
|
|
3958
|
+
if (!this.hasChildren()) {
|
|
3959
|
+
return this.constructor.inputSchema();
|
|
3960
|
+
}
|
|
3961
|
+
const baseSchema = super.inputSchema();
|
|
3962
|
+
if (typeof baseSchema === "boolean")
|
|
3963
|
+
return baseSchema;
|
|
3964
|
+
const wc = this.config.extras?.whileConfig;
|
|
3965
|
+
if (!wc?.iterationInputConfig) {
|
|
3966
|
+
return baseSchema;
|
|
3967
|
+
}
|
|
3968
|
+
const properties = { ...baseSchema.properties || {} };
|
|
3969
|
+
for (const [key, propConfig] of Object.entries(wc.iterationInputConfig)) {
|
|
3970
|
+
if (propConfig.mode === "array" && properties[key]) {
|
|
3971
|
+
const scalarSchema = properties[key];
|
|
3972
|
+
properties[key] = {
|
|
3973
|
+
anyOf: [scalarSchema, { type: "array", items: scalarSchema }]
|
|
3974
|
+
};
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
return {
|
|
3978
|
+
...baseSchema,
|
|
3979
|
+
properties
|
|
3980
|
+
};
|
|
3981
|
+
}
|
|
3982
|
+
static inputSchema() {
|
|
3983
|
+
return {
|
|
3984
|
+
type: "object",
|
|
3985
|
+
properties: {},
|
|
3986
|
+
additionalProperties: true
|
|
3987
|
+
};
|
|
3988
|
+
}
|
|
3989
|
+
static outputSchema() {
|
|
3990
|
+
return {
|
|
3991
|
+
type: "object",
|
|
3992
|
+
properties: {
|
|
3993
|
+
_iterations: {
|
|
3994
|
+
type: "number",
|
|
3995
|
+
title: "Iterations",
|
|
3996
|
+
description: "Number of iterations executed"
|
|
3997
|
+
}
|
|
3998
|
+
},
|
|
3999
|
+
additionalProperties: true
|
|
4000
|
+
};
|
|
4001
|
+
}
|
|
4002
|
+
outputSchema() {
|
|
4003
|
+
if (!this.hasChildren()) {
|
|
4004
|
+
return this.constructor.outputSchema();
|
|
4005
|
+
}
|
|
4006
|
+
const tasks = this.subGraph.getTasks();
|
|
4007
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
4008
|
+
if (endingNodes.length === 0) {
|
|
4009
|
+
return this.constructor.outputSchema();
|
|
4010
|
+
}
|
|
4011
|
+
const properties = {
|
|
4012
|
+
_iterations: {
|
|
4013
|
+
type: "number",
|
|
4014
|
+
title: "Iterations",
|
|
4015
|
+
description: "Number of iterations executed"
|
|
4016
|
+
}
|
|
4017
|
+
};
|
|
4018
|
+
for (const task of endingNodes) {
|
|
4019
|
+
const taskOutputSchema = task.outputSchema();
|
|
4020
|
+
if (typeof taskOutputSchema === "boolean")
|
|
4021
|
+
continue;
|
|
4022
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
4023
|
+
for (const [key, schema] of Object.entries(taskProperties)) {
|
|
4024
|
+
if (!properties[key]) {
|
|
4025
|
+
properties[key] = schema;
|
|
4026
|
+
}
|
|
4027
|
+
}
|
|
4028
|
+
}
|
|
4029
|
+
return {
|
|
4030
|
+
type: "object",
|
|
4031
|
+
properties,
|
|
4032
|
+
additionalProperties: false
|
|
4033
|
+
};
|
|
4034
|
+
}
|
|
4035
|
+
}
|
|
4036
|
+
Workflow.prototype.while = CreateLoopWorkflow(WhileTask);
|
|
4037
|
+
Workflow.prototype.endWhile = CreateEndLoopWorkflow("endWhile");
|
|
4038
|
+
|
|
4039
|
+
// src/task/iterationSchema.ts
|
|
4040
|
+
function isFlexibleSchema(schema) {
|
|
4041
|
+
if (typeof schema === "boolean")
|
|
4042
|
+
return false;
|
|
4043
|
+
const variants = schema.oneOf ?? schema.anyOf;
|
|
4044
|
+
const arr = Array.isArray(variants) ? variants : undefined;
|
|
4045
|
+
if (!arr || arr.length !== 2)
|
|
4046
|
+
return false;
|
|
4047
|
+
let hasScalar = false;
|
|
4048
|
+
let hasArray = false;
|
|
4049
|
+
for (const variant of arr) {
|
|
4050
|
+
if (typeof variant !== "object")
|
|
4051
|
+
continue;
|
|
4052
|
+
const v = variant;
|
|
4053
|
+
if (v.type === "array" || "items" in v) {
|
|
4054
|
+
hasArray = true;
|
|
4055
|
+
} else {
|
|
4056
|
+
hasScalar = true;
|
|
4057
|
+
}
|
|
4058
|
+
}
|
|
4059
|
+
return hasScalar && hasArray;
|
|
4060
|
+
}
|
|
4061
|
+
function isStrictArraySchema(schema) {
|
|
4062
|
+
if (typeof schema === "boolean")
|
|
4063
|
+
return false;
|
|
4064
|
+
const s = schema;
|
|
4065
|
+
return s.type === "array" && !isFlexibleSchema(schema);
|
|
4066
|
+
}
|
|
4067
|
+
function getInputModeFromSchema(schema) {
|
|
4068
|
+
if (isFlexibleSchema(schema))
|
|
4069
|
+
return "flexible";
|
|
4070
|
+
if (isStrictArraySchema(schema))
|
|
4071
|
+
return "array";
|
|
4072
|
+
return "scalar";
|
|
4073
|
+
}
|
|
4074
|
+
function getIterationContextSchemaForType(taskType) {
|
|
4075
|
+
if (taskType === "MapTask" || taskType === "ReduceTask") {
|
|
4076
|
+
return ITERATOR_CONTEXT_SCHEMA;
|
|
4077
|
+
}
|
|
4078
|
+
if (taskType === "WhileTask") {
|
|
4079
|
+
return WHILE_CONTEXT_SCHEMA;
|
|
4080
|
+
}
|
|
4081
|
+
return;
|
|
4082
|
+
}
|
|
4083
|
+
function addIterationContextToSchema(existingSchema, parentTaskType) {
|
|
4084
|
+
const contextSchema = getIterationContextSchemaForType(parentTaskType);
|
|
4085
|
+
if (!contextSchema) {
|
|
4086
|
+
return existingSchema ?? { type: "object", properties: {} };
|
|
4087
|
+
}
|
|
4088
|
+
const baseProperties = existingSchema && typeof existingSchema !== "boolean" && existingSchema.properties && typeof existingSchema.properties !== "boolean" ? existingSchema.properties : {};
|
|
4089
|
+
const contextProperties = typeof contextSchema !== "boolean" && contextSchema.properties && typeof contextSchema.properties !== "boolean" ? contextSchema.properties : {};
|
|
4090
|
+
return {
|
|
4091
|
+
type: "object",
|
|
4092
|
+
properties: {
|
|
4093
|
+
...baseProperties,
|
|
4094
|
+
...contextProperties
|
|
4095
|
+
}
|
|
4096
|
+
};
|
|
4097
|
+
}
|
|
4098
|
+
function isIterationProperty(schema) {
|
|
4099
|
+
if (!schema || typeof schema === "boolean")
|
|
4100
|
+
return false;
|
|
4101
|
+
return schema["x-ui-iteration"] === true;
|
|
4102
|
+
}
|
|
4103
|
+
function filterIterationProperties(schema) {
|
|
4104
|
+
if (!schema || typeof schema === "boolean")
|
|
4105
|
+
return schema;
|
|
4106
|
+
const props = schema.properties;
|
|
4107
|
+
if (!props || typeof props === "boolean")
|
|
4108
|
+
return schema;
|
|
4109
|
+
const filteredProps = {};
|
|
4110
|
+
for (const [key, propSchema] of Object.entries(props)) {
|
|
4111
|
+
if (!isIterationProperty(propSchema)) {
|
|
4112
|
+
filteredProps[key] = propSchema;
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
if (Object.keys(filteredProps).length === 0) {
|
|
4116
|
+
return { type: "object", properties: {} };
|
|
4117
|
+
}
|
|
4118
|
+
return { ...schema, properties: filteredProps };
|
|
4119
|
+
}
|
|
4120
|
+
function extractIterationProperties(schema) {
|
|
4121
|
+
if (!schema || typeof schema === "boolean")
|
|
4122
|
+
return;
|
|
4123
|
+
const props = schema.properties;
|
|
4124
|
+
if (!props || typeof props === "boolean")
|
|
4125
|
+
return;
|
|
4126
|
+
const iterProps = {};
|
|
4127
|
+
for (const [key, propSchema] of Object.entries(props)) {
|
|
4128
|
+
if (isIterationProperty(propSchema)) {
|
|
4129
|
+
iterProps[key] = propSchema;
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
if (Object.keys(iterProps).length === 0)
|
|
4133
|
+
return;
|
|
4134
|
+
return { type: "object", properties: iterProps };
|
|
4135
|
+
}
|
|
4136
|
+
function removeIterationProperties(schema) {
|
|
4137
|
+
return filterIterationProperties(schema);
|
|
4138
|
+
}
|
|
4139
|
+
function mergeChainedOutputToInput(inputSchema, outputSchema) {
|
|
4140
|
+
const baseSchema = filterIterationProperties(inputSchema) ?? {
|
|
4141
|
+
type: "object",
|
|
4142
|
+
properties: {}
|
|
4143
|
+
};
|
|
4144
|
+
if (!outputSchema || typeof outputSchema === "boolean") {
|
|
4145
|
+
return baseSchema;
|
|
4146
|
+
}
|
|
4147
|
+
const outProps = outputSchema.properties;
|
|
4148
|
+
if (!outProps || typeof outProps === "boolean") {
|
|
4149
|
+
return baseSchema;
|
|
4150
|
+
}
|
|
4151
|
+
const baseProps = typeof baseSchema !== "boolean" && baseSchema.properties && typeof baseSchema.properties !== "boolean" ? baseSchema.properties : {};
|
|
4152
|
+
const mergedProperties = { ...baseProps };
|
|
4153
|
+
for (const [key, propSchema] of Object.entries(outProps)) {
|
|
4154
|
+
if (typeof propSchema === "object" && propSchema !== null) {
|
|
4155
|
+
mergedProperties[key] = { ...propSchema, "x-ui-iteration": true };
|
|
4156
|
+
}
|
|
4157
|
+
}
|
|
4158
|
+
return {
|
|
4159
|
+
type: "object",
|
|
4160
|
+
properties: mergedProperties
|
|
4161
|
+
};
|
|
4162
|
+
}
|
|
4163
|
+
function buildIterationInputSchema(innerSchema, config) {
|
|
4164
|
+
if (!innerSchema || typeof innerSchema === "boolean") {
|
|
4165
|
+
return { type: "object", properties: {} };
|
|
4166
|
+
}
|
|
4167
|
+
const innerProps = innerSchema.properties;
|
|
4168
|
+
if (!innerProps || typeof innerProps === "boolean") {
|
|
4169
|
+
return { type: "object", properties: {} };
|
|
4170
|
+
}
|
|
4171
|
+
const properties = {};
|
|
4172
|
+
const propsRecord = innerProps;
|
|
4173
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4174
|
+
if (typeof propSchema === "boolean")
|
|
4175
|
+
continue;
|
|
4176
|
+
if (propSchema["x-ui-iteration"]) {
|
|
4177
|
+
continue;
|
|
4178
|
+
}
|
|
4179
|
+
const originalProps = propSchema;
|
|
4180
|
+
const metadata = {};
|
|
4181
|
+
for (const metaKey of Object.keys(originalProps)) {
|
|
4182
|
+
if (metaKey === "title" || metaKey === "description" || metaKey.startsWith("x-")) {
|
|
4183
|
+
metadata[metaKey] = originalProps[metaKey];
|
|
4184
|
+
}
|
|
4185
|
+
}
|
|
4186
|
+
const baseSchema = extractBaseSchema(propSchema);
|
|
4187
|
+
const propConfig = config?.[key];
|
|
4188
|
+
const mode = propConfig?.mode ?? "flexible";
|
|
4189
|
+
const base = propConfig?.baseSchema ?? baseSchema;
|
|
4190
|
+
let wrappedSchema;
|
|
4191
|
+
switch (mode) {
|
|
4192
|
+
case "array":
|
|
4193
|
+
wrappedSchema = createArraySchema(base);
|
|
4194
|
+
break;
|
|
4195
|
+
case "scalar":
|
|
4196
|
+
wrappedSchema = base;
|
|
4197
|
+
break;
|
|
4198
|
+
case "flexible":
|
|
4199
|
+
default:
|
|
4200
|
+
wrappedSchema = createFlexibleSchema(base);
|
|
4201
|
+
break;
|
|
4202
|
+
}
|
|
4203
|
+
if (Object.keys(metadata).length > 0 && typeof wrappedSchema === "object") {
|
|
4204
|
+
properties[key] = { ...metadata, ...wrappedSchema };
|
|
4205
|
+
} else {
|
|
4206
|
+
properties[key] = wrappedSchema;
|
|
4207
|
+
}
|
|
4208
|
+
}
|
|
4209
|
+
return {
|
|
4210
|
+
type: "object",
|
|
4211
|
+
properties
|
|
4212
|
+
};
|
|
4213
|
+
}
|
|
4214
|
+
function findArrayPorts(schema) {
|
|
4215
|
+
if (!schema || typeof schema === "boolean")
|
|
4216
|
+
return [];
|
|
4217
|
+
const props = schema.properties;
|
|
4218
|
+
if (!props || typeof props === "boolean")
|
|
4219
|
+
return [];
|
|
4220
|
+
const arrayPorts = [];
|
|
4221
|
+
const propsRecord = props;
|
|
4222
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4223
|
+
if (typeof propSchema === "boolean")
|
|
4224
|
+
continue;
|
|
4225
|
+
if (propSchema.type === "array") {
|
|
4226
|
+
arrayPorts.push(key);
|
|
4227
|
+
}
|
|
4228
|
+
}
|
|
4229
|
+
return arrayPorts;
|
|
4230
|
+
}
|
|
4231
|
+
function wrapSchemaInArray(schema) {
|
|
4232
|
+
if (!schema || typeof schema === "boolean")
|
|
4233
|
+
return schema;
|
|
4234
|
+
const props = schema.properties;
|
|
4235
|
+
if (!props || typeof props === "boolean")
|
|
4236
|
+
return schema;
|
|
4237
|
+
const propsRecord = props;
|
|
4238
|
+
const wrappedProperties = {};
|
|
4239
|
+
for (const [key, propSchema] of Object.entries(propsRecord)) {
|
|
4240
|
+
wrappedProperties[key] = {
|
|
4241
|
+
type: "array",
|
|
4242
|
+
items: propSchema
|
|
4243
|
+
};
|
|
4244
|
+
}
|
|
4245
|
+
return {
|
|
4246
|
+
type: "object",
|
|
4247
|
+
properties: wrappedProperties
|
|
4248
|
+
};
|
|
4249
|
+
}
|
|
4250
|
+
// src/task/JobQueueFactory.ts
|
|
4251
|
+
import {
|
|
4252
|
+
JobQueueClient,
|
|
4253
|
+
JobQueueServer
|
|
4254
|
+
} from "@workglow/job-queue";
|
|
4255
|
+
import { InMemoryQueueStorage } from "@workglow/storage";
|
|
4256
|
+
import { createServiceToken as createServiceToken2, globalServiceRegistry as globalServiceRegistry3 } from "@workglow/util";
|
|
4257
|
+
var JOB_QUEUE_FACTORY = createServiceToken2("taskgraph.jobQueueFactory");
|
|
4258
|
+
var defaultJobQueueFactory = async ({
|
|
4259
|
+
queueName,
|
|
4260
|
+
jobClass,
|
|
4261
|
+
options
|
|
4262
|
+
}) => {
|
|
4263
|
+
const storage = options?.storage ?? new InMemoryQueueStorage(queueName);
|
|
4264
|
+
await storage.setupDatabase();
|
|
4265
|
+
const server = new JobQueueServer(jobClass, {
|
|
4266
|
+
storage,
|
|
4267
|
+
queueName,
|
|
4268
|
+
limiter: options?.limiter,
|
|
4269
|
+
workerCount: options?.workerCount,
|
|
4270
|
+
pollIntervalMs: options?.pollIntervalMs,
|
|
4271
|
+
deleteAfterCompletionMs: options?.deleteAfterCompletionMs,
|
|
4272
|
+
deleteAfterFailureMs: options?.deleteAfterFailureMs,
|
|
4273
|
+
deleteAfterDisabledMs: options?.deleteAfterDisabledMs,
|
|
4274
|
+
cleanupIntervalMs: options?.cleanupIntervalMs
|
|
4275
|
+
});
|
|
4276
|
+
const client = new JobQueueClient({
|
|
4277
|
+
storage,
|
|
4278
|
+
queueName
|
|
4279
|
+
});
|
|
4280
|
+
client.attach(server);
|
|
4281
|
+
return { server, client, storage };
|
|
4282
|
+
};
|
|
4283
|
+
function registerJobQueueFactory(factory) {
|
|
4284
|
+
globalServiceRegistry3.registerInstance(JOB_QUEUE_FACTORY, factory);
|
|
4285
|
+
}
|
|
4286
|
+
function createJobQueueFactoryWithOptions(defaultOptions = {}) {
|
|
4287
|
+
return async ({
|
|
4288
|
+
queueName,
|
|
4289
|
+
jobClass,
|
|
4290
|
+
options
|
|
4291
|
+
}) => {
|
|
4292
|
+
const mergedOptions = {
|
|
4293
|
+
...defaultOptions,
|
|
4294
|
+
...options ?? {}
|
|
4295
|
+
};
|
|
4296
|
+
const storage = mergedOptions.storage ?? new InMemoryQueueStorage(queueName);
|
|
4297
|
+
await storage.setupDatabase();
|
|
4298
|
+
const server = new JobQueueServer(jobClass, {
|
|
4299
|
+
storage,
|
|
4300
|
+
queueName,
|
|
4301
|
+
limiter: mergedOptions.limiter,
|
|
4302
|
+
workerCount: mergedOptions.workerCount,
|
|
4303
|
+
pollIntervalMs: mergedOptions.pollIntervalMs,
|
|
4304
|
+
deleteAfterCompletionMs: mergedOptions.deleteAfterCompletionMs,
|
|
4305
|
+
deleteAfterFailureMs: mergedOptions.deleteAfterFailureMs,
|
|
4306
|
+
deleteAfterDisabledMs: mergedOptions.deleteAfterDisabledMs,
|
|
4307
|
+
cleanupIntervalMs: mergedOptions.cleanupIntervalMs
|
|
4308
|
+
});
|
|
4309
|
+
const client = new JobQueueClient({
|
|
4310
|
+
storage,
|
|
4311
|
+
queueName
|
|
4312
|
+
});
|
|
4313
|
+
client.attach(server);
|
|
4314
|
+
return { server, client, storage };
|
|
4315
|
+
};
|
|
4316
|
+
}
|
|
4317
|
+
function getJobQueueFactory() {
|
|
4318
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
4319
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
4320
|
+
}
|
|
4321
|
+
return globalServiceRegistry3.get(JOB_QUEUE_FACTORY);
|
|
4322
|
+
}
|
|
4323
|
+
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
4324
|
+
registerJobQueueFactory(defaultJobQueueFactory);
|
|
4325
|
+
}
|
|
4326
|
+
// src/task/JobQueueTask.ts
|
|
4327
|
+
import { Job as Job2 } from "@workglow/job-queue";
|
|
4328
|
+
|
|
4329
|
+
// src/task/TaskQueueRegistry.ts
|
|
4330
|
+
var taskQueueRegistry = null;
|
|
4331
|
+
|
|
4332
|
+
class TaskQueueRegistry {
|
|
4333
|
+
queues = new Map;
|
|
4334
|
+
registerQueue(queue) {
|
|
4335
|
+
const queueName = queue.server.queueName;
|
|
4336
|
+
if (this.queues.has(queueName)) {
|
|
4337
|
+
throw new Error(`Queue with name ${queueName} already exists`);
|
|
4338
|
+
}
|
|
4339
|
+
this.queues.set(queueName, queue);
|
|
4340
|
+
}
|
|
4341
|
+
getQueue(queueName) {
|
|
4342
|
+
return this.queues.get(queueName);
|
|
4343
|
+
}
|
|
4344
|
+
startQueues() {
|
|
4345
|
+
for (const queue of this.queues.values()) {
|
|
4346
|
+
queue.server.start();
|
|
4347
|
+
}
|
|
4348
|
+
return this;
|
|
4349
|
+
}
|
|
4350
|
+
stopQueues() {
|
|
4351
|
+
for (const queue of this.queues.values()) {
|
|
4352
|
+
queue.server.stop();
|
|
4353
|
+
}
|
|
4354
|
+
return this;
|
|
4355
|
+
}
|
|
4356
|
+
clearQueues() {
|
|
4357
|
+
for (const queue of this.queues.values()) {
|
|
4358
|
+
queue.storage.deleteAll();
|
|
4359
|
+
}
|
|
4360
|
+
return this;
|
|
4361
|
+
}
|
|
4362
|
+
}
|
|
4363
|
+
function getTaskQueueRegistry() {
|
|
4364
|
+
if (!taskQueueRegistry) {
|
|
4365
|
+
taskQueueRegistry = new TaskQueueRegistry;
|
|
2640
4366
|
}
|
|
2641
4367
|
return taskQueueRegistry;
|
|
2642
4368
|
}
|
|
@@ -2781,6 +4507,155 @@ class JobQueueTask extends GraphAsTask {
|
|
|
2781
4507
|
super.abort();
|
|
2782
4508
|
}
|
|
2783
4509
|
}
|
|
4510
|
+
// src/task/MapTask.ts
|
|
4511
|
+
class MapTask extends IteratorTask {
|
|
4512
|
+
static type = "MapTask";
|
|
4513
|
+
static category = "Flow Control";
|
|
4514
|
+
static title = "Map";
|
|
4515
|
+
static description = "Transforms array inputs by running a workflow per item";
|
|
4516
|
+
static compoundMerge = PROPERTY_ARRAY;
|
|
4517
|
+
static inputSchema() {
|
|
4518
|
+
return {
|
|
4519
|
+
type: "object",
|
|
4520
|
+
properties: {},
|
|
4521
|
+
additionalProperties: true
|
|
4522
|
+
};
|
|
4523
|
+
}
|
|
4524
|
+
static outputSchema() {
|
|
4525
|
+
return {
|
|
4526
|
+
type: "object",
|
|
4527
|
+
properties: {},
|
|
4528
|
+
additionalProperties: true
|
|
4529
|
+
};
|
|
4530
|
+
}
|
|
4531
|
+
get preserveOrder() {
|
|
4532
|
+
return this.config.preserveOrder ?? true;
|
|
4533
|
+
}
|
|
4534
|
+
get flatten() {
|
|
4535
|
+
return this.config.flatten ?? false;
|
|
4536
|
+
}
|
|
4537
|
+
preserveIterationOrder() {
|
|
4538
|
+
return this.preserveOrder;
|
|
4539
|
+
}
|
|
4540
|
+
getEmptyResult() {
|
|
4541
|
+
const schema = this.outputSchema();
|
|
4542
|
+
if (typeof schema === "boolean") {
|
|
4543
|
+
return {};
|
|
4544
|
+
}
|
|
4545
|
+
const result = {};
|
|
4546
|
+
for (const key of Object.keys(schema.properties || {})) {
|
|
4547
|
+
result[key] = [];
|
|
4548
|
+
}
|
|
4549
|
+
return result;
|
|
4550
|
+
}
|
|
4551
|
+
outputSchema() {
|
|
4552
|
+
if (!this.hasChildren()) {
|
|
4553
|
+
return this.constructor.outputSchema();
|
|
4554
|
+
}
|
|
4555
|
+
return this.getWrappedOutputSchema();
|
|
4556
|
+
}
|
|
4557
|
+
collectResults(results) {
|
|
4558
|
+
const collected = super.collectResults(results);
|
|
4559
|
+
if (!this.flatten || typeof collected !== "object" || collected === null) {
|
|
4560
|
+
return collected;
|
|
4561
|
+
}
|
|
4562
|
+
const flattened = {};
|
|
4563
|
+
for (const [key, value] of Object.entries(collected)) {
|
|
4564
|
+
if (Array.isArray(value)) {
|
|
4565
|
+
flattened[key] = value.flat();
|
|
4566
|
+
} else {
|
|
4567
|
+
flattened[key] = value;
|
|
4568
|
+
}
|
|
4569
|
+
}
|
|
4570
|
+
return flattened;
|
|
4571
|
+
}
|
|
4572
|
+
}
|
|
4573
|
+
queueMicrotask(() => {
|
|
4574
|
+
Workflow.prototype.map = CreateLoopWorkflow(MapTask);
|
|
4575
|
+
Workflow.prototype.endMap = CreateEndLoopWorkflow("endMap");
|
|
4576
|
+
});
|
|
4577
|
+
// src/task/ReduceTask.ts
|
|
4578
|
+
class ReduceTask extends IteratorTask {
|
|
4579
|
+
static type = "ReduceTask";
|
|
4580
|
+
static category = "Flow Control";
|
|
4581
|
+
static title = "Reduce";
|
|
4582
|
+
static description = "Processes iterated inputs sequentially with an accumulator (fold)";
|
|
4583
|
+
constructor(input = {}, config = {}) {
|
|
4584
|
+
const reduceConfig = {
|
|
4585
|
+
...config,
|
|
4586
|
+
concurrencyLimit: 1,
|
|
4587
|
+
batchSize: 1
|
|
4588
|
+
};
|
|
4589
|
+
super(input, reduceConfig);
|
|
4590
|
+
}
|
|
4591
|
+
get initialValue() {
|
|
4592
|
+
return this.config.initialValue ?? {};
|
|
4593
|
+
}
|
|
4594
|
+
isReduceTask() {
|
|
4595
|
+
return true;
|
|
4596
|
+
}
|
|
4597
|
+
getInitialAccumulator() {
|
|
4598
|
+
const value = this.initialValue;
|
|
4599
|
+
if (Array.isArray(value)) {
|
|
4600
|
+
return [...value];
|
|
4601
|
+
}
|
|
4602
|
+
if (value && typeof value === "object") {
|
|
4603
|
+
return { ...value };
|
|
4604
|
+
}
|
|
4605
|
+
return value;
|
|
4606
|
+
}
|
|
4607
|
+
buildIterationRunInput(analysis, index, iterationCount, extraInput = {}) {
|
|
4608
|
+
return super.buildIterationRunInput(analysis, index, iterationCount, {
|
|
4609
|
+
accumulator: extraInput.accumulator
|
|
4610
|
+
});
|
|
4611
|
+
}
|
|
4612
|
+
getEmptyResult() {
|
|
4613
|
+
return this.getInitialAccumulator();
|
|
4614
|
+
}
|
|
4615
|
+
static inputSchema() {
|
|
4616
|
+
return {
|
|
4617
|
+
type: "object",
|
|
4618
|
+
properties: {},
|
|
4619
|
+
additionalProperties: true
|
|
4620
|
+
};
|
|
4621
|
+
}
|
|
4622
|
+
static outputSchema() {
|
|
4623
|
+
return {
|
|
4624
|
+
type: "object",
|
|
4625
|
+
properties: {},
|
|
4626
|
+
additionalProperties: true
|
|
4627
|
+
};
|
|
4628
|
+
}
|
|
4629
|
+
outputSchema() {
|
|
4630
|
+
if (!this.hasChildren()) {
|
|
4631
|
+
return this.constructor.outputSchema();
|
|
4632
|
+
}
|
|
4633
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
4634
|
+
if (endingNodes.length === 0) {
|
|
4635
|
+
return this.constructor.outputSchema();
|
|
4636
|
+
}
|
|
4637
|
+
const properties = {};
|
|
4638
|
+
for (const task of endingNodes) {
|
|
4639
|
+
const taskOutputSchema = task.outputSchema();
|
|
4640
|
+
if (typeof taskOutputSchema === "boolean")
|
|
4641
|
+
continue;
|
|
4642
|
+
for (const [key, schema] of Object.entries(taskOutputSchema.properties || {})) {
|
|
4643
|
+
if (!properties[key]) {
|
|
4644
|
+
properties[key] = schema;
|
|
4645
|
+
}
|
|
4646
|
+
}
|
|
4647
|
+
}
|
|
4648
|
+
return {
|
|
4649
|
+
type: "object",
|
|
4650
|
+
properties,
|
|
4651
|
+
additionalProperties: false
|
|
4652
|
+
};
|
|
4653
|
+
}
|
|
4654
|
+
}
|
|
4655
|
+
queueMicrotask(() => {
|
|
4656
|
+
Workflow.prototype.reduce = CreateLoopWorkflow(ReduceTask);
|
|
4657
|
+
Workflow.prototype.endReduce = CreateEndLoopWorkflow("endReduce");
|
|
4658
|
+
});
|
|
2784
4659
|
// src/task/TaskRegistry.ts
|
|
2785
4660
|
var taskConstructors = new Map;
|
|
2786
4661
|
function registerTask(baseClass) {
|
|
@@ -2850,7 +4725,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
2850
4725
|
};
|
|
2851
4726
|
// src/task/index.ts
|
|
2852
4727
|
var registerBaseTasks = () => {
|
|
2853
|
-
const tasks = [ConditionalTask,
|
|
4728
|
+
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
2854
4729
|
tasks.map(TaskRegistry.registerTask);
|
|
2855
4730
|
return tasks;
|
|
2856
4731
|
};
|
|
@@ -3011,25 +4886,53 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
3011
4886
|
}
|
|
3012
4887
|
}
|
|
3013
4888
|
export {
|
|
4889
|
+
wrapSchemaInArray,
|
|
3014
4890
|
setTaskQueueRegistry,
|
|
3015
4891
|
serialGraph,
|
|
4892
|
+
schemaAcceptsArray,
|
|
3016
4893
|
resolveSchemaInputs,
|
|
4894
|
+
removeIterationProperties,
|
|
3017
4895
|
registerJobQueueFactory,
|
|
3018
4896
|
registerBaseTasks,
|
|
3019
4897
|
pipe,
|
|
3020
4898
|
parallel,
|
|
4899
|
+
mergeChainedOutputToInput,
|
|
4900
|
+
isTaskStreamable,
|
|
4901
|
+
isStrictArraySchema,
|
|
4902
|
+
isIterationProperty,
|
|
4903
|
+
isFlexibleSchema,
|
|
4904
|
+
hasVectorOutput,
|
|
4905
|
+
hasVectorLikeInput,
|
|
3021
4906
|
getTaskQueueRegistry,
|
|
4907
|
+
getPortStreamMode,
|
|
4908
|
+
getOutputStreamMode,
|
|
4909
|
+
getNestedValue,
|
|
3022
4910
|
getLastTask,
|
|
3023
4911
|
getJobQueueFactory,
|
|
4912
|
+
getIterationContextSchemaForType,
|
|
4913
|
+
getInputModeFromSchema,
|
|
4914
|
+
findArrayPorts,
|
|
4915
|
+
filterIterationProperties,
|
|
4916
|
+
extractIterationProperties,
|
|
4917
|
+
extractBaseSchema,
|
|
4918
|
+
evaluateCondition,
|
|
3024
4919
|
ensureTask,
|
|
4920
|
+
edgeNeedsAccumulation,
|
|
3025
4921
|
createTaskFromGraphJSON,
|
|
3026
4922
|
createTaskFromDependencyJSON,
|
|
3027
4923
|
createJobQueueFactoryWithOptions,
|
|
3028
4924
|
createGraphFromGraphJSON,
|
|
3029
4925
|
createGraphFromDependencyJSON,
|
|
4926
|
+
createFlexibleSchema,
|
|
4927
|
+
createArraySchema,
|
|
3030
4928
|
connect,
|
|
4929
|
+
buildIterationInputSchema,
|
|
4930
|
+
addIterationContextToSchema,
|
|
3031
4931
|
WorkflowError,
|
|
3032
4932
|
Workflow,
|
|
4933
|
+
WhileTaskRunner,
|
|
4934
|
+
WhileTask,
|
|
4935
|
+
WHILE_CONTEXT_SCHEMA,
|
|
3033
4936
|
TaskStatus,
|
|
3034
4937
|
TaskRegistry,
|
|
3035
4938
|
TaskQueueRegistry,
|
|
@@ -3052,10 +4955,15 @@ export {
|
|
|
3052
4955
|
Task,
|
|
3053
4956
|
TASK_OUTPUT_REPOSITORY,
|
|
3054
4957
|
TASK_GRAPH_REPOSITORY,
|
|
4958
|
+
ReduceTask,
|
|
3055
4959
|
PROPERTY_ARRAY,
|
|
4960
|
+
MapTask,
|
|
3056
4961
|
JobTaskFailedError,
|
|
3057
4962
|
JobQueueTask,
|
|
3058
4963
|
JOB_QUEUE_FACTORY,
|
|
4964
|
+
IteratorTaskRunner,
|
|
4965
|
+
IteratorTask,
|
|
4966
|
+
ITERATOR_CONTEXT_SCHEMA,
|
|
3059
4967
|
GraphAsTaskRunner,
|
|
3060
4968
|
GraphAsTask,
|
|
3061
4969
|
GRAPH_RESULT_ARRAY,
|
|
@@ -3066,7 +4974,10 @@ export {
|
|
|
3066
4974
|
DATAFLOW_ERROR_PORT,
|
|
3067
4975
|
DATAFLOW_ALL_PORTS,
|
|
3068
4976
|
CreateWorkflow,
|
|
4977
|
+
CreateLoopWorkflow,
|
|
4978
|
+
CreateEndLoopWorkflow,
|
|
4979
|
+
CreateAdaptiveWorkflow,
|
|
3069
4980
|
ConditionalTask
|
|
3070
4981
|
};
|
|
3071
4982
|
|
|
3072
|
-
//# debugId=
|
|
4983
|
+
//# debugId=C8B7F47334D5CDD864756E2164756E21
|