@workglow/task-graph 0.0.101 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1083 -275
- package/dist/browser.js.map +26 -23
- package/dist/bun.js +1083 -275
- package/dist/bun.js.map +26 -23
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1083 -275
- package/dist/node.js.map +26 -23
- package/dist/task/ConditionalTask.d.ts +11 -2
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +22 -13
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +6 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +22 -8
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +21 -3
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +11 -1
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +11 -1
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +25 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +12 -3
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +11 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +11 -1
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task/iterationSchema.d.ts +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/node.js
CHANGED
|
@@ -14,15 +14,29 @@ var TaskStatus = {
|
|
|
14
14
|
var TaskConfigSchema = {
|
|
15
15
|
type: "object",
|
|
16
16
|
properties: {
|
|
17
|
-
id: {
|
|
17
|
+
id: {
|
|
18
|
+
"x-ui-hidden": true
|
|
19
|
+
},
|
|
18
20
|
title: { type: "string" },
|
|
19
21
|
description: { type: "string" },
|
|
20
22
|
cacheable: { type: "boolean" },
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
24
|
+
inputSchema: {
|
|
25
|
+
type: "object",
|
|
26
|
+
properties: {},
|
|
27
|
+
additionalProperties: true,
|
|
28
|
+
"x-ui-hidden": true
|
|
29
|
+
},
|
|
30
|
+
outputSchema: {
|
|
31
|
+
type: "object",
|
|
32
|
+
properties: {},
|
|
33
|
+
additionalProperties: true,
|
|
34
|
+
"x-ui-hidden": true
|
|
35
|
+
},
|
|
23
36
|
extras: {
|
|
24
37
|
type: "object",
|
|
25
|
-
additionalProperties: true
|
|
38
|
+
additionalProperties: true,
|
|
39
|
+
"x-ui-hidden": true
|
|
26
40
|
}
|
|
27
41
|
},
|
|
28
42
|
additionalProperties: false
|
|
@@ -227,8 +241,352 @@ class DataflowArrow extends Dataflow {
|
|
|
227
241
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
228
242
|
}
|
|
229
243
|
}
|
|
244
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
245
|
+
import { uuid4 } from "@workglow/util";
|
|
246
|
+
function calculateNodeDepths(graph) {
|
|
247
|
+
const depths = new Map;
|
|
248
|
+
const tasks = graph.getTasks();
|
|
249
|
+
for (const task of tasks) {
|
|
250
|
+
depths.set(task.id, 0);
|
|
251
|
+
}
|
|
252
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
253
|
+
for (const task of sortedTasks) {
|
|
254
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
255
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
256
|
+
for (const targetTask of targetTasks) {
|
|
257
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
258
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return depths;
|
|
262
|
+
}
|
|
263
|
+
function computeGraphInputSchema(graph, options) {
|
|
264
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
265
|
+
const properties = {};
|
|
266
|
+
const required = [];
|
|
267
|
+
const propertyOrigins = {};
|
|
268
|
+
const tasks = graph.getTasks();
|
|
269
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
270
|
+
for (const task of startingNodes) {
|
|
271
|
+
const taskInputSchema = task.inputSchema();
|
|
272
|
+
if (typeof taskInputSchema === "boolean") {
|
|
273
|
+
if (taskInputSchema === false) {
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (taskInputSchema === true) {
|
|
277
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
282
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
283
|
+
if (!properties[inputName]) {
|
|
284
|
+
properties[inputName] = inputProp;
|
|
285
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
286
|
+
required.push(inputName);
|
|
287
|
+
}
|
|
288
|
+
if (trackOrigins) {
|
|
289
|
+
propertyOrigins[inputName] = [task.id];
|
|
290
|
+
}
|
|
291
|
+
} else if (trackOrigins) {
|
|
292
|
+
propertyOrigins[inputName].push(task.id);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
297
|
+
for (const task of tasks) {
|
|
298
|
+
if (sourceIds.has(task.id))
|
|
299
|
+
continue;
|
|
300
|
+
const taskInputSchema = task.inputSchema();
|
|
301
|
+
if (typeof taskInputSchema === "boolean")
|
|
302
|
+
continue;
|
|
303
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
304
|
+
if (requiredKeys.size === 0)
|
|
305
|
+
continue;
|
|
306
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
307
|
+
for (const key of requiredKeys) {
|
|
308
|
+
if (connectedPorts.has(key))
|
|
309
|
+
continue;
|
|
310
|
+
if (properties[key]) {
|
|
311
|
+
if (trackOrigins) {
|
|
312
|
+
propertyOrigins[key].push(task.id);
|
|
313
|
+
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
317
|
+
continue;
|
|
318
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
319
|
+
if (!prop || typeof prop === "boolean")
|
|
320
|
+
continue;
|
|
321
|
+
properties[key] = prop;
|
|
322
|
+
if (!required.includes(key)) {
|
|
323
|
+
required.push(key);
|
|
324
|
+
}
|
|
325
|
+
if (trackOrigins) {
|
|
326
|
+
propertyOrigins[key] = [task.id];
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (trackOrigins) {
|
|
331
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
332
|
+
const prop = properties[propName];
|
|
333
|
+
if (!prop || typeof prop === "boolean")
|
|
334
|
+
continue;
|
|
335
|
+
if (origins.length === 1) {
|
|
336
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
337
|
+
} else {
|
|
338
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
type: "object",
|
|
344
|
+
properties,
|
|
345
|
+
...required.length > 0 ? { required } : {},
|
|
346
|
+
additionalProperties: false
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function computeGraphOutputSchema(graph, options) {
|
|
350
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
351
|
+
const properties = {};
|
|
352
|
+
const required = [];
|
|
353
|
+
const propertyOrigins = {};
|
|
354
|
+
const tasks = graph.getTasks();
|
|
355
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
356
|
+
const depths = calculateNodeDepths(graph);
|
|
357
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
358
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
359
|
+
const propertyCount = {};
|
|
360
|
+
const propertySchema = {};
|
|
361
|
+
for (const task of lastLevelNodes) {
|
|
362
|
+
const taskOutputSchema = task.outputSchema();
|
|
363
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
364
|
+
if (taskOutputSchema === false) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (taskOutputSchema === true) {
|
|
368
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
373
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
374
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
375
|
+
if (!propertySchema[outputName]) {
|
|
376
|
+
propertySchema[outputName] = outputProp;
|
|
377
|
+
}
|
|
378
|
+
if (trackOrigins) {
|
|
379
|
+
if (!propertyOrigins[outputName]) {
|
|
380
|
+
propertyOrigins[outputName] = [task.id];
|
|
381
|
+
} else {
|
|
382
|
+
propertyOrigins[outputName].push(task.id);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
388
|
+
const outputProp = propertySchema[outputName];
|
|
389
|
+
if (lastLevelNodes.length === 1) {
|
|
390
|
+
properties[outputName] = outputProp;
|
|
391
|
+
} else {
|
|
392
|
+
properties[outputName] = {
|
|
393
|
+
type: "array",
|
|
394
|
+
items: outputProp
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
if (trackOrigins) {
|
|
399
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
400
|
+
const prop = properties[propName];
|
|
401
|
+
if (!prop || typeof prop === "boolean")
|
|
402
|
+
continue;
|
|
403
|
+
if (origins.length === 1) {
|
|
404
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
405
|
+
} else {
|
|
406
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
return {
|
|
411
|
+
type: "object",
|
|
412
|
+
properties,
|
|
413
|
+
...required.length > 0 ? { required } : {},
|
|
414
|
+
additionalProperties: false
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
function stripOriginAnnotations(schema) {
|
|
418
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
419
|
+
return schema;
|
|
420
|
+
const properties = schema.properties;
|
|
421
|
+
if (!properties)
|
|
422
|
+
return schema;
|
|
423
|
+
const strippedProperties = {};
|
|
424
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
425
|
+
if (!prop || typeof prop !== "object") {
|
|
426
|
+
strippedProperties[key] = prop;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
430
|
+
strippedProperties[key] = rest;
|
|
431
|
+
}
|
|
432
|
+
return { ...schema, properties: strippedProperties };
|
|
433
|
+
}
|
|
434
|
+
function getOriginTaskIds(prop) {
|
|
435
|
+
if (prop["x-source-task-ids"]) {
|
|
436
|
+
return prop["x-source-task-ids"];
|
|
437
|
+
}
|
|
438
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
439
|
+
return [prop["x-source-task-id"]];
|
|
440
|
+
}
|
|
441
|
+
return [];
|
|
442
|
+
}
|
|
443
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
444
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
445
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
446
|
+
if (hasInputTask && hasOutputTask) {
|
|
447
|
+
return json;
|
|
448
|
+
}
|
|
449
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
450
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const prependTasks = [];
|
|
452
|
+
const appendTasks = [];
|
|
453
|
+
const inputDataflows = [];
|
|
454
|
+
const outputDataflows = [];
|
|
455
|
+
if (!hasInputTask && inputSchema) {
|
|
456
|
+
const inputTaskId = uuid4();
|
|
457
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
458
|
+
prependTasks.push({
|
|
459
|
+
id: inputTaskId,
|
|
460
|
+
type: "InputTask",
|
|
461
|
+
config: {
|
|
462
|
+
inputSchema: strippedInputSchema,
|
|
463
|
+
outputSchema: strippedInputSchema
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
467
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
468
|
+
if (!prop || typeof prop === "boolean")
|
|
469
|
+
continue;
|
|
470
|
+
const origins = getOriginTaskIds(prop);
|
|
471
|
+
for (const originId of origins) {
|
|
472
|
+
inputDataflows.push({
|
|
473
|
+
sourceTaskId: inputTaskId,
|
|
474
|
+
sourceTaskPortId: propName,
|
|
475
|
+
targetTaskId: originId,
|
|
476
|
+
targetTaskPortId: propName
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
if (!hasOutputTask && outputSchema) {
|
|
483
|
+
const outputTaskId = uuid4();
|
|
484
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
485
|
+
appendTasks.push({
|
|
486
|
+
id: outputTaskId,
|
|
487
|
+
type: "OutputTask",
|
|
488
|
+
config: {
|
|
489
|
+
inputSchema: strippedOutputSchema,
|
|
490
|
+
outputSchema: strippedOutputSchema
|
|
491
|
+
}
|
|
492
|
+
});
|
|
493
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
494
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
495
|
+
if (!prop || typeof prop === "boolean")
|
|
496
|
+
continue;
|
|
497
|
+
const origins = getOriginTaskIds(prop);
|
|
498
|
+
for (const originId of origins) {
|
|
499
|
+
outputDataflows.push({
|
|
500
|
+
sourceTaskId: originId,
|
|
501
|
+
sourceTaskPortId: propName,
|
|
502
|
+
targetTaskId: outputTaskId,
|
|
503
|
+
targetTaskPortId: propName
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return {
|
|
510
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
511
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
515
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
516
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
517
|
+
if (hasInputTask && hasOutputTask) {
|
|
518
|
+
return items;
|
|
519
|
+
}
|
|
520
|
+
const prependItems = [];
|
|
521
|
+
const appendItems = [];
|
|
522
|
+
if (!hasInputTask) {
|
|
523
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
524
|
+
const inputTaskId = uuid4();
|
|
525
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
526
|
+
prependItems.push({
|
|
527
|
+
id: inputTaskId,
|
|
528
|
+
type: "InputTask",
|
|
529
|
+
config: {
|
|
530
|
+
inputSchema: strippedInputSchema,
|
|
531
|
+
outputSchema: strippedInputSchema
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
535
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
536
|
+
if (!prop || typeof prop === "boolean")
|
|
537
|
+
continue;
|
|
538
|
+
const origins = getOriginTaskIds(prop);
|
|
539
|
+
for (const originId of origins) {
|
|
540
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
541
|
+
if (!targetItem)
|
|
542
|
+
continue;
|
|
543
|
+
if (!targetItem.dependencies) {
|
|
544
|
+
targetItem.dependencies = {};
|
|
545
|
+
}
|
|
546
|
+
const existing = targetItem.dependencies[propName];
|
|
547
|
+
const dep = { id: inputTaskId, output: propName };
|
|
548
|
+
if (!existing) {
|
|
549
|
+
targetItem.dependencies[propName] = dep;
|
|
550
|
+
} else if (Array.isArray(existing)) {
|
|
551
|
+
existing.push(dep);
|
|
552
|
+
} else {
|
|
553
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (!hasOutputTask) {
|
|
560
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
561
|
+
const outputTaskId = uuid4();
|
|
562
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
563
|
+
const outputDependencies = {};
|
|
564
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
565
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
566
|
+
if (!prop || typeof prop === "boolean")
|
|
567
|
+
continue;
|
|
568
|
+
const origins = getOriginTaskIds(prop);
|
|
569
|
+
if (origins.length === 1) {
|
|
570
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
571
|
+
} else if (origins.length > 1) {
|
|
572
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
appendItems.push({
|
|
577
|
+
id: outputTaskId,
|
|
578
|
+
type: "OutputTask",
|
|
579
|
+
config: {
|
|
580
|
+
inputSchema: strippedOutputSchema,
|
|
581
|
+
outputSchema: strippedOutputSchema
|
|
582
|
+
},
|
|
583
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
return [...prependItems, ...items, ...appendItems];
|
|
587
|
+
}
|
|
230
588
|
// src/task-graph/TaskGraph.ts
|
|
231
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
589
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
232
590
|
|
|
233
591
|
// src/task/GraphAsTask.ts
|
|
234
592
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -236,9 +594,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
236
594
|
// src/task-graph/TaskGraphRunner.ts
|
|
237
595
|
import {
|
|
238
596
|
collectPropertyValues,
|
|
597
|
+
getLogger as getLogger3,
|
|
239
598
|
globalServiceRegistry as globalServiceRegistry2,
|
|
240
599
|
ServiceRegistry as ServiceRegistry2,
|
|
241
|
-
uuid4 as
|
|
600
|
+
uuid4 as uuid43
|
|
242
601
|
} from "@workglow/util";
|
|
243
602
|
|
|
244
603
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -271,6 +630,9 @@ class TaskOutputRepository {
|
|
|
271
630
|
}
|
|
272
631
|
}
|
|
273
632
|
|
|
633
|
+
// src/task/ConditionalTask.ts
|
|
634
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
635
|
+
|
|
274
636
|
// src/task/ConditionUtils.ts
|
|
275
637
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
276
638
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -343,7 +705,7 @@ import {
|
|
|
343
705
|
compileSchema,
|
|
344
706
|
deepEqual,
|
|
345
707
|
EventEmitter as EventEmitter3,
|
|
346
|
-
uuid4
|
|
708
|
+
uuid4 as uuid42
|
|
347
709
|
} from "@workglow/util";
|
|
348
710
|
|
|
349
711
|
// src/task/TaskError.ts
|
|
@@ -377,6 +739,13 @@ class TaskAbortedError extends TaskError {
|
|
|
377
739
|
}
|
|
378
740
|
}
|
|
379
741
|
|
|
742
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
743
|
+
static type = "TaskTimeoutError";
|
|
744
|
+
constructor(timeoutMs) {
|
|
745
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
380
749
|
class TaskFailedError extends TaskError {
|
|
381
750
|
static type = "TaskFailedError";
|
|
382
751
|
constructor(message = "Task failed") {
|
|
@@ -408,7 +777,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
408
777
|
}
|
|
409
778
|
|
|
410
779
|
// src/task/TaskRunner.ts
|
|
411
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
780
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
412
781
|
|
|
413
782
|
// src/task/InputResolver.ts
|
|
414
783
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -472,7 +841,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
472
841
|
if (!prop || typeof prop === "boolean")
|
|
473
842
|
return "none";
|
|
474
843
|
const xStream = prop["x-stream"];
|
|
475
|
-
if (xStream === "append" || xStream === "replace")
|
|
844
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
476
845
|
return xStream;
|
|
477
846
|
return "none";
|
|
478
847
|
}
|
|
@@ -487,7 +856,7 @@ function getStreamingPorts(schema) {
|
|
|
487
856
|
if (!prop || typeof prop === "boolean")
|
|
488
857
|
continue;
|
|
489
858
|
const xStream = prop["x-stream"];
|
|
490
|
-
if (xStream === "append" || xStream === "replace") {
|
|
859
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
491
860
|
result.push({ port: name, mode: xStream });
|
|
492
861
|
}
|
|
493
862
|
}
|
|
@@ -531,6 +900,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
531
900
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
532
901
|
return sourceMode !== targetMode;
|
|
533
902
|
}
|
|
903
|
+
function getObjectPortId(schema) {
|
|
904
|
+
if (typeof schema === "boolean")
|
|
905
|
+
return;
|
|
906
|
+
const props = schema.properties;
|
|
907
|
+
if (!props)
|
|
908
|
+
return;
|
|
909
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
910
|
+
if (!prop || typeof prop === "boolean")
|
|
911
|
+
continue;
|
|
912
|
+
if (prop["x-stream"] === "object")
|
|
913
|
+
return name;
|
|
914
|
+
}
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
function getStructuredOutputSchemas(schema) {
|
|
918
|
+
const result = new Map;
|
|
919
|
+
if (typeof schema === "boolean")
|
|
920
|
+
return result;
|
|
921
|
+
const props = schema.properties;
|
|
922
|
+
if (!props)
|
|
923
|
+
return result;
|
|
924
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
925
|
+
if (!prop || typeof prop === "boolean")
|
|
926
|
+
continue;
|
|
927
|
+
if (prop["x-structured-output"] === true) {
|
|
928
|
+
result.set(name, prop);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
return result;
|
|
932
|
+
}
|
|
933
|
+
function hasStructuredOutput(schema) {
|
|
934
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
935
|
+
}
|
|
534
936
|
|
|
535
937
|
// src/task/TaskRunner.ts
|
|
536
938
|
class TaskRunner {
|
|
@@ -541,12 +943,17 @@ class TaskRunner {
|
|
|
541
943
|
outputCache;
|
|
542
944
|
registry = globalServiceRegistry;
|
|
543
945
|
inputStreams;
|
|
946
|
+
timeoutTimer;
|
|
947
|
+
pendingTimeoutError;
|
|
544
948
|
shouldAccumulate = true;
|
|
545
949
|
constructor(task) {
|
|
546
950
|
this.task = task;
|
|
547
951
|
this.own = this.own.bind(this);
|
|
548
952
|
this.handleProgress = this.handleProgress.bind(this);
|
|
549
953
|
}
|
|
954
|
+
get timerLabel() {
|
|
955
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
956
|
+
}
|
|
550
957
|
async run(overrides = {}, config = {}) {
|
|
551
958
|
await this.handleStart(config);
|
|
552
959
|
try {
|
|
@@ -593,7 +1000,7 @@ class TaskRunner {
|
|
|
593
1000
|
return this.task.runOutputData;
|
|
594
1001
|
} catch (err) {
|
|
595
1002
|
await this.handleError(err);
|
|
596
|
-
throw err;
|
|
1003
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
597
1004
|
}
|
|
598
1005
|
}
|
|
599
1006
|
async runReactive(overrides = {}) {
|
|
@@ -650,7 +1057,14 @@ class TaskRunner {
|
|
|
650
1057
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
651
1058
|
}
|
|
652
1059
|
}
|
|
1060
|
+
if (streamMode === "object") {
|
|
1061
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1062
|
+
if (ports.length === 0) {
|
|
1063
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
653
1066
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1067
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
654
1068
|
let chunkCount = 0;
|
|
655
1069
|
let finalOutput;
|
|
656
1070
|
this.task.emit("stream_start");
|
|
@@ -681,6 +1095,13 @@ class TaskRunner {
|
|
|
681
1095
|
break;
|
|
682
1096
|
}
|
|
683
1097
|
case "object-delta": {
|
|
1098
|
+
if (accumulatedObjects) {
|
|
1099
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1100
|
+
}
|
|
1101
|
+
this.task.runOutputData = {
|
|
1102
|
+
...this.task.runOutputData,
|
|
1103
|
+
[event.port]: event.objectDelta
|
|
1104
|
+
};
|
|
684
1105
|
this.task.emit("stream_chunk", event);
|
|
685
1106
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
686
1107
|
await this.handleProgress(progress);
|
|
@@ -693,11 +1114,18 @@ class TaskRunner {
|
|
|
693
1114
|
break;
|
|
694
1115
|
}
|
|
695
1116
|
case "finish": {
|
|
696
|
-
if (accumulated) {
|
|
1117
|
+
if (accumulated || accumulatedObjects) {
|
|
697
1118
|
const merged = { ...event.data || {} };
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
1119
|
+
if (accumulated) {
|
|
1120
|
+
for (const [port, text] of accumulated) {
|
|
1121
|
+
if (text.length > 0)
|
|
1122
|
+
merged[port] = text;
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
if (accumulatedObjects) {
|
|
1126
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1127
|
+
merged[port] = obj;
|
|
1128
|
+
}
|
|
701
1129
|
}
|
|
702
1130
|
finalOutput = merged;
|
|
703
1131
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -743,12 +1171,20 @@ class TaskRunner {
|
|
|
743
1171
|
this.outputCache = cache;
|
|
744
1172
|
}
|
|
745
1173
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1174
|
+
const timeout = this.task.config.timeout;
|
|
1175
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1176
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1177
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1178
|
+
this.abort();
|
|
1179
|
+
}, timeout);
|
|
1180
|
+
}
|
|
746
1181
|
if (config.updateProgress) {
|
|
747
1182
|
this.updateProgress = config.updateProgress;
|
|
748
1183
|
}
|
|
749
1184
|
if (config.registry) {
|
|
750
1185
|
this.registry = config.registry;
|
|
751
1186
|
}
|
|
1187
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
752
1188
|
this.task.emit("start");
|
|
753
1189
|
this.task.emit("status", this.task.status);
|
|
754
1190
|
}
|
|
@@ -756,12 +1192,21 @@ class TaskRunner {
|
|
|
756
1192
|
async handleStartReactive() {
|
|
757
1193
|
this.reactiveRunning = true;
|
|
758
1194
|
}
|
|
1195
|
+
clearTimeoutTimer() {
|
|
1196
|
+
if (this.timeoutTimer !== undefined) {
|
|
1197
|
+
clearTimeout(this.timeoutTimer);
|
|
1198
|
+
this.timeoutTimer = undefined;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
759
1201
|
async handleAbort() {
|
|
760
1202
|
if (this.task.status === TaskStatus.ABORTING)
|
|
761
1203
|
return;
|
|
1204
|
+
this.clearTimeoutTimer();
|
|
1205
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
762
1206
|
this.task.status = TaskStatus.ABORTING;
|
|
763
1207
|
this.task.progress = 100;
|
|
764
|
-
this.task.error = new TaskAbortedError;
|
|
1208
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1209
|
+
this.pendingTimeoutError = undefined;
|
|
765
1210
|
this.task.emit("abort", this.task.error);
|
|
766
1211
|
this.task.emit("status", this.task.status);
|
|
767
1212
|
}
|
|
@@ -771,6 +1216,9 @@ class TaskRunner {
|
|
|
771
1216
|
async handleComplete() {
|
|
772
1217
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
773
1218
|
return;
|
|
1219
|
+
this.clearTimeoutTimer();
|
|
1220
|
+
this.pendingTimeoutError = undefined;
|
|
1221
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
774
1222
|
this.task.completedAt = new Date;
|
|
775
1223
|
this.task.progress = 100;
|
|
776
1224
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -799,6 +1247,9 @@ class TaskRunner {
|
|
|
799
1247
|
return this.handleAbort();
|
|
800
1248
|
if (this.task.status === TaskStatus.FAILED)
|
|
801
1249
|
return;
|
|
1250
|
+
this.clearTimeoutTimer();
|
|
1251
|
+
this.pendingTimeoutError = undefined;
|
|
1252
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
802
1253
|
if (this.task.hasChildren()) {
|
|
803
1254
|
this.task.subGraph.abort();
|
|
804
1255
|
}
|
|
@@ -829,6 +1280,7 @@ class Task {
|
|
|
829
1280
|
static cacheable = true;
|
|
830
1281
|
static hasDynamicSchemas = false;
|
|
831
1282
|
static passthroughInputsToOutputs = false;
|
|
1283
|
+
static customizable = false;
|
|
832
1284
|
static inputSchema() {
|
|
833
1285
|
return {
|
|
834
1286
|
type: "object",
|
|
@@ -902,6 +1354,9 @@ class Task {
|
|
|
902
1354
|
runInputData = {};
|
|
903
1355
|
runOutputData = {};
|
|
904
1356
|
config;
|
|
1357
|
+
get id() {
|
|
1358
|
+
return this.config.id;
|
|
1359
|
+
}
|
|
905
1360
|
runConfig = {};
|
|
906
1361
|
status = TaskStatus.PENDING;
|
|
907
1362
|
progress = 0;
|
|
@@ -923,9 +1378,11 @@ class Task {
|
|
|
923
1378
|
this.resetInputData();
|
|
924
1379
|
const title = this.constructor.title || undefined;
|
|
925
1380
|
const baseConfig = Object.assign({
|
|
926
|
-
id: uuid4(),
|
|
927
1381
|
...title ? { title } : {}
|
|
928
1382
|
}, config);
|
|
1383
|
+
if (baseConfig.id === undefined) {
|
|
1384
|
+
baseConfig.id = uuid42();
|
|
1385
|
+
}
|
|
929
1386
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
930
1387
|
this.runConfig = runConfig;
|
|
931
1388
|
}
|
|
@@ -935,7 +1392,7 @@ class Task {
|
|
|
935
1392
|
return {};
|
|
936
1393
|
}
|
|
937
1394
|
try {
|
|
938
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1395
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
939
1396
|
const defaultData = compiledSchema.getData(undefined, {
|
|
940
1397
|
addOptionalProps: true,
|
|
941
1398
|
removeInvalidData: false,
|
|
@@ -1053,7 +1510,7 @@ class Task {
|
|
|
1053
1510
|
continue;
|
|
1054
1511
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1055
1512
|
if (isArray) {
|
|
1056
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1513
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1057
1514
|
const newitems = [...existingItems];
|
|
1058
1515
|
const overrideItem = overrides[inputId];
|
|
1059
1516
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1109,25 +1566,29 @@ class Task {
|
|
|
1109
1566
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1110
1567
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1111
1568
|
}
|
|
1112
|
-
static
|
|
1113
|
-
static getConfigSchemaNode(type) {
|
|
1569
|
+
static getConfigSchemaNode() {
|
|
1114
1570
|
const schema = this.configSchema();
|
|
1115
1571
|
if (!schema)
|
|
1116
1572
|
return;
|
|
1117
|
-
if (!
|
|
1573
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1118
1574
|
try {
|
|
1119
1575
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1120
|
-
|
|
1576
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1577
|
+
value: schemaNode,
|
|
1578
|
+
writable: true,
|
|
1579
|
+
configurable: true,
|
|
1580
|
+
enumerable: false
|
|
1581
|
+
});
|
|
1121
1582
|
} catch (error) {
|
|
1122
1583
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1123
1584
|
return;
|
|
1124
1585
|
}
|
|
1125
1586
|
}
|
|
1126
|
-
return this.
|
|
1587
|
+
return this.__compiledConfigSchema;
|
|
1127
1588
|
}
|
|
1128
1589
|
validateAndApplyConfigDefaults(config) {
|
|
1129
1590
|
const ctor = this.constructor;
|
|
1130
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1591
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1131
1592
|
if (!schemaNode)
|
|
1132
1593
|
return config;
|
|
1133
1594
|
const result = schemaNode.validate(config);
|
|
@@ -1140,7 +1601,6 @@ class Task {
|
|
|
1140
1601
|
}
|
|
1141
1602
|
return config;
|
|
1142
1603
|
}
|
|
1143
|
-
static _inputSchemaNode = new Map;
|
|
1144
1604
|
static generateInputSchemaNode(schema) {
|
|
1145
1605
|
if (typeof schema === "boolean") {
|
|
1146
1606
|
if (schema === false) {
|
|
@@ -1150,24 +1610,41 @@ class Task {
|
|
|
1150
1610
|
}
|
|
1151
1611
|
return compileSchema(schema);
|
|
1152
1612
|
}
|
|
1153
|
-
static getInputSchemaNode(
|
|
1154
|
-
if (!
|
|
1613
|
+
static getInputSchemaNode() {
|
|
1614
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1155
1615
|
const dataPortSchema = this.inputSchema();
|
|
1156
1616
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1157
1617
|
try {
|
|
1158
|
-
|
|
1618
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1619
|
+
value: schemaNode,
|
|
1620
|
+
writable: true,
|
|
1621
|
+
configurable: true,
|
|
1622
|
+
enumerable: false
|
|
1623
|
+
});
|
|
1159
1624
|
} catch (error) {
|
|
1160
1625
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1161
|
-
|
|
1626
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1627
|
+
value: compileSchema({}),
|
|
1628
|
+
writable: true,
|
|
1629
|
+
configurable: true,
|
|
1630
|
+
enumerable: false
|
|
1631
|
+
});
|
|
1162
1632
|
}
|
|
1163
1633
|
}
|
|
1164
|
-
return this.
|
|
1634
|
+
return this.__compiledInputSchema;
|
|
1165
1635
|
}
|
|
1166
|
-
getInputSchemaNode(
|
|
1167
|
-
return this.constructor.getInputSchemaNode(
|
|
1636
|
+
getInputSchemaNode() {
|
|
1637
|
+
return this.constructor.getInputSchemaNode();
|
|
1168
1638
|
}
|
|
1169
1639
|
async validateInput(input) {
|
|
1170
|
-
const
|
|
1640
|
+
const ctor = this.constructor;
|
|
1641
|
+
let schemaNode;
|
|
1642
|
+
if (ctor.hasDynamicSchemas) {
|
|
1643
|
+
const instanceSchema = this.inputSchema();
|
|
1644
|
+
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1645
|
+
} else {
|
|
1646
|
+
schemaNode = this.getInputSchemaNode();
|
|
1647
|
+
}
|
|
1171
1648
|
const result = schemaNode.validate(input);
|
|
1172
1649
|
if (!result.valid) {
|
|
1173
1650
|
const errorMessages = result.errors.map((e) => {
|
|
@@ -1178,9 +1655,6 @@ class Task {
|
|
|
1178
1655
|
}
|
|
1179
1656
|
return true;
|
|
1180
1657
|
}
|
|
1181
|
-
id() {
|
|
1182
|
-
return this.config.id;
|
|
1183
|
-
}
|
|
1184
1658
|
stripSymbols(obj) {
|
|
1185
1659
|
if (obj === null || obj === undefined) {
|
|
1186
1660
|
return obj;
|
|
@@ -1202,14 +1676,15 @@ class Task {
|
|
|
1202
1676
|
}
|
|
1203
1677
|
return obj;
|
|
1204
1678
|
}
|
|
1205
|
-
toJSON() {
|
|
1679
|
+
toJSON(_options) {
|
|
1206
1680
|
const extras = this.config.extras;
|
|
1207
1681
|
const json = this.stripSymbols({
|
|
1208
|
-
id: this.
|
|
1682
|
+
id: this.id,
|
|
1209
1683
|
type: this.type,
|
|
1210
1684
|
defaults: this.defaults,
|
|
1211
1685
|
config: {
|
|
1212
1686
|
...this.config.title ? { title: this.config.title } : {},
|
|
1687
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1213
1688
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1214
1689
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1215
1690
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1217,8 +1692,8 @@ class Task {
|
|
|
1217
1692
|
});
|
|
1218
1693
|
return json;
|
|
1219
1694
|
}
|
|
1220
|
-
toDependencyJSON() {
|
|
1221
|
-
const json = this.toJSON();
|
|
1695
|
+
toDependencyJSON(options) {
|
|
1696
|
+
const json = this.toJSON(options);
|
|
1222
1697
|
return json;
|
|
1223
1698
|
}
|
|
1224
1699
|
hasChildren() {
|
|
@@ -1248,7 +1723,7 @@ class Task {
|
|
|
1248
1723
|
this.subGraph.removeDataflow(dataflow);
|
|
1249
1724
|
}
|
|
1250
1725
|
for (const child of this.subGraph.getTasks()) {
|
|
1251
|
-
this.subGraph.removeTask(child.
|
|
1726
|
+
this.subGraph.removeTask(child.id);
|
|
1252
1727
|
}
|
|
1253
1728
|
}
|
|
1254
1729
|
this.events.emit("regenerate");
|
|
@@ -1339,7 +1814,7 @@ class ConditionalTask extends Task {
|
|
|
1339
1814
|
}
|
|
1340
1815
|
}
|
|
1341
1816
|
} catch (error) {
|
|
1342
|
-
|
|
1817
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1343
1818
|
}
|
|
1344
1819
|
}
|
|
1345
1820
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1504,7 +1979,7 @@ class DependencyBasedScheduler {
|
|
|
1504
1979
|
if (task.status === TaskStatus.DISABLED) {
|
|
1505
1980
|
return false;
|
|
1506
1981
|
}
|
|
1507
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
1982
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1508
1983
|
if (sourceDataflows.length > 0) {
|
|
1509
1984
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1510
1985
|
if (allIncomingDisabled) {
|
|
@@ -1631,6 +2106,10 @@ class TaskGraphRunner {
|
|
|
1631
2106
|
graph.outputCache = outputCache;
|
|
1632
2107
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1633
2108
|
}
|
|
2109
|
+
runId = "";
|
|
2110
|
+
get timerLabel() {
|
|
2111
|
+
return `graph:${this.runId}`;
|
|
2112
|
+
}
|
|
1634
2113
|
async runGraph(input = {}, config) {
|
|
1635
2114
|
await this.handleStart(config);
|
|
1636
2115
|
const results = [];
|
|
@@ -1643,25 +2122,33 @@ class TaskGraphRunner {
|
|
|
1643
2122
|
if (this.failedTaskErrors.size > 0) {
|
|
1644
2123
|
break;
|
|
1645
2124
|
}
|
|
1646
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2125
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1647
2126
|
const runAsync = async () => {
|
|
2127
|
+
let errorRouted = false;
|
|
1648
2128
|
try {
|
|
1649
2129
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1650
2130
|
const taskPromise = this.runTask(task, taskInput);
|
|
1651
|
-
this.inProgressTasks.set(task.
|
|
2131
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1652
2132
|
const taskResult = await taskPromise;
|
|
1653
|
-
if (this.graph.getTargetDataflows(task.
|
|
2133
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1654
2134
|
results.push(taskResult);
|
|
1655
2135
|
}
|
|
1656
2136
|
} catch (error2) {
|
|
1657
|
-
this.
|
|
2137
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2138
|
+
errorRouted = true;
|
|
2139
|
+
this.pushErrorOutputToEdges(task);
|
|
2140
|
+
} else {
|
|
2141
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2142
|
+
}
|
|
1658
2143
|
} finally {
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
2144
|
+
if (!errorRouted) {
|
|
2145
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2146
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2147
|
+
}
|
|
2148
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1662
2149
|
}
|
|
1663
2150
|
};
|
|
1664
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2151
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1665
2152
|
}
|
|
1666
2153
|
} catch (err) {
|
|
1667
2154
|
error = err;
|
|
@@ -1685,7 +2172,7 @@ class TaskGraphRunner {
|
|
|
1685
2172
|
const results = [];
|
|
1686
2173
|
try {
|
|
1687
2174
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1688
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2175
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1689
2176
|
if (task.status === TaskStatus.PENDING) {
|
|
1690
2177
|
task.resetInputData();
|
|
1691
2178
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1693,9 +2180,9 @@ class TaskGraphRunner {
|
|
|
1693
2180
|
const taskInput = isRootTask ? input : {};
|
|
1694
2181
|
const taskResult = await task.runReactive(taskInput);
|
|
1695
2182
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1696
|
-
if (this.graph.getTargetDataflows(task.
|
|
2183
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1697
2184
|
results.push({
|
|
1698
|
-
id: task.
|
|
2185
|
+
id: task.id,
|
|
1699
2186
|
type: task.constructor.runtype || task.constructor.type,
|
|
1700
2187
|
data: taskResult
|
|
1701
2188
|
});
|
|
@@ -1715,7 +2202,7 @@ class TaskGraphRunner {
|
|
|
1715
2202
|
await this.handleDisable();
|
|
1716
2203
|
}
|
|
1717
2204
|
filterInputForTask(task, input) {
|
|
1718
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2205
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1719
2206
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1720
2207
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1721
2208
|
const filteredInput = {};
|
|
@@ -1754,13 +2241,13 @@ class TaskGraphRunner {
|
|
|
1754
2241
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1755
2242
|
}
|
|
1756
2243
|
copyInputFromEdgesToNode(task) {
|
|
1757
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2244
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1758
2245
|
for (const dataflow of dataflows) {
|
|
1759
2246
|
this.addInputData(task, dataflow.getPortData());
|
|
1760
2247
|
}
|
|
1761
2248
|
}
|
|
1762
2249
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1763
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2250
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1764
2251
|
for (const dataflow of dataflows) {
|
|
1765
2252
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1766
2253
|
if (compatibility === "static") {
|
|
@@ -1775,7 +2262,7 @@ class TaskGraphRunner {
|
|
|
1775
2262
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1776
2263
|
if (!node?.config?.id)
|
|
1777
2264
|
return;
|
|
1778
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2265
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1779
2266
|
const effectiveStatus = status ?? node.status;
|
|
1780
2267
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1781
2268
|
const branches = node.config.branches ?? [];
|
|
@@ -1806,10 +2293,31 @@ class TaskGraphRunner {
|
|
|
1806
2293
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1807
2294
|
if (!node?.config?.id)
|
|
1808
2295
|
return;
|
|
1809
|
-
graph.getTargetDataflows(node.
|
|
2296
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1810
2297
|
dataflow.error = node.error;
|
|
1811
2298
|
});
|
|
1812
2299
|
}
|
|
2300
|
+
hasErrorOutputEdges(task) {
|
|
2301
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2302
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2303
|
+
}
|
|
2304
|
+
pushErrorOutputToEdges(task) {
|
|
2305
|
+
const taskError = task.error;
|
|
2306
|
+
const errorData = {
|
|
2307
|
+
error: taskError?.message ?? "Unknown error",
|
|
2308
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2309
|
+
};
|
|
2310
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2311
|
+
for (const df of dataflows) {
|
|
2312
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2313
|
+
df.value = errorData;
|
|
2314
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2315
|
+
} else {
|
|
2316
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
this.propagateDisabledStatus(this.graph);
|
|
2320
|
+
}
|
|
1813
2321
|
propagateDisabledStatus(graph) {
|
|
1814
2322
|
let changed = true;
|
|
1815
2323
|
while (changed) {
|
|
@@ -1818,7 +2326,7 @@ class TaskGraphRunner {
|
|
|
1818
2326
|
if (task.status !== TaskStatus.PENDING) {
|
|
1819
2327
|
continue;
|
|
1820
2328
|
}
|
|
1821
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2329
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1822
2330
|
if (incomingDataflows.length === 0) {
|
|
1823
2331
|
continue;
|
|
1824
2332
|
}
|
|
@@ -1829,10 +2337,10 @@ class TaskGraphRunner {
|
|
|
1829
2337
|
task.completedAt = new Date;
|
|
1830
2338
|
task.emit("disabled");
|
|
1831
2339
|
task.emit("status", task.status);
|
|
1832
|
-
graph.getTargetDataflows(task.
|
|
2340
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1833
2341
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1834
2342
|
});
|
|
1835
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2343
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1836
2344
|
changed = true;
|
|
1837
2345
|
}
|
|
1838
2346
|
}
|
|
@@ -1841,7 +2349,7 @@ class TaskGraphRunner {
|
|
|
1841
2349
|
taskNeedsAccumulation(task) {
|
|
1842
2350
|
if (this.outputCache)
|
|
1843
2351
|
return true;
|
|
1844
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2352
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1845
2353
|
if (outEdges.length === 0)
|
|
1846
2354
|
return this.accumulateLeafOutputs;
|
|
1847
2355
|
const outSchema = task.outputSchema();
|
|
@@ -1864,7 +2372,7 @@ class TaskGraphRunner {
|
|
|
1864
2372
|
async runTask(task, input) {
|
|
1865
2373
|
const isStreamable = isTaskStreamable(task);
|
|
1866
2374
|
if (isStreamable) {
|
|
1867
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2375
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1868
2376
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1869
2377
|
if (streamingEdges.length > 0) {
|
|
1870
2378
|
const inputStreams = new Map;
|
|
@@ -1889,13 +2397,13 @@ class TaskGraphRunner {
|
|
|
1889
2397
|
});
|
|
1890
2398
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1891
2399
|
return {
|
|
1892
|
-
id: task.
|
|
2400
|
+
id: task.id,
|
|
1893
2401
|
type: task.constructor.runtype || task.constructor.type,
|
|
1894
2402
|
data: results
|
|
1895
2403
|
};
|
|
1896
2404
|
}
|
|
1897
2405
|
async awaitStreamInputs(task) {
|
|
1898
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2406
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1899
2407
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1900
2408
|
if (streamPromises.length > 0) {
|
|
1901
2409
|
await Promise.all(streamPromises);
|
|
@@ -1910,17 +2418,17 @@ class TaskGraphRunner {
|
|
|
1910
2418
|
streamingNotified = true;
|
|
1911
2419
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1912
2420
|
this.pushStreamToEdges(task, streamMode);
|
|
1913
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2421
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1914
2422
|
}
|
|
1915
2423
|
};
|
|
1916
2424
|
const onStreamStart = () => {
|
|
1917
|
-
this.graph.emit("task_stream_start", task.
|
|
2425
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1918
2426
|
};
|
|
1919
2427
|
const onStreamChunk = (event) => {
|
|
1920
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2428
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1921
2429
|
};
|
|
1922
2430
|
const onStreamEnd = (output) => {
|
|
1923
|
-
this.graph.emit("task_stream_end", task.
|
|
2431
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1924
2432
|
};
|
|
1925
2433
|
task.on("status", onStatus);
|
|
1926
2434
|
task.on("stream_start", onStreamStart);
|
|
@@ -1935,7 +2443,7 @@ class TaskGraphRunner {
|
|
|
1935
2443
|
});
|
|
1936
2444
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1937
2445
|
return {
|
|
1938
|
-
id: task.
|
|
2446
|
+
id: task.id,
|
|
1939
2447
|
type: task.constructor.runtype || task.constructor.type,
|
|
1940
2448
|
data: results
|
|
1941
2449
|
};
|
|
@@ -1973,7 +2481,7 @@ class TaskGraphRunner {
|
|
|
1973
2481
|
});
|
|
1974
2482
|
}
|
|
1975
2483
|
pushStreamToEdges(task, streamMode) {
|
|
1976
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2484
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1977
2485
|
if (targetDataflows.length === 0)
|
|
1978
2486
|
return;
|
|
1979
2487
|
const groups = new Map;
|
|
@@ -2064,11 +2572,15 @@ class TaskGraphRunner {
|
|
|
2064
2572
|
this.abortController?.abort();
|
|
2065
2573
|
}, { once: true });
|
|
2066
2574
|
}
|
|
2067
|
-
this.
|
|
2575
|
+
this.runId = uuid43();
|
|
2576
|
+
this.resetGraph(this.graph, this.runId);
|
|
2068
2577
|
this.processScheduler.reset();
|
|
2069
2578
|
this.inProgressTasks.clear();
|
|
2070
2579
|
this.inProgressFunctions.clear();
|
|
2071
2580
|
this.failedTaskErrors.clear();
|
|
2581
|
+
const logger = getLogger3();
|
|
2582
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2583
|
+
logger.time(this.timerLabel);
|
|
2072
2584
|
this.graph.emit("start");
|
|
2073
2585
|
}
|
|
2074
2586
|
async handleStartReactive() {
|
|
@@ -2080,6 +2592,9 @@ class TaskGraphRunner {
|
|
|
2080
2592
|
}
|
|
2081
2593
|
async handleComplete() {
|
|
2082
2594
|
this.running = false;
|
|
2595
|
+
const logger = getLogger3();
|
|
2596
|
+
logger.timeEnd(this.timerLabel);
|
|
2597
|
+
logger.groupEnd();
|
|
2083
2598
|
this.graph.emit("complete");
|
|
2084
2599
|
}
|
|
2085
2600
|
async handleCompleteReactive() {
|
|
@@ -2092,6 +2607,9 @@ class TaskGraphRunner {
|
|
|
2092
2607
|
}
|
|
2093
2608
|
}));
|
|
2094
2609
|
this.running = false;
|
|
2610
|
+
const logger = getLogger3();
|
|
2611
|
+
logger.timeEnd(this.timerLabel);
|
|
2612
|
+
logger.groupEnd();
|
|
2095
2613
|
this.graph.emit("error", error);
|
|
2096
2614
|
}
|
|
2097
2615
|
async handleErrorReactive() {
|
|
@@ -2104,6 +2622,9 @@ class TaskGraphRunner {
|
|
|
2104
2622
|
}
|
|
2105
2623
|
});
|
|
2106
2624
|
this.running = false;
|
|
2625
|
+
const logger = getLogger3();
|
|
2626
|
+
logger.timeEnd(this.timerLabel);
|
|
2627
|
+
logger.groupEnd();
|
|
2107
2628
|
this.graph.emit("abort");
|
|
2108
2629
|
}
|
|
2109
2630
|
async handleAbortReactive() {
|
|
@@ -2179,7 +2700,7 @@ var graphAsTaskConfigSchema = {
|
|
|
2179
2700
|
type: "object",
|
|
2180
2701
|
properties: {
|
|
2181
2702
|
...TaskConfigSchema["properties"],
|
|
2182
|
-
compoundMerge: { type: "string" }
|
|
2703
|
+
compoundMerge: { type: "string", "x-ui-hidden": true }
|
|
2183
2704
|
},
|
|
2184
2705
|
additionalProperties: false
|
|
2185
2706
|
};
|
|
@@ -2218,118 +2739,27 @@ class GraphAsTask extends Task {
|
|
|
2218
2739
|
if (!this.hasChildren()) {
|
|
2219
2740
|
return this.constructor.inputSchema();
|
|
2220
2741
|
}
|
|
2221
|
-
|
|
2222
|
-
const required = [];
|
|
2223
|
-
const tasks = this.subGraph.getTasks();
|
|
2224
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2225
|
-
for (const task of startingNodes) {
|
|
2226
|
-
const taskInputSchema = task.inputSchema();
|
|
2227
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2228
|
-
if (taskInputSchema === false) {
|
|
2229
|
-
continue;
|
|
2230
|
-
}
|
|
2231
|
-
if (taskInputSchema === true) {
|
|
2232
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2233
|
-
continue;
|
|
2234
|
-
}
|
|
2235
|
-
}
|
|
2236
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2237
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2238
|
-
if (!properties[inputName]) {
|
|
2239
|
-
properties[inputName] = inputProp;
|
|
2240
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2241
|
-
required.push(inputName);
|
|
2242
|
-
}
|
|
2243
|
-
}
|
|
2244
|
-
}
|
|
2245
|
-
}
|
|
2246
|
-
return {
|
|
2247
|
-
type: "object",
|
|
2248
|
-
properties,
|
|
2249
|
-
...required.length > 0 ? { required } : {},
|
|
2250
|
-
additionalProperties: false
|
|
2251
|
-
};
|
|
2742
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2252
2743
|
}
|
|
2253
2744
|
_inputSchemaNode;
|
|
2254
|
-
getInputSchemaNode(
|
|
2745
|
+
getInputSchemaNode() {
|
|
2255
2746
|
if (!this._inputSchemaNode) {
|
|
2256
|
-
const dataPortSchema = this.inputSchema();
|
|
2257
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2258
2747
|
try {
|
|
2748
|
+
const dataPortSchema = this.inputSchema();
|
|
2749
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2259
2750
|
this._inputSchemaNode = schemaNode;
|
|
2260
2751
|
} catch (error) {
|
|
2261
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2752
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2262
2753
|
this._inputSchemaNode = compileSchema2({});
|
|
2263
2754
|
}
|
|
2264
2755
|
}
|
|
2265
2756
|
return this._inputSchemaNode;
|
|
2266
2757
|
}
|
|
2267
|
-
calculateNodeDepths() {
|
|
2268
|
-
const depths = new Map;
|
|
2269
|
-
const tasks = this.subGraph.getTasks();
|
|
2270
|
-
for (const task of tasks) {
|
|
2271
|
-
depths.set(task.config.id, 0);
|
|
2272
|
-
}
|
|
2273
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2274
|
-
for (const task of sortedTasks) {
|
|
2275
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2276
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2277
|
-
for (const targetTask of targetTasks) {
|
|
2278
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2279
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2280
|
-
}
|
|
2281
|
-
}
|
|
2282
|
-
return depths;
|
|
2283
|
-
}
|
|
2284
2758
|
outputSchema() {
|
|
2285
2759
|
if (!this.hasChildren()) {
|
|
2286
2760
|
return this.constructor.outputSchema();
|
|
2287
2761
|
}
|
|
2288
|
-
|
|
2289
|
-
const required = [];
|
|
2290
|
-
const tasks = this.subGraph.getTasks();
|
|
2291
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2292
|
-
const depths = this.calculateNodeDepths();
|
|
2293
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2294
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2295
|
-
const propertyCount = {};
|
|
2296
|
-
const propertySchema = {};
|
|
2297
|
-
for (const task of lastLevelNodes) {
|
|
2298
|
-
const taskOutputSchema = task.outputSchema();
|
|
2299
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2300
|
-
if (taskOutputSchema === false) {
|
|
2301
|
-
continue;
|
|
2302
|
-
}
|
|
2303
|
-
if (taskOutputSchema === true) {
|
|
2304
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2305
|
-
continue;
|
|
2306
|
-
}
|
|
2307
|
-
}
|
|
2308
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2309
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2310
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2311
|
-
if (!propertySchema[outputName]) {
|
|
2312
|
-
propertySchema[outputName] = outputProp;
|
|
2313
|
-
}
|
|
2314
|
-
}
|
|
2315
|
-
}
|
|
2316
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2317
|
-
const outputProp = propertySchema[outputName];
|
|
2318
|
-
if (lastLevelNodes.length === 1) {
|
|
2319
|
-
properties[outputName] = outputProp;
|
|
2320
|
-
} else {
|
|
2321
|
-
properties[outputName] = {
|
|
2322
|
-
type: "array",
|
|
2323
|
-
items: outputProp
|
|
2324
|
-
};
|
|
2325
|
-
}
|
|
2326
|
-
}
|
|
2327
|
-
return {
|
|
2328
|
-
type: "object",
|
|
2329
|
-
properties,
|
|
2330
|
-
...required.length > 0 ? { required } : {},
|
|
2331
|
-
additionalProperties: false
|
|
2332
|
-
};
|
|
2762
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2333
2763
|
}
|
|
2334
2764
|
resetInputData() {
|
|
2335
2765
|
super.resetInputData();
|
|
@@ -2364,8 +2794,8 @@ class GraphAsTask extends Task {
|
|
|
2364
2794
|
const endingNodeIds = new Set;
|
|
2365
2795
|
const tasks = this.subGraph.getTasks();
|
|
2366
2796
|
for (const task of tasks) {
|
|
2367
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2368
|
-
endingNodeIds.add(task.
|
|
2797
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2798
|
+
endingNodeIds.add(task.id);
|
|
2369
2799
|
}
|
|
2370
2800
|
}
|
|
2371
2801
|
const eventQueue = [];
|
|
@@ -2409,32 +2839,36 @@ class GraphAsTask extends Task {
|
|
|
2409
2839
|
this._inputSchemaNode = undefined;
|
|
2410
2840
|
this.events.emit("regenerate");
|
|
2411
2841
|
}
|
|
2412
|
-
toJSON() {
|
|
2413
|
-
let json = super.toJSON();
|
|
2842
|
+
toJSON(options) {
|
|
2843
|
+
let json = super.toJSON(options);
|
|
2414
2844
|
const hasChildren = this.hasChildren();
|
|
2415
2845
|
if (hasChildren) {
|
|
2416
2846
|
json = {
|
|
2417
2847
|
...json,
|
|
2418
2848
|
merge: this.compoundMerge,
|
|
2419
|
-
subgraph: this.subGraph.toJSON()
|
|
2849
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2420
2850
|
};
|
|
2421
2851
|
}
|
|
2422
2852
|
return json;
|
|
2423
2853
|
}
|
|
2424
|
-
toDependencyJSON() {
|
|
2425
|
-
const json = this.toJSON();
|
|
2854
|
+
toDependencyJSON(options) {
|
|
2855
|
+
const json = this.toJSON(options);
|
|
2426
2856
|
if (this.hasChildren()) {
|
|
2427
2857
|
if ("subgraph" in json) {
|
|
2428
2858
|
delete json.subgraph;
|
|
2429
2859
|
}
|
|
2430
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2860
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2431
2861
|
}
|
|
2432
2862
|
return json;
|
|
2433
2863
|
}
|
|
2434
2864
|
}
|
|
2435
2865
|
|
|
2436
2866
|
// src/task-graph/Workflow.ts
|
|
2437
|
-
import {
|
|
2867
|
+
import {
|
|
2868
|
+
EventEmitter as EventEmitter4,
|
|
2869
|
+
getLogger as getLogger4,
|
|
2870
|
+
uuid4 as uuid44
|
|
2871
|
+
} from "@workglow/util";
|
|
2438
2872
|
function CreateWorkflow(taskClass) {
|
|
2439
2873
|
return Workflow.createWorkflow(taskClass);
|
|
2440
2874
|
}
|
|
@@ -2535,43 +2969,48 @@ class Workflow {
|
|
|
2535
2969
|
const helper = function(input = {}, config = {}) {
|
|
2536
2970
|
this._error = "";
|
|
2537
2971
|
const parent = getLastTask(this);
|
|
2538
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
2972
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2539
2973
|
if (this._dataFlows.length > 0) {
|
|
2540
2974
|
this._dataFlows.forEach((dataflow) => {
|
|
2541
2975
|
const taskSchema = task.inputSchema();
|
|
2542
2976
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2543
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2544
|
-
|
|
2977
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
2978
|
+
getLogger4().error(this._error);
|
|
2545
2979
|
return;
|
|
2546
2980
|
}
|
|
2547
|
-
dataflow.targetTaskId = task.
|
|
2981
|
+
dataflow.targetTaskId = task.id;
|
|
2548
2982
|
this.graph.addDataflow(dataflow);
|
|
2549
2983
|
});
|
|
2550
2984
|
this._dataFlows = [];
|
|
2551
2985
|
}
|
|
2552
|
-
if (parent
|
|
2986
|
+
if (parent) {
|
|
2553
2987
|
const nodes = this._graph.getTasks();
|
|
2554
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
2988
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2555
2989
|
const earlierTasks = [];
|
|
2556
2990
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2557
2991
|
earlierTasks.push(nodes[i]);
|
|
2558
2992
|
}
|
|
2559
2993
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2994
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2560
2995
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2561
2996
|
providedInputKeys,
|
|
2997
|
+
connectedInputKeys,
|
|
2562
2998
|
earlierTasks
|
|
2563
2999
|
});
|
|
2564
3000
|
if (result.error) {
|
|
2565
3001
|
if (this.isLoopBuilder) {
|
|
2566
3002
|
this._error = result.error;
|
|
2567
|
-
|
|
3003
|
+
getLogger4().warn(this._error);
|
|
2568
3004
|
} else {
|
|
2569
3005
|
this._error = result.error + " Task not added.";
|
|
2570
|
-
|
|
2571
|
-
this.graph.removeTask(task.
|
|
3006
|
+
getLogger4().error(this._error);
|
|
3007
|
+
this.graph.removeTask(task.id);
|
|
2572
3008
|
}
|
|
2573
3009
|
}
|
|
2574
3010
|
}
|
|
3011
|
+
if (!this._error) {
|
|
3012
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3013
|
+
}
|
|
2575
3014
|
return this;
|
|
2576
3015
|
};
|
|
2577
3016
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2651,18 +3090,18 @@ class Workflow {
|
|
|
2651
3090
|
const nodes = this._graph.getTasks();
|
|
2652
3091
|
if (nodes.length === 0) {
|
|
2653
3092
|
this._error = "No tasks to remove";
|
|
2654
|
-
|
|
3093
|
+
getLogger4().error(this._error);
|
|
2655
3094
|
return this;
|
|
2656
3095
|
}
|
|
2657
3096
|
const lastNode = nodes[nodes.length - 1];
|
|
2658
|
-
this._graph.removeTask(lastNode.
|
|
3097
|
+
this._graph.removeTask(lastNode.id);
|
|
2659
3098
|
return this;
|
|
2660
3099
|
}
|
|
2661
|
-
toJSON() {
|
|
2662
|
-
return this._graph.toJSON();
|
|
3100
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3101
|
+
return this._graph.toJSON(options);
|
|
2663
3102
|
}
|
|
2664
|
-
toDependencyJSON() {
|
|
2665
|
-
return this._graph.toDependencyJSON();
|
|
3103
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3104
|
+
return this._graph.toDependencyJSON(options);
|
|
2666
3105
|
}
|
|
2667
3106
|
pipe(...args) {
|
|
2668
3107
|
return pipe(args, this);
|
|
@@ -2682,25 +3121,40 @@ class Workflow {
|
|
|
2682
3121
|
if (-index > nodes.length) {
|
|
2683
3122
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2684
3123
|
this._error = errorMsg;
|
|
2685
|
-
|
|
3124
|
+
getLogger4().error(this._error);
|
|
2686
3125
|
throw new WorkflowError(errorMsg);
|
|
2687
3126
|
}
|
|
2688
3127
|
const lastNode = nodes[nodes.length + index];
|
|
2689
3128
|
const outputSchema = lastNode.outputSchema();
|
|
2690
3129
|
if (typeof outputSchema === "boolean") {
|
|
2691
3130
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2692
|
-
const errorMsg = `Task ${lastNode.
|
|
3131
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2693
3132
|
this._error = errorMsg;
|
|
2694
|
-
|
|
3133
|
+
getLogger4().error(this._error);
|
|
2695
3134
|
throw new WorkflowError(errorMsg);
|
|
2696
3135
|
}
|
|
2697
3136
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2698
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3137
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2699
3138
|
this._error = errorMsg;
|
|
2700
|
-
|
|
3139
|
+
getLogger4().error(this._error);
|
|
2701
3140
|
throw new WorkflowError(errorMsg);
|
|
2702
3141
|
}
|
|
2703
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3142
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3143
|
+
return this;
|
|
3144
|
+
}
|
|
3145
|
+
onError(handler) {
|
|
3146
|
+
this._error = "";
|
|
3147
|
+
const parent = getLastTask(this);
|
|
3148
|
+
if (!parent) {
|
|
3149
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3150
|
+
getLogger4().error(this._error);
|
|
3151
|
+
throw new WorkflowError(this._error);
|
|
3152
|
+
}
|
|
3153
|
+
const handlerTask = ensureTask(handler);
|
|
3154
|
+
this.graph.addTask(handlerTask);
|
|
3155
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3156
|
+
this.graph.addDataflow(dataflow);
|
|
3157
|
+
this.events.emit("changed", handlerTask.id);
|
|
2704
3158
|
return this;
|
|
2705
3159
|
}
|
|
2706
3160
|
toTaskGraph() {
|
|
@@ -2785,16 +3239,16 @@ class Workflow {
|
|
|
2785
3239
|
addLoopTask(taskClass, config = {}) {
|
|
2786
3240
|
this._error = "";
|
|
2787
3241
|
const parent = getLastTask(this);
|
|
2788
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3242
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2789
3243
|
if (this._dataFlows.length > 0) {
|
|
2790
3244
|
this._dataFlows.forEach((dataflow) => {
|
|
2791
3245
|
const taskSchema = task.inputSchema();
|
|
2792
3246
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2793
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2794
|
-
|
|
3247
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3248
|
+
getLogger4().error(this._error);
|
|
2795
3249
|
return;
|
|
2796
3250
|
}
|
|
2797
|
-
dataflow.targetTaskId = task.
|
|
3251
|
+
dataflow.targetTaskId = task.id;
|
|
2798
3252
|
this.graph.addDataflow(dataflow);
|
|
2799
3253
|
});
|
|
2800
3254
|
this._dataFlows = [];
|
|
@@ -2809,9 +3263,9 @@ class Workflow {
|
|
|
2809
3263
|
if (!pending)
|
|
2810
3264
|
return;
|
|
2811
3265
|
const { parent, iteratorTask } = pending;
|
|
2812
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3266
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2813
3267
|
const nodes = this._graph.getTasks();
|
|
2814
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3268
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2815
3269
|
const earlierTasks = [];
|
|
2816
3270
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2817
3271
|
earlierTasks.push(nodes[i]);
|
|
@@ -2821,8 +3275,81 @@ class Workflow {
|
|
|
2821
3275
|
});
|
|
2822
3276
|
if (result.error) {
|
|
2823
3277
|
this._error = result.error + " Task not added.";
|
|
2824
|
-
|
|
2825
|
-
this.graph.removeTask(iteratorTask.
|
|
3278
|
+
getLogger4().error(this._error);
|
|
3279
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3284
|
+
const tasks = graph.getTasks();
|
|
3285
|
+
for (const task of tasks) {
|
|
3286
|
+
if (task.type === "InputTask") {
|
|
3287
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3288
|
+
if (outgoing.length === 0)
|
|
3289
|
+
continue;
|
|
3290
|
+
const properties = {};
|
|
3291
|
+
const required = [];
|
|
3292
|
+
for (const df of outgoing) {
|
|
3293
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3294
|
+
if (!targetTask)
|
|
3295
|
+
continue;
|
|
3296
|
+
const targetSchema = targetTask.inputSchema();
|
|
3297
|
+
if (typeof targetSchema === "boolean")
|
|
3298
|
+
continue;
|
|
3299
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3300
|
+
if (prop && typeof prop !== "boolean") {
|
|
3301
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3302
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3303
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3304
|
+
required.push(df.sourceTaskPortId);
|
|
3305
|
+
}
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
const schema = {
|
|
3310
|
+
type: "object",
|
|
3311
|
+
properties,
|
|
3312
|
+
...required.length > 0 ? { required } : {},
|
|
3313
|
+
additionalProperties: false
|
|
3314
|
+
};
|
|
3315
|
+
task.config = {
|
|
3316
|
+
...task.config,
|
|
3317
|
+
inputSchema: schema,
|
|
3318
|
+
outputSchema: schema
|
|
3319
|
+
};
|
|
3320
|
+
}
|
|
3321
|
+
if (task.type === "OutputTask") {
|
|
3322
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3323
|
+
if (incoming.length === 0)
|
|
3324
|
+
continue;
|
|
3325
|
+
const properties = {};
|
|
3326
|
+
const required = [];
|
|
3327
|
+
for (const df of incoming) {
|
|
3328
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3329
|
+
if (!sourceTask)
|
|
3330
|
+
continue;
|
|
3331
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3332
|
+
if (typeof sourceSchema === "boolean")
|
|
3333
|
+
continue;
|
|
3334
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3335
|
+
if (prop && typeof prop !== "boolean") {
|
|
3336
|
+
properties[df.targetTaskPortId] = prop;
|
|
3337
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3338
|
+
required.push(df.targetTaskPortId);
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
const schema = {
|
|
3343
|
+
type: "object",
|
|
3344
|
+
properties,
|
|
3345
|
+
...required.length > 0 ? { required } : {},
|
|
3346
|
+
additionalProperties: false
|
|
3347
|
+
};
|
|
3348
|
+
task.config = {
|
|
3349
|
+
...task.config,
|
|
3350
|
+
inputSchema: schema,
|
|
3351
|
+
outputSchema: schema
|
|
3352
|
+
};
|
|
2826
3353
|
}
|
|
2827
3354
|
}
|
|
2828
3355
|
}
|
|
@@ -2832,6 +3359,7 @@ class Workflow {
|
|
|
2832
3359
|
const sourceSchema = sourceTask.outputSchema();
|
|
2833
3360
|
const targetSchema = targetTask.inputSchema();
|
|
2834
3361
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3362
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2835
3363
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2836
3364
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2837
3365
|
const formats = new Set;
|
|
@@ -2907,18 +3435,33 @@ class Workflow {
|
|
|
2907
3435
|
if (typeof fromSchema === "object") {
|
|
2908
3436
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2909
3437
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3438
|
+
if (matches.has(fromOutputPortId))
|
|
3439
|
+
continue;
|
|
2910
3440
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2911
3441
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2912
3442
|
}
|
|
2913
3443
|
return;
|
|
2914
3444
|
}
|
|
2915
3445
|
}
|
|
3446
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3447
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3448
|
+
if (matches.has(toInputPortId))
|
|
3449
|
+
continue;
|
|
3450
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3451
|
+
continue;
|
|
3452
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3453
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3454
|
+
}
|
|
3455
|
+
return;
|
|
3456
|
+
}
|
|
2916
3457
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2917
3458
|
return;
|
|
2918
3459
|
}
|
|
2919
3460
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2920
3461
|
if (matches.has(toInputPortId))
|
|
2921
3462
|
continue;
|
|
3463
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3464
|
+
continue;
|
|
2922
3465
|
const candidates = [];
|
|
2923
3466
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2924
3467
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2938,22 +3481,32 @@ class Workflow {
|
|
|
2938
3481
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2939
3482
|
}
|
|
2940
3483
|
};
|
|
2941
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3484
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2942
3485
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2943
3486
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2944
3487
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2945
3488
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2946
3489
|
});
|
|
2947
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3490
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2948
3491
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2949
3492
|
});
|
|
2950
3493
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2951
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3494
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2952
3495
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2953
3496
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2954
3497
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2955
3498
|
const earlierTask = earlierTasks[i];
|
|
2956
3499
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3500
|
+
if (earlierTask.type === "InputTask") {
|
|
3501
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3502
|
+
if (matches.has(requiredInputId))
|
|
3503
|
+
continue;
|
|
3504
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3505
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3506
|
+
}
|
|
3507
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3508
|
+
continue;
|
|
3509
|
+
}
|
|
2957
3510
|
const makeMatchFromEarlier = (comparator) => {
|
|
2958
3511
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2959
3512
|
return;
|
|
@@ -2963,7 +3516,7 @@ class Workflow {
|
|
|
2963
3516
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2964
3517
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2965
3518
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2966
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3519
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2967
3520
|
}
|
|
2968
3521
|
}
|
|
2969
3522
|
}
|
|
@@ -2989,6 +3542,10 @@ class Workflow {
|
|
|
2989
3542
|
};
|
|
2990
3543
|
}
|
|
2991
3544
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3545
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3546
|
+
if (existingTargetConnections.length > 0) {
|
|
3547
|
+
return { matches, unmatchedRequired: [] };
|
|
3548
|
+
}
|
|
2992
3549
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
2993
3550
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
2994
3551
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3111,7 +3668,7 @@ function getLastTask(workflow) {
|
|
|
3111
3668
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3112
3669
|
}
|
|
3113
3670
|
function connect(source, target, workflow) {
|
|
3114
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3671
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3115
3672
|
}
|
|
3116
3673
|
function pipe(args, workflow = new Workflow) {
|
|
3117
3674
|
let previousTask = getLastTask(workflow);
|
|
@@ -3167,7 +3724,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3167
3724
|
// src/task-graph/TaskGraph.ts
|
|
3168
3725
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3169
3726
|
constructor() {
|
|
3170
|
-
super((task) => task.
|
|
3727
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3171
3728
|
}
|
|
3172
3729
|
}
|
|
3173
3730
|
|
|
@@ -3264,18 +3821,22 @@ class TaskGraph {
|
|
|
3264
3821
|
return this._dag.removeNode(taskId);
|
|
3265
3822
|
}
|
|
3266
3823
|
resetGraph() {
|
|
3267
|
-
this.runner.resetGraph(this,
|
|
3824
|
+
this.runner.resetGraph(this, uuid45());
|
|
3268
3825
|
}
|
|
3269
|
-
toJSON() {
|
|
3270
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3826
|
+
toJSON(options) {
|
|
3827
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3271
3828
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3272
|
-
|
|
3829
|
+
let json = {
|
|
3273
3830
|
tasks,
|
|
3274
3831
|
dataflows
|
|
3275
3832
|
};
|
|
3833
|
+
if (options?.withBoundaryNodes) {
|
|
3834
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3835
|
+
}
|
|
3836
|
+
return json;
|
|
3276
3837
|
}
|
|
3277
|
-
toDependencyJSON() {
|
|
3278
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3838
|
+
toDependencyJSON(options) {
|
|
3839
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3279
3840
|
this.getDataflows().forEach((df) => {
|
|
3280
3841
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3281
3842
|
if (!target.dependencies) {
|
|
@@ -3301,6 +3862,9 @@ class TaskGraph {
|
|
|
3301
3862
|
}
|
|
3302
3863
|
}
|
|
3303
3864
|
});
|
|
3865
|
+
if (options?.withBoundaryNodes) {
|
|
3866
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3867
|
+
}
|
|
3304
3868
|
return tasks;
|
|
3305
3869
|
}
|
|
3306
3870
|
get events() {
|
|
@@ -3319,7 +3883,7 @@ class TaskGraph {
|
|
|
3319
3883
|
const tasks = this.getTasks();
|
|
3320
3884
|
tasks.forEach((task) => {
|
|
3321
3885
|
const unsub = task.subscribe("status", (status) => {
|
|
3322
|
-
callback(task.
|
|
3886
|
+
callback(task.id, status);
|
|
3323
3887
|
});
|
|
3324
3888
|
unsubscribes.push(unsub);
|
|
3325
3889
|
});
|
|
@@ -3328,7 +3892,7 @@ class TaskGraph {
|
|
|
3328
3892
|
if (!task || typeof task.subscribe !== "function")
|
|
3329
3893
|
return;
|
|
3330
3894
|
const unsub = task.subscribe("status", (status) => {
|
|
3331
|
-
callback(task.
|
|
3895
|
+
callback(task.id, status);
|
|
3332
3896
|
});
|
|
3333
3897
|
unsubscribes.push(unsub);
|
|
3334
3898
|
};
|
|
@@ -3343,7 +3907,7 @@ class TaskGraph {
|
|
|
3343
3907
|
const tasks = this.getTasks();
|
|
3344
3908
|
tasks.forEach((task) => {
|
|
3345
3909
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3346
|
-
callback(task.
|
|
3910
|
+
callback(task.id, progress, message, ...args);
|
|
3347
3911
|
});
|
|
3348
3912
|
unsubscribes.push(unsub);
|
|
3349
3913
|
});
|
|
@@ -3352,7 +3916,7 @@ class TaskGraph {
|
|
|
3352
3916
|
if (!task || typeof task.subscribe !== "function")
|
|
3353
3917
|
return;
|
|
3354
3918
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3355
|
-
callback(task.
|
|
3919
|
+
callback(task.id, progress, message, ...args);
|
|
3356
3920
|
});
|
|
3357
3921
|
unsubscribes.push(unsub);
|
|
3358
3922
|
};
|
|
@@ -3437,7 +4001,7 @@ class TaskGraph {
|
|
|
3437
4001
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3438
4002
|
const edges = [];
|
|
3439
4003
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3440
|
-
edges.push(new Dataflow(tasks[i].
|
|
4004
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3441
4005
|
}
|
|
3442
4006
|
return edges;
|
|
3443
4007
|
}
|
|
@@ -3447,9 +4011,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3447
4011
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3448
4012
|
return graph;
|
|
3449
4013
|
}
|
|
4014
|
+
// src/task/FallbackTaskRunner.ts
|
|
4015
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4016
|
+
async executeTask(input) {
|
|
4017
|
+
if (this.task.fallbackMode === "data") {
|
|
4018
|
+
return this.executeDataFallback(input);
|
|
4019
|
+
}
|
|
4020
|
+
return this.executeTaskFallback(input);
|
|
4021
|
+
}
|
|
4022
|
+
async executeTaskReactive(input, output) {
|
|
4023
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4024
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4025
|
+
}
|
|
4026
|
+
async executeTaskFallback(input) {
|
|
4027
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4028
|
+
if (tasks.length === 0) {
|
|
4029
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4030
|
+
}
|
|
4031
|
+
const errors = [];
|
|
4032
|
+
const totalAttempts = tasks.length;
|
|
4033
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4034
|
+
if (this.abortController?.signal.aborted) {
|
|
4035
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4036
|
+
}
|
|
4037
|
+
const alternativeTask = tasks[i];
|
|
4038
|
+
const attemptNumber = i + 1;
|
|
4039
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4040
|
+
try {
|
|
4041
|
+
this.resetTask(alternativeTask);
|
|
4042
|
+
const result = await alternativeTask.run(input);
|
|
4043
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4044
|
+
return await this.executeTaskReactive(input, result);
|
|
4045
|
+
} catch (error) {
|
|
4046
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4047
|
+
throw error;
|
|
4048
|
+
}
|
|
4049
|
+
errors.push({ task: alternativeTask, error });
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
4052
|
+
throw this.buildAggregateError(errors, "task");
|
|
4053
|
+
}
|
|
4054
|
+
async executeDataFallback(input) {
|
|
4055
|
+
const alternatives = this.task.alternatives;
|
|
4056
|
+
if (alternatives.length === 0) {
|
|
4057
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4058
|
+
}
|
|
4059
|
+
const errors = [];
|
|
4060
|
+
const totalAttempts = alternatives.length;
|
|
4061
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4062
|
+
if (this.abortController?.signal.aborted) {
|
|
4063
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4064
|
+
}
|
|
4065
|
+
const alternative = alternatives[i];
|
|
4066
|
+
const attemptNumber = i + 1;
|
|
4067
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4068
|
+
try {
|
|
4069
|
+
this.resetSubgraph();
|
|
4070
|
+
const mergedInput = { ...input, ...alternative };
|
|
4071
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4072
|
+
parentSignal: this.abortController?.signal,
|
|
4073
|
+
outputCache: this.outputCache
|
|
4074
|
+
});
|
|
4075
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4076
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4077
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4078
|
+
} catch (error) {
|
|
4079
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4080
|
+
throw error;
|
|
4081
|
+
}
|
|
4082
|
+
errors.push({ alternative, error });
|
|
4083
|
+
}
|
|
4084
|
+
}
|
|
4085
|
+
throw this.buildAggregateError(errors, "data");
|
|
4086
|
+
}
|
|
4087
|
+
resetTask(task) {
|
|
4088
|
+
task.status = TaskStatus.PENDING;
|
|
4089
|
+
task.progress = 0;
|
|
4090
|
+
task.error = undefined;
|
|
4091
|
+
task.completedAt = undefined;
|
|
4092
|
+
task.startedAt = undefined;
|
|
4093
|
+
task.resetInputData();
|
|
4094
|
+
}
|
|
4095
|
+
resetSubgraph() {
|
|
4096
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4097
|
+
this.resetTask(task);
|
|
4098
|
+
}
|
|
4099
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4100
|
+
dataflow.reset();
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
buildAggregateError(errors, mode) {
|
|
4104
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4105
|
+
const details = errors.map((e, i) => {
|
|
4106
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4107
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4108
|
+
}).join(`
|
|
4109
|
+
`);
|
|
4110
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4111
|
+
${details}`);
|
|
4112
|
+
}
|
|
4113
|
+
}
|
|
4114
|
+
|
|
4115
|
+
// src/task/FallbackTask.ts
|
|
4116
|
+
var fallbackTaskConfigSchema = {
|
|
4117
|
+
type: "object",
|
|
4118
|
+
properties: {
|
|
4119
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4120
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4121
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4122
|
+
},
|
|
4123
|
+
additionalProperties: false
|
|
4124
|
+
};
|
|
4125
|
+
|
|
4126
|
+
class FallbackTask extends GraphAsTask {
|
|
4127
|
+
static type = "FallbackTask";
|
|
4128
|
+
static category = "Flow Control";
|
|
4129
|
+
static title = "Fallback";
|
|
4130
|
+
static description = "Try alternatives until one succeeds";
|
|
4131
|
+
static hasDynamicSchemas = true;
|
|
4132
|
+
static configSchema() {
|
|
4133
|
+
return fallbackTaskConfigSchema;
|
|
4134
|
+
}
|
|
4135
|
+
get runner() {
|
|
4136
|
+
if (!this._runner) {
|
|
4137
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4138
|
+
}
|
|
4139
|
+
return this._runner;
|
|
4140
|
+
}
|
|
4141
|
+
get fallbackMode() {
|
|
4142
|
+
return this.config?.fallbackMode ?? "task";
|
|
4143
|
+
}
|
|
4144
|
+
get alternatives() {
|
|
4145
|
+
return this.config?.alternatives ?? [];
|
|
4146
|
+
}
|
|
4147
|
+
inputSchema() {
|
|
4148
|
+
if (!this.hasChildren()) {
|
|
4149
|
+
return this.constructor.inputSchema();
|
|
4150
|
+
}
|
|
4151
|
+
if (this.fallbackMode === "data") {
|
|
4152
|
+
return super.inputSchema();
|
|
4153
|
+
}
|
|
4154
|
+
const properties = {};
|
|
4155
|
+
const tasks = this.subGraph.getTasks();
|
|
4156
|
+
for (const task of tasks) {
|
|
4157
|
+
const taskInputSchema = task.inputSchema();
|
|
4158
|
+
if (typeof taskInputSchema === "boolean")
|
|
4159
|
+
continue;
|
|
4160
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4161
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4162
|
+
if (!properties[inputName]) {
|
|
4163
|
+
properties[inputName] = inputProp;
|
|
4164
|
+
}
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
return {
|
|
4168
|
+
type: "object",
|
|
4169
|
+
properties,
|
|
4170
|
+
additionalProperties: true
|
|
4171
|
+
};
|
|
4172
|
+
}
|
|
4173
|
+
outputSchema() {
|
|
4174
|
+
if (!this.hasChildren()) {
|
|
4175
|
+
return this.constructor.outputSchema();
|
|
4176
|
+
}
|
|
4177
|
+
const tasks = this.subGraph.getTasks();
|
|
4178
|
+
if (tasks.length === 0) {
|
|
4179
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4180
|
+
}
|
|
4181
|
+
if (this.fallbackMode === "task") {
|
|
4182
|
+
const firstTask = tasks[0];
|
|
4183
|
+
return firstTask.outputSchema();
|
|
4184
|
+
}
|
|
4185
|
+
return super.outputSchema();
|
|
4186
|
+
}
|
|
4187
|
+
toJSON() {
|
|
4188
|
+
const json = super.toJSON();
|
|
4189
|
+
return {
|
|
4190
|
+
...json,
|
|
4191
|
+
config: {
|
|
4192
|
+
..."config" in json ? json.config : {},
|
|
4193
|
+
fallbackMode: this.fallbackMode,
|
|
4194
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4195
|
+
}
|
|
4196
|
+
};
|
|
4197
|
+
}
|
|
4198
|
+
}
|
|
4199
|
+
queueMicrotask(() => {
|
|
4200
|
+
Workflow.prototype.fallback = function() {
|
|
4201
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4202
|
+
};
|
|
4203
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4204
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4205
|
+
return this.addLoopTask(FallbackTask, {
|
|
4206
|
+
fallbackMode: "data",
|
|
4207
|
+
alternatives
|
|
4208
|
+
});
|
|
4209
|
+
};
|
|
4210
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4211
|
+
});
|
|
3450
4212
|
// src/task/IteratorTaskRunner.ts
|
|
3451
4213
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3452
|
-
subGraphRunChain = Promise.resolve();
|
|
3453
4214
|
async executeTask(input) {
|
|
3454
4215
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3455
4216
|
if (analysis.iterationCount === 0) {
|
|
@@ -3471,13 +4232,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3471
4232
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3472
4233
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3473
4234
|
const completionOrderResults = [];
|
|
4235
|
+
let completedCount = 0;
|
|
3474
4236
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3475
4237
|
if (this.abortController?.signal.aborted) {
|
|
3476
4238
|
break;
|
|
3477
4239
|
}
|
|
3478
4240
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3479
4241
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3480
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4242
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4243
|
+
completedCount++;
|
|
4244
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4245
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4246
|
+
});
|
|
3481
4247
|
for (const { index, result } of batchResults) {
|
|
3482
4248
|
if (result === undefined)
|
|
3483
4249
|
continue;
|
|
@@ -3487,8 +4253,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3487
4253
|
completionOrderResults.push(result);
|
|
3488
4254
|
}
|
|
3489
4255
|
}
|
|
3490
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3491
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3492
4256
|
}
|
|
3493
4257
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3494
4258
|
return this.task.collectResults(collected);
|
|
@@ -3510,7 +4274,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3510
4274
|
}
|
|
3511
4275
|
return accumulator;
|
|
3512
4276
|
}
|
|
3513
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4277
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3514
4278
|
const results = [];
|
|
3515
4279
|
let cursor = 0;
|
|
3516
4280
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3528,33 +4292,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3528
4292
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3529
4293
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3530
4294
|
results.push({ index, result });
|
|
4295
|
+
await onItemComplete?.();
|
|
3531
4296
|
}
|
|
3532
4297
|
});
|
|
3533
4298
|
await Promise.all(workers);
|
|
3534
4299
|
return results;
|
|
3535
4300
|
}
|
|
4301
|
+
cloneGraph(graph) {
|
|
4302
|
+
const clone = new TaskGraph;
|
|
4303
|
+
for (const task of graph.getTasks()) {
|
|
4304
|
+
const ctor = task.constructor;
|
|
4305
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4306
|
+
if (task.hasChildren()) {
|
|
4307
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4308
|
+
}
|
|
4309
|
+
clone.addTask(newTask);
|
|
4310
|
+
}
|
|
4311
|
+
for (const df of graph.getDataflows()) {
|
|
4312
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4313
|
+
}
|
|
4314
|
+
return clone;
|
|
4315
|
+
}
|
|
3536
4316
|
async executeSubgraphIteration(input) {
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
4317
|
+
if (this.abortController?.signal.aborted) {
|
|
4318
|
+
return;
|
|
4319
|
+
}
|
|
4320
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4321
|
+
const results = await graphClone.run(input, {
|
|
4322
|
+
parentSignal: this.abortController?.signal,
|
|
4323
|
+
outputCache: this.outputCache
|
|
3541
4324
|
});
|
|
3542
|
-
|
|
3543
|
-
|
|
3544
|
-
if (this.abortController?.signal.aborted) {
|
|
3545
|
-
return;
|
|
3546
|
-
}
|
|
3547
|
-
const results = await this.task.subGraph.run(input, {
|
|
3548
|
-
parentSignal: this.abortController?.signal,
|
|
3549
|
-
outputCache: this.outputCache
|
|
3550
|
-
});
|
|
3551
|
-
if (results.length === 0) {
|
|
3552
|
-
return;
|
|
3553
|
-
}
|
|
3554
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3555
|
-
} finally {
|
|
3556
|
-
releaseTurn?.();
|
|
4325
|
+
if (results.length === 0) {
|
|
4326
|
+
return;
|
|
3557
4327
|
}
|
|
4328
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3558
4329
|
}
|
|
3559
4330
|
}
|
|
3560
4331
|
|
|
@@ -3635,23 +4406,17 @@ function inferIterationFromSchema(schema) {
|
|
|
3635
4406
|
return false;
|
|
3636
4407
|
}
|
|
3637
4408
|
function createFlexibleSchema(baseSchema) {
|
|
3638
|
-
if (typeof baseSchema === "boolean")
|
|
3639
|
-
return baseSchema;
|
|
3640
4409
|
return {
|
|
3641
4410
|
anyOf: [baseSchema, { type: "array", items: baseSchema }]
|
|
3642
4411
|
};
|
|
3643
4412
|
}
|
|
3644
4413
|
function createArraySchema(baseSchema) {
|
|
3645
|
-
if (typeof baseSchema === "boolean")
|
|
3646
|
-
return baseSchema;
|
|
3647
4414
|
return {
|
|
3648
4415
|
type: "array",
|
|
3649
4416
|
items: baseSchema
|
|
3650
4417
|
};
|
|
3651
4418
|
}
|
|
3652
4419
|
function extractBaseSchema(schema) {
|
|
3653
|
-
if (typeof schema === "boolean")
|
|
3654
|
-
return schema;
|
|
3655
4420
|
const schemaType = schema.type;
|
|
3656
4421
|
if (schemaType === "array" && schema.items) {
|
|
3657
4422
|
return schema.items;
|
|
@@ -3844,7 +4609,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3844
4609
|
const tasks = this.subGraph.getTasks();
|
|
3845
4610
|
if (tasks.length === 0)
|
|
3846
4611
|
return;
|
|
3847
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4612
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3848
4613
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3849
4614
|
const properties = {};
|
|
3850
4615
|
const required = [];
|
|
@@ -3871,6 +4636,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3871
4636
|
}
|
|
3872
4637
|
}
|
|
3873
4638
|
}
|
|
4639
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4640
|
+
for (const task of tasks) {
|
|
4641
|
+
if (sourceIds.has(task.id))
|
|
4642
|
+
continue;
|
|
4643
|
+
const inputSchema = task.inputSchema();
|
|
4644
|
+
if (typeof inputSchema === "boolean")
|
|
4645
|
+
continue;
|
|
4646
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4647
|
+
if (requiredKeys.size === 0)
|
|
4648
|
+
continue;
|
|
4649
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4650
|
+
for (const key of requiredKeys) {
|
|
4651
|
+
if (connectedPorts.has(key))
|
|
4652
|
+
continue;
|
|
4653
|
+
if (properties[key])
|
|
4654
|
+
continue;
|
|
4655
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4656
|
+
continue;
|
|
4657
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4658
|
+
if (!prop || typeof prop === "boolean")
|
|
4659
|
+
continue;
|
|
4660
|
+
properties[key] = prop;
|
|
4661
|
+
if (!required.includes(key)) {
|
|
4662
|
+
required.push(key);
|
|
4663
|
+
}
|
|
4664
|
+
}
|
|
4665
|
+
}
|
|
3874
4666
|
return {
|
|
3875
4667
|
type: "object",
|
|
3876
4668
|
properties,
|
|
@@ -4005,7 +4797,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4005
4797
|
if (!this.hasChildren()) {
|
|
4006
4798
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4007
4799
|
}
|
|
4008
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4800
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4009
4801
|
if (endingNodes.length === 0) {
|
|
4010
4802
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4011
4803
|
}
|
|
@@ -4215,8 +5007,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4215
5007
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4216
5008
|
}
|
|
4217
5009
|
this._currentIteration++;
|
|
4218
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4219
|
-
await context.updateProgress(progress, `
|
|
5010
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5011
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4220
5012
|
}
|
|
4221
5013
|
return currentOutput;
|
|
4222
5014
|
}
|
|
@@ -4259,8 +5051,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4259
5051
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4260
5052
|
}
|
|
4261
5053
|
this._currentIteration++;
|
|
4262
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4263
|
-
await context.updateProgress(progress, `
|
|
5054
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5055
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4264
5056
|
}
|
|
4265
5057
|
yield { type: "finish", data: currentOutput };
|
|
4266
5058
|
}
|
|
@@ -4336,7 +5128,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4336
5128
|
return this.constructor.outputSchema();
|
|
4337
5129
|
}
|
|
4338
5130
|
const tasks = this.subGraph.getTasks();
|
|
4339
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5131
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4340
5132
|
if (endingNodes.length === 0) {
|
|
4341
5133
|
return this.constructor.outputSchema();
|
|
4342
5134
|
}
|
|
@@ -4711,7 +5503,11 @@ var jobQueueTaskConfigSchema = {
|
|
|
4711
5503
|
type: "object",
|
|
4712
5504
|
properties: {
|
|
4713
5505
|
...graphAsTaskConfigSchema["properties"],
|
|
4714
|
-
queue: {
|
|
5506
|
+
queue: {
|
|
5507
|
+
oneOf: [{ type: "boolean" }, { type: "string" }],
|
|
5508
|
+
description: "Queue handling: false=run inline, true=use default, string=explicit queue name",
|
|
5509
|
+
"x-ui-hidden": true
|
|
5510
|
+
}
|
|
4715
5511
|
},
|
|
4716
5512
|
additionalProperties: false
|
|
4717
5513
|
};
|
|
@@ -4999,7 +5795,7 @@ class ReduceTask extends IteratorTask {
|
|
|
4999
5795
|
if (!this.hasChildren()) {
|
|
5000
5796
|
return this.constructor.outputSchema();
|
|
5001
5797
|
}
|
|
5002
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5798
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5003
5799
|
if (endingNodes.length === 0) {
|
|
5004
5800
|
return this.constructor.outputSchema();
|
|
5005
5801
|
}
|
|
@@ -5037,14 +5833,14 @@ var TaskRegistry = {
|
|
|
5037
5833
|
};
|
|
5038
5834
|
|
|
5039
5835
|
// src/task/TaskJSON.ts
|
|
5040
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5836
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5041
5837
|
if (!item.id)
|
|
5042
5838
|
throw new TaskJSONError("Task id required");
|
|
5043
5839
|
if (!item.type)
|
|
5044
5840
|
throw new TaskJSONError("Task type required");
|
|
5045
5841
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5046
5842
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5047
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5843
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5048
5844
|
if (!taskClass)
|
|
5049
5845
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5050
5846
|
const taskConfig = {
|
|
@@ -5071,20 +5867,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5071
5867
|
}
|
|
5072
5868
|
return subGraph;
|
|
5073
5869
|
};
|
|
5074
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5075
|
-
const task = createSingleTaskFromJSON(item);
|
|
5870
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5871
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5076
5872
|
if (item.subgraph) {
|
|
5077
5873
|
if (!(task instanceof GraphAsTask)) {
|
|
5078
5874
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5079
5875
|
}
|
|
5080
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5876
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5081
5877
|
}
|
|
5082
5878
|
return task;
|
|
5083
5879
|
};
|
|
5084
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5880
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5085
5881
|
const subGraph = new TaskGraph;
|
|
5086
5882
|
for (const subitem of graphJsonObj.tasks) {
|
|
5087
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5883
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5088
5884
|
}
|
|
5089
5885
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5090
5886
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5093,7 +5889,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5093
5889
|
};
|
|
5094
5890
|
// src/task/index.ts
|
|
5095
5891
|
var registerBaseTasks = () => {
|
|
5096
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5892
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5097
5893
|
tasks.map(TaskRegistry.registerTask);
|
|
5098
5894
|
return tasks;
|
|
5099
5895
|
};
|
|
@@ -5276,11 +6072,14 @@ export {
|
|
|
5276
6072
|
isFlexibleSchema,
|
|
5277
6073
|
hasVectorOutput,
|
|
5278
6074
|
hasVectorLikeInput,
|
|
6075
|
+
hasStructuredOutput,
|
|
5279
6076
|
graphAsTaskConfigSchema,
|
|
5280
6077
|
getTaskQueueRegistry,
|
|
6078
|
+
getStructuredOutputSchemas,
|
|
5281
6079
|
getStreamingPorts,
|
|
5282
6080
|
getPortStreamMode,
|
|
5283
6081
|
getOutputStreamMode,
|
|
6082
|
+
getObjectPortId,
|
|
5284
6083
|
getNestedValue,
|
|
5285
6084
|
getLastTask,
|
|
5286
6085
|
getJobQueueFactory,
|
|
@@ -5289,6 +6088,7 @@ export {
|
|
|
5289
6088
|
getAppendPortId,
|
|
5290
6089
|
findArrayPorts,
|
|
5291
6090
|
filterIterationProperties,
|
|
6091
|
+
fallbackTaskConfigSchema,
|
|
5292
6092
|
extractIterationProperties,
|
|
5293
6093
|
extractBaseSchema,
|
|
5294
6094
|
evaluateCondition,
|
|
@@ -5303,13 +6103,19 @@ export {
|
|
|
5303
6103
|
createArraySchema,
|
|
5304
6104
|
connect,
|
|
5305
6105
|
conditionalTaskConfigSchema,
|
|
6106
|
+
computeGraphOutputSchema,
|
|
6107
|
+
computeGraphInputSchema,
|
|
6108
|
+
calculateNodeDepths,
|
|
5306
6109
|
buildIterationInputSchema,
|
|
5307
6110
|
addIterationContextToSchema,
|
|
6111
|
+
addBoundaryNodesToGraphJson,
|
|
6112
|
+
addBoundaryNodesToDependencyJson,
|
|
5308
6113
|
WorkflowError,
|
|
5309
6114
|
Workflow,
|
|
5310
6115
|
WhileTaskRunner,
|
|
5311
6116
|
WhileTask,
|
|
5312
6117
|
WHILE_CONTEXT_SCHEMA,
|
|
6118
|
+
TaskTimeoutError,
|
|
5313
6119
|
TaskStatus,
|
|
5314
6120
|
TaskRegistry,
|
|
5315
6121
|
TaskQueueRegistry,
|
|
@@ -5345,6 +6151,8 @@ export {
|
|
|
5345
6151
|
GraphAsTaskRunner,
|
|
5346
6152
|
GraphAsTask,
|
|
5347
6153
|
GRAPH_RESULT_ARRAY,
|
|
6154
|
+
FallbackTaskRunner,
|
|
6155
|
+
FallbackTask,
|
|
5348
6156
|
EventTaskGraphToDagMapping,
|
|
5349
6157
|
EventDagToTaskGraphMapping,
|
|
5350
6158
|
DataflowArrow,
|
|
@@ -5358,4 +6166,4 @@ export {
|
|
|
5358
6166
|
ConditionalTask
|
|
5359
6167
|
};
|
|
5360
6168
|
|
|
5361
|
-
//# debugId=
|
|
6169
|
+
//# debugId=2492C1E27B2BED5064756E2164756E21
|