@workglow/task-graph 0.0.101 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1083 -275
- package/dist/browser.js.map +26 -23
- package/dist/bun.js +1083 -275
- package/dist/bun.js.map +26 -23
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1083 -275
- package/dist/node.js.map +26 -23
- package/dist/task/ConditionalTask.d.ts +11 -2
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +22 -13
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +6 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +22 -8
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +21 -3
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +11 -1
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +11 -1
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +25 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +12 -3
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +11 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +11 -1
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task/iterationSchema.d.ts +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/bun.js
CHANGED
|
@@ -15,15 +15,29 @@ var TaskStatus = {
|
|
|
15
15
|
var TaskConfigSchema = {
|
|
16
16
|
type: "object",
|
|
17
17
|
properties: {
|
|
18
|
-
id: {
|
|
18
|
+
id: {
|
|
19
|
+
"x-ui-hidden": true
|
|
20
|
+
},
|
|
19
21
|
title: { type: "string" },
|
|
20
22
|
description: { type: "string" },
|
|
21
23
|
cacheable: { type: "boolean" },
|
|
22
|
-
|
|
23
|
-
|
|
24
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
25
|
+
inputSchema: {
|
|
26
|
+
type: "object",
|
|
27
|
+
properties: {},
|
|
28
|
+
additionalProperties: true,
|
|
29
|
+
"x-ui-hidden": true
|
|
30
|
+
},
|
|
31
|
+
outputSchema: {
|
|
32
|
+
type: "object",
|
|
33
|
+
properties: {},
|
|
34
|
+
additionalProperties: true,
|
|
35
|
+
"x-ui-hidden": true
|
|
36
|
+
},
|
|
24
37
|
extras: {
|
|
25
38
|
type: "object",
|
|
26
|
-
additionalProperties: true
|
|
39
|
+
additionalProperties: true,
|
|
40
|
+
"x-ui-hidden": true
|
|
27
41
|
}
|
|
28
42
|
},
|
|
29
43
|
additionalProperties: false
|
|
@@ -228,8 +242,352 @@ class DataflowArrow extends Dataflow {
|
|
|
228
242
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
229
243
|
}
|
|
230
244
|
}
|
|
245
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
246
|
+
import { uuid4 } from "@workglow/util";
|
|
247
|
+
function calculateNodeDepths(graph) {
|
|
248
|
+
const depths = new Map;
|
|
249
|
+
const tasks = graph.getTasks();
|
|
250
|
+
for (const task of tasks) {
|
|
251
|
+
depths.set(task.id, 0);
|
|
252
|
+
}
|
|
253
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
254
|
+
for (const task of sortedTasks) {
|
|
255
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
256
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
257
|
+
for (const targetTask of targetTasks) {
|
|
258
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
259
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return depths;
|
|
263
|
+
}
|
|
264
|
+
function computeGraphInputSchema(graph, options) {
|
|
265
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
266
|
+
const properties = {};
|
|
267
|
+
const required = [];
|
|
268
|
+
const propertyOrigins = {};
|
|
269
|
+
const tasks = graph.getTasks();
|
|
270
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
271
|
+
for (const task of startingNodes) {
|
|
272
|
+
const taskInputSchema = task.inputSchema();
|
|
273
|
+
if (typeof taskInputSchema === "boolean") {
|
|
274
|
+
if (taskInputSchema === false) {
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (taskInputSchema === true) {
|
|
278
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
283
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
284
|
+
if (!properties[inputName]) {
|
|
285
|
+
properties[inputName] = inputProp;
|
|
286
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
287
|
+
required.push(inputName);
|
|
288
|
+
}
|
|
289
|
+
if (trackOrigins) {
|
|
290
|
+
propertyOrigins[inputName] = [task.id];
|
|
291
|
+
}
|
|
292
|
+
} else if (trackOrigins) {
|
|
293
|
+
propertyOrigins[inputName].push(task.id);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
298
|
+
for (const task of tasks) {
|
|
299
|
+
if (sourceIds.has(task.id))
|
|
300
|
+
continue;
|
|
301
|
+
const taskInputSchema = task.inputSchema();
|
|
302
|
+
if (typeof taskInputSchema === "boolean")
|
|
303
|
+
continue;
|
|
304
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
305
|
+
if (requiredKeys.size === 0)
|
|
306
|
+
continue;
|
|
307
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
308
|
+
for (const key of requiredKeys) {
|
|
309
|
+
if (connectedPorts.has(key))
|
|
310
|
+
continue;
|
|
311
|
+
if (properties[key]) {
|
|
312
|
+
if (trackOrigins) {
|
|
313
|
+
propertyOrigins[key].push(task.id);
|
|
314
|
+
}
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
318
|
+
continue;
|
|
319
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
320
|
+
if (!prop || typeof prop === "boolean")
|
|
321
|
+
continue;
|
|
322
|
+
properties[key] = prop;
|
|
323
|
+
if (!required.includes(key)) {
|
|
324
|
+
required.push(key);
|
|
325
|
+
}
|
|
326
|
+
if (trackOrigins) {
|
|
327
|
+
propertyOrigins[key] = [task.id];
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (trackOrigins) {
|
|
332
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
333
|
+
const prop = properties[propName];
|
|
334
|
+
if (!prop || typeof prop === "boolean")
|
|
335
|
+
continue;
|
|
336
|
+
if (origins.length === 1) {
|
|
337
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
338
|
+
} else {
|
|
339
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return {
|
|
344
|
+
type: "object",
|
|
345
|
+
properties,
|
|
346
|
+
...required.length > 0 ? { required } : {},
|
|
347
|
+
additionalProperties: false
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
function computeGraphOutputSchema(graph, options) {
|
|
351
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
352
|
+
const properties = {};
|
|
353
|
+
const required = [];
|
|
354
|
+
const propertyOrigins = {};
|
|
355
|
+
const tasks = graph.getTasks();
|
|
356
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
357
|
+
const depths = calculateNodeDepths(graph);
|
|
358
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
359
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
360
|
+
const propertyCount = {};
|
|
361
|
+
const propertySchema = {};
|
|
362
|
+
for (const task of lastLevelNodes) {
|
|
363
|
+
const taskOutputSchema = task.outputSchema();
|
|
364
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
365
|
+
if (taskOutputSchema === false) {
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (taskOutputSchema === true) {
|
|
369
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
374
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
375
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
376
|
+
if (!propertySchema[outputName]) {
|
|
377
|
+
propertySchema[outputName] = outputProp;
|
|
378
|
+
}
|
|
379
|
+
if (trackOrigins) {
|
|
380
|
+
if (!propertyOrigins[outputName]) {
|
|
381
|
+
propertyOrigins[outputName] = [task.id];
|
|
382
|
+
} else {
|
|
383
|
+
propertyOrigins[outputName].push(task.id);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
389
|
+
const outputProp = propertySchema[outputName];
|
|
390
|
+
if (lastLevelNodes.length === 1) {
|
|
391
|
+
properties[outputName] = outputProp;
|
|
392
|
+
} else {
|
|
393
|
+
properties[outputName] = {
|
|
394
|
+
type: "array",
|
|
395
|
+
items: outputProp
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (trackOrigins) {
|
|
400
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
401
|
+
const prop = properties[propName];
|
|
402
|
+
if (!prop || typeof prop === "boolean")
|
|
403
|
+
continue;
|
|
404
|
+
if (origins.length === 1) {
|
|
405
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
406
|
+
} else {
|
|
407
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return {
|
|
412
|
+
type: "object",
|
|
413
|
+
properties,
|
|
414
|
+
...required.length > 0 ? { required } : {},
|
|
415
|
+
additionalProperties: false
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
function stripOriginAnnotations(schema) {
|
|
419
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
420
|
+
return schema;
|
|
421
|
+
const properties = schema.properties;
|
|
422
|
+
if (!properties)
|
|
423
|
+
return schema;
|
|
424
|
+
const strippedProperties = {};
|
|
425
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
426
|
+
if (!prop || typeof prop !== "object") {
|
|
427
|
+
strippedProperties[key] = prop;
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
431
|
+
strippedProperties[key] = rest;
|
|
432
|
+
}
|
|
433
|
+
return { ...schema, properties: strippedProperties };
|
|
434
|
+
}
|
|
435
|
+
function getOriginTaskIds(prop) {
|
|
436
|
+
if (prop["x-source-task-ids"]) {
|
|
437
|
+
return prop["x-source-task-ids"];
|
|
438
|
+
}
|
|
439
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
440
|
+
return [prop["x-source-task-id"]];
|
|
441
|
+
}
|
|
442
|
+
return [];
|
|
443
|
+
}
|
|
444
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
445
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
446
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
447
|
+
if (hasInputTask && hasOutputTask) {
|
|
448
|
+
return json;
|
|
449
|
+
}
|
|
450
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
452
|
+
const prependTasks = [];
|
|
453
|
+
const appendTasks = [];
|
|
454
|
+
const inputDataflows = [];
|
|
455
|
+
const outputDataflows = [];
|
|
456
|
+
if (!hasInputTask && inputSchema) {
|
|
457
|
+
const inputTaskId = uuid4();
|
|
458
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
459
|
+
prependTasks.push({
|
|
460
|
+
id: inputTaskId,
|
|
461
|
+
type: "InputTask",
|
|
462
|
+
config: {
|
|
463
|
+
inputSchema: strippedInputSchema,
|
|
464
|
+
outputSchema: strippedInputSchema
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
468
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
469
|
+
if (!prop || typeof prop === "boolean")
|
|
470
|
+
continue;
|
|
471
|
+
const origins = getOriginTaskIds(prop);
|
|
472
|
+
for (const originId of origins) {
|
|
473
|
+
inputDataflows.push({
|
|
474
|
+
sourceTaskId: inputTaskId,
|
|
475
|
+
sourceTaskPortId: propName,
|
|
476
|
+
targetTaskId: originId,
|
|
477
|
+
targetTaskPortId: propName
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
if (!hasOutputTask && outputSchema) {
|
|
484
|
+
const outputTaskId = uuid4();
|
|
485
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
486
|
+
appendTasks.push({
|
|
487
|
+
id: outputTaskId,
|
|
488
|
+
type: "OutputTask",
|
|
489
|
+
config: {
|
|
490
|
+
inputSchema: strippedOutputSchema,
|
|
491
|
+
outputSchema: strippedOutputSchema
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
495
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
496
|
+
if (!prop || typeof prop === "boolean")
|
|
497
|
+
continue;
|
|
498
|
+
const origins = getOriginTaskIds(prop);
|
|
499
|
+
for (const originId of origins) {
|
|
500
|
+
outputDataflows.push({
|
|
501
|
+
sourceTaskId: originId,
|
|
502
|
+
sourceTaskPortId: propName,
|
|
503
|
+
targetTaskId: outputTaskId,
|
|
504
|
+
targetTaskPortId: propName
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return {
|
|
511
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
512
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
516
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
517
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
518
|
+
if (hasInputTask && hasOutputTask) {
|
|
519
|
+
return items;
|
|
520
|
+
}
|
|
521
|
+
const prependItems = [];
|
|
522
|
+
const appendItems = [];
|
|
523
|
+
if (!hasInputTask) {
|
|
524
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
525
|
+
const inputTaskId = uuid4();
|
|
526
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
527
|
+
prependItems.push({
|
|
528
|
+
id: inputTaskId,
|
|
529
|
+
type: "InputTask",
|
|
530
|
+
config: {
|
|
531
|
+
inputSchema: strippedInputSchema,
|
|
532
|
+
outputSchema: strippedInputSchema
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
536
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
537
|
+
if (!prop || typeof prop === "boolean")
|
|
538
|
+
continue;
|
|
539
|
+
const origins = getOriginTaskIds(prop);
|
|
540
|
+
for (const originId of origins) {
|
|
541
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
542
|
+
if (!targetItem)
|
|
543
|
+
continue;
|
|
544
|
+
if (!targetItem.dependencies) {
|
|
545
|
+
targetItem.dependencies = {};
|
|
546
|
+
}
|
|
547
|
+
const existing = targetItem.dependencies[propName];
|
|
548
|
+
const dep = { id: inputTaskId, output: propName };
|
|
549
|
+
if (!existing) {
|
|
550
|
+
targetItem.dependencies[propName] = dep;
|
|
551
|
+
} else if (Array.isArray(existing)) {
|
|
552
|
+
existing.push(dep);
|
|
553
|
+
} else {
|
|
554
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
if (!hasOutputTask) {
|
|
561
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
562
|
+
const outputTaskId = uuid4();
|
|
563
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
564
|
+
const outputDependencies = {};
|
|
565
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
566
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
567
|
+
if (!prop || typeof prop === "boolean")
|
|
568
|
+
continue;
|
|
569
|
+
const origins = getOriginTaskIds(prop);
|
|
570
|
+
if (origins.length === 1) {
|
|
571
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
572
|
+
} else if (origins.length > 1) {
|
|
573
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
appendItems.push({
|
|
578
|
+
id: outputTaskId,
|
|
579
|
+
type: "OutputTask",
|
|
580
|
+
config: {
|
|
581
|
+
inputSchema: strippedOutputSchema,
|
|
582
|
+
outputSchema: strippedOutputSchema
|
|
583
|
+
},
|
|
584
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
return [...prependItems, ...items, ...appendItems];
|
|
588
|
+
}
|
|
231
589
|
// src/task-graph/TaskGraph.ts
|
|
232
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
590
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
233
591
|
|
|
234
592
|
// src/task/GraphAsTask.ts
|
|
235
593
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -237,9 +595,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
237
595
|
// src/task-graph/TaskGraphRunner.ts
|
|
238
596
|
import {
|
|
239
597
|
collectPropertyValues,
|
|
598
|
+
getLogger as getLogger3,
|
|
240
599
|
globalServiceRegistry as globalServiceRegistry2,
|
|
241
600
|
ServiceRegistry as ServiceRegistry2,
|
|
242
|
-
uuid4 as
|
|
601
|
+
uuid4 as uuid43
|
|
243
602
|
} from "@workglow/util";
|
|
244
603
|
|
|
245
604
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -272,6 +631,9 @@ class TaskOutputRepository {
|
|
|
272
631
|
}
|
|
273
632
|
}
|
|
274
633
|
|
|
634
|
+
// src/task/ConditionalTask.ts
|
|
635
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
636
|
+
|
|
275
637
|
// src/task/ConditionUtils.ts
|
|
276
638
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
277
639
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -344,7 +706,7 @@ import {
|
|
|
344
706
|
compileSchema,
|
|
345
707
|
deepEqual,
|
|
346
708
|
EventEmitter as EventEmitter3,
|
|
347
|
-
uuid4
|
|
709
|
+
uuid4 as uuid42
|
|
348
710
|
} from "@workglow/util";
|
|
349
711
|
|
|
350
712
|
// src/task/TaskError.ts
|
|
@@ -378,6 +740,13 @@ class TaskAbortedError extends TaskError {
|
|
|
378
740
|
}
|
|
379
741
|
}
|
|
380
742
|
|
|
743
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
744
|
+
static type = "TaskTimeoutError";
|
|
745
|
+
constructor(timeoutMs) {
|
|
746
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
381
750
|
class TaskFailedError extends TaskError {
|
|
382
751
|
static type = "TaskFailedError";
|
|
383
752
|
constructor(message = "Task failed") {
|
|
@@ -409,7 +778,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
409
778
|
}
|
|
410
779
|
|
|
411
780
|
// src/task/TaskRunner.ts
|
|
412
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
781
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
413
782
|
|
|
414
783
|
// src/task/InputResolver.ts
|
|
415
784
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -473,7 +842,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
473
842
|
if (!prop || typeof prop === "boolean")
|
|
474
843
|
return "none";
|
|
475
844
|
const xStream = prop["x-stream"];
|
|
476
|
-
if (xStream === "append" || xStream === "replace")
|
|
845
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
477
846
|
return xStream;
|
|
478
847
|
return "none";
|
|
479
848
|
}
|
|
@@ -488,7 +857,7 @@ function getStreamingPorts(schema) {
|
|
|
488
857
|
if (!prop || typeof prop === "boolean")
|
|
489
858
|
continue;
|
|
490
859
|
const xStream = prop["x-stream"];
|
|
491
|
-
if (xStream === "append" || xStream === "replace") {
|
|
860
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
492
861
|
result.push({ port: name, mode: xStream });
|
|
493
862
|
}
|
|
494
863
|
}
|
|
@@ -532,6 +901,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
532
901
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
533
902
|
return sourceMode !== targetMode;
|
|
534
903
|
}
|
|
904
|
+
function getObjectPortId(schema) {
|
|
905
|
+
if (typeof schema === "boolean")
|
|
906
|
+
return;
|
|
907
|
+
const props = schema.properties;
|
|
908
|
+
if (!props)
|
|
909
|
+
return;
|
|
910
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
911
|
+
if (!prop || typeof prop === "boolean")
|
|
912
|
+
continue;
|
|
913
|
+
if (prop["x-stream"] === "object")
|
|
914
|
+
return name;
|
|
915
|
+
}
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
function getStructuredOutputSchemas(schema) {
|
|
919
|
+
const result = new Map;
|
|
920
|
+
if (typeof schema === "boolean")
|
|
921
|
+
return result;
|
|
922
|
+
const props = schema.properties;
|
|
923
|
+
if (!props)
|
|
924
|
+
return result;
|
|
925
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
926
|
+
if (!prop || typeof prop === "boolean")
|
|
927
|
+
continue;
|
|
928
|
+
if (prop["x-structured-output"] === true) {
|
|
929
|
+
result.set(name, prop);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
return result;
|
|
933
|
+
}
|
|
934
|
+
function hasStructuredOutput(schema) {
|
|
935
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
936
|
+
}
|
|
535
937
|
|
|
536
938
|
// src/task/TaskRunner.ts
|
|
537
939
|
class TaskRunner {
|
|
@@ -542,12 +944,17 @@ class TaskRunner {
|
|
|
542
944
|
outputCache;
|
|
543
945
|
registry = globalServiceRegistry;
|
|
544
946
|
inputStreams;
|
|
947
|
+
timeoutTimer;
|
|
948
|
+
pendingTimeoutError;
|
|
545
949
|
shouldAccumulate = true;
|
|
546
950
|
constructor(task) {
|
|
547
951
|
this.task = task;
|
|
548
952
|
this.own = this.own.bind(this);
|
|
549
953
|
this.handleProgress = this.handleProgress.bind(this);
|
|
550
954
|
}
|
|
955
|
+
get timerLabel() {
|
|
956
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
957
|
+
}
|
|
551
958
|
async run(overrides = {}, config = {}) {
|
|
552
959
|
await this.handleStart(config);
|
|
553
960
|
try {
|
|
@@ -594,7 +1001,7 @@ class TaskRunner {
|
|
|
594
1001
|
return this.task.runOutputData;
|
|
595
1002
|
} catch (err) {
|
|
596
1003
|
await this.handleError(err);
|
|
597
|
-
throw err;
|
|
1004
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
598
1005
|
}
|
|
599
1006
|
}
|
|
600
1007
|
async runReactive(overrides = {}) {
|
|
@@ -651,7 +1058,14 @@ class TaskRunner {
|
|
|
651
1058
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
652
1059
|
}
|
|
653
1060
|
}
|
|
1061
|
+
if (streamMode === "object") {
|
|
1062
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1063
|
+
if (ports.length === 0) {
|
|
1064
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
654
1067
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1068
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
655
1069
|
let chunkCount = 0;
|
|
656
1070
|
let finalOutput;
|
|
657
1071
|
this.task.emit("stream_start");
|
|
@@ -682,6 +1096,13 @@ class TaskRunner {
|
|
|
682
1096
|
break;
|
|
683
1097
|
}
|
|
684
1098
|
case "object-delta": {
|
|
1099
|
+
if (accumulatedObjects) {
|
|
1100
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1101
|
+
}
|
|
1102
|
+
this.task.runOutputData = {
|
|
1103
|
+
...this.task.runOutputData,
|
|
1104
|
+
[event.port]: event.objectDelta
|
|
1105
|
+
};
|
|
685
1106
|
this.task.emit("stream_chunk", event);
|
|
686
1107
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
687
1108
|
await this.handleProgress(progress);
|
|
@@ -694,11 +1115,18 @@ class TaskRunner {
|
|
|
694
1115
|
break;
|
|
695
1116
|
}
|
|
696
1117
|
case "finish": {
|
|
697
|
-
if (accumulated) {
|
|
1118
|
+
if (accumulated || accumulatedObjects) {
|
|
698
1119
|
const merged = { ...event.data || {} };
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
1120
|
+
if (accumulated) {
|
|
1121
|
+
for (const [port, text] of accumulated) {
|
|
1122
|
+
if (text.length > 0)
|
|
1123
|
+
merged[port] = text;
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
if (accumulatedObjects) {
|
|
1127
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1128
|
+
merged[port] = obj;
|
|
1129
|
+
}
|
|
702
1130
|
}
|
|
703
1131
|
finalOutput = merged;
|
|
704
1132
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -744,12 +1172,20 @@ class TaskRunner {
|
|
|
744
1172
|
this.outputCache = cache;
|
|
745
1173
|
}
|
|
746
1174
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1175
|
+
const timeout = this.task.config.timeout;
|
|
1176
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1177
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1178
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1179
|
+
this.abort();
|
|
1180
|
+
}, timeout);
|
|
1181
|
+
}
|
|
747
1182
|
if (config.updateProgress) {
|
|
748
1183
|
this.updateProgress = config.updateProgress;
|
|
749
1184
|
}
|
|
750
1185
|
if (config.registry) {
|
|
751
1186
|
this.registry = config.registry;
|
|
752
1187
|
}
|
|
1188
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
753
1189
|
this.task.emit("start");
|
|
754
1190
|
this.task.emit("status", this.task.status);
|
|
755
1191
|
}
|
|
@@ -757,12 +1193,21 @@ class TaskRunner {
|
|
|
757
1193
|
async handleStartReactive() {
|
|
758
1194
|
this.reactiveRunning = true;
|
|
759
1195
|
}
|
|
1196
|
+
clearTimeoutTimer() {
|
|
1197
|
+
if (this.timeoutTimer !== undefined) {
|
|
1198
|
+
clearTimeout(this.timeoutTimer);
|
|
1199
|
+
this.timeoutTimer = undefined;
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
760
1202
|
async handleAbort() {
|
|
761
1203
|
if (this.task.status === TaskStatus.ABORTING)
|
|
762
1204
|
return;
|
|
1205
|
+
this.clearTimeoutTimer();
|
|
1206
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
763
1207
|
this.task.status = TaskStatus.ABORTING;
|
|
764
1208
|
this.task.progress = 100;
|
|
765
|
-
this.task.error = new TaskAbortedError;
|
|
1209
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1210
|
+
this.pendingTimeoutError = undefined;
|
|
766
1211
|
this.task.emit("abort", this.task.error);
|
|
767
1212
|
this.task.emit("status", this.task.status);
|
|
768
1213
|
}
|
|
@@ -772,6 +1217,9 @@ class TaskRunner {
|
|
|
772
1217
|
async handleComplete() {
|
|
773
1218
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
774
1219
|
return;
|
|
1220
|
+
this.clearTimeoutTimer();
|
|
1221
|
+
this.pendingTimeoutError = undefined;
|
|
1222
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
775
1223
|
this.task.completedAt = new Date;
|
|
776
1224
|
this.task.progress = 100;
|
|
777
1225
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -800,6 +1248,9 @@ class TaskRunner {
|
|
|
800
1248
|
return this.handleAbort();
|
|
801
1249
|
if (this.task.status === TaskStatus.FAILED)
|
|
802
1250
|
return;
|
|
1251
|
+
this.clearTimeoutTimer();
|
|
1252
|
+
this.pendingTimeoutError = undefined;
|
|
1253
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
803
1254
|
if (this.task.hasChildren()) {
|
|
804
1255
|
this.task.subGraph.abort();
|
|
805
1256
|
}
|
|
@@ -830,6 +1281,7 @@ class Task {
|
|
|
830
1281
|
static cacheable = true;
|
|
831
1282
|
static hasDynamicSchemas = false;
|
|
832
1283
|
static passthroughInputsToOutputs = false;
|
|
1284
|
+
static customizable = false;
|
|
833
1285
|
static inputSchema() {
|
|
834
1286
|
return {
|
|
835
1287
|
type: "object",
|
|
@@ -903,6 +1355,9 @@ class Task {
|
|
|
903
1355
|
runInputData = {};
|
|
904
1356
|
runOutputData = {};
|
|
905
1357
|
config;
|
|
1358
|
+
get id() {
|
|
1359
|
+
return this.config.id;
|
|
1360
|
+
}
|
|
906
1361
|
runConfig = {};
|
|
907
1362
|
status = TaskStatus.PENDING;
|
|
908
1363
|
progress = 0;
|
|
@@ -924,9 +1379,11 @@ class Task {
|
|
|
924
1379
|
this.resetInputData();
|
|
925
1380
|
const title = this.constructor.title || undefined;
|
|
926
1381
|
const baseConfig = Object.assign({
|
|
927
|
-
id: uuid4(),
|
|
928
1382
|
...title ? { title } : {}
|
|
929
1383
|
}, config);
|
|
1384
|
+
if (baseConfig.id === undefined) {
|
|
1385
|
+
baseConfig.id = uuid42();
|
|
1386
|
+
}
|
|
930
1387
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
931
1388
|
this.runConfig = runConfig;
|
|
932
1389
|
}
|
|
@@ -936,7 +1393,7 @@ class Task {
|
|
|
936
1393
|
return {};
|
|
937
1394
|
}
|
|
938
1395
|
try {
|
|
939
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1396
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
940
1397
|
const defaultData = compiledSchema.getData(undefined, {
|
|
941
1398
|
addOptionalProps: true,
|
|
942
1399
|
removeInvalidData: false,
|
|
@@ -1054,7 +1511,7 @@ class Task {
|
|
|
1054
1511
|
continue;
|
|
1055
1512
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1056
1513
|
if (isArray) {
|
|
1057
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1514
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1058
1515
|
const newitems = [...existingItems];
|
|
1059
1516
|
const overrideItem = overrides[inputId];
|
|
1060
1517
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1110,25 +1567,29 @@ class Task {
|
|
|
1110
1567
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1111
1568
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1112
1569
|
}
|
|
1113
|
-
static
|
|
1114
|
-
static getConfigSchemaNode(type) {
|
|
1570
|
+
static getConfigSchemaNode() {
|
|
1115
1571
|
const schema = this.configSchema();
|
|
1116
1572
|
if (!schema)
|
|
1117
1573
|
return;
|
|
1118
|
-
if (!
|
|
1574
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1119
1575
|
try {
|
|
1120
1576
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1121
|
-
|
|
1577
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1578
|
+
value: schemaNode,
|
|
1579
|
+
writable: true,
|
|
1580
|
+
configurable: true,
|
|
1581
|
+
enumerable: false
|
|
1582
|
+
});
|
|
1122
1583
|
} catch (error) {
|
|
1123
1584
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1124
1585
|
return;
|
|
1125
1586
|
}
|
|
1126
1587
|
}
|
|
1127
|
-
return this.
|
|
1588
|
+
return this.__compiledConfigSchema;
|
|
1128
1589
|
}
|
|
1129
1590
|
validateAndApplyConfigDefaults(config) {
|
|
1130
1591
|
const ctor = this.constructor;
|
|
1131
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1592
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1132
1593
|
if (!schemaNode)
|
|
1133
1594
|
return config;
|
|
1134
1595
|
const result = schemaNode.validate(config);
|
|
@@ -1141,7 +1602,6 @@ class Task {
|
|
|
1141
1602
|
}
|
|
1142
1603
|
return config;
|
|
1143
1604
|
}
|
|
1144
|
-
static _inputSchemaNode = new Map;
|
|
1145
1605
|
static generateInputSchemaNode(schema) {
|
|
1146
1606
|
if (typeof schema === "boolean") {
|
|
1147
1607
|
if (schema === false) {
|
|
@@ -1151,24 +1611,41 @@ class Task {
|
|
|
1151
1611
|
}
|
|
1152
1612
|
return compileSchema(schema);
|
|
1153
1613
|
}
|
|
1154
|
-
static getInputSchemaNode(
|
|
1155
|
-
if (!
|
|
1614
|
+
static getInputSchemaNode() {
|
|
1615
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1156
1616
|
const dataPortSchema = this.inputSchema();
|
|
1157
1617
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1158
1618
|
try {
|
|
1159
|
-
|
|
1619
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1620
|
+
value: schemaNode,
|
|
1621
|
+
writable: true,
|
|
1622
|
+
configurable: true,
|
|
1623
|
+
enumerable: false
|
|
1624
|
+
});
|
|
1160
1625
|
} catch (error) {
|
|
1161
1626
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1162
|
-
|
|
1627
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1628
|
+
value: compileSchema({}),
|
|
1629
|
+
writable: true,
|
|
1630
|
+
configurable: true,
|
|
1631
|
+
enumerable: false
|
|
1632
|
+
});
|
|
1163
1633
|
}
|
|
1164
1634
|
}
|
|
1165
|
-
return this.
|
|
1635
|
+
return this.__compiledInputSchema;
|
|
1166
1636
|
}
|
|
1167
|
-
getInputSchemaNode(
|
|
1168
|
-
return this.constructor.getInputSchemaNode(
|
|
1637
|
+
getInputSchemaNode() {
|
|
1638
|
+
return this.constructor.getInputSchemaNode();
|
|
1169
1639
|
}
|
|
1170
1640
|
async validateInput(input) {
|
|
1171
|
-
const
|
|
1641
|
+
const ctor = this.constructor;
|
|
1642
|
+
let schemaNode;
|
|
1643
|
+
if (ctor.hasDynamicSchemas) {
|
|
1644
|
+
const instanceSchema = this.inputSchema();
|
|
1645
|
+
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1646
|
+
} else {
|
|
1647
|
+
schemaNode = this.getInputSchemaNode();
|
|
1648
|
+
}
|
|
1172
1649
|
const result = schemaNode.validate(input);
|
|
1173
1650
|
if (!result.valid) {
|
|
1174
1651
|
const errorMessages = result.errors.map((e) => {
|
|
@@ -1179,9 +1656,6 @@ class Task {
|
|
|
1179
1656
|
}
|
|
1180
1657
|
return true;
|
|
1181
1658
|
}
|
|
1182
|
-
id() {
|
|
1183
|
-
return this.config.id;
|
|
1184
|
-
}
|
|
1185
1659
|
stripSymbols(obj) {
|
|
1186
1660
|
if (obj === null || obj === undefined) {
|
|
1187
1661
|
return obj;
|
|
@@ -1203,14 +1677,15 @@ class Task {
|
|
|
1203
1677
|
}
|
|
1204
1678
|
return obj;
|
|
1205
1679
|
}
|
|
1206
|
-
toJSON() {
|
|
1680
|
+
toJSON(_options) {
|
|
1207
1681
|
const extras = this.config.extras;
|
|
1208
1682
|
const json = this.stripSymbols({
|
|
1209
|
-
id: this.
|
|
1683
|
+
id: this.id,
|
|
1210
1684
|
type: this.type,
|
|
1211
1685
|
defaults: this.defaults,
|
|
1212
1686
|
config: {
|
|
1213
1687
|
...this.config.title ? { title: this.config.title } : {},
|
|
1688
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1214
1689
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1215
1690
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1216
1691
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1218,8 +1693,8 @@ class Task {
|
|
|
1218
1693
|
});
|
|
1219
1694
|
return json;
|
|
1220
1695
|
}
|
|
1221
|
-
toDependencyJSON() {
|
|
1222
|
-
const json = this.toJSON();
|
|
1696
|
+
toDependencyJSON(options) {
|
|
1697
|
+
const json = this.toJSON(options);
|
|
1223
1698
|
return json;
|
|
1224
1699
|
}
|
|
1225
1700
|
hasChildren() {
|
|
@@ -1249,7 +1724,7 @@ class Task {
|
|
|
1249
1724
|
this.subGraph.removeDataflow(dataflow);
|
|
1250
1725
|
}
|
|
1251
1726
|
for (const child of this.subGraph.getTasks()) {
|
|
1252
|
-
this.subGraph.removeTask(child.
|
|
1727
|
+
this.subGraph.removeTask(child.id);
|
|
1253
1728
|
}
|
|
1254
1729
|
}
|
|
1255
1730
|
this.events.emit("regenerate");
|
|
@@ -1340,7 +1815,7 @@ class ConditionalTask extends Task {
|
|
|
1340
1815
|
}
|
|
1341
1816
|
}
|
|
1342
1817
|
} catch (error) {
|
|
1343
|
-
|
|
1818
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1344
1819
|
}
|
|
1345
1820
|
}
|
|
1346
1821
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1505,7 +1980,7 @@ class DependencyBasedScheduler {
|
|
|
1505
1980
|
if (task.status === TaskStatus.DISABLED) {
|
|
1506
1981
|
return false;
|
|
1507
1982
|
}
|
|
1508
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
1983
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1509
1984
|
if (sourceDataflows.length > 0) {
|
|
1510
1985
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1511
1986
|
if (allIncomingDisabled) {
|
|
@@ -1632,6 +2107,10 @@ class TaskGraphRunner {
|
|
|
1632
2107
|
graph.outputCache = outputCache;
|
|
1633
2108
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1634
2109
|
}
|
|
2110
|
+
runId = "";
|
|
2111
|
+
get timerLabel() {
|
|
2112
|
+
return `graph:${this.runId}`;
|
|
2113
|
+
}
|
|
1635
2114
|
async runGraph(input = {}, config) {
|
|
1636
2115
|
await this.handleStart(config);
|
|
1637
2116
|
const results = [];
|
|
@@ -1644,25 +2123,33 @@ class TaskGraphRunner {
|
|
|
1644
2123
|
if (this.failedTaskErrors.size > 0) {
|
|
1645
2124
|
break;
|
|
1646
2125
|
}
|
|
1647
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2126
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1648
2127
|
const runAsync = async () => {
|
|
2128
|
+
let errorRouted = false;
|
|
1649
2129
|
try {
|
|
1650
2130
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1651
2131
|
const taskPromise = this.runTask(task, taskInput);
|
|
1652
|
-
this.inProgressTasks.set(task.
|
|
2132
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1653
2133
|
const taskResult = await taskPromise;
|
|
1654
|
-
if (this.graph.getTargetDataflows(task.
|
|
2134
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1655
2135
|
results.push(taskResult);
|
|
1656
2136
|
}
|
|
1657
2137
|
} catch (error2) {
|
|
1658
|
-
this.
|
|
2138
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2139
|
+
errorRouted = true;
|
|
2140
|
+
this.pushErrorOutputToEdges(task);
|
|
2141
|
+
} else {
|
|
2142
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2143
|
+
}
|
|
1659
2144
|
} finally {
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
2145
|
+
if (!errorRouted) {
|
|
2146
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2147
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2148
|
+
}
|
|
2149
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1663
2150
|
}
|
|
1664
2151
|
};
|
|
1665
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2152
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1666
2153
|
}
|
|
1667
2154
|
} catch (err) {
|
|
1668
2155
|
error = err;
|
|
@@ -1686,7 +2173,7 @@ class TaskGraphRunner {
|
|
|
1686
2173
|
const results = [];
|
|
1687
2174
|
try {
|
|
1688
2175
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1689
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2176
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1690
2177
|
if (task.status === TaskStatus.PENDING) {
|
|
1691
2178
|
task.resetInputData();
|
|
1692
2179
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1694,9 +2181,9 @@ class TaskGraphRunner {
|
|
|
1694
2181
|
const taskInput = isRootTask ? input : {};
|
|
1695
2182
|
const taskResult = await task.runReactive(taskInput);
|
|
1696
2183
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1697
|
-
if (this.graph.getTargetDataflows(task.
|
|
2184
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1698
2185
|
results.push({
|
|
1699
|
-
id: task.
|
|
2186
|
+
id: task.id,
|
|
1700
2187
|
type: task.constructor.runtype || task.constructor.type,
|
|
1701
2188
|
data: taskResult
|
|
1702
2189
|
});
|
|
@@ -1716,7 +2203,7 @@ class TaskGraphRunner {
|
|
|
1716
2203
|
await this.handleDisable();
|
|
1717
2204
|
}
|
|
1718
2205
|
filterInputForTask(task, input) {
|
|
1719
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2206
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1720
2207
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1721
2208
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1722
2209
|
const filteredInput = {};
|
|
@@ -1755,13 +2242,13 @@ class TaskGraphRunner {
|
|
|
1755
2242
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1756
2243
|
}
|
|
1757
2244
|
copyInputFromEdgesToNode(task) {
|
|
1758
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2245
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1759
2246
|
for (const dataflow of dataflows) {
|
|
1760
2247
|
this.addInputData(task, dataflow.getPortData());
|
|
1761
2248
|
}
|
|
1762
2249
|
}
|
|
1763
2250
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1764
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2251
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1765
2252
|
for (const dataflow of dataflows) {
|
|
1766
2253
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1767
2254
|
if (compatibility === "static") {
|
|
@@ -1776,7 +2263,7 @@ class TaskGraphRunner {
|
|
|
1776
2263
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1777
2264
|
if (!node?.config?.id)
|
|
1778
2265
|
return;
|
|
1779
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2266
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1780
2267
|
const effectiveStatus = status ?? node.status;
|
|
1781
2268
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1782
2269
|
const branches = node.config.branches ?? [];
|
|
@@ -1807,10 +2294,31 @@ class TaskGraphRunner {
|
|
|
1807
2294
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1808
2295
|
if (!node?.config?.id)
|
|
1809
2296
|
return;
|
|
1810
|
-
graph.getTargetDataflows(node.
|
|
2297
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1811
2298
|
dataflow.error = node.error;
|
|
1812
2299
|
});
|
|
1813
2300
|
}
|
|
2301
|
+
hasErrorOutputEdges(task) {
|
|
2302
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2303
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2304
|
+
}
|
|
2305
|
+
pushErrorOutputToEdges(task) {
|
|
2306
|
+
const taskError = task.error;
|
|
2307
|
+
const errorData = {
|
|
2308
|
+
error: taskError?.message ?? "Unknown error",
|
|
2309
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2310
|
+
};
|
|
2311
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2312
|
+
for (const df of dataflows) {
|
|
2313
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2314
|
+
df.value = errorData;
|
|
2315
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2316
|
+
} else {
|
|
2317
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
this.propagateDisabledStatus(this.graph);
|
|
2321
|
+
}
|
|
1814
2322
|
propagateDisabledStatus(graph) {
|
|
1815
2323
|
let changed = true;
|
|
1816
2324
|
while (changed) {
|
|
@@ -1819,7 +2327,7 @@ class TaskGraphRunner {
|
|
|
1819
2327
|
if (task.status !== TaskStatus.PENDING) {
|
|
1820
2328
|
continue;
|
|
1821
2329
|
}
|
|
1822
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2330
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1823
2331
|
if (incomingDataflows.length === 0) {
|
|
1824
2332
|
continue;
|
|
1825
2333
|
}
|
|
@@ -1830,10 +2338,10 @@ class TaskGraphRunner {
|
|
|
1830
2338
|
task.completedAt = new Date;
|
|
1831
2339
|
task.emit("disabled");
|
|
1832
2340
|
task.emit("status", task.status);
|
|
1833
|
-
graph.getTargetDataflows(task.
|
|
2341
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1834
2342
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1835
2343
|
});
|
|
1836
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2344
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1837
2345
|
changed = true;
|
|
1838
2346
|
}
|
|
1839
2347
|
}
|
|
@@ -1842,7 +2350,7 @@ class TaskGraphRunner {
|
|
|
1842
2350
|
taskNeedsAccumulation(task) {
|
|
1843
2351
|
if (this.outputCache)
|
|
1844
2352
|
return true;
|
|
1845
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2353
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1846
2354
|
if (outEdges.length === 0)
|
|
1847
2355
|
return this.accumulateLeafOutputs;
|
|
1848
2356
|
const outSchema = task.outputSchema();
|
|
@@ -1865,7 +2373,7 @@ class TaskGraphRunner {
|
|
|
1865
2373
|
async runTask(task, input) {
|
|
1866
2374
|
const isStreamable = isTaskStreamable(task);
|
|
1867
2375
|
if (isStreamable) {
|
|
1868
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2376
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1869
2377
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1870
2378
|
if (streamingEdges.length > 0) {
|
|
1871
2379
|
const inputStreams = new Map;
|
|
@@ -1890,13 +2398,13 @@ class TaskGraphRunner {
|
|
|
1890
2398
|
});
|
|
1891
2399
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1892
2400
|
return {
|
|
1893
|
-
id: task.
|
|
2401
|
+
id: task.id,
|
|
1894
2402
|
type: task.constructor.runtype || task.constructor.type,
|
|
1895
2403
|
data: results
|
|
1896
2404
|
};
|
|
1897
2405
|
}
|
|
1898
2406
|
async awaitStreamInputs(task) {
|
|
1899
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2407
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1900
2408
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1901
2409
|
if (streamPromises.length > 0) {
|
|
1902
2410
|
await Promise.all(streamPromises);
|
|
@@ -1911,17 +2419,17 @@ class TaskGraphRunner {
|
|
|
1911
2419
|
streamingNotified = true;
|
|
1912
2420
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1913
2421
|
this.pushStreamToEdges(task, streamMode);
|
|
1914
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2422
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1915
2423
|
}
|
|
1916
2424
|
};
|
|
1917
2425
|
const onStreamStart = () => {
|
|
1918
|
-
this.graph.emit("task_stream_start", task.
|
|
2426
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1919
2427
|
};
|
|
1920
2428
|
const onStreamChunk = (event) => {
|
|
1921
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2429
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1922
2430
|
};
|
|
1923
2431
|
const onStreamEnd = (output) => {
|
|
1924
|
-
this.graph.emit("task_stream_end", task.
|
|
2432
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1925
2433
|
};
|
|
1926
2434
|
task.on("status", onStatus);
|
|
1927
2435
|
task.on("stream_start", onStreamStart);
|
|
@@ -1936,7 +2444,7 @@ class TaskGraphRunner {
|
|
|
1936
2444
|
});
|
|
1937
2445
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1938
2446
|
return {
|
|
1939
|
-
id: task.
|
|
2447
|
+
id: task.id,
|
|
1940
2448
|
type: task.constructor.runtype || task.constructor.type,
|
|
1941
2449
|
data: results
|
|
1942
2450
|
};
|
|
@@ -1974,7 +2482,7 @@ class TaskGraphRunner {
|
|
|
1974
2482
|
});
|
|
1975
2483
|
}
|
|
1976
2484
|
pushStreamToEdges(task, streamMode) {
|
|
1977
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2485
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1978
2486
|
if (targetDataflows.length === 0)
|
|
1979
2487
|
return;
|
|
1980
2488
|
const groups = new Map;
|
|
@@ -2065,11 +2573,15 @@ class TaskGraphRunner {
|
|
|
2065
2573
|
this.abortController?.abort();
|
|
2066
2574
|
}, { once: true });
|
|
2067
2575
|
}
|
|
2068
|
-
this.
|
|
2576
|
+
this.runId = uuid43();
|
|
2577
|
+
this.resetGraph(this.graph, this.runId);
|
|
2069
2578
|
this.processScheduler.reset();
|
|
2070
2579
|
this.inProgressTasks.clear();
|
|
2071
2580
|
this.inProgressFunctions.clear();
|
|
2072
2581
|
this.failedTaskErrors.clear();
|
|
2582
|
+
const logger = getLogger3();
|
|
2583
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2584
|
+
logger.time(this.timerLabel);
|
|
2073
2585
|
this.graph.emit("start");
|
|
2074
2586
|
}
|
|
2075
2587
|
async handleStartReactive() {
|
|
@@ -2081,6 +2593,9 @@ class TaskGraphRunner {
|
|
|
2081
2593
|
}
|
|
2082
2594
|
async handleComplete() {
|
|
2083
2595
|
this.running = false;
|
|
2596
|
+
const logger = getLogger3();
|
|
2597
|
+
logger.timeEnd(this.timerLabel);
|
|
2598
|
+
logger.groupEnd();
|
|
2084
2599
|
this.graph.emit("complete");
|
|
2085
2600
|
}
|
|
2086
2601
|
async handleCompleteReactive() {
|
|
@@ -2093,6 +2608,9 @@ class TaskGraphRunner {
|
|
|
2093
2608
|
}
|
|
2094
2609
|
}));
|
|
2095
2610
|
this.running = false;
|
|
2611
|
+
const logger = getLogger3();
|
|
2612
|
+
logger.timeEnd(this.timerLabel);
|
|
2613
|
+
logger.groupEnd();
|
|
2096
2614
|
this.graph.emit("error", error);
|
|
2097
2615
|
}
|
|
2098
2616
|
async handleErrorReactive() {
|
|
@@ -2105,6 +2623,9 @@ class TaskGraphRunner {
|
|
|
2105
2623
|
}
|
|
2106
2624
|
});
|
|
2107
2625
|
this.running = false;
|
|
2626
|
+
const logger = getLogger3();
|
|
2627
|
+
logger.timeEnd(this.timerLabel);
|
|
2628
|
+
logger.groupEnd();
|
|
2108
2629
|
this.graph.emit("abort");
|
|
2109
2630
|
}
|
|
2110
2631
|
async handleAbortReactive() {
|
|
@@ -2180,7 +2701,7 @@ var graphAsTaskConfigSchema = {
|
|
|
2180
2701
|
type: "object",
|
|
2181
2702
|
properties: {
|
|
2182
2703
|
...TaskConfigSchema["properties"],
|
|
2183
|
-
compoundMerge: { type: "string" }
|
|
2704
|
+
compoundMerge: { type: "string", "x-ui-hidden": true }
|
|
2184
2705
|
},
|
|
2185
2706
|
additionalProperties: false
|
|
2186
2707
|
};
|
|
@@ -2219,118 +2740,27 @@ class GraphAsTask extends Task {
|
|
|
2219
2740
|
if (!this.hasChildren()) {
|
|
2220
2741
|
return this.constructor.inputSchema();
|
|
2221
2742
|
}
|
|
2222
|
-
|
|
2223
|
-
const required = [];
|
|
2224
|
-
const tasks = this.subGraph.getTasks();
|
|
2225
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2226
|
-
for (const task of startingNodes) {
|
|
2227
|
-
const taskInputSchema = task.inputSchema();
|
|
2228
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2229
|
-
if (taskInputSchema === false) {
|
|
2230
|
-
continue;
|
|
2231
|
-
}
|
|
2232
|
-
if (taskInputSchema === true) {
|
|
2233
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2234
|
-
continue;
|
|
2235
|
-
}
|
|
2236
|
-
}
|
|
2237
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2238
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2239
|
-
if (!properties[inputName]) {
|
|
2240
|
-
properties[inputName] = inputProp;
|
|
2241
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2242
|
-
required.push(inputName);
|
|
2243
|
-
}
|
|
2244
|
-
}
|
|
2245
|
-
}
|
|
2246
|
-
}
|
|
2247
|
-
return {
|
|
2248
|
-
type: "object",
|
|
2249
|
-
properties,
|
|
2250
|
-
...required.length > 0 ? { required } : {},
|
|
2251
|
-
additionalProperties: false
|
|
2252
|
-
};
|
|
2743
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2253
2744
|
}
|
|
2254
2745
|
_inputSchemaNode;
|
|
2255
|
-
getInputSchemaNode(
|
|
2746
|
+
getInputSchemaNode() {
|
|
2256
2747
|
if (!this._inputSchemaNode) {
|
|
2257
|
-
const dataPortSchema = this.inputSchema();
|
|
2258
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2259
2748
|
try {
|
|
2749
|
+
const dataPortSchema = this.inputSchema();
|
|
2750
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2260
2751
|
this._inputSchemaNode = schemaNode;
|
|
2261
2752
|
} catch (error) {
|
|
2262
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2753
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2263
2754
|
this._inputSchemaNode = compileSchema2({});
|
|
2264
2755
|
}
|
|
2265
2756
|
}
|
|
2266
2757
|
return this._inputSchemaNode;
|
|
2267
2758
|
}
|
|
2268
|
-
calculateNodeDepths() {
|
|
2269
|
-
const depths = new Map;
|
|
2270
|
-
const tasks = this.subGraph.getTasks();
|
|
2271
|
-
for (const task of tasks) {
|
|
2272
|
-
depths.set(task.config.id, 0);
|
|
2273
|
-
}
|
|
2274
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2275
|
-
for (const task of sortedTasks) {
|
|
2276
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2277
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2278
|
-
for (const targetTask of targetTasks) {
|
|
2279
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2280
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
return depths;
|
|
2284
|
-
}
|
|
2285
2759
|
outputSchema() {
|
|
2286
2760
|
if (!this.hasChildren()) {
|
|
2287
2761
|
return this.constructor.outputSchema();
|
|
2288
2762
|
}
|
|
2289
|
-
|
|
2290
|
-
const required = [];
|
|
2291
|
-
const tasks = this.subGraph.getTasks();
|
|
2292
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2293
|
-
const depths = this.calculateNodeDepths();
|
|
2294
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2295
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2296
|
-
const propertyCount = {};
|
|
2297
|
-
const propertySchema = {};
|
|
2298
|
-
for (const task of lastLevelNodes) {
|
|
2299
|
-
const taskOutputSchema = task.outputSchema();
|
|
2300
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2301
|
-
if (taskOutputSchema === false) {
|
|
2302
|
-
continue;
|
|
2303
|
-
}
|
|
2304
|
-
if (taskOutputSchema === true) {
|
|
2305
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2306
|
-
continue;
|
|
2307
|
-
}
|
|
2308
|
-
}
|
|
2309
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2310
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2311
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2312
|
-
if (!propertySchema[outputName]) {
|
|
2313
|
-
propertySchema[outputName] = outputProp;
|
|
2314
|
-
}
|
|
2315
|
-
}
|
|
2316
|
-
}
|
|
2317
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2318
|
-
const outputProp = propertySchema[outputName];
|
|
2319
|
-
if (lastLevelNodes.length === 1) {
|
|
2320
|
-
properties[outputName] = outputProp;
|
|
2321
|
-
} else {
|
|
2322
|
-
properties[outputName] = {
|
|
2323
|
-
type: "array",
|
|
2324
|
-
items: outputProp
|
|
2325
|
-
};
|
|
2326
|
-
}
|
|
2327
|
-
}
|
|
2328
|
-
return {
|
|
2329
|
-
type: "object",
|
|
2330
|
-
properties,
|
|
2331
|
-
...required.length > 0 ? { required } : {},
|
|
2332
|
-
additionalProperties: false
|
|
2333
|
-
};
|
|
2763
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2334
2764
|
}
|
|
2335
2765
|
resetInputData() {
|
|
2336
2766
|
super.resetInputData();
|
|
@@ -2365,8 +2795,8 @@ class GraphAsTask extends Task {
|
|
|
2365
2795
|
const endingNodeIds = new Set;
|
|
2366
2796
|
const tasks = this.subGraph.getTasks();
|
|
2367
2797
|
for (const task of tasks) {
|
|
2368
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2369
|
-
endingNodeIds.add(task.
|
|
2798
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2799
|
+
endingNodeIds.add(task.id);
|
|
2370
2800
|
}
|
|
2371
2801
|
}
|
|
2372
2802
|
const eventQueue = [];
|
|
@@ -2410,32 +2840,36 @@ class GraphAsTask extends Task {
|
|
|
2410
2840
|
this._inputSchemaNode = undefined;
|
|
2411
2841
|
this.events.emit("regenerate");
|
|
2412
2842
|
}
|
|
2413
|
-
toJSON() {
|
|
2414
|
-
let json = super.toJSON();
|
|
2843
|
+
toJSON(options) {
|
|
2844
|
+
let json = super.toJSON(options);
|
|
2415
2845
|
const hasChildren = this.hasChildren();
|
|
2416
2846
|
if (hasChildren) {
|
|
2417
2847
|
json = {
|
|
2418
2848
|
...json,
|
|
2419
2849
|
merge: this.compoundMerge,
|
|
2420
|
-
subgraph: this.subGraph.toJSON()
|
|
2850
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2421
2851
|
};
|
|
2422
2852
|
}
|
|
2423
2853
|
return json;
|
|
2424
2854
|
}
|
|
2425
|
-
toDependencyJSON() {
|
|
2426
|
-
const json = this.toJSON();
|
|
2855
|
+
toDependencyJSON(options) {
|
|
2856
|
+
const json = this.toJSON(options);
|
|
2427
2857
|
if (this.hasChildren()) {
|
|
2428
2858
|
if ("subgraph" in json) {
|
|
2429
2859
|
delete json.subgraph;
|
|
2430
2860
|
}
|
|
2431
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2861
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2432
2862
|
}
|
|
2433
2863
|
return json;
|
|
2434
2864
|
}
|
|
2435
2865
|
}
|
|
2436
2866
|
|
|
2437
2867
|
// src/task-graph/Workflow.ts
|
|
2438
|
-
import {
|
|
2868
|
+
import {
|
|
2869
|
+
EventEmitter as EventEmitter4,
|
|
2870
|
+
getLogger as getLogger4,
|
|
2871
|
+
uuid4 as uuid44
|
|
2872
|
+
} from "@workglow/util";
|
|
2439
2873
|
function CreateWorkflow(taskClass) {
|
|
2440
2874
|
return Workflow.createWorkflow(taskClass);
|
|
2441
2875
|
}
|
|
@@ -2536,43 +2970,48 @@ class Workflow {
|
|
|
2536
2970
|
const helper = function(input = {}, config = {}) {
|
|
2537
2971
|
this._error = "";
|
|
2538
2972
|
const parent = getLastTask(this);
|
|
2539
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
2973
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2540
2974
|
if (this._dataFlows.length > 0) {
|
|
2541
2975
|
this._dataFlows.forEach((dataflow) => {
|
|
2542
2976
|
const taskSchema = task.inputSchema();
|
|
2543
2977
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2544
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2545
|
-
|
|
2978
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
2979
|
+
getLogger4().error(this._error);
|
|
2546
2980
|
return;
|
|
2547
2981
|
}
|
|
2548
|
-
dataflow.targetTaskId = task.
|
|
2982
|
+
dataflow.targetTaskId = task.id;
|
|
2549
2983
|
this.graph.addDataflow(dataflow);
|
|
2550
2984
|
});
|
|
2551
2985
|
this._dataFlows = [];
|
|
2552
2986
|
}
|
|
2553
|
-
if (parent
|
|
2987
|
+
if (parent) {
|
|
2554
2988
|
const nodes = this._graph.getTasks();
|
|
2555
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
2989
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2556
2990
|
const earlierTasks = [];
|
|
2557
2991
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2558
2992
|
earlierTasks.push(nodes[i]);
|
|
2559
2993
|
}
|
|
2560
2994
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2995
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2561
2996
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2562
2997
|
providedInputKeys,
|
|
2998
|
+
connectedInputKeys,
|
|
2563
2999
|
earlierTasks
|
|
2564
3000
|
});
|
|
2565
3001
|
if (result.error) {
|
|
2566
3002
|
if (this.isLoopBuilder) {
|
|
2567
3003
|
this._error = result.error;
|
|
2568
|
-
|
|
3004
|
+
getLogger4().warn(this._error);
|
|
2569
3005
|
} else {
|
|
2570
3006
|
this._error = result.error + " Task not added.";
|
|
2571
|
-
|
|
2572
|
-
this.graph.removeTask(task.
|
|
3007
|
+
getLogger4().error(this._error);
|
|
3008
|
+
this.graph.removeTask(task.id);
|
|
2573
3009
|
}
|
|
2574
3010
|
}
|
|
2575
3011
|
}
|
|
3012
|
+
if (!this._error) {
|
|
3013
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3014
|
+
}
|
|
2576
3015
|
return this;
|
|
2577
3016
|
};
|
|
2578
3017
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2652,18 +3091,18 @@ class Workflow {
|
|
|
2652
3091
|
const nodes = this._graph.getTasks();
|
|
2653
3092
|
if (nodes.length === 0) {
|
|
2654
3093
|
this._error = "No tasks to remove";
|
|
2655
|
-
|
|
3094
|
+
getLogger4().error(this._error);
|
|
2656
3095
|
return this;
|
|
2657
3096
|
}
|
|
2658
3097
|
const lastNode = nodes[nodes.length - 1];
|
|
2659
|
-
this._graph.removeTask(lastNode.
|
|
3098
|
+
this._graph.removeTask(lastNode.id);
|
|
2660
3099
|
return this;
|
|
2661
3100
|
}
|
|
2662
|
-
toJSON() {
|
|
2663
|
-
return this._graph.toJSON();
|
|
3101
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3102
|
+
return this._graph.toJSON(options);
|
|
2664
3103
|
}
|
|
2665
|
-
toDependencyJSON() {
|
|
2666
|
-
return this._graph.toDependencyJSON();
|
|
3104
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3105
|
+
return this._graph.toDependencyJSON(options);
|
|
2667
3106
|
}
|
|
2668
3107
|
pipe(...args) {
|
|
2669
3108
|
return pipe(args, this);
|
|
@@ -2683,25 +3122,40 @@ class Workflow {
|
|
|
2683
3122
|
if (-index > nodes.length) {
|
|
2684
3123
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2685
3124
|
this._error = errorMsg;
|
|
2686
|
-
|
|
3125
|
+
getLogger4().error(this._error);
|
|
2687
3126
|
throw new WorkflowError(errorMsg);
|
|
2688
3127
|
}
|
|
2689
3128
|
const lastNode = nodes[nodes.length + index];
|
|
2690
3129
|
const outputSchema = lastNode.outputSchema();
|
|
2691
3130
|
if (typeof outputSchema === "boolean") {
|
|
2692
3131
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2693
|
-
const errorMsg = `Task ${lastNode.
|
|
3132
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2694
3133
|
this._error = errorMsg;
|
|
2695
|
-
|
|
3134
|
+
getLogger4().error(this._error);
|
|
2696
3135
|
throw new WorkflowError(errorMsg);
|
|
2697
3136
|
}
|
|
2698
3137
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2699
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3138
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2700
3139
|
this._error = errorMsg;
|
|
2701
|
-
|
|
3140
|
+
getLogger4().error(this._error);
|
|
2702
3141
|
throw new WorkflowError(errorMsg);
|
|
2703
3142
|
}
|
|
2704
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3143
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3144
|
+
return this;
|
|
3145
|
+
}
|
|
3146
|
+
onError(handler) {
|
|
3147
|
+
this._error = "";
|
|
3148
|
+
const parent = getLastTask(this);
|
|
3149
|
+
if (!parent) {
|
|
3150
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3151
|
+
getLogger4().error(this._error);
|
|
3152
|
+
throw new WorkflowError(this._error);
|
|
3153
|
+
}
|
|
3154
|
+
const handlerTask = ensureTask(handler);
|
|
3155
|
+
this.graph.addTask(handlerTask);
|
|
3156
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3157
|
+
this.graph.addDataflow(dataflow);
|
|
3158
|
+
this.events.emit("changed", handlerTask.id);
|
|
2705
3159
|
return this;
|
|
2706
3160
|
}
|
|
2707
3161
|
toTaskGraph() {
|
|
@@ -2786,16 +3240,16 @@ class Workflow {
|
|
|
2786
3240
|
addLoopTask(taskClass, config = {}) {
|
|
2787
3241
|
this._error = "";
|
|
2788
3242
|
const parent = getLastTask(this);
|
|
2789
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3243
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2790
3244
|
if (this._dataFlows.length > 0) {
|
|
2791
3245
|
this._dataFlows.forEach((dataflow) => {
|
|
2792
3246
|
const taskSchema = task.inputSchema();
|
|
2793
3247
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2794
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2795
|
-
|
|
3248
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3249
|
+
getLogger4().error(this._error);
|
|
2796
3250
|
return;
|
|
2797
3251
|
}
|
|
2798
|
-
dataflow.targetTaskId = task.
|
|
3252
|
+
dataflow.targetTaskId = task.id;
|
|
2799
3253
|
this.graph.addDataflow(dataflow);
|
|
2800
3254
|
});
|
|
2801
3255
|
this._dataFlows = [];
|
|
@@ -2810,9 +3264,9 @@ class Workflow {
|
|
|
2810
3264
|
if (!pending)
|
|
2811
3265
|
return;
|
|
2812
3266
|
const { parent, iteratorTask } = pending;
|
|
2813
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3267
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2814
3268
|
const nodes = this._graph.getTasks();
|
|
2815
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3269
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2816
3270
|
const earlierTasks = [];
|
|
2817
3271
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2818
3272
|
earlierTasks.push(nodes[i]);
|
|
@@ -2822,8 +3276,81 @@ class Workflow {
|
|
|
2822
3276
|
});
|
|
2823
3277
|
if (result.error) {
|
|
2824
3278
|
this._error = result.error + " Task not added.";
|
|
2825
|
-
|
|
2826
|
-
this.graph.removeTask(iteratorTask.
|
|
3279
|
+
getLogger4().error(this._error);
|
|
3280
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3285
|
+
const tasks = graph.getTasks();
|
|
3286
|
+
for (const task of tasks) {
|
|
3287
|
+
if (task.type === "InputTask") {
|
|
3288
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3289
|
+
if (outgoing.length === 0)
|
|
3290
|
+
continue;
|
|
3291
|
+
const properties = {};
|
|
3292
|
+
const required = [];
|
|
3293
|
+
for (const df of outgoing) {
|
|
3294
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3295
|
+
if (!targetTask)
|
|
3296
|
+
continue;
|
|
3297
|
+
const targetSchema = targetTask.inputSchema();
|
|
3298
|
+
if (typeof targetSchema === "boolean")
|
|
3299
|
+
continue;
|
|
3300
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3301
|
+
if (prop && typeof prop !== "boolean") {
|
|
3302
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3303
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3304
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3305
|
+
required.push(df.sourceTaskPortId);
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
}
|
|
3310
|
+
const schema = {
|
|
3311
|
+
type: "object",
|
|
3312
|
+
properties,
|
|
3313
|
+
...required.length > 0 ? { required } : {},
|
|
3314
|
+
additionalProperties: false
|
|
3315
|
+
};
|
|
3316
|
+
task.config = {
|
|
3317
|
+
...task.config,
|
|
3318
|
+
inputSchema: schema,
|
|
3319
|
+
outputSchema: schema
|
|
3320
|
+
};
|
|
3321
|
+
}
|
|
3322
|
+
if (task.type === "OutputTask") {
|
|
3323
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3324
|
+
if (incoming.length === 0)
|
|
3325
|
+
continue;
|
|
3326
|
+
const properties = {};
|
|
3327
|
+
const required = [];
|
|
3328
|
+
for (const df of incoming) {
|
|
3329
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3330
|
+
if (!sourceTask)
|
|
3331
|
+
continue;
|
|
3332
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3333
|
+
if (typeof sourceSchema === "boolean")
|
|
3334
|
+
continue;
|
|
3335
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3336
|
+
if (prop && typeof prop !== "boolean") {
|
|
3337
|
+
properties[df.targetTaskPortId] = prop;
|
|
3338
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3339
|
+
required.push(df.targetTaskPortId);
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
}
|
|
3343
|
+
const schema = {
|
|
3344
|
+
type: "object",
|
|
3345
|
+
properties,
|
|
3346
|
+
...required.length > 0 ? { required } : {},
|
|
3347
|
+
additionalProperties: false
|
|
3348
|
+
};
|
|
3349
|
+
task.config = {
|
|
3350
|
+
...task.config,
|
|
3351
|
+
inputSchema: schema,
|
|
3352
|
+
outputSchema: schema
|
|
3353
|
+
};
|
|
2827
3354
|
}
|
|
2828
3355
|
}
|
|
2829
3356
|
}
|
|
@@ -2833,6 +3360,7 @@ class Workflow {
|
|
|
2833
3360
|
const sourceSchema = sourceTask.outputSchema();
|
|
2834
3361
|
const targetSchema = targetTask.inputSchema();
|
|
2835
3362
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3363
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2836
3364
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2837
3365
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2838
3366
|
const formats = new Set;
|
|
@@ -2908,18 +3436,33 @@ class Workflow {
|
|
|
2908
3436
|
if (typeof fromSchema === "object") {
|
|
2909
3437
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2910
3438
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3439
|
+
if (matches.has(fromOutputPortId))
|
|
3440
|
+
continue;
|
|
2911
3441
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2912
3442
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2913
3443
|
}
|
|
2914
3444
|
return;
|
|
2915
3445
|
}
|
|
2916
3446
|
}
|
|
3447
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3448
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3449
|
+
if (matches.has(toInputPortId))
|
|
3450
|
+
continue;
|
|
3451
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3452
|
+
continue;
|
|
3453
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3454
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3455
|
+
}
|
|
3456
|
+
return;
|
|
3457
|
+
}
|
|
2917
3458
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2918
3459
|
return;
|
|
2919
3460
|
}
|
|
2920
3461
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2921
3462
|
if (matches.has(toInputPortId))
|
|
2922
3463
|
continue;
|
|
3464
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3465
|
+
continue;
|
|
2923
3466
|
const candidates = [];
|
|
2924
3467
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2925
3468
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2939,22 +3482,32 @@ class Workflow {
|
|
|
2939
3482
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2940
3483
|
}
|
|
2941
3484
|
};
|
|
2942
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3485
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2943
3486
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2944
3487
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2945
3488
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2946
3489
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2947
3490
|
});
|
|
2948
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3491
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2949
3492
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2950
3493
|
});
|
|
2951
3494
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2952
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3495
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2953
3496
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2954
3497
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2955
3498
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2956
3499
|
const earlierTask = earlierTasks[i];
|
|
2957
3500
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3501
|
+
if (earlierTask.type === "InputTask") {
|
|
3502
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3503
|
+
if (matches.has(requiredInputId))
|
|
3504
|
+
continue;
|
|
3505
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3506
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3507
|
+
}
|
|
3508
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3509
|
+
continue;
|
|
3510
|
+
}
|
|
2958
3511
|
const makeMatchFromEarlier = (comparator) => {
|
|
2959
3512
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2960
3513
|
return;
|
|
@@ -2964,7 +3517,7 @@ class Workflow {
|
|
|
2964
3517
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2965
3518
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2966
3519
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2967
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3520
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2968
3521
|
}
|
|
2969
3522
|
}
|
|
2970
3523
|
}
|
|
@@ -2990,6 +3543,10 @@ class Workflow {
|
|
|
2990
3543
|
};
|
|
2991
3544
|
}
|
|
2992
3545
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3546
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3547
|
+
if (existingTargetConnections.length > 0) {
|
|
3548
|
+
return { matches, unmatchedRequired: [] };
|
|
3549
|
+
}
|
|
2993
3550
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
2994
3551
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
2995
3552
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3112,7 +3669,7 @@ function getLastTask(workflow) {
|
|
|
3112
3669
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3113
3670
|
}
|
|
3114
3671
|
function connect(source, target, workflow) {
|
|
3115
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3672
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3116
3673
|
}
|
|
3117
3674
|
function pipe(args, workflow = new Workflow) {
|
|
3118
3675
|
let previousTask = getLastTask(workflow);
|
|
@@ -3168,7 +3725,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3168
3725
|
// src/task-graph/TaskGraph.ts
|
|
3169
3726
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3170
3727
|
constructor() {
|
|
3171
|
-
super((task) => task.
|
|
3728
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3172
3729
|
}
|
|
3173
3730
|
}
|
|
3174
3731
|
|
|
@@ -3265,18 +3822,22 @@ class TaskGraph {
|
|
|
3265
3822
|
return this._dag.removeNode(taskId);
|
|
3266
3823
|
}
|
|
3267
3824
|
resetGraph() {
|
|
3268
|
-
this.runner.resetGraph(this,
|
|
3825
|
+
this.runner.resetGraph(this, uuid45());
|
|
3269
3826
|
}
|
|
3270
|
-
toJSON() {
|
|
3271
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3827
|
+
toJSON(options) {
|
|
3828
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3272
3829
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3273
|
-
|
|
3830
|
+
let json = {
|
|
3274
3831
|
tasks,
|
|
3275
3832
|
dataflows
|
|
3276
3833
|
};
|
|
3834
|
+
if (options?.withBoundaryNodes) {
|
|
3835
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3836
|
+
}
|
|
3837
|
+
return json;
|
|
3277
3838
|
}
|
|
3278
|
-
toDependencyJSON() {
|
|
3279
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3839
|
+
toDependencyJSON(options) {
|
|
3840
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3280
3841
|
this.getDataflows().forEach((df) => {
|
|
3281
3842
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3282
3843
|
if (!target.dependencies) {
|
|
@@ -3302,6 +3863,9 @@ class TaskGraph {
|
|
|
3302
3863
|
}
|
|
3303
3864
|
}
|
|
3304
3865
|
});
|
|
3866
|
+
if (options?.withBoundaryNodes) {
|
|
3867
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3868
|
+
}
|
|
3305
3869
|
return tasks;
|
|
3306
3870
|
}
|
|
3307
3871
|
get events() {
|
|
@@ -3320,7 +3884,7 @@ class TaskGraph {
|
|
|
3320
3884
|
const tasks = this.getTasks();
|
|
3321
3885
|
tasks.forEach((task) => {
|
|
3322
3886
|
const unsub = task.subscribe("status", (status) => {
|
|
3323
|
-
callback(task.
|
|
3887
|
+
callback(task.id, status);
|
|
3324
3888
|
});
|
|
3325
3889
|
unsubscribes.push(unsub);
|
|
3326
3890
|
});
|
|
@@ -3329,7 +3893,7 @@ class TaskGraph {
|
|
|
3329
3893
|
if (!task || typeof task.subscribe !== "function")
|
|
3330
3894
|
return;
|
|
3331
3895
|
const unsub = task.subscribe("status", (status) => {
|
|
3332
|
-
callback(task.
|
|
3896
|
+
callback(task.id, status);
|
|
3333
3897
|
});
|
|
3334
3898
|
unsubscribes.push(unsub);
|
|
3335
3899
|
};
|
|
@@ -3344,7 +3908,7 @@ class TaskGraph {
|
|
|
3344
3908
|
const tasks = this.getTasks();
|
|
3345
3909
|
tasks.forEach((task) => {
|
|
3346
3910
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3347
|
-
callback(task.
|
|
3911
|
+
callback(task.id, progress, message, ...args);
|
|
3348
3912
|
});
|
|
3349
3913
|
unsubscribes.push(unsub);
|
|
3350
3914
|
});
|
|
@@ -3353,7 +3917,7 @@ class TaskGraph {
|
|
|
3353
3917
|
if (!task || typeof task.subscribe !== "function")
|
|
3354
3918
|
return;
|
|
3355
3919
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3356
|
-
callback(task.
|
|
3920
|
+
callback(task.id, progress, message, ...args);
|
|
3357
3921
|
});
|
|
3358
3922
|
unsubscribes.push(unsub);
|
|
3359
3923
|
};
|
|
@@ -3438,7 +4002,7 @@ class TaskGraph {
|
|
|
3438
4002
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3439
4003
|
const edges = [];
|
|
3440
4004
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3441
|
-
edges.push(new Dataflow(tasks[i].
|
|
4005
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3442
4006
|
}
|
|
3443
4007
|
return edges;
|
|
3444
4008
|
}
|
|
@@ -3448,9 +4012,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3448
4012
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3449
4013
|
return graph;
|
|
3450
4014
|
}
|
|
4015
|
+
// src/task/FallbackTaskRunner.ts
|
|
4016
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4017
|
+
async executeTask(input) {
|
|
4018
|
+
if (this.task.fallbackMode === "data") {
|
|
4019
|
+
return this.executeDataFallback(input);
|
|
4020
|
+
}
|
|
4021
|
+
return this.executeTaskFallback(input);
|
|
4022
|
+
}
|
|
4023
|
+
async executeTaskReactive(input, output) {
|
|
4024
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4025
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4026
|
+
}
|
|
4027
|
+
async executeTaskFallback(input) {
|
|
4028
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4029
|
+
if (tasks.length === 0) {
|
|
4030
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4031
|
+
}
|
|
4032
|
+
const errors = [];
|
|
4033
|
+
const totalAttempts = tasks.length;
|
|
4034
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4035
|
+
if (this.abortController?.signal.aborted) {
|
|
4036
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4037
|
+
}
|
|
4038
|
+
const alternativeTask = tasks[i];
|
|
4039
|
+
const attemptNumber = i + 1;
|
|
4040
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4041
|
+
try {
|
|
4042
|
+
this.resetTask(alternativeTask);
|
|
4043
|
+
const result = await alternativeTask.run(input);
|
|
4044
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4045
|
+
return await this.executeTaskReactive(input, result);
|
|
4046
|
+
} catch (error) {
|
|
4047
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4048
|
+
throw error;
|
|
4049
|
+
}
|
|
4050
|
+
errors.push({ task: alternativeTask, error });
|
|
4051
|
+
}
|
|
4052
|
+
}
|
|
4053
|
+
throw this.buildAggregateError(errors, "task");
|
|
4054
|
+
}
|
|
4055
|
+
async executeDataFallback(input) {
|
|
4056
|
+
const alternatives = this.task.alternatives;
|
|
4057
|
+
if (alternatives.length === 0) {
|
|
4058
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4059
|
+
}
|
|
4060
|
+
const errors = [];
|
|
4061
|
+
const totalAttempts = alternatives.length;
|
|
4062
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4063
|
+
if (this.abortController?.signal.aborted) {
|
|
4064
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4065
|
+
}
|
|
4066
|
+
const alternative = alternatives[i];
|
|
4067
|
+
const attemptNumber = i + 1;
|
|
4068
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4069
|
+
try {
|
|
4070
|
+
this.resetSubgraph();
|
|
4071
|
+
const mergedInput = { ...input, ...alternative };
|
|
4072
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4073
|
+
parentSignal: this.abortController?.signal,
|
|
4074
|
+
outputCache: this.outputCache
|
|
4075
|
+
});
|
|
4076
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4077
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4078
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4079
|
+
} catch (error) {
|
|
4080
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4081
|
+
throw error;
|
|
4082
|
+
}
|
|
4083
|
+
errors.push({ alternative, error });
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
throw this.buildAggregateError(errors, "data");
|
|
4087
|
+
}
|
|
4088
|
+
resetTask(task) {
|
|
4089
|
+
task.status = TaskStatus.PENDING;
|
|
4090
|
+
task.progress = 0;
|
|
4091
|
+
task.error = undefined;
|
|
4092
|
+
task.completedAt = undefined;
|
|
4093
|
+
task.startedAt = undefined;
|
|
4094
|
+
task.resetInputData();
|
|
4095
|
+
}
|
|
4096
|
+
resetSubgraph() {
|
|
4097
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4098
|
+
this.resetTask(task);
|
|
4099
|
+
}
|
|
4100
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4101
|
+
dataflow.reset();
|
|
4102
|
+
}
|
|
4103
|
+
}
|
|
4104
|
+
buildAggregateError(errors, mode) {
|
|
4105
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4106
|
+
const details = errors.map((e, i) => {
|
|
4107
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4108
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4109
|
+
}).join(`
|
|
4110
|
+
`);
|
|
4111
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4112
|
+
${details}`);
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
|
|
4116
|
+
// src/task/FallbackTask.ts
|
|
4117
|
+
var fallbackTaskConfigSchema = {
|
|
4118
|
+
type: "object",
|
|
4119
|
+
properties: {
|
|
4120
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4121
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4122
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4123
|
+
},
|
|
4124
|
+
additionalProperties: false
|
|
4125
|
+
};
|
|
4126
|
+
|
|
4127
|
+
class FallbackTask extends GraphAsTask {
|
|
4128
|
+
static type = "FallbackTask";
|
|
4129
|
+
static category = "Flow Control";
|
|
4130
|
+
static title = "Fallback";
|
|
4131
|
+
static description = "Try alternatives until one succeeds";
|
|
4132
|
+
static hasDynamicSchemas = true;
|
|
4133
|
+
static configSchema() {
|
|
4134
|
+
return fallbackTaskConfigSchema;
|
|
4135
|
+
}
|
|
4136
|
+
get runner() {
|
|
4137
|
+
if (!this._runner) {
|
|
4138
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4139
|
+
}
|
|
4140
|
+
return this._runner;
|
|
4141
|
+
}
|
|
4142
|
+
get fallbackMode() {
|
|
4143
|
+
return this.config?.fallbackMode ?? "task";
|
|
4144
|
+
}
|
|
4145
|
+
get alternatives() {
|
|
4146
|
+
return this.config?.alternatives ?? [];
|
|
4147
|
+
}
|
|
4148
|
+
inputSchema() {
|
|
4149
|
+
if (!this.hasChildren()) {
|
|
4150
|
+
return this.constructor.inputSchema();
|
|
4151
|
+
}
|
|
4152
|
+
if (this.fallbackMode === "data") {
|
|
4153
|
+
return super.inputSchema();
|
|
4154
|
+
}
|
|
4155
|
+
const properties = {};
|
|
4156
|
+
const tasks = this.subGraph.getTasks();
|
|
4157
|
+
for (const task of tasks) {
|
|
4158
|
+
const taskInputSchema = task.inputSchema();
|
|
4159
|
+
if (typeof taskInputSchema === "boolean")
|
|
4160
|
+
continue;
|
|
4161
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4162
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4163
|
+
if (!properties[inputName]) {
|
|
4164
|
+
properties[inputName] = inputProp;
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
return {
|
|
4169
|
+
type: "object",
|
|
4170
|
+
properties,
|
|
4171
|
+
additionalProperties: true
|
|
4172
|
+
};
|
|
4173
|
+
}
|
|
4174
|
+
outputSchema() {
|
|
4175
|
+
if (!this.hasChildren()) {
|
|
4176
|
+
return this.constructor.outputSchema();
|
|
4177
|
+
}
|
|
4178
|
+
const tasks = this.subGraph.getTasks();
|
|
4179
|
+
if (tasks.length === 0) {
|
|
4180
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4181
|
+
}
|
|
4182
|
+
if (this.fallbackMode === "task") {
|
|
4183
|
+
const firstTask = tasks[0];
|
|
4184
|
+
return firstTask.outputSchema();
|
|
4185
|
+
}
|
|
4186
|
+
return super.outputSchema();
|
|
4187
|
+
}
|
|
4188
|
+
toJSON() {
|
|
4189
|
+
const json = super.toJSON();
|
|
4190
|
+
return {
|
|
4191
|
+
...json,
|
|
4192
|
+
config: {
|
|
4193
|
+
..."config" in json ? json.config : {},
|
|
4194
|
+
fallbackMode: this.fallbackMode,
|
|
4195
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4196
|
+
}
|
|
4197
|
+
};
|
|
4198
|
+
}
|
|
4199
|
+
}
|
|
4200
|
+
queueMicrotask(() => {
|
|
4201
|
+
Workflow.prototype.fallback = function() {
|
|
4202
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4203
|
+
};
|
|
4204
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4205
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4206
|
+
return this.addLoopTask(FallbackTask, {
|
|
4207
|
+
fallbackMode: "data",
|
|
4208
|
+
alternatives
|
|
4209
|
+
});
|
|
4210
|
+
};
|
|
4211
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4212
|
+
});
|
|
3451
4213
|
// src/task/IteratorTaskRunner.ts
|
|
3452
4214
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3453
|
-
subGraphRunChain = Promise.resolve();
|
|
3454
4215
|
async executeTask(input) {
|
|
3455
4216
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3456
4217
|
if (analysis.iterationCount === 0) {
|
|
@@ -3472,13 +4233,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3472
4233
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3473
4234
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3474
4235
|
const completionOrderResults = [];
|
|
4236
|
+
let completedCount = 0;
|
|
3475
4237
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3476
4238
|
if (this.abortController?.signal.aborted) {
|
|
3477
4239
|
break;
|
|
3478
4240
|
}
|
|
3479
4241
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3480
4242
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3481
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4243
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4244
|
+
completedCount++;
|
|
4245
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4246
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4247
|
+
});
|
|
3482
4248
|
for (const { index, result } of batchResults) {
|
|
3483
4249
|
if (result === undefined)
|
|
3484
4250
|
continue;
|
|
@@ -3488,8 +4254,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3488
4254
|
completionOrderResults.push(result);
|
|
3489
4255
|
}
|
|
3490
4256
|
}
|
|
3491
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3492
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3493
4257
|
}
|
|
3494
4258
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3495
4259
|
return this.task.collectResults(collected);
|
|
@@ -3511,7 +4275,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3511
4275
|
}
|
|
3512
4276
|
return accumulator;
|
|
3513
4277
|
}
|
|
3514
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4278
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3515
4279
|
const results = [];
|
|
3516
4280
|
let cursor = 0;
|
|
3517
4281
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3529,33 +4293,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3529
4293
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3530
4294
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3531
4295
|
results.push({ index, result });
|
|
4296
|
+
await onItemComplete?.();
|
|
3532
4297
|
}
|
|
3533
4298
|
});
|
|
3534
4299
|
await Promise.all(workers);
|
|
3535
4300
|
return results;
|
|
3536
4301
|
}
|
|
4302
|
+
cloneGraph(graph) {
|
|
4303
|
+
const clone = new TaskGraph;
|
|
4304
|
+
for (const task of graph.getTasks()) {
|
|
4305
|
+
const ctor = task.constructor;
|
|
4306
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4307
|
+
if (task.hasChildren()) {
|
|
4308
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4309
|
+
}
|
|
4310
|
+
clone.addTask(newTask);
|
|
4311
|
+
}
|
|
4312
|
+
for (const df of graph.getDataflows()) {
|
|
4313
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4314
|
+
}
|
|
4315
|
+
return clone;
|
|
4316
|
+
}
|
|
3537
4317
|
async executeSubgraphIteration(input) {
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
4318
|
+
if (this.abortController?.signal.aborted) {
|
|
4319
|
+
return;
|
|
4320
|
+
}
|
|
4321
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4322
|
+
const results = await graphClone.run(input, {
|
|
4323
|
+
parentSignal: this.abortController?.signal,
|
|
4324
|
+
outputCache: this.outputCache
|
|
3542
4325
|
});
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
if (this.abortController?.signal.aborted) {
|
|
3546
|
-
return;
|
|
3547
|
-
}
|
|
3548
|
-
const results = await this.task.subGraph.run(input, {
|
|
3549
|
-
parentSignal: this.abortController?.signal,
|
|
3550
|
-
outputCache: this.outputCache
|
|
3551
|
-
});
|
|
3552
|
-
if (results.length === 0) {
|
|
3553
|
-
return;
|
|
3554
|
-
}
|
|
3555
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3556
|
-
} finally {
|
|
3557
|
-
releaseTurn?.();
|
|
4326
|
+
if (results.length === 0) {
|
|
4327
|
+
return;
|
|
3558
4328
|
}
|
|
4329
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3559
4330
|
}
|
|
3560
4331
|
}
|
|
3561
4332
|
|
|
@@ -3636,23 +4407,17 @@ function inferIterationFromSchema(schema) {
|
|
|
3636
4407
|
return false;
|
|
3637
4408
|
}
|
|
3638
4409
|
function createFlexibleSchema(baseSchema) {
|
|
3639
|
-
if (typeof baseSchema === "boolean")
|
|
3640
|
-
return baseSchema;
|
|
3641
4410
|
return {
|
|
3642
4411
|
anyOf: [baseSchema, { type: "array", items: baseSchema }]
|
|
3643
4412
|
};
|
|
3644
4413
|
}
|
|
3645
4414
|
function createArraySchema(baseSchema) {
|
|
3646
|
-
if (typeof baseSchema === "boolean")
|
|
3647
|
-
return baseSchema;
|
|
3648
4415
|
return {
|
|
3649
4416
|
type: "array",
|
|
3650
4417
|
items: baseSchema
|
|
3651
4418
|
};
|
|
3652
4419
|
}
|
|
3653
4420
|
function extractBaseSchema(schema) {
|
|
3654
|
-
if (typeof schema === "boolean")
|
|
3655
|
-
return schema;
|
|
3656
4421
|
const schemaType = schema.type;
|
|
3657
4422
|
if (schemaType === "array" && schema.items) {
|
|
3658
4423
|
return schema.items;
|
|
@@ -3845,7 +4610,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3845
4610
|
const tasks = this.subGraph.getTasks();
|
|
3846
4611
|
if (tasks.length === 0)
|
|
3847
4612
|
return;
|
|
3848
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4613
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3849
4614
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3850
4615
|
const properties = {};
|
|
3851
4616
|
const required = [];
|
|
@@ -3872,6 +4637,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3872
4637
|
}
|
|
3873
4638
|
}
|
|
3874
4639
|
}
|
|
4640
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4641
|
+
for (const task of tasks) {
|
|
4642
|
+
if (sourceIds.has(task.id))
|
|
4643
|
+
continue;
|
|
4644
|
+
const inputSchema = task.inputSchema();
|
|
4645
|
+
if (typeof inputSchema === "boolean")
|
|
4646
|
+
continue;
|
|
4647
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4648
|
+
if (requiredKeys.size === 0)
|
|
4649
|
+
continue;
|
|
4650
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4651
|
+
for (const key of requiredKeys) {
|
|
4652
|
+
if (connectedPorts.has(key))
|
|
4653
|
+
continue;
|
|
4654
|
+
if (properties[key])
|
|
4655
|
+
continue;
|
|
4656
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4657
|
+
continue;
|
|
4658
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4659
|
+
if (!prop || typeof prop === "boolean")
|
|
4660
|
+
continue;
|
|
4661
|
+
properties[key] = prop;
|
|
4662
|
+
if (!required.includes(key)) {
|
|
4663
|
+
required.push(key);
|
|
4664
|
+
}
|
|
4665
|
+
}
|
|
4666
|
+
}
|
|
3875
4667
|
return {
|
|
3876
4668
|
type: "object",
|
|
3877
4669
|
properties,
|
|
@@ -4006,7 +4798,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4006
4798
|
if (!this.hasChildren()) {
|
|
4007
4799
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4008
4800
|
}
|
|
4009
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4801
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4010
4802
|
if (endingNodes.length === 0) {
|
|
4011
4803
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4012
4804
|
}
|
|
@@ -4216,8 +5008,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4216
5008
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4217
5009
|
}
|
|
4218
5010
|
this._currentIteration++;
|
|
4219
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4220
|
-
await context.updateProgress(progress, `
|
|
5011
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5012
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4221
5013
|
}
|
|
4222
5014
|
return currentOutput;
|
|
4223
5015
|
}
|
|
@@ -4260,8 +5052,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4260
5052
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4261
5053
|
}
|
|
4262
5054
|
this._currentIteration++;
|
|
4263
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4264
|
-
await context.updateProgress(progress, `
|
|
5055
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5056
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4265
5057
|
}
|
|
4266
5058
|
yield { type: "finish", data: currentOutput };
|
|
4267
5059
|
}
|
|
@@ -4337,7 +5129,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4337
5129
|
return this.constructor.outputSchema();
|
|
4338
5130
|
}
|
|
4339
5131
|
const tasks = this.subGraph.getTasks();
|
|
4340
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5132
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4341
5133
|
if (endingNodes.length === 0) {
|
|
4342
5134
|
return this.constructor.outputSchema();
|
|
4343
5135
|
}
|
|
@@ -4712,7 +5504,11 @@ var jobQueueTaskConfigSchema = {
|
|
|
4712
5504
|
type: "object",
|
|
4713
5505
|
properties: {
|
|
4714
5506
|
...graphAsTaskConfigSchema["properties"],
|
|
4715
|
-
queue: {
|
|
5507
|
+
queue: {
|
|
5508
|
+
oneOf: [{ type: "boolean" }, { type: "string" }],
|
|
5509
|
+
description: "Queue handling: false=run inline, true=use default, string=explicit queue name",
|
|
5510
|
+
"x-ui-hidden": true
|
|
5511
|
+
}
|
|
4716
5512
|
},
|
|
4717
5513
|
additionalProperties: false
|
|
4718
5514
|
};
|
|
@@ -5000,7 +5796,7 @@ class ReduceTask extends IteratorTask {
|
|
|
5000
5796
|
if (!this.hasChildren()) {
|
|
5001
5797
|
return this.constructor.outputSchema();
|
|
5002
5798
|
}
|
|
5003
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5799
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5004
5800
|
if (endingNodes.length === 0) {
|
|
5005
5801
|
return this.constructor.outputSchema();
|
|
5006
5802
|
}
|
|
@@ -5038,14 +5834,14 @@ var TaskRegistry = {
|
|
|
5038
5834
|
};
|
|
5039
5835
|
|
|
5040
5836
|
// src/task/TaskJSON.ts
|
|
5041
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5837
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5042
5838
|
if (!item.id)
|
|
5043
5839
|
throw new TaskJSONError("Task id required");
|
|
5044
5840
|
if (!item.type)
|
|
5045
5841
|
throw new TaskJSONError("Task type required");
|
|
5046
5842
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5047
5843
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5048
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5844
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5049
5845
|
if (!taskClass)
|
|
5050
5846
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5051
5847
|
const taskConfig = {
|
|
@@ -5072,20 +5868,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5072
5868
|
}
|
|
5073
5869
|
return subGraph;
|
|
5074
5870
|
};
|
|
5075
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5076
|
-
const task = createSingleTaskFromJSON(item);
|
|
5871
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5872
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5077
5873
|
if (item.subgraph) {
|
|
5078
5874
|
if (!(task instanceof GraphAsTask)) {
|
|
5079
5875
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5080
5876
|
}
|
|
5081
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5877
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5082
5878
|
}
|
|
5083
5879
|
return task;
|
|
5084
5880
|
};
|
|
5085
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5881
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5086
5882
|
const subGraph = new TaskGraph;
|
|
5087
5883
|
for (const subitem of graphJsonObj.tasks) {
|
|
5088
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5884
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5089
5885
|
}
|
|
5090
5886
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5091
5887
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5094,7 +5890,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5094
5890
|
};
|
|
5095
5891
|
// src/task/index.ts
|
|
5096
5892
|
var registerBaseTasks = () => {
|
|
5097
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5893
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5098
5894
|
tasks.map(TaskRegistry.registerTask);
|
|
5099
5895
|
return tasks;
|
|
5100
5896
|
};
|
|
@@ -5277,11 +6073,14 @@ export {
|
|
|
5277
6073
|
isFlexibleSchema,
|
|
5278
6074
|
hasVectorOutput,
|
|
5279
6075
|
hasVectorLikeInput,
|
|
6076
|
+
hasStructuredOutput,
|
|
5280
6077
|
graphAsTaskConfigSchema,
|
|
5281
6078
|
getTaskQueueRegistry,
|
|
6079
|
+
getStructuredOutputSchemas,
|
|
5282
6080
|
getStreamingPorts,
|
|
5283
6081
|
getPortStreamMode,
|
|
5284
6082
|
getOutputStreamMode,
|
|
6083
|
+
getObjectPortId,
|
|
5285
6084
|
getNestedValue,
|
|
5286
6085
|
getLastTask,
|
|
5287
6086
|
getJobQueueFactory,
|
|
@@ -5290,6 +6089,7 @@ export {
|
|
|
5290
6089
|
getAppendPortId,
|
|
5291
6090
|
findArrayPorts,
|
|
5292
6091
|
filterIterationProperties,
|
|
6092
|
+
fallbackTaskConfigSchema,
|
|
5293
6093
|
extractIterationProperties,
|
|
5294
6094
|
extractBaseSchema,
|
|
5295
6095
|
evaluateCondition,
|
|
@@ -5304,13 +6104,19 @@ export {
|
|
|
5304
6104
|
createArraySchema,
|
|
5305
6105
|
connect,
|
|
5306
6106
|
conditionalTaskConfigSchema,
|
|
6107
|
+
computeGraphOutputSchema,
|
|
6108
|
+
computeGraphInputSchema,
|
|
6109
|
+
calculateNodeDepths,
|
|
5307
6110
|
buildIterationInputSchema,
|
|
5308
6111
|
addIterationContextToSchema,
|
|
6112
|
+
addBoundaryNodesToGraphJson,
|
|
6113
|
+
addBoundaryNodesToDependencyJson,
|
|
5309
6114
|
WorkflowError,
|
|
5310
6115
|
Workflow,
|
|
5311
6116
|
WhileTaskRunner,
|
|
5312
6117
|
WhileTask,
|
|
5313
6118
|
WHILE_CONTEXT_SCHEMA,
|
|
6119
|
+
TaskTimeoutError,
|
|
5314
6120
|
TaskStatus,
|
|
5315
6121
|
TaskRegistry,
|
|
5316
6122
|
TaskQueueRegistry,
|
|
@@ -5346,6 +6152,8 @@ export {
|
|
|
5346
6152
|
GraphAsTaskRunner,
|
|
5347
6153
|
GraphAsTask,
|
|
5348
6154
|
GRAPH_RESULT_ARRAY,
|
|
6155
|
+
FallbackTaskRunner,
|
|
6156
|
+
FallbackTask,
|
|
5349
6157
|
EventTaskGraphToDagMapping,
|
|
5350
6158
|
EventDagToTaskGraphMapping,
|
|
5351
6159
|
DataflowArrow,
|
|
@@ -5359,4 +6167,4 @@ export {
|
|
|
5359
6167
|
ConditionalTask
|
|
5360
6168
|
};
|
|
5361
6169
|
|
|
5362
|
-
//# debugId=
|
|
6170
|
+
//# debugId=8E9465F7037F297B64756E2164756E21
|