@workglow/task-graph 0.0.102 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1052 -263
- package/dist/browser.js.map +24 -21
- package/dist/bun.js +1052 -263
- package/dist/bun.js.map +24 -21
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1052 -263
- package/dist/node.js.map +24 -21
- package/dist/task/ConditionalTask.d.ts +5 -1
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +15 -12
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +5 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +9 -1
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +4 -0
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +20 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +7 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +5 -0
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/bun.js
CHANGED
|
@@ -21,6 +21,7 @@ var TaskConfigSchema = {
|
|
|
21
21
|
title: { type: "string" },
|
|
22
22
|
description: { type: "string" },
|
|
23
23
|
cacheable: { type: "boolean" },
|
|
24
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
24
25
|
inputSchema: {
|
|
25
26
|
type: "object",
|
|
26
27
|
properties: {},
|
|
@@ -241,8 +242,352 @@ class DataflowArrow extends Dataflow {
|
|
|
241
242
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
242
243
|
}
|
|
243
244
|
}
|
|
245
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
246
|
+
import { uuid4 } from "@workglow/util";
|
|
247
|
+
function calculateNodeDepths(graph) {
|
|
248
|
+
const depths = new Map;
|
|
249
|
+
const tasks = graph.getTasks();
|
|
250
|
+
for (const task of tasks) {
|
|
251
|
+
depths.set(task.id, 0);
|
|
252
|
+
}
|
|
253
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
254
|
+
for (const task of sortedTasks) {
|
|
255
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
256
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
257
|
+
for (const targetTask of targetTasks) {
|
|
258
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
259
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return depths;
|
|
263
|
+
}
|
|
264
|
+
function computeGraphInputSchema(graph, options) {
|
|
265
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
266
|
+
const properties = {};
|
|
267
|
+
const required = [];
|
|
268
|
+
const propertyOrigins = {};
|
|
269
|
+
const tasks = graph.getTasks();
|
|
270
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
271
|
+
for (const task of startingNodes) {
|
|
272
|
+
const taskInputSchema = task.inputSchema();
|
|
273
|
+
if (typeof taskInputSchema === "boolean") {
|
|
274
|
+
if (taskInputSchema === false) {
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (taskInputSchema === true) {
|
|
278
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
283
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
284
|
+
if (!properties[inputName]) {
|
|
285
|
+
properties[inputName] = inputProp;
|
|
286
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
287
|
+
required.push(inputName);
|
|
288
|
+
}
|
|
289
|
+
if (trackOrigins) {
|
|
290
|
+
propertyOrigins[inputName] = [task.id];
|
|
291
|
+
}
|
|
292
|
+
} else if (trackOrigins) {
|
|
293
|
+
propertyOrigins[inputName].push(task.id);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
298
|
+
for (const task of tasks) {
|
|
299
|
+
if (sourceIds.has(task.id))
|
|
300
|
+
continue;
|
|
301
|
+
const taskInputSchema = task.inputSchema();
|
|
302
|
+
if (typeof taskInputSchema === "boolean")
|
|
303
|
+
continue;
|
|
304
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
305
|
+
if (requiredKeys.size === 0)
|
|
306
|
+
continue;
|
|
307
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
308
|
+
for (const key of requiredKeys) {
|
|
309
|
+
if (connectedPorts.has(key))
|
|
310
|
+
continue;
|
|
311
|
+
if (properties[key]) {
|
|
312
|
+
if (trackOrigins) {
|
|
313
|
+
propertyOrigins[key].push(task.id);
|
|
314
|
+
}
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
318
|
+
continue;
|
|
319
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
320
|
+
if (!prop || typeof prop === "boolean")
|
|
321
|
+
continue;
|
|
322
|
+
properties[key] = prop;
|
|
323
|
+
if (!required.includes(key)) {
|
|
324
|
+
required.push(key);
|
|
325
|
+
}
|
|
326
|
+
if (trackOrigins) {
|
|
327
|
+
propertyOrigins[key] = [task.id];
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (trackOrigins) {
|
|
332
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
333
|
+
const prop = properties[propName];
|
|
334
|
+
if (!prop || typeof prop === "boolean")
|
|
335
|
+
continue;
|
|
336
|
+
if (origins.length === 1) {
|
|
337
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
338
|
+
} else {
|
|
339
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return {
|
|
344
|
+
type: "object",
|
|
345
|
+
properties,
|
|
346
|
+
...required.length > 0 ? { required } : {},
|
|
347
|
+
additionalProperties: false
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
function computeGraphOutputSchema(graph, options) {
|
|
351
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
352
|
+
const properties = {};
|
|
353
|
+
const required = [];
|
|
354
|
+
const propertyOrigins = {};
|
|
355
|
+
const tasks = graph.getTasks();
|
|
356
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
357
|
+
const depths = calculateNodeDepths(graph);
|
|
358
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
359
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
360
|
+
const propertyCount = {};
|
|
361
|
+
const propertySchema = {};
|
|
362
|
+
for (const task of lastLevelNodes) {
|
|
363
|
+
const taskOutputSchema = task.outputSchema();
|
|
364
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
365
|
+
if (taskOutputSchema === false) {
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (taskOutputSchema === true) {
|
|
369
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
374
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
375
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
376
|
+
if (!propertySchema[outputName]) {
|
|
377
|
+
propertySchema[outputName] = outputProp;
|
|
378
|
+
}
|
|
379
|
+
if (trackOrigins) {
|
|
380
|
+
if (!propertyOrigins[outputName]) {
|
|
381
|
+
propertyOrigins[outputName] = [task.id];
|
|
382
|
+
} else {
|
|
383
|
+
propertyOrigins[outputName].push(task.id);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
389
|
+
const outputProp = propertySchema[outputName];
|
|
390
|
+
if (lastLevelNodes.length === 1) {
|
|
391
|
+
properties[outputName] = outputProp;
|
|
392
|
+
} else {
|
|
393
|
+
properties[outputName] = {
|
|
394
|
+
type: "array",
|
|
395
|
+
items: outputProp
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (trackOrigins) {
|
|
400
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
401
|
+
const prop = properties[propName];
|
|
402
|
+
if (!prop || typeof prop === "boolean")
|
|
403
|
+
continue;
|
|
404
|
+
if (origins.length === 1) {
|
|
405
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
406
|
+
} else {
|
|
407
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return {
|
|
412
|
+
type: "object",
|
|
413
|
+
properties,
|
|
414
|
+
...required.length > 0 ? { required } : {},
|
|
415
|
+
additionalProperties: false
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
function stripOriginAnnotations(schema) {
|
|
419
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
420
|
+
return schema;
|
|
421
|
+
const properties = schema.properties;
|
|
422
|
+
if (!properties)
|
|
423
|
+
return schema;
|
|
424
|
+
const strippedProperties = {};
|
|
425
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
426
|
+
if (!prop || typeof prop !== "object") {
|
|
427
|
+
strippedProperties[key] = prop;
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
431
|
+
strippedProperties[key] = rest;
|
|
432
|
+
}
|
|
433
|
+
return { ...schema, properties: strippedProperties };
|
|
434
|
+
}
|
|
435
|
+
function getOriginTaskIds(prop) {
|
|
436
|
+
if (prop["x-source-task-ids"]) {
|
|
437
|
+
return prop["x-source-task-ids"];
|
|
438
|
+
}
|
|
439
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
440
|
+
return [prop["x-source-task-id"]];
|
|
441
|
+
}
|
|
442
|
+
return [];
|
|
443
|
+
}
|
|
444
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
445
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
446
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
447
|
+
if (hasInputTask && hasOutputTask) {
|
|
448
|
+
return json;
|
|
449
|
+
}
|
|
450
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
452
|
+
const prependTasks = [];
|
|
453
|
+
const appendTasks = [];
|
|
454
|
+
const inputDataflows = [];
|
|
455
|
+
const outputDataflows = [];
|
|
456
|
+
if (!hasInputTask && inputSchema) {
|
|
457
|
+
const inputTaskId = uuid4();
|
|
458
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
459
|
+
prependTasks.push({
|
|
460
|
+
id: inputTaskId,
|
|
461
|
+
type: "InputTask",
|
|
462
|
+
config: {
|
|
463
|
+
inputSchema: strippedInputSchema,
|
|
464
|
+
outputSchema: strippedInputSchema
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
468
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
469
|
+
if (!prop || typeof prop === "boolean")
|
|
470
|
+
continue;
|
|
471
|
+
const origins = getOriginTaskIds(prop);
|
|
472
|
+
for (const originId of origins) {
|
|
473
|
+
inputDataflows.push({
|
|
474
|
+
sourceTaskId: inputTaskId,
|
|
475
|
+
sourceTaskPortId: propName,
|
|
476
|
+
targetTaskId: originId,
|
|
477
|
+
targetTaskPortId: propName
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
if (!hasOutputTask && outputSchema) {
|
|
484
|
+
const outputTaskId = uuid4();
|
|
485
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
486
|
+
appendTasks.push({
|
|
487
|
+
id: outputTaskId,
|
|
488
|
+
type: "OutputTask",
|
|
489
|
+
config: {
|
|
490
|
+
inputSchema: strippedOutputSchema,
|
|
491
|
+
outputSchema: strippedOutputSchema
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
495
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
496
|
+
if (!prop || typeof prop === "boolean")
|
|
497
|
+
continue;
|
|
498
|
+
const origins = getOriginTaskIds(prop);
|
|
499
|
+
for (const originId of origins) {
|
|
500
|
+
outputDataflows.push({
|
|
501
|
+
sourceTaskId: originId,
|
|
502
|
+
sourceTaskPortId: propName,
|
|
503
|
+
targetTaskId: outputTaskId,
|
|
504
|
+
targetTaskPortId: propName
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return {
|
|
511
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
512
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
516
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
517
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
518
|
+
if (hasInputTask && hasOutputTask) {
|
|
519
|
+
return items;
|
|
520
|
+
}
|
|
521
|
+
const prependItems = [];
|
|
522
|
+
const appendItems = [];
|
|
523
|
+
if (!hasInputTask) {
|
|
524
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
525
|
+
const inputTaskId = uuid4();
|
|
526
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
527
|
+
prependItems.push({
|
|
528
|
+
id: inputTaskId,
|
|
529
|
+
type: "InputTask",
|
|
530
|
+
config: {
|
|
531
|
+
inputSchema: strippedInputSchema,
|
|
532
|
+
outputSchema: strippedInputSchema
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
536
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
537
|
+
if (!prop || typeof prop === "boolean")
|
|
538
|
+
continue;
|
|
539
|
+
const origins = getOriginTaskIds(prop);
|
|
540
|
+
for (const originId of origins) {
|
|
541
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
542
|
+
if (!targetItem)
|
|
543
|
+
continue;
|
|
544
|
+
if (!targetItem.dependencies) {
|
|
545
|
+
targetItem.dependencies = {};
|
|
546
|
+
}
|
|
547
|
+
const existing = targetItem.dependencies[propName];
|
|
548
|
+
const dep = { id: inputTaskId, output: propName };
|
|
549
|
+
if (!existing) {
|
|
550
|
+
targetItem.dependencies[propName] = dep;
|
|
551
|
+
} else if (Array.isArray(existing)) {
|
|
552
|
+
existing.push(dep);
|
|
553
|
+
} else {
|
|
554
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
if (!hasOutputTask) {
|
|
561
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
562
|
+
const outputTaskId = uuid4();
|
|
563
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
564
|
+
const outputDependencies = {};
|
|
565
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
566
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
567
|
+
if (!prop || typeof prop === "boolean")
|
|
568
|
+
continue;
|
|
569
|
+
const origins = getOriginTaskIds(prop);
|
|
570
|
+
if (origins.length === 1) {
|
|
571
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
572
|
+
} else if (origins.length > 1) {
|
|
573
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
appendItems.push({
|
|
578
|
+
id: outputTaskId,
|
|
579
|
+
type: "OutputTask",
|
|
580
|
+
config: {
|
|
581
|
+
inputSchema: strippedOutputSchema,
|
|
582
|
+
outputSchema: strippedOutputSchema
|
|
583
|
+
},
|
|
584
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
return [...prependItems, ...items, ...appendItems];
|
|
588
|
+
}
|
|
244
589
|
// src/task-graph/TaskGraph.ts
|
|
245
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
590
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
246
591
|
|
|
247
592
|
// src/task/GraphAsTask.ts
|
|
248
593
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -250,9 +595,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
250
595
|
// src/task-graph/TaskGraphRunner.ts
|
|
251
596
|
import {
|
|
252
597
|
collectPropertyValues,
|
|
598
|
+
getLogger as getLogger3,
|
|
253
599
|
globalServiceRegistry as globalServiceRegistry2,
|
|
254
600
|
ServiceRegistry as ServiceRegistry2,
|
|
255
|
-
uuid4 as
|
|
601
|
+
uuid4 as uuid43
|
|
256
602
|
} from "@workglow/util";
|
|
257
603
|
|
|
258
604
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -285,6 +631,9 @@ class TaskOutputRepository {
|
|
|
285
631
|
}
|
|
286
632
|
}
|
|
287
633
|
|
|
634
|
+
// src/task/ConditionalTask.ts
|
|
635
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
636
|
+
|
|
288
637
|
// src/task/ConditionUtils.ts
|
|
289
638
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
290
639
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -357,7 +706,7 @@ import {
|
|
|
357
706
|
compileSchema,
|
|
358
707
|
deepEqual,
|
|
359
708
|
EventEmitter as EventEmitter3,
|
|
360
|
-
uuid4
|
|
709
|
+
uuid4 as uuid42
|
|
361
710
|
} from "@workglow/util";
|
|
362
711
|
|
|
363
712
|
// src/task/TaskError.ts
|
|
@@ -391,6 +740,13 @@ class TaskAbortedError extends TaskError {
|
|
|
391
740
|
}
|
|
392
741
|
}
|
|
393
742
|
|
|
743
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
744
|
+
static type = "TaskTimeoutError";
|
|
745
|
+
constructor(timeoutMs) {
|
|
746
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
394
750
|
class TaskFailedError extends TaskError {
|
|
395
751
|
static type = "TaskFailedError";
|
|
396
752
|
constructor(message = "Task failed") {
|
|
@@ -422,7 +778,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
422
778
|
}
|
|
423
779
|
|
|
424
780
|
// src/task/TaskRunner.ts
|
|
425
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
781
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
426
782
|
|
|
427
783
|
// src/task/InputResolver.ts
|
|
428
784
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -486,7 +842,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
486
842
|
if (!prop || typeof prop === "boolean")
|
|
487
843
|
return "none";
|
|
488
844
|
const xStream = prop["x-stream"];
|
|
489
|
-
if (xStream === "append" || xStream === "replace")
|
|
845
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
490
846
|
return xStream;
|
|
491
847
|
return "none";
|
|
492
848
|
}
|
|
@@ -501,7 +857,7 @@ function getStreamingPorts(schema) {
|
|
|
501
857
|
if (!prop || typeof prop === "boolean")
|
|
502
858
|
continue;
|
|
503
859
|
const xStream = prop["x-stream"];
|
|
504
|
-
if (xStream === "append" || xStream === "replace") {
|
|
860
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
505
861
|
result.push({ port: name, mode: xStream });
|
|
506
862
|
}
|
|
507
863
|
}
|
|
@@ -545,6 +901,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
545
901
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
546
902
|
return sourceMode !== targetMode;
|
|
547
903
|
}
|
|
904
|
+
function getObjectPortId(schema) {
|
|
905
|
+
if (typeof schema === "boolean")
|
|
906
|
+
return;
|
|
907
|
+
const props = schema.properties;
|
|
908
|
+
if (!props)
|
|
909
|
+
return;
|
|
910
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
911
|
+
if (!prop || typeof prop === "boolean")
|
|
912
|
+
continue;
|
|
913
|
+
if (prop["x-stream"] === "object")
|
|
914
|
+
return name;
|
|
915
|
+
}
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
function getStructuredOutputSchemas(schema) {
|
|
919
|
+
const result = new Map;
|
|
920
|
+
if (typeof schema === "boolean")
|
|
921
|
+
return result;
|
|
922
|
+
const props = schema.properties;
|
|
923
|
+
if (!props)
|
|
924
|
+
return result;
|
|
925
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
926
|
+
if (!prop || typeof prop === "boolean")
|
|
927
|
+
continue;
|
|
928
|
+
if (prop["x-structured-output"] === true) {
|
|
929
|
+
result.set(name, prop);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
return result;
|
|
933
|
+
}
|
|
934
|
+
function hasStructuredOutput(schema) {
|
|
935
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
936
|
+
}
|
|
548
937
|
|
|
549
938
|
// src/task/TaskRunner.ts
|
|
550
939
|
class TaskRunner {
|
|
@@ -555,12 +944,17 @@ class TaskRunner {
|
|
|
555
944
|
outputCache;
|
|
556
945
|
registry = globalServiceRegistry;
|
|
557
946
|
inputStreams;
|
|
947
|
+
timeoutTimer;
|
|
948
|
+
pendingTimeoutError;
|
|
558
949
|
shouldAccumulate = true;
|
|
559
950
|
constructor(task) {
|
|
560
951
|
this.task = task;
|
|
561
952
|
this.own = this.own.bind(this);
|
|
562
953
|
this.handleProgress = this.handleProgress.bind(this);
|
|
563
954
|
}
|
|
955
|
+
get timerLabel() {
|
|
956
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
957
|
+
}
|
|
564
958
|
async run(overrides = {}, config = {}) {
|
|
565
959
|
await this.handleStart(config);
|
|
566
960
|
try {
|
|
@@ -607,7 +1001,7 @@ class TaskRunner {
|
|
|
607
1001
|
return this.task.runOutputData;
|
|
608
1002
|
} catch (err) {
|
|
609
1003
|
await this.handleError(err);
|
|
610
|
-
throw err;
|
|
1004
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
611
1005
|
}
|
|
612
1006
|
}
|
|
613
1007
|
async runReactive(overrides = {}) {
|
|
@@ -664,7 +1058,14 @@ class TaskRunner {
|
|
|
664
1058
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
665
1059
|
}
|
|
666
1060
|
}
|
|
1061
|
+
if (streamMode === "object") {
|
|
1062
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1063
|
+
if (ports.length === 0) {
|
|
1064
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
667
1067
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1068
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
668
1069
|
let chunkCount = 0;
|
|
669
1070
|
let finalOutput;
|
|
670
1071
|
this.task.emit("stream_start");
|
|
@@ -695,6 +1096,13 @@ class TaskRunner {
|
|
|
695
1096
|
break;
|
|
696
1097
|
}
|
|
697
1098
|
case "object-delta": {
|
|
1099
|
+
if (accumulatedObjects) {
|
|
1100
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1101
|
+
}
|
|
1102
|
+
this.task.runOutputData = {
|
|
1103
|
+
...this.task.runOutputData,
|
|
1104
|
+
[event.port]: event.objectDelta
|
|
1105
|
+
};
|
|
698
1106
|
this.task.emit("stream_chunk", event);
|
|
699
1107
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
700
1108
|
await this.handleProgress(progress);
|
|
@@ -707,11 +1115,18 @@ class TaskRunner {
|
|
|
707
1115
|
break;
|
|
708
1116
|
}
|
|
709
1117
|
case "finish": {
|
|
710
|
-
if (accumulated) {
|
|
1118
|
+
if (accumulated || accumulatedObjects) {
|
|
711
1119
|
const merged = { ...event.data || {} };
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
1120
|
+
if (accumulated) {
|
|
1121
|
+
for (const [port, text] of accumulated) {
|
|
1122
|
+
if (text.length > 0)
|
|
1123
|
+
merged[port] = text;
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
if (accumulatedObjects) {
|
|
1127
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1128
|
+
merged[port] = obj;
|
|
1129
|
+
}
|
|
715
1130
|
}
|
|
716
1131
|
finalOutput = merged;
|
|
717
1132
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -757,12 +1172,20 @@ class TaskRunner {
|
|
|
757
1172
|
this.outputCache = cache;
|
|
758
1173
|
}
|
|
759
1174
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1175
|
+
const timeout = this.task.config.timeout;
|
|
1176
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1177
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1178
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1179
|
+
this.abort();
|
|
1180
|
+
}, timeout);
|
|
1181
|
+
}
|
|
760
1182
|
if (config.updateProgress) {
|
|
761
1183
|
this.updateProgress = config.updateProgress;
|
|
762
1184
|
}
|
|
763
1185
|
if (config.registry) {
|
|
764
1186
|
this.registry = config.registry;
|
|
765
1187
|
}
|
|
1188
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
766
1189
|
this.task.emit("start");
|
|
767
1190
|
this.task.emit("status", this.task.status);
|
|
768
1191
|
}
|
|
@@ -770,12 +1193,21 @@ class TaskRunner {
|
|
|
770
1193
|
async handleStartReactive() {
|
|
771
1194
|
this.reactiveRunning = true;
|
|
772
1195
|
}
|
|
1196
|
+
clearTimeoutTimer() {
|
|
1197
|
+
if (this.timeoutTimer !== undefined) {
|
|
1198
|
+
clearTimeout(this.timeoutTimer);
|
|
1199
|
+
this.timeoutTimer = undefined;
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
773
1202
|
async handleAbort() {
|
|
774
1203
|
if (this.task.status === TaskStatus.ABORTING)
|
|
775
1204
|
return;
|
|
1205
|
+
this.clearTimeoutTimer();
|
|
1206
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
776
1207
|
this.task.status = TaskStatus.ABORTING;
|
|
777
1208
|
this.task.progress = 100;
|
|
778
|
-
this.task.error = new TaskAbortedError;
|
|
1209
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1210
|
+
this.pendingTimeoutError = undefined;
|
|
779
1211
|
this.task.emit("abort", this.task.error);
|
|
780
1212
|
this.task.emit("status", this.task.status);
|
|
781
1213
|
}
|
|
@@ -785,6 +1217,9 @@ class TaskRunner {
|
|
|
785
1217
|
async handleComplete() {
|
|
786
1218
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
787
1219
|
return;
|
|
1220
|
+
this.clearTimeoutTimer();
|
|
1221
|
+
this.pendingTimeoutError = undefined;
|
|
1222
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
788
1223
|
this.task.completedAt = new Date;
|
|
789
1224
|
this.task.progress = 100;
|
|
790
1225
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -813,6 +1248,9 @@ class TaskRunner {
|
|
|
813
1248
|
return this.handleAbort();
|
|
814
1249
|
if (this.task.status === TaskStatus.FAILED)
|
|
815
1250
|
return;
|
|
1251
|
+
this.clearTimeoutTimer();
|
|
1252
|
+
this.pendingTimeoutError = undefined;
|
|
1253
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
816
1254
|
if (this.task.hasChildren()) {
|
|
817
1255
|
this.task.subGraph.abort();
|
|
818
1256
|
}
|
|
@@ -917,6 +1355,9 @@ class Task {
|
|
|
917
1355
|
runInputData = {};
|
|
918
1356
|
runOutputData = {};
|
|
919
1357
|
config;
|
|
1358
|
+
get id() {
|
|
1359
|
+
return this.config.id;
|
|
1360
|
+
}
|
|
920
1361
|
runConfig = {};
|
|
921
1362
|
status = TaskStatus.PENDING;
|
|
922
1363
|
progress = 0;
|
|
@@ -938,9 +1379,11 @@ class Task {
|
|
|
938
1379
|
this.resetInputData();
|
|
939
1380
|
const title = this.constructor.title || undefined;
|
|
940
1381
|
const baseConfig = Object.assign({
|
|
941
|
-
id: uuid4(),
|
|
942
1382
|
...title ? { title } : {}
|
|
943
1383
|
}, config);
|
|
1384
|
+
if (baseConfig.id === undefined) {
|
|
1385
|
+
baseConfig.id = uuid42();
|
|
1386
|
+
}
|
|
944
1387
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
945
1388
|
this.runConfig = runConfig;
|
|
946
1389
|
}
|
|
@@ -950,7 +1393,7 @@ class Task {
|
|
|
950
1393
|
return {};
|
|
951
1394
|
}
|
|
952
1395
|
try {
|
|
953
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1396
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
954
1397
|
const defaultData = compiledSchema.getData(undefined, {
|
|
955
1398
|
addOptionalProps: true,
|
|
956
1399
|
removeInvalidData: false,
|
|
@@ -1068,7 +1511,7 @@ class Task {
|
|
|
1068
1511
|
continue;
|
|
1069
1512
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1070
1513
|
if (isArray) {
|
|
1071
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1514
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1072
1515
|
const newitems = [...existingItems];
|
|
1073
1516
|
const overrideItem = overrides[inputId];
|
|
1074
1517
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1124,25 +1567,29 @@ class Task {
|
|
|
1124
1567
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1125
1568
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1126
1569
|
}
|
|
1127
|
-
static
|
|
1128
|
-
static getConfigSchemaNode(type) {
|
|
1570
|
+
static getConfigSchemaNode() {
|
|
1129
1571
|
const schema = this.configSchema();
|
|
1130
1572
|
if (!schema)
|
|
1131
1573
|
return;
|
|
1132
|
-
if (!
|
|
1574
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1133
1575
|
try {
|
|
1134
1576
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1135
|
-
|
|
1577
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1578
|
+
value: schemaNode,
|
|
1579
|
+
writable: true,
|
|
1580
|
+
configurable: true,
|
|
1581
|
+
enumerable: false
|
|
1582
|
+
});
|
|
1136
1583
|
} catch (error) {
|
|
1137
1584
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1138
1585
|
return;
|
|
1139
1586
|
}
|
|
1140
1587
|
}
|
|
1141
|
-
return this.
|
|
1588
|
+
return this.__compiledConfigSchema;
|
|
1142
1589
|
}
|
|
1143
1590
|
validateAndApplyConfigDefaults(config) {
|
|
1144
1591
|
const ctor = this.constructor;
|
|
1145
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1592
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1146
1593
|
if (!schemaNode)
|
|
1147
1594
|
return config;
|
|
1148
1595
|
const result = schemaNode.validate(config);
|
|
@@ -1155,7 +1602,6 @@ class Task {
|
|
|
1155
1602
|
}
|
|
1156
1603
|
return config;
|
|
1157
1604
|
}
|
|
1158
|
-
static _inputSchemaNode = new Map;
|
|
1159
1605
|
static generateInputSchemaNode(schema) {
|
|
1160
1606
|
if (typeof schema === "boolean") {
|
|
1161
1607
|
if (schema === false) {
|
|
@@ -1165,21 +1611,31 @@ class Task {
|
|
|
1165
1611
|
}
|
|
1166
1612
|
return compileSchema(schema);
|
|
1167
1613
|
}
|
|
1168
|
-
static getInputSchemaNode(
|
|
1169
|
-
if (!
|
|
1614
|
+
static getInputSchemaNode() {
|
|
1615
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1170
1616
|
const dataPortSchema = this.inputSchema();
|
|
1171
1617
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1172
1618
|
try {
|
|
1173
|
-
|
|
1619
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1620
|
+
value: schemaNode,
|
|
1621
|
+
writable: true,
|
|
1622
|
+
configurable: true,
|
|
1623
|
+
enumerable: false
|
|
1624
|
+
});
|
|
1174
1625
|
} catch (error) {
|
|
1175
1626
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1176
|
-
|
|
1627
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1628
|
+
value: compileSchema({}),
|
|
1629
|
+
writable: true,
|
|
1630
|
+
configurable: true,
|
|
1631
|
+
enumerable: false
|
|
1632
|
+
});
|
|
1177
1633
|
}
|
|
1178
1634
|
}
|
|
1179
|
-
return this.
|
|
1635
|
+
return this.__compiledInputSchema;
|
|
1180
1636
|
}
|
|
1181
|
-
getInputSchemaNode(
|
|
1182
|
-
return this.constructor.getInputSchemaNode(
|
|
1637
|
+
getInputSchemaNode() {
|
|
1638
|
+
return this.constructor.getInputSchemaNode();
|
|
1183
1639
|
}
|
|
1184
1640
|
async validateInput(input) {
|
|
1185
1641
|
const ctor = this.constructor;
|
|
@@ -1188,7 +1644,7 @@ class Task {
|
|
|
1188
1644
|
const instanceSchema = this.inputSchema();
|
|
1189
1645
|
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1190
1646
|
} else {
|
|
1191
|
-
schemaNode = this.getInputSchemaNode(
|
|
1647
|
+
schemaNode = this.getInputSchemaNode();
|
|
1192
1648
|
}
|
|
1193
1649
|
const result = schemaNode.validate(input);
|
|
1194
1650
|
if (!result.valid) {
|
|
@@ -1200,9 +1656,6 @@ class Task {
|
|
|
1200
1656
|
}
|
|
1201
1657
|
return true;
|
|
1202
1658
|
}
|
|
1203
|
-
id() {
|
|
1204
|
-
return this.config.id;
|
|
1205
|
-
}
|
|
1206
1659
|
stripSymbols(obj) {
|
|
1207
1660
|
if (obj === null || obj === undefined) {
|
|
1208
1661
|
return obj;
|
|
@@ -1224,14 +1677,15 @@ class Task {
|
|
|
1224
1677
|
}
|
|
1225
1678
|
return obj;
|
|
1226
1679
|
}
|
|
1227
|
-
toJSON() {
|
|
1680
|
+
toJSON(_options) {
|
|
1228
1681
|
const extras = this.config.extras;
|
|
1229
1682
|
const json = this.stripSymbols({
|
|
1230
|
-
id: this.
|
|
1683
|
+
id: this.id,
|
|
1231
1684
|
type: this.type,
|
|
1232
1685
|
defaults: this.defaults,
|
|
1233
1686
|
config: {
|
|
1234
1687
|
...this.config.title ? { title: this.config.title } : {},
|
|
1688
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1235
1689
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1236
1690
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1237
1691
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1239,8 +1693,8 @@ class Task {
|
|
|
1239
1693
|
});
|
|
1240
1694
|
return json;
|
|
1241
1695
|
}
|
|
1242
|
-
toDependencyJSON() {
|
|
1243
|
-
const json = this.toJSON();
|
|
1696
|
+
toDependencyJSON(options) {
|
|
1697
|
+
const json = this.toJSON(options);
|
|
1244
1698
|
return json;
|
|
1245
1699
|
}
|
|
1246
1700
|
hasChildren() {
|
|
@@ -1270,7 +1724,7 @@ class Task {
|
|
|
1270
1724
|
this.subGraph.removeDataflow(dataflow);
|
|
1271
1725
|
}
|
|
1272
1726
|
for (const child of this.subGraph.getTasks()) {
|
|
1273
|
-
this.subGraph.removeTask(child.
|
|
1727
|
+
this.subGraph.removeTask(child.id);
|
|
1274
1728
|
}
|
|
1275
1729
|
}
|
|
1276
1730
|
this.events.emit("regenerate");
|
|
@@ -1361,7 +1815,7 @@ class ConditionalTask extends Task {
|
|
|
1361
1815
|
}
|
|
1362
1816
|
}
|
|
1363
1817
|
} catch (error) {
|
|
1364
|
-
|
|
1818
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1365
1819
|
}
|
|
1366
1820
|
}
|
|
1367
1821
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1526,7 +1980,7 @@ class DependencyBasedScheduler {
|
|
|
1526
1980
|
if (task.status === TaskStatus.DISABLED) {
|
|
1527
1981
|
return false;
|
|
1528
1982
|
}
|
|
1529
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
1983
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1530
1984
|
if (sourceDataflows.length > 0) {
|
|
1531
1985
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1532
1986
|
if (allIncomingDisabled) {
|
|
@@ -1653,6 +2107,10 @@ class TaskGraphRunner {
|
|
|
1653
2107
|
graph.outputCache = outputCache;
|
|
1654
2108
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1655
2109
|
}
|
|
2110
|
+
runId = "";
|
|
2111
|
+
get timerLabel() {
|
|
2112
|
+
return `graph:${this.runId}`;
|
|
2113
|
+
}
|
|
1656
2114
|
async runGraph(input = {}, config) {
|
|
1657
2115
|
await this.handleStart(config);
|
|
1658
2116
|
const results = [];
|
|
@@ -1665,25 +2123,33 @@ class TaskGraphRunner {
|
|
|
1665
2123
|
if (this.failedTaskErrors.size > 0) {
|
|
1666
2124
|
break;
|
|
1667
2125
|
}
|
|
1668
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2126
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1669
2127
|
const runAsync = async () => {
|
|
2128
|
+
let errorRouted = false;
|
|
1670
2129
|
try {
|
|
1671
2130
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1672
2131
|
const taskPromise = this.runTask(task, taskInput);
|
|
1673
|
-
this.inProgressTasks.set(task.
|
|
2132
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1674
2133
|
const taskResult = await taskPromise;
|
|
1675
|
-
if (this.graph.getTargetDataflows(task.
|
|
2134
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1676
2135
|
results.push(taskResult);
|
|
1677
2136
|
}
|
|
1678
2137
|
} catch (error2) {
|
|
1679
|
-
this.
|
|
2138
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2139
|
+
errorRouted = true;
|
|
2140
|
+
this.pushErrorOutputToEdges(task);
|
|
2141
|
+
} else {
|
|
2142
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2143
|
+
}
|
|
1680
2144
|
} finally {
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
2145
|
+
if (!errorRouted) {
|
|
2146
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2147
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2148
|
+
}
|
|
2149
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1684
2150
|
}
|
|
1685
2151
|
};
|
|
1686
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2152
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1687
2153
|
}
|
|
1688
2154
|
} catch (err) {
|
|
1689
2155
|
error = err;
|
|
@@ -1707,7 +2173,7 @@ class TaskGraphRunner {
|
|
|
1707
2173
|
const results = [];
|
|
1708
2174
|
try {
|
|
1709
2175
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1710
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2176
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1711
2177
|
if (task.status === TaskStatus.PENDING) {
|
|
1712
2178
|
task.resetInputData();
|
|
1713
2179
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1715,9 +2181,9 @@ class TaskGraphRunner {
|
|
|
1715
2181
|
const taskInput = isRootTask ? input : {};
|
|
1716
2182
|
const taskResult = await task.runReactive(taskInput);
|
|
1717
2183
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1718
|
-
if (this.graph.getTargetDataflows(task.
|
|
2184
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1719
2185
|
results.push({
|
|
1720
|
-
id: task.
|
|
2186
|
+
id: task.id,
|
|
1721
2187
|
type: task.constructor.runtype || task.constructor.type,
|
|
1722
2188
|
data: taskResult
|
|
1723
2189
|
});
|
|
@@ -1737,7 +2203,7 @@ class TaskGraphRunner {
|
|
|
1737
2203
|
await this.handleDisable();
|
|
1738
2204
|
}
|
|
1739
2205
|
filterInputForTask(task, input) {
|
|
1740
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2206
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1741
2207
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1742
2208
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1743
2209
|
const filteredInput = {};
|
|
@@ -1776,13 +2242,13 @@ class TaskGraphRunner {
|
|
|
1776
2242
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1777
2243
|
}
|
|
1778
2244
|
copyInputFromEdgesToNode(task) {
|
|
1779
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2245
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1780
2246
|
for (const dataflow of dataflows) {
|
|
1781
2247
|
this.addInputData(task, dataflow.getPortData());
|
|
1782
2248
|
}
|
|
1783
2249
|
}
|
|
1784
2250
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1785
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2251
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1786
2252
|
for (const dataflow of dataflows) {
|
|
1787
2253
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1788
2254
|
if (compatibility === "static") {
|
|
@@ -1797,7 +2263,7 @@ class TaskGraphRunner {
|
|
|
1797
2263
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1798
2264
|
if (!node?.config?.id)
|
|
1799
2265
|
return;
|
|
1800
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2266
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1801
2267
|
const effectiveStatus = status ?? node.status;
|
|
1802
2268
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1803
2269
|
const branches = node.config.branches ?? [];
|
|
@@ -1828,10 +2294,31 @@ class TaskGraphRunner {
|
|
|
1828
2294
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1829
2295
|
if (!node?.config?.id)
|
|
1830
2296
|
return;
|
|
1831
|
-
graph.getTargetDataflows(node.
|
|
2297
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1832
2298
|
dataflow.error = node.error;
|
|
1833
2299
|
});
|
|
1834
2300
|
}
|
|
2301
|
+
hasErrorOutputEdges(task) {
|
|
2302
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2303
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2304
|
+
}
|
|
2305
|
+
pushErrorOutputToEdges(task) {
|
|
2306
|
+
const taskError = task.error;
|
|
2307
|
+
const errorData = {
|
|
2308
|
+
error: taskError?.message ?? "Unknown error",
|
|
2309
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2310
|
+
};
|
|
2311
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2312
|
+
for (const df of dataflows) {
|
|
2313
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2314
|
+
df.value = errorData;
|
|
2315
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2316
|
+
} else {
|
|
2317
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
this.propagateDisabledStatus(this.graph);
|
|
2321
|
+
}
|
|
1835
2322
|
propagateDisabledStatus(graph) {
|
|
1836
2323
|
let changed = true;
|
|
1837
2324
|
while (changed) {
|
|
@@ -1840,7 +2327,7 @@ class TaskGraphRunner {
|
|
|
1840
2327
|
if (task.status !== TaskStatus.PENDING) {
|
|
1841
2328
|
continue;
|
|
1842
2329
|
}
|
|
1843
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2330
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1844
2331
|
if (incomingDataflows.length === 0) {
|
|
1845
2332
|
continue;
|
|
1846
2333
|
}
|
|
@@ -1851,10 +2338,10 @@ class TaskGraphRunner {
|
|
|
1851
2338
|
task.completedAt = new Date;
|
|
1852
2339
|
task.emit("disabled");
|
|
1853
2340
|
task.emit("status", task.status);
|
|
1854
|
-
graph.getTargetDataflows(task.
|
|
2341
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1855
2342
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1856
2343
|
});
|
|
1857
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2344
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1858
2345
|
changed = true;
|
|
1859
2346
|
}
|
|
1860
2347
|
}
|
|
@@ -1863,7 +2350,7 @@ class TaskGraphRunner {
|
|
|
1863
2350
|
taskNeedsAccumulation(task) {
|
|
1864
2351
|
if (this.outputCache)
|
|
1865
2352
|
return true;
|
|
1866
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2353
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1867
2354
|
if (outEdges.length === 0)
|
|
1868
2355
|
return this.accumulateLeafOutputs;
|
|
1869
2356
|
const outSchema = task.outputSchema();
|
|
@@ -1886,7 +2373,7 @@ class TaskGraphRunner {
|
|
|
1886
2373
|
async runTask(task, input) {
|
|
1887
2374
|
const isStreamable = isTaskStreamable(task);
|
|
1888
2375
|
if (isStreamable) {
|
|
1889
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2376
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1890
2377
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1891
2378
|
if (streamingEdges.length > 0) {
|
|
1892
2379
|
const inputStreams = new Map;
|
|
@@ -1911,13 +2398,13 @@ class TaskGraphRunner {
|
|
|
1911
2398
|
});
|
|
1912
2399
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1913
2400
|
return {
|
|
1914
|
-
id: task.
|
|
2401
|
+
id: task.id,
|
|
1915
2402
|
type: task.constructor.runtype || task.constructor.type,
|
|
1916
2403
|
data: results
|
|
1917
2404
|
};
|
|
1918
2405
|
}
|
|
1919
2406
|
async awaitStreamInputs(task) {
|
|
1920
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2407
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1921
2408
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1922
2409
|
if (streamPromises.length > 0) {
|
|
1923
2410
|
await Promise.all(streamPromises);
|
|
@@ -1932,17 +2419,17 @@ class TaskGraphRunner {
|
|
|
1932
2419
|
streamingNotified = true;
|
|
1933
2420
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1934
2421
|
this.pushStreamToEdges(task, streamMode);
|
|
1935
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2422
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1936
2423
|
}
|
|
1937
2424
|
};
|
|
1938
2425
|
const onStreamStart = () => {
|
|
1939
|
-
this.graph.emit("task_stream_start", task.
|
|
2426
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1940
2427
|
};
|
|
1941
2428
|
const onStreamChunk = (event) => {
|
|
1942
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2429
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1943
2430
|
};
|
|
1944
2431
|
const onStreamEnd = (output) => {
|
|
1945
|
-
this.graph.emit("task_stream_end", task.
|
|
2432
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1946
2433
|
};
|
|
1947
2434
|
task.on("status", onStatus);
|
|
1948
2435
|
task.on("stream_start", onStreamStart);
|
|
@@ -1957,7 +2444,7 @@ class TaskGraphRunner {
|
|
|
1957
2444
|
});
|
|
1958
2445
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1959
2446
|
return {
|
|
1960
|
-
id: task.
|
|
2447
|
+
id: task.id,
|
|
1961
2448
|
type: task.constructor.runtype || task.constructor.type,
|
|
1962
2449
|
data: results
|
|
1963
2450
|
};
|
|
@@ -1995,7 +2482,7 @@ class TaskGraphRunner {
|
|
|
1995
2482
|
});
|
|
1996
2483
|
}
|
|
1997
2484
|
pushStreamToEdges(task, streamMode) {
|
|
1998
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2485
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1999
2486
|
if (targetDataflows.length === 0)
|
|
2000
2487
|
return;
|
|
2001
2488
|
const groups = new Map;
|
|
@@ -2086,11 +2573,15 @@ class TaskGraphRunner {
|
|
|
2086
2573
|
this.abortController?.abort();
|
|
2087
2574
|
}, { once: true });
|
|
2088
2575
|
}
|
|
2089
|
-
this.
|
|
2576
|
+
this.runId = uuid43();
|
|
2577
|
+
this.resetGraph(this.graph, this.runId);
|
|
2090
2578
|
this.processScheduler.reset();
|
|
2091
2579
|
this.inProgressTasks.clear();
|
|
2092
2580
|
this.inProgressFunctions.clear();
|
|
2093
2581
|
this.failedTaskErrors.clear();
|
|
2582
|
+
const logger = getLogger3();
|
|
2583
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2584
|
+
logger.time(this.timerLabel);
|
|
2094
2585
|
this.graph.emit("start");
|
|
2095
2586
|
}
|
|
2096
2587
|
async handleStartReactive() {
|
|
@@ -2102,6 +2593,9 @@ class TaskGraphRunner {
|
|
|
2102
2593
|
}
|
|
2103
2594
|
async handleComplete() {
|
|
2104
2595
|
this.running = false;
|
|
2596
|
+
const logger = getLogger3();
|
|
2597
|
+
logger.timeEnd(this.timerLabel);
|
|
2598
|
+
logger.groupEnd();
|
|
2105
2599
|
this.graph.emit("complete");
|
|
2106
2600
|
}
|
|
2107
2601
|
async handleCompleteReactive() {
|
|
@@ -2114,6 +2608,9 @@ class TaskGraphRunner {
|
|
|
2114
2608
|
}
|
|
2115
2609
|
}));
|
|
2116
2610
|
this.running = false;
|
|
2611
|
+
const logger = getLogger3();
|
|
2612
|
+
logger.timeEnd(this.timerLabel);
|
|
2613
|
+
logger.groupEnd();
|
|
2117
2614
|
this.graph.emit("error", error);
|
|
2118
2615
|
}
|
|
2119
2616
|
async handleErrorReactive() {
|
|
@@ -2126,6 +2623,9 @@ class TaskGraphRunner {
|
|
|
2126
2623
|
}
|
|
2127
2624
|
});
|
|
2128
2625
|
this.running = false;
|
|
2626
|
+
const logger = getLogger3();
|
|
2627
|
+
logger.timeEnd(this.timerLabel);
|
|
2628
|
+
logger.groupEnd();
|
|
2129
2629
|
this.graph.emit("abort");
|
|
2130
2630
|
}
|
|
2131
2631
|
async handleAbortReactive() {
|
|
@@ -2240,118 +2740,27 @@ class GraphAsTask extends Task {
|
|
|
2240
2740
|
if (!this.hasChildren()) {
|
|
2241
2741
|
return this.constructor.inputSchema();
|
|
2242
2742
|
}
|
|
2243
|
-
|
|
2244
|
-
const required = [];
|
|
2245
|
-
const tasks = this.subGraph.getTasks();
|
|
2246
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2247
|
-
for (const task of startingNodes) {
|
|
2248
|
-
const taskInputSchema = task.inputSchema();
|
|
2249
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2250
|
-
if (taskInputSchema === false) {
|
|
2251
|
-
continue;
|
|
2252
|
-
}
|
|
2253
|
-
if (taskInputSchema === true) {
|
|
2254
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2255
|
-
continue;
|
|
2256
|
-
}
|
|
2257
|
-
}
|
|
2258
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2259
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2260
|
-
if (!properties[inputName]) {
|
|
2261
|
-
properties[inputName] = inputProp;
|
|
2262
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2263
|
-
required.push(inputName);
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
}
|
|
2268
|
-
return {
|
|
2269
|
-
type: "object",
|
|
2270
|
-
properties,
|
|
2271
|
-
...required.length > 0 ? { required } : {},
|
|
2272
|
-
additionalProperties: false
|
|
2273
|
-
};
|
|
2743
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2274
2744
|
}
|
|
2275
2745
|
_inputSchemaNode;
|
|
2276
|
-
getInputSchemaNode(
|
|
2746
|
+
getInputSchemaNode() {
|
|
2277
2747
|
if (!this._inputSchemaNode) {
|
|
2278
|
-
const dataPortSchema = this.inputSchema();
|
|
2279
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2280
2748
|
try {
|
|
2749
|
+
const dataPortSchema = this.inputSchema();
|
|
2750
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2281
2751
|
this._inputSchemaNode = schemaNode;
|
|
2282
2752
|
} catch (error) {
|
|
2283
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2753
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2284
2754
|
this._inputSchemaNode = compileSchema2({});
|
|
2285
2755
|
}
|
|
2286
2756
|
}
|
|
2287
2757
|
return this._inputSchemaNode;
|
|
2288
2758
|
}
|
|
2289
|
-
calculateNodeDepths() {
|
|
2290
|
-
const depths = new Map;
|
|
2291
|
-
const tasks = this.subGraph.getTasks();
|
|
2292
|
-
for (const task of tasks) {
|
|
2293
|
-
depths.set(task.config.id, 0);
|
|
2294
|
-
}
|
|
2295
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2296
|
-
for (const task of sortedTasks) {
|
|
2297
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2298
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2299
|
-
for (const targetTask of targetTasks) {
|
|
2300
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2301
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2302
|
-
}
|
|
2303
|
-
}
|
|
2304
|
-
return depths;
|
|
2305
|
-
}
|
|
2306
2759
|
outputSchema() {
|
|
2307
2760
|
if (!this.hasChildren()) {
|
|
2308
2761
|
return this.constructor.outputSchema();
|
|
2309
2762
|
}
|
|
2310
|
-
|
|
2311
|
-
const required = [];
|
|
2312
|
-
const tasks = this.subGraph.getTasks();
|
|
2313
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2314
|
-
const depths = this.calculateNodeDepths();
|
|
2315
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2316
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2317
|
-
const propertyCount = {};
|
|
2318
|
-
const propertySchema = {};
|
|
2319
|
-
for (const task of lastLevelNodes) {
|
|
2320
|
-
const taskOutputSchema = task.outputSchema();
|
|
2321
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2322
|
-
if (taskOutputSchema === false) {
|
|
2323
|
-
continue;
|
|
2324
|
-
}
|
|
2325
|
-
if (taskOutputSchema === true) {
|
|
2326
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2327
|
-
continue;
|
|
2328
|
-
}
|
|
2329
|
-
}
|
|
2330
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2331
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2332
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2333
|
-
if (!propertySchema[outputName]) {
|
|
2334
|
-
propertySchema[outputName] = outputProp;
|
|
2335
|
-
}
|
|
2336
|
-
}
|
|
2337
|
-
}
|
|
2338
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2339
|
-
const outputProp = propertySchema[outputName];
|
|
2340
|
-
if (lastLevelNodes.length === 1) {
|
|
2341
|
-
properties[outputName] = outputProp;
|
|
2342
|
-
} else {
|
|
2343
|
-
properties[outputName] = {
|
|
2344
|
-
type: "array",
|
|
2345
|
-
items: outputProp
|
|
2346
|
-
};
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
return {
|
|
2350
|
-
type: "object",
|
|
2351
|
-
properties,
|
|
2352
|
-
...required.length > 0 ? { required } : {},
|
|
2353
|
-
additionalProperties: false
|
|
2354
|
-
};
|
|
2763
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2355
2764
|
}
|
|
2356
2765
|
resetInputData() {
|
|
2357
2766
|
super.resetInputData();
|
|
@@ -2386,8 +2795,8 @@ class GraphAsTask extends Task {
|
|
|
2386
2795
|
const endingNodeIds = new Set;
|
|
2387
2796
|
const tasks = this.subGraph.getTasks();
|
|
2388
2797
|
for (const task of tasks) {
|
|
2389
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2390
|
-
endingNodeIds.add(task.
|
|
2798
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2799
|
+
endingNodeIds.add(task.id);
|
|
2391
2800
|
}
|
|
2392
2801
|
}
|
|
2393
2802
|
const eventQueue = [];
|
|
@@ -2431,32 +2840,36 @@ class GraphAsTask extends Task {
|
|
|
2431
2840
|
this._inputSchemaNode = undefined;
|
|
2432
2841
|
this.events.emit("regenerate");
|
|
2433
2842
|
}
|
|
2434
|
-
toJSON() {
|
|
2435
|
-
let json = super.toJSON();
|
|
2843
|
+
toJSON(options) {
|
|
2844
|
+
let json = super.toJSON(options);
|
|
2436
2845
|
const hasChildren = this.hasChildren();
|
|
2437
2846
|
if (hasChildren) {
|
|
2438
2847
|
json = {
|
|
2439
2848
|
...json,
|
|
2440
2849
|
merge: this.compoundMerge,
|
|
2441
|
-
subgraph: this.subGraph.toJSON()
|
|
2850
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2442
2851
|
};
|
|
2443
2852
|
}
|
|
2444
2853
|
return json;
|
|
2445
2854
|
}
|
|
2446
|
-
toDependencyJSON() {
|
|
2447
|
-
const json = this.toJSON();
|
|
2855
|
+
toDependencyJSON(options) {
|
|
2856
|
+
const json = this.toJSON(options);
|
|
2448
2857
|
if (this.hasChildren()) {
|
|
2449
2858
|
if ("subgraph" in json) {
|
|
2450
2859
|
delete json.subgraph;
|
|
2451
2860
|
}
|
|
2452
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2861
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2453
2862
|
}
|
|
2454
2863
|
return json;
|
|
2455
2864
|
}
|
|
2456
2865
|
}
|
|
2457
2866
|
|
|
2458
2867
|
// src/task-graph/Workflow.ts
|
|
2459
|
-
import {
|
|
2868
|
+
import {
|
|
2869
|
+
EventEmitter as EventEmitter4,
|
|
2870
|
+
getLogger as getLogger4,
|
|
2871
|
+
uuid4 as uuid44
|
|
2872
|
+
} from "@workglow/util";
|
|
2460
2873
|
function CreateWorkflow(taskClass) {
|
|
2461
2874
|
return Workflow.createWorkflow(taskClass);
|
|
2462
2875
|
}
|
|
@@ -2557,43 +2970,48 @@ class Workflow {
|
|
|
2557
2970
|
const helper = function(input = {}, config = {}) {
|
|
2558
2971
|
this._error = "";
|
|
2559
2972
|
const parent = getLastTask(this);
|
|
2560
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
2973
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2561
2974
|
if (this._dataFlows.length > 0) {
|
|
2562
2975
|
this._dataFlows.forEach((dataflow) => {
|
|
2563
2976
|
const taskSchema = task.inputSchema();
|
|
2564
2977
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2565
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2566
|
-
|
|
2978
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
2979
|
+
getLogger4().error(this._error);
|
|
2567
2980
|
return;
|
|
2568
2981
|
}
|
|
2569
|
-
dataflow.targetTaskId = task.
|
|
2982
|
+
dataflow.targetTaskId = task.id;
|
|
2570
2983
|
this.graph.addDataflow(dataflow);
|
|
2571
2984
|
});
|
|
2572
2985
|
this._dataFlows = [];
|
|
2573
2986
|
}
|
|
2574
|
-
if (parent
|
|
2987
|
+
if (parent) {
|
|
2575
2988
|
const nodes = this._graph.getTasks();
|
|
2576
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
2989
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2577
2990
|
const earlierTasks = [];
|
|
2578
2991
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2579
2992
|
earlierTasks.push(nodes[i]);
|
|
2580
2993
|
}
|
|
2581
2994
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2995
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2582
2996
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2583
2997
|
providedInputKeys,
|
|
2998
|
+
connectedInputKeys,
|
|
2584
2999
|
earlierTasks
|
|
2585
3000
|
});
|
|
2586
3001
|
if (result.error) {
|
|
2587
3002
|
if (this.isLoopBuilder) {
|
|
2588
3003
|
this._error = result.error;
|
|
2589
|
-
|
|
3004
|
+
getLogger4().warn(this._error);
|
|
2590
3005
|
} else {
|
|
2591
3006
|
this._error = result.error + " Task not added.";
|
|
2592
|
-
|
|
2593
|
-
this.graph.removeTask(task.
|
|
3007
|
+
getLogger4().error(this._error);
|
|
3008
|
+
this.graph.removeTask(task.id);
|
|
2594
3009
|
}
|
|
2595
3010
|
}
|
|
2596
3011
|
}
|
|
3012
|
+
if (!this._error) {
|
|
3013
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3014
|
+
}
|
|
2597
3015
|
return this;
|
|
2598
3016
|
};
|
|
2599
3017
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2673,18 +3091,18 @@ class Workflow {
|
|
|
2673
3091
|
const nodes = this._graph.getTasks();
|
|
2674
3092
|
if (nodes.length === 0) {
|
|
2675
3093
|
this._error = "No tasks to remove";
|
|
2676
|
-
|
|
3094
|
+
getLogger4().error(this._error);
|
|
2677
3095
|
return this;
|
|
2678
3096
|
}
|
|
2679
3097
|
const lastNode = nodes[nodes.length - 1];
|
|
2680
|
-
this._graph.removeTask(lastNode.
|
|
3098
|
+
this._graph.removeTask(lastNode.id);
|
|
2681
3099
|
return this;
|
|
2682
3100
|
}
|
|
2683
|
-
toJSON() {
|
|
2684
|
-
return this._graph.toJSON();
|
|
3101
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3102
|
+
return this._graph.toJSON(options);
|
|
2685
3103
|
}
|
|
2686
|
-
toDependencyJSON() {
|
|
2687
|
-
return this._graph.toDependencyJSON();
|
|
3104
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3105
|
+
return this._graph.toDependencyJSON(options);
|
|
2688
3106
|
}
|
|
2689
3107
|
pipe(...args) {
|
|
2690
3108
|
return pipe(args, this);
|
|
@@ -2704,25 +3122,40 @@ class Workflow {
|
|
|
2704
3122
|
if (-index > nodes.length) {
|
|
2705
3123
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2706
3124
|
this._error = errorMsg;
|
|
2707
|
-
|
|
3125
|
+
getLogger4().error(this._error);
|
|
2708
3126
|
throw new WorkflowError(errorMsg);
|
|
2709
3127
|
}
|
|
2710
3128
|
const lastNode = nodes[nodes.length + index];
|
|
2711
3129
|
const outputSchema = lastNode.outputSchema();
|
|
2712
3130
|
if (typeof outputSchema === "boolean") {
|
|
2713
3131
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2714
|
-
const errorMsg = `Task ${lastNode.
|
|
3132
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2715
3133
|
this._error = errorMsg;
|
|
2716
|
-
|
|
3134
|
+
getLogger4().error(this._error);
|
|
2717
3135
|
throw new WorkflowError(errorMsg);
|
|
2718
3136
|
}
|
|
2719
3137
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2720
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3138
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2721
3139
|
this._error = errorMsg;
|
|
2722
|
-
|
|
3140
|
+
getLogger4().error(this._error);
|
|
2723
3141
|
throw new WorkflowError(errorMsg);
|
|
2724
3142
|
}
|
|
2725
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3143
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3144
|
+
return this;
|
|
3145
|
+
}
|
|
3146
|
+
onError(handler) {
|
|
3147
|
+
this._error = "";
|
|
3148
|
+
const parent = getLastTask(this);
|
|
3149
|
+
if (!parent) {
|
|
3150
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3151
|
+
getLogger4().error(this._error);
|
|
3152
|
+
throw new WorkflowError(this._error);
|
|
3153
|
+
}
|
|
3154
|
+
const handlerTask = ensureTask(handler);
|
|
3155
|
+
this.graph.addTask(handlerTask);
|
|
3156
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3157
|
+
this.graph.addDataflow(dataflow);
|
|
3158
|
+
this.events.emit("changed", handlerTask.id);
|
|
2726
3159
|
return this;
|
|
2727
3160
|
}
|
|
2728
3161
|
toTaskGraph() {
|
|
@@ -2807,16 +3240,16 @@ class Workflow {
|
|
|
2807
3240
|
addLoopTask(taskClass, config = {}) {
|
|
2808
3241
|
this._error = "";
|
|
2809
3242
|
const parent = getLastTask(this);
|
|
2810
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3243
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2811
3244
|
if (this._dataFlows.length > 0) {
|
|
2812
3245
|
this._dataFlows.forEach((dataflow) => {
|
|
2813
3246
|
const taskSchema = task.inputSchema();
|
|
2814
3247
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2815
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2816
|
-
|
|
3248
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3249
|
+
getLogger4().error(this._error);
|
|
2817
3250
|
return;
|
|
2818
3251
|
}
|
|
2819
|
-
dataflow.targetTaskId = task.
|
|
3252
|
+
dataflow.targetTaskId = task.id;
|
|
2820
3253
|
this.graph.addDataflow(dataflow);
|
|
2821
3254
|
});
|
|
2822
3255
|
this._dataFlows = [];
|
|
@@ -2831,9 +3264,9 @@ class Workflow {
|
|
|
2831
3264
|
if (!pending)
|
|
2832
3265
|
return;
|
|
2833
3266
|
const { parent, iteratorTask } = pending;
|
|
2834
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3267
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2835
3268
|
const nodes = this._graph.getTasks();
|
|
2836
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3269
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2837
3270
|
const earlierTasks = [];
|
|
2838
3271
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2839
3272
|
earlierTasks.push(nodes[i]);
|
|
@@ -2843,8 +3276,81 @@ class Workflow {
|
|
|
2843
3276
|
});
|
|
2844
3277
|
if (result.error) {
|
|
2845
3278
|
this._error = result.error + " Task not added.";
|
|
2846
|
-
|
|
2847
|
-
this.graph.removeTask(iteratorTask.
|
|
3279
|
+
getLogger4().error(this._error);
|
|
3280
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3285
|
+
const tasks = graph.getTasks();
|
|
3286
|
+
for (const task of tasks) {
|
|
3287
|
+
if (task.type === "InputTask") {
|
|
3288
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3289
|
+
if (outgoing.length === 0)
|
|
3290
|
+
continue;
|
|
3291
|
+
const properties = {};
|
|
3292
|
+
const required = [];
|
|
3293
|
+
for (const df of outgoing) {
|
|
3294
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3295
|
+
if (!targetTask)
|
|
3296
|
+
continue;
|
|
3297
|
+
const targetSchema = targetTask.inputSchema();
|
|
3298
|
+
if (typeof targetSchema === "boolean")
|
|
3299
|
+
continue;
|
|
3300
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3301
|
+
if (prop && typeof prop !== "boolean") {
|
|
3302
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3303
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3304
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3305
|
+
required.push(df.sourceTaskPortId);
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
}
|
|
3310
|
+
const schema = {
|
|
3311
|
+
type: "object",
|
|
3312
|
+
properties,
|
|
3313
|
+
...required.length > 0 ? { required } : {},
|
|
3314
|
+
additionalProperties: false
|
|
3315
|
+
};
|
|
3316
|
+
task.config = {
|
|
3317
|
+
...task.config,
|
|
3318
|
+
inputSchema: schema,
|
|
3319
|
+
outputSchema: schema
|
|
3320
|
+
};
|
|
3321
|
+
}
|
|
3322
|
+
if (task.type === "OutputTask") {
|
|
3323
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3324
|
+
if (incoming.length === 0)
|
|
3325
|
+
continue;
|
|
3326
|
+
const properties = {};
|
|
3327
|
+
const required = [];
|
|
3328
|
+
for (const df of incoming) {
|
|
3329
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3330
|
+
if (!sourceTask)
|
|
3331
|
+
continue;
|
|
3332
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3333
|
+
if (typeof sourceSchema === "boolean")
|
|
3334
|
+
continue;
|
|
3335
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3336
|
+
if (prop && typeof prop !== "boolean") {
|
|
3337
|
+
properties[df.targetTaskPortId] = prop;
|
|
3338
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3339
|
+
required.push(df.targetTaskPortId);
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
}
|
|
3343
|
+
const schema = {
|
|
3344
|
+
type: "object",
|
|
3345
|
+
properties,
|
|
3346
|
+
...required.length > 0 ? { required } : {},
|
|
3347
|
+
additionalProperties: false
|
|
3348
|
+
};
|
|
3349
|
+
task.config = {
|
|
3350
|
+
...task.config,
|
|
3351
|
+
inputSchema: schema,
|
|
3352
|
+
outputSchema: schema
|
|
3353
|
+
};
|
|
2848
3354
|
}
|
|
2849
3355
|
}
|
|
2850
3356
|
}
|
|
@@ -2854,6 +3360,7 @@ class Workflow {
|
|
|
2854
3360
|
const sourceSchema = sourceTask.outputSchema();
|
|
2855
3361
|
const targetSchema = targetTask.inputSchema();
|
|
2856
3362
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3363
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2857
3364
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2858
3365
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2859
3366
|
const formats = new Set;
|
|
@@ -2929,18 +3436,33 @@ class Workflow {
|
|
|
2929
3436
|
if (typeof fromSchema === "object") {
|
|
2930
3437
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2931
3438
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3439
|
+
if (matches.has(fromOutputPortId))
|
|
3440
|
+
continue;
|
|
2932
3441
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2933
3442
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2934
3443
|
}
|
|
2935
3444
|
return;
|
|
2936
3445
|
}
|
|
2937
3446
|
}
|
|
3447
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3448
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3449
|
+
if (matches.has(toInputPortId))
|
|
3450
|
+
continue;
|
|
3451
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3452
|
+
continue;
|
|
3453
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3454
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3455
|
+
}
|
|
3456
|
+
return;
|
|
3457
|
+
}
|
|
2938
3458
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2939
3459
|
return;
|
|
2940
3460
|
}
|
|
2941
3461
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2942
3462
|
if (matches.has(toInputPortId))
|
|
2943
3463
|
continue;
|
|
3464
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3465
|
+
continue;
|
|
2944
3466
|
const candidates = [];
|
|
2945
3467
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2946
3468
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2960,22 +3482,32 @@ class Workflow {
|
|
|
2960
3482
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2961
3483
|
}
|
|
2962
3484
|
};
|
|
2963
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3485
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2964
3486
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2965
3487
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2966
3488
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2967
3489
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2968
3490
|
});
|
|
2969
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3491
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2970
3492
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2971
3493
|
});
|
|
2972
3494
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2973
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3495
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2974
3496
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2975
3497
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2976
3498
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2977
3499
|
const earlierTask = earlierTasks[i];
|
|
2978
3500
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3501
|
+
if (earlierTask.type === "InputTask") {
|
|
3502
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3503
|
+
if (matches.has(requiredInputId))
|
|
3504
|
+
continue;
|
|
3505
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3506
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3507
|
+
}
|
|
3508
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3509
|
+
continue;
|
|
3510
|
+
}
|
|
2979
3511
|
const makeMatchFromEarlier = (comparator) => {
|
|
2980
3512
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2981
3513
|
return;
|
|
@@ -2985,7 +3517,7 @@ class Workflow {
|
|
|
2985
3517
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2986
3518
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2987
3519
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2988
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3520
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2989
3521
|
}
|
|
2990
3522
|
}
|
|
2991
3523
|
}
|
|
@@ -3011,6 +3543,10 @@ class Workflow {
|
|
|
3011
3543
|
};
|
|
3012
3544
|
}
|
|
3013
3545
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3546
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3547
|
+
if (existingTargetConnections.length > 0) {
|
|
3548
|
+
return { matches, unmatchedRequired: [] };
|
|
3549
|
+
}
|
|
3014
3550
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
3015
3551
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
3016
3552
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3133,7 +3669,7 @@ function getLastTask(workflow) {
|
|
|
3133
3669
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3134
3670
|
}
|
|
3135
3671
|
function connect(source, target, workflow) {
|
|
3136
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3672
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3137
3673
|
}
|
|
3138
3674
|
function pipe(args, workflow = new Workflow) {
|
|
3139
3675
|
let previousTask = getLastTask(workflow);
|
|
@@ -3189,7 +3725,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3189
3725
|
// src/task-graph/TaskGraph.ts
|
|
3190
3726
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3191
3727
|
constructor() {
|
|
3192
|
-
super((task) => task.
|
|
3728
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3193
3729
|
}
|
|
3194
3730
|
}
|
|
3195
3731
|
|
|
@@ -3286,18 +3822,22 @@ class TaskGraph {
|
|
|
3286
3822
|
return this._dag.removeNode(taskId);
|
|
3287
3823
|
}
|
|
3288
3824
|
resetGraph() {
|
|
3289
|
-
this.runner.resetGraph(this,
|
|
3825
|
+
this.runner.resetGraph(this, uuid45());
|
|
3290
3826
|
}
|
|
3291
|
-
toJSON() {
|
|
3292
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3827
|
+
toJSON(options) {
|
|
3828
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3293
3829
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3294
|
-
|
|
3830
|
+
let json = {
|
|
3295
3831
|
tasks,
|
|
3296
3832
|
dataflows
|
|
3297
3833
|
};
|
|
3834
|
+
if (options?.withBoundaryNodes) {
|
|
3835
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3836
|
+
}
|
|
3837
|
+
return json;
|
|
3298
3838
|
}
|
|
3299
|
-
toDependencyJSON() {
|
|
3300
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3839
|
+
toDependencyJSON(options) {
|
|
3840
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3301
3841
|
this.getDataflows().forEach((df) => {
|
|
3302
3842
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3303
3843
|
if (!target.dependencies) {
|
|
@@ -3323,6 +3863,9 @@ class TaskGraph {
|
|
|
3323
3863
|
}
|
|
3324
3864
|
}
|
|
3325
3865
|
});
|
|
3866
|
+
if (options?.withBoundaryNodes) {
|
|
3867
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3868
|
+
}
|
|
3326
3869
|
return tasks;
|
|
3327
3870
|
}
|
|
3328
3871
|
get events() {
|
|
@@ -3341,7 +3884,7 @@ class TaskGraph {
|
|
|
3341
3884
|
const tasks = this.getTasks();
|
|
3342
3885
|
tasks.forEach((task) => {
|
|
3343
3886
|
const unsub = task.subscribe("status", (status) => {
|
|
3344
|
-
callback(task.
|
|
3887
|
+
callback(task.id, status);
|
|
3345
3888
|
});
|
|
3346
3889
|
unsubscribes.push(unsub);
|
|
3347
3890
|
});
|
|
@@ -3350,7 +3893,7 @@ class TaskGraph {
|
|
|
3350
3893
|
if (!task || typeof task.subscribe !== "function")
|
|
3351
3894
|
return;
|
|
3352
3895
|
const unsub = task.subscribe("status", (status) => {
|
|
3353
|
-
callback(task.
|
|
3896
|
+
callback(task.id, status);
|
|
3354
3897
|
});
|
|
3355
3898
|
unsubscribes.push(unsub);
|
|
3356
3899
|
};
|
|
@@ -3365,7 +3908,7 @@ class TaskGraph {
|
|
|
3365
3908
|
const tasks = this.getTasks();
|
|
3366
3909
|
tasks.forEach((task) => {
|
|
3367
3910
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3368
|
-
callback(task.
|
|
3911
|
+
callback(task.id, progress, message, ...args);
|
|
3369
3912
|
});
|
|
3370
3913
|
unsubscribes.push(unsub);
|
|
3371
3914
|
});
|
|
@@ -3374,7 +3917,7 @@ class TaskGraph {
|
|
|
3374
3917
|
if (!task || typeof task.subscribe !== "function")
|
|
3375
3918
|
return;
|
|
3376
3919
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3377
|
-
callback(task.
|
|
3920
|
+
callback(task.id, progress, message, ...args);
|
|
3378
3921
|
});
|
|
3379
3922
|
unsubscribes.push(unsub);
|
|
3380
3923
|
};
|
|
@@ -3459,7 +4002,7 @@ class TaskGraph {
|
|
|
3459
4002
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3460
4003
|
const edges = [];
|
|
3461
4004
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3462
|
-
edges.push(new Dataflow(tasks[i].
|
|
4005
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3463
4006
|
}
|
|
3464
4007
|
return edges;
|
|
3465
4008
|
}
|
|
@@ -3469,9 +4012,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3469
4012
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3470
4013
|
return graph;
|
|
3471
4014
|
}
|
|
4015
|
+
// src/task/FallbackTaskRunner.ts
|
|
4016
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4017
|
+
async executeTask(input) {
|
|
4018
|
+
if (this.task.fallbackMode === "data") {
|
|
4019
|
+
return this.executeDataFallback(input);
|
|
4020
|
+
}
|
|
4021
|
+
return this.executeTaskFallback(input);
|
|
4022
|
+
}
|
|
4023
|
+
async executeTaskReactive(input, output) {
|
|
4024
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4025
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4026
|
+
}
|
|
4027
|
+
async executeTaskFallback(input) {
|
|
4028
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4029
|
+
if (tasks.length === 0) {
|
|
4030
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4031
|
+
}
|
|
4032
|
+
const errors = [];
|
|
4033
|
+
const totalAttempts = tasks.length;
|
|
4034
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4035
|
+
if (this.abortController?.signal.aborted) {
|
|
4036
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4037
|
+
}
|
|
4038
|
+
const alternativeTask = tasks[i];
|
|
4039
|
+
const attemptNumber = i + 1;
|
|
4040
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4041
|
+
try {
|
|
4042
|
+
this.resetTask(alternativeTask);
|
|
4043
|
+
const result = await alternativeTask.run(input);
|
|
4044
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4045
|
+
return await this.executeTaskReactive(input, result);
|
|
4046
|
+
} catch (error) {
|
|
4047
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4048
|
+
throw error;
|
|
4049
|
+
}
|
|
4050
|
+
errors.push({ task: alternativeTask, error });
|
|
4051
|
+
}
|
|
4052
|
+
}
|
|
4053
|
+
throw this.buildAggregateError(errors, "task");
|
|
4054
|
+
}
|
|
4055
|
+
async executeDataFallback(input) {
|
|
4056
|
+
const alternatives = this.task.alternatives;
|
|
4057
|
+
if (alternatives.length === 0) {
|
|
4058
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4059
|
+
}
|
|
4060
|
+
const errors = [];
|
|
4061
|
+
const totalAttempts = alternatives.length;
|
|
4062
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4063
|
+
if (this.abortController?.signal.aborted) {
|
|
4064
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4065
|
+
}
|
|
4066
|
+
const alternative = alternatives[i];
|
|
4067
|
+
const attemptNumber = i + 1;
|
|
4068
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4069
|
+
try {
|
|
4070
|
+
this.resetSubgraph();
|
|
4071
|
+
const mergedInput = { ...input, ...alternative };
|
|
4072
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4073
|
+
parentSignal: this.abortController?.signal,
|
|
4074
|
+
outputCache: this.outputCache
|
|
4075
|
+
});
|
|
4076
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4077
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4078
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4079
|
+
} catch (error) {
|
|
4080
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4081
|
+
throw error;
|
|
4082
|
+
}
|
|
4083
|
+
errors.push({ alternative, error });
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
throw this.buildAggregateError(errors, "data");
|
|
4087
|
+
}
|
|
4088
|
+
resetTask(task) {
|
|
4089
|
+
task.status = TaskStatus.PENDING;
|
|
4090
|
+
task.progress = 0;
|
|
4091
|
+
task.error = undefined;
|
|
4092
|
+
task.completedAt = undefined;
|
|
4093
|
+
task.startedAt = undefined;
|
|
4094
|
+
task.resetInputData();
|
|
4095
|
+
}
|
|
4096
|
+
resetSubgraph() {
|
|
4097
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4098
|
+
this.resetTask(task);
|
|
4099
|
+
}
|
|
4100
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4101
|
+
dataflow.reset();
|
|
4102
|
+
}
|
|
4103
|
+
}
|
|
4104
|
+
buildAggregateError(errors, mode) {
|
|
4105
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4106
|
+
const details = errors.map((e, i) => {
|
|
4107
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4108
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4109
|
+
}).join(`
|
|
4110
|
+
`);
|
|
4111
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4112
|
+
${details}`);
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
|
|
4116
|
+
// src/task/FallbackTask.ts
|
|
4117
|
+
var fallbackTaskConfigSchema = {
|
|
4118
|
+
type: "object",
|
|
4119
|
+
properties: {
|
|
4120
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4121
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4122
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4123
|
+
},
|
|
4124
|
+
additionalProperties: false
|
|
4125
|
+
};
|
|
4126
|
+
|
|
4127
|
+
class FallbackTask extends GraphAsTask {
|
|
4128
|
+
static type = "FallbackTask";
|
|
4129
|
+
static category = "Flow Control";
|
|
4130
|
+
static title = "Fallback";
|
|
4131
|
+
static description = "Try alternatives until one succeeds";
|
|
4132
|
+
static hasDynamicSchemas = true;
|
|
4133
|
+
static configSchema() {
|
|
4134
|
+
return fallbackTaskConfigSchema;
|
|
4135
|
+
}
|
|
4136
|
+
get runner() {
|
|
4137
|
+
if (!this._runner) {
|
|
4138
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4139
|
+
}
|
|
4140
|
+
return this._runner;
|
|
4141
|
+
}
|
|
4142
|
+
get fallbackMode() {
|
|
4143
|
+
return this.config?.fallbackMode ?? "task";
|
|
4144
|
+
}
|
|
4145
|
+
get alternatives() {
|
|
4146
|
+
return this.config?.alternatives ?? [];
|
|
4147
|
+
}
|
|
4148
|
+
inputSchema() {
|
|
4149
|
+
if (!this.hasChildren()) {
|
|
4150
|
+
return this.constructor.inputSchema();
|
|
4151
|
+
}
|
|
4152
|
+
if (this.fallbackMode === "data") {
|
|
4153
|
+
return super.inputSchema();
|
|
4154
|
+
}
|
|
4155
|
+
const properties = {};
|
|
4156
|
+
const tasks = this.subGraph.getTasks();
|
|
4157
|
+
for (const task of tasks) {
|
|
4158
|
+
const taskInputSchema = task.inputSchema();
|
|
4159
|
+
if (typeof taskInputSchema === "boolean")
|
|
4160
|
+
continue;
|
|
4161
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4162
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4163
|
+
if (!properties[inputName]) {
|
|
4164
|
+
properties[inputName] = inputProp;
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
return {
|
|
4169
|
+
type: "object",
|
|
4170
|
+
properties,
|
|
4171
|
+
additionalProperties: true
|
|
4172
|
+
};
|
|
4173
|
+
}
|
|
4174
|
+
outputSchema() {
|
|
4175
|
+
if (!this.hasChildren()) {
|
|
4176
|
+
return this.constructor.outputSchema();
|
|
4177
|
+
}
|
|
4178
|
+
const tasks = this.subGraph.getTasks();
|
|
4179
|
+
if (tasks.length === 0) {
|
|
4180
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4181
|
+
}
|
|
4182
|
+
if (this.fallbackMode === "task") {
|
|
4183
|
+
const firstTask = tasks[0];
|
|
4184
|
+
return firstTask.outputSchema();
|
|
4185
|
+
}
|
|
4186
|
+
return super.outputSchema();
|
|
4187
|
+
}
|
|
4188
|
+
toJSON() {
|
|
4189
|
+
const json = super.toJSON();
|
|
4190
|
+
return {
|
|
4191
|
+
...json,
|
|
4192
|
+
config: {
|
|
4193
|
+
..."config" in json ? json.config : {},
|
|
4194
|
+
fallbackMode: this.fallbackMode,
|
|
4195
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4196
|
+
}
|
|
4197
|
+
};
|
|
4198
|
+
}
|
|
4199
|
+
}
|
|
4200
|
+
queueMicrotask(() => {
|
|
4201
|
+
Workflow.prototype.fallback = function() {
|
|
4202
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4203
|
+
};
|
|
4204
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4205
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4206
|
+
return this.addLoopTask(FallbackTask, {
|
|
4207
|
+
fallbackMode: "data",
|
|
4208
|
+
alternatives
|
|
4209
|
+
});
|
|
4210
|
+
};
|
|
4211
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4212
|
+
});
|
|
3472
4213
|
// src/task/IteratorTaskRunner.ts
|
|
3473
4214
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3474
|
-
subGraphRunChain = Promise.resolve();
|
|
3475
4215
|
async executeTask(input) {
|
|
3476
4216
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3477
4217
|
if (analysis.iterationCount === 0) {
|
|
@@ -3493,13 +4233,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3493
4233
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3494
4234
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3495
4235
|
const completionOrderResults = [];
|
|
4236
|
+
let completedCount = 0;
|
|
3496
4237
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3497
4238
|
if (this.abortController?.signal.aborted) {
|
|
3498
4239
|
break;
|
|
3499
4240
|
}
|
|
3500
4241
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3501
4242
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3502
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4243
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4244
|
+
completedCount++;
|
|
4245
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4246
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4247
|
+
});
|
|
3503
4248
|
for (const { index, result } of batchResults) {
|
|
3504
4249
|
if (result === undefined)
|
|
3505
4250
|
continue;
|
|
@@ -3509,8 +4254,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3509
4254
|
completionOrderResults.push(result);
|
|
3510
4255
|
}
|
|
3511
4256
|
}
|
|
3512
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3513
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3514
4257
|
}
|
|
3515
4258
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3516
4259
|
return this.task.collectResults(collected);
|
|
@@ -3532,7 +4275,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3532
4275
|
}
|
|
3533
4276
|
return accumulator;
|
|
3534
4277
|
}
|
|
3535
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4278
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3536
4279
|
const results = [];
|
|
3537
4280
|
let cursor = 0;
|
|
3538
4281
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3550,33 +4293,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3550
4293
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3551
4294
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3552
4295
|
results.push({ index, result });
|
|
4296
|
+
await onItemComplete?.();
|
|
3553
4297
|
}
|
|
3554
4298
|
});
|
|
3555
4299
|
await Promise.all(workers);
|
|
3556
4300
|
return results;
|
|
3557
4301
|
}
|
|
4302
|
+
cloneGraph(graph) {
|
|
4303
|
+
const clone = new TaskGraph;
|
|
4304
|
+
for (const task of graph.getTasks()) {
|
|
4305
|
+
const ctor = task.constructor;
|
|
4306
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4307
|
+
if (task.hasChildren()) {
|
|
4308
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4309
|
+
}
|
|
4310
|
+
clone.addTask(newTask);
|
|
4311
|
+
}
|
|
4312
|
+
for (const df of graph.getDataflows()) {
|
|
4313
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4314
|
+
}
|
|
4315
|
+
return clone;
|
|
4316
|
+
}
|
|
3558
4317
|
async executeSubgraphIteration(input) {
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
4318
|
+
if (this.abortController?.signal.aborted) {
|
|
4319
|
+
return;
|
|
4320
|
+
}
|
|
4321
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4322
|
+
const results = await graphClone.run(input, {
|
|
4323
|
+
parentSignal: this.abortController?.signal,
|
|
4324
|
+
outputCache: this.outputCache
|
|
3563
4325
|
});
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
if (this.abortController?.signal.aborted) {
|
|
3567
|
-
return;
|
|
3568
|
-
}
|
|
3569
|
-
const results = await this.task.subGraph.run(input, {
|
|
3570
|
-
parentSignal: this.abortController?.signal,
|
|
3571
|
-
outputCache: this.outputCache
|
|
3572
|
-
});
|
|
3573
|
-
if (results.length === 0) {
|
|
3574
|
-
return;
|
|
3575
|
-
}
|
|
3576
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3577
|
-
} finally {
|
|
3578
|
-
releaseTurn?.();
|
|
4326
|
+
if (results.length === 0) {
|
|
4327
|
+
return;
|
|
3579
4328
|
}
|
|
4329
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3580
4330
|
}
|
|
3581
4331
|
}
|
|
3582
4332
|
|
|
@@ -3860,7 +4610,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3860
4610
|
const tasks = this.subGraph.getTasks();
|
|
3861
4611
|
if (tasks.length === 0)
|
|
3862
4612
|
return;
|
|
3863
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4613
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3864
4614
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3865
4615
|
const properties = {};
|
|
3866
4616
|
const required = [];
|
|
@@ -3887,6 +4637,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3887
4637
|
}
|
|
3888
4638
|
}
|
|
3889
4639
|
}
|
|
4640
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4641
|
+
for (const task of tasks) {
|
|
4642
|
+
if (sourceIds.has(task.id))
|
|
4643
|
+
continue;
|
|
4644
|
+
const inputSchema = task.inputSchema();
|
|
4645
|
+
if (typeof inputSchema === "boolean")
|
|
4646
|
+
continue;
|
|
4647
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4648
|
+
if (requiredKeys.size === 0)
|
|
4649
|
+
continue;
|
|
4650
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4651
|
+
for (const key of requiredKeys) {
|
|
4652
|
+
if (connectedPorts.has(key))
|
|
4653
|
+
continue;
|
|
4654
|
+
if (properties[key])
|
|
4655
|
+
continue;
|
|
4656
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4657
|
+
continue;
|
|
4658
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4659
|
+
if (!prop || typeof prop === "boolean")
|
|
4660
|
+
continue;
|
|
4661
|
+
properties[key] = prop;
|
|
4662
|
+
if (!required.includes(key)) {
|
|
4663
|
+
required.push(key);
|
|
4664
|
+
}
|
|
4665
|
+
}
|
|
4666
|
+
}
|
|
3890
4667
|
return {
|
|
3891
4668
|
type: "object",
|
|
3892
4669
|
properties,
|
|
@@ -4021,7 +4798,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4021
4798
|
if (!this.hasChildren()) {
|
|
4022
4799
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4023
4800
|
}
|
|
4024
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4801
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4025
4802
|
if (endingNodes.length === 0) {
|
|
4026
4803
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4027
4804
|
}
|
|
@@ -4231,8 +5008,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4231
5008
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4232
5009
|
}
|
|
4233
5010
|
this._currentIteration++;
|
|
4234
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4235
|
-
await context.updateProgress(progress, `
|
|
5011
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5012
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4236
5013
|
}
|
|
4237
5014
|
return currentOutput;
|
|
4238
5015
|
}
|
|
@@ -4275,8 +5052,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4275
5052
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4276
5053
|
}
|
|
4277
5054
|
this._currentIteration++;
|
|
4278
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4279
|
-
await context.updateProgress(progress, `
|
|
5055
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5056
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4280
5057
|
}
|
|
4281
5058
|
yield { type: "finish", data: currentOutput };
|
|
4282
5059
|
}
|
|
@@ -4352,7 +5129,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4352
5129
|
return this.constructor.outputSchema();
|
|
4353
5130
|
}
|
|
4354
5131
|
const tasks = this.subGraph.getTasks();
|
|
4355
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5132
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4356
5133
|
if (endingNodes.length === 0) {
|
|
4357
5134
|
return this.constructor.outputSchema();
|
|
4358
5135
|
}
|
|
@@ -5019,7 +5796,7 @@ class ReduceTask extends IteratorTask {
|
|
|
5019
5796
|
if (!this.hasChildren()) {
|
|
5020
5797
|
return this.constructor.outputSchema();
|
|
5021
5798
|
}
|
|
5022
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5799
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5023
5800
|
if (endingNodes.length === 0) {
|
|
5024
5801
|
return this.constructor.outputSchema();
|
|
5025
5802
|
}
|
|
@@ -5057,14 +5834,14 @@ var TaskRegistry = {
|
|
|
5057
5834
|
};
|
|
5058
5835
|
|
|
5059
5836
|
// src/task/TaskJSON.ts
|
|
5060
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5837
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5061
5838
|
if (!item.id)
|
|
5062
5839
|
throw new TaskJSONError("Task id required");
|
|
5063
5840
|
if (!item.type)
|
|
5064
5841
|
throw new TaskJSONError("Task type required");
|
|
5065
5842
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5066
5843
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5067
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5844
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5068
5845
|
if (!taskClass)
|
|
5069
5846
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5070
5847
|
const taskConfig = {
|
|
@@ -5091,20 +5868,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5091
5868
|
}
|
|
5092
5869
|
return subGraph;
|
|
5093
5870
|
};
|
|
5094
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5095
|
-
const task = createSingleTaskFromJSON(item);
|
|
5871
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5872
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5096
5873
|
if (item.subgraph) {
|
|
5097
5874
|
if (!(task instanceof GraphAsTask)) {
|
|
5098
5875
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5099
5876
|
}
|
|
5100
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5877
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5101
5878
|
}
|
|
5102
5879
|
return task;
|
|
5103
5880
|
};
|
|
5104
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5881
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5105
5882
|
const subGraph = new TaskGraph;
|
|
5106
5883
|
for (const subitem of graphJsonObj.tasks) {
|
|
5107
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5884
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5108
5885
|
}
|
|
5109
5886
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5110
5887
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5113,7 +5890,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5113
5890
|
};
|
|
5114
5891
|
// src/task/index.ts
|
|
5115
5892
|
var registerBaseTasks = () => {
|
|
5116
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5893
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5117
5894
|
tasks.map(TaskRegistry.registerTask);
|
|
5118
5895
|
return tasks;
|
|
5119
5896
|
};
|
|
@@ -5296,11 +6073,14 @@ export {
|
|
|
5296
6073
|
isFlexibleSchema,
|
|
5297
6074
|
hasVectorOutput,
|
|
5298
6075
|
hasVectorLikeInput,
|
|
6076
|
+
hasStructuredOutput,
|
|
5299
6077
|
graphAsTaskConfigSchema,
|
|
5300
6078
|
getTaskQueueRegistry,
|
|
6079
|
+
getStructuredOutputSchemas,
|
|
5301
6080
|
getStreamingPorts,
|
|
5302
6081
|
getPortStreamMode,
|
|
5303
6082
|
getOutputStreamMode,
|
|
6083
|
+
getObjectPortId,
|
|
5304
6084
|
getNestedValue,
|
|
5305
6085
|
getLastTask,
|
|
5306
6086
|
getJobQueueFactory,
|
|
@@ -5309,6 +6089,7 @@ export {
|
|
|
5309
6089
|
getAppendPortId,
|
|
5310
6090
|
findArrayPorts,
|
|
5311
6091
|
filterIterationProperties,
|
|
6092
|
+
fallbackTaskConfigSchema,
|
|
5312
6093
|
extractIterationProperties,
|
|
5313
6094
|
extractBaseSchema,
|
|
5314
6095
|
evaluateCondition,
|
|
@@ -5323,13 +6104,19 @@ export {
|
|
|
5323
6104
|
createArraySchema,
|
|
5324
6105
|
connect,
|
|
5325
6106
|
conditionalTaskConfigSchema,
|
|
6107
|
+
computeGraphOutputSchema,
|
|
6108
|
+
computeGraphInputSchema,
|
|
6109
|
+
calculateNodeDepths,
|
|
5326
6110
|
buildIterationInputSchema,
|
|
5327
6111
|
addIterationContextToSchema,
|
|
6112
|
+
addBoundaryNodesToGraphJson,
|
|
6113
|
+
addBoundaryNodesToDependencyJson,
|
|
5328
6114
|
WorkflowError,
|
|
5329
6115
|
Workflow,
|
|
5330
6116
|
WhileTaskRunner,
|
|
5331
6117
|
WhileTask,
|
|
5332
6118
|
WHILE_CONTEXT_SCHEMA,
|
|
6119
|
+
TaskTimeoutError,
|
|
5333
6120
|
TaskStatus,
|
|
5334
6121
|
TaskRegistry,
|
|
5335
6122
|
TaskQueueRegistry,
|
|
@@ -5365,6 +6152,8 @@ export {
|
|
|
5365
6152
|
GraphAsTaskRunner,
|
|
5366
6153
|
GraphAsTask,
|
|
5367
6154
|
GRAPH_RESULT_ARRAY,
|
|
6155
|
+
FallbackTaskRunner,
|
|
6156
|
+
FallbackTask,
|
|
5368
6157
|
EventTaskGraphToDagMapping,
|
|
5369
6158
|
EventDagToTaskGraphMapping,
|
|
5370
6159
|
DataflowArrow,
|
|
@@ -5378,4 +6167,4 @@ export {
|
|
|
5378
6167
|
ConditionalTask
|
|
5379
6168
|
};
|
|
5380
6169
|
|
|
5381
|
-
//# debugId=
|
|
6170
|
+
//# debugId=8E9465F7037F297B64756E2164756E21
|