@workglow/task-graph 0.0.102 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1052 -263
- package/dist/browser.js.map +24 -21
- package/dist/bun.js +1052 -263
- package/dist/bun.js.map +24 -21
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1052 -263
- package/dist/node.js.map +24 -21
- package/dist/task/ConditionalTask.d.ts +5 -1
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +15 -12
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +5 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +9 -1
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +4 -0
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +20 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +7 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +5 -0
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/browser.js
CHANGED
|
@@ -20,6 +20,7 @@ var TaskConfigSchema = {
|
|
|
20
20
|
title: { type: "string" },
|
|
21
21
|
description: { type: "string" },
|
|
22
22
|
cacheable: { type: "boolean" },
|
|
23
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
23
24
|
inputSchema: {
|
|
24
25
|
type: "object",
|
|
25
26
|
properties: {},
|
|
@@ -240,8 +241,352 @@ class DataflowArrow extends Dataflow {
|
|
|
240
241
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
241
242
|
}
|
|
242
243
|
}
|
|
244
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
245
|
+
import { uuid4 } from "@workglow/util";
|
|
246
|
+
function calculateNodeDepths(graph) {
|
|
247
|
+
const depths = new Map;
|
|
248
|
+
const tasks = graph.getTasks();
|
|
249
|
+
for (const task of tasks) {
|
|
250
|
+
depths.set(task.id, 0);
|
|
251
|
+
}
|
|
252
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
253
|
+
for (const task of sortedTasks) {
|
|
254
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
255
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
256
|
+
for (const targetTask of targetTasks) {
|
|
257
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
258
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return depths;
|
|
262
|
+
}
|
|
263
|
+
function computeGraphInputSchema(graph, options) {
|
|
264
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
265
|
+
const properties = {};
|
|
266
|
+
const required = [];
|
|
267
|
+
const propertyOrigins = {};
|
|
268
|
+
const tasks = graph.getTasks();
|
|
269
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
270
|
+
for (const task of startingNodes) {
|
|
271
|
+
const taskInputSchema = task.inputSchema();
|
|
272
|
+
if (typeof taskInputSchema === "boolean") {
|
|
273
|
+
if (taskInputSchema === false) {
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (taskInputSchema === true) {
|
|
277
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
282
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
283
|
+
if (!properties[inputName]) {
|
|
284
|
+
properties[inputName] = inputProp;
|
|
285
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
286
|
+
required.push(inputName);
|
|
287
|
+
}
|
|
288
|
+
if (trackOrigins) {
|
|
289
|
+
propertyOrigins[inputName] = [task.id];
|
|
290
|
+
}
|
|
291
|
+
} else if (trackOrigins) {
|
|
292
|
+
propertyOrigins[inputName].push(task.id);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
297
|
+
for (const task of tasks) {
|
|
298
|
+
if (sourceIds.has(task.id))
|
|
299
|
+
continue;
|
|
300
|
+
const taskInputSchema = task.inputSchema();
|
|
301
|
+
if (typeof taskInputSchema === "boolean")
|
|
302
|
+
continue;
|
|
303
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
304
|
+
if (requiredKeys.size === 0)
|
|
305
|
+
continue;
|
|
306
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
307
|
+
for (const key of requiredKeys) {
|
|
308
|
+
if (connectedPorts.has(key))
|
|
309
|
+
continue;
|
|
310
|
+
if (properties[key]) {
|
|
311
|
+
if (trackOrigins) {
|
|
312
|
+
propertyOrigins[key].push(task.id);
|
|
313
|
+
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
317
|
+
continue;
|
|
318
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
319
|
+
if (!prop || typeof prop === "boolean")
|
|
320
|
+
continue;
|
|
321
|
+
properties[key] = prop;
|
|
322
|
+
if (!required.includes(key)) {
|
|
323
|
+
required.push(key);
|
|
324
|
+
}
|
|
325
|
+
if (trackOrigins) {
|
|
326
|
+
propertyOrigins[key] = [task.id];
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (trackOrigins) {
|
|
331
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
332
|
+
const prop = properties[propName];
|
|
333
|
+
if (!prop || typeof prop === "boolean")
|
|
334
|
+
continue;
|
|
335
|
+
if (origins.length === 1) {
|
|
336
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
337
|
+
} else {
|
|
338
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
type: "object",
|
|
344
|
+
properties,
|
|
345
|
+
...required.length > 0 ? { required } : {},
|
|
346
|
+
additionalProperties: false
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function computeGraphOutputSchema(graph, options) {
|
|
350
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
351
|
+
const properties = {};
|
|
352
|
+
const required = [];
|
|
353
|
+
const propertyOrigins = {};
|
|
354
|
+
const tasks = graph.getTasks();
|
|
355
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
356
|
+
const depths = calculateNodeDepths(graph);
|
|
357
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
358
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
359
|
+
const propertyCount = {};
|
|
360
|
+
const propertySchema = {};
|
|
361
|
+
for (const task of lastLevelNodes) {
|
|
362
|
+
const taskOutputSchema = task.outputSchema();
|
|
363
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
364
|
+
if (taskOutputSchema === false) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (taskOutputSchema === true) {
|
|
368
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
373
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
374
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
375
|
+
if (!propertySchema[outputName]) {
|
|
376
|
+
propertySchema[outputName] = outputProp;
|
|
377
|
+
}
|
|
378
|
+
if (trackOrigins) {
|
|
379
|
+
if (!propertyOrigins[outputName]) {
|
|
380
|
+
propertyOrigins[outputName] = [task.id];
|
|
381
|
+
} else {
|
|
382
|
+
propertyOrigins[outputName].push(task.id);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
388
|
+
const outputProp = propertySchema[outputName];
|
|
389
|
+
if (lastLevelNodes.length === 1) {
|
|
390
|
+
properties[outputName] = outputProp;
|
|
391
|
+
} else {
|
|
392
|
+
properties[outputName] = {
|
|
393
|
+
type: "array",
|
|
394
|
+
items: outputProp
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
if (trackOrigins) {
|
|
399
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
400
|
+
const prop = properties[propName];
|
|
401
|
+
if (!prop || typeof prop === "boolean")
|
|
402
|
+
continue;
|
|
403
|
+
if (origins.length === 1) {
|
|
404
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
405
|
+
} else {
|
|
406
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
return {
|
|
411
|
+
type: "object",
|
|
412
|
+
properties,
|
|
413
|
+
...required.length > 0 ? { required } : {},
|
|
414
|
+
additionalProperties: false
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
function stripOriginAnnotations(schema) {
|
|
418
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
419
|
+
return schema;
|
|
420
|
+
const properties = schema.properties;
|
|
421
|
+
if (!properties)
|
|
422
|
+
return schema;
|
|
423
|
+
const strippedProperties = {};
|
|
424
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
425
|
+
if (!prop || typeof prop !== "object") {
|
|
426
|
+
strippedProperties[key] = prop;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
430
|
+
strippedProperties[key] = rest;
|
|
431
|
+
}
|
|
432
|
+
return { ...schema, properties: strippedProperties };
|
|
433
|
+
}
|
|
434
|
+
function getOriginTaskIds(prop) {
|
|
435
|
+
if (prop["x-source-task-ids"]) {
|
|
436
|
+
return prop["x-source-task-ids"];
|
|
437
|
+
}
|
|
438
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
439
|
+
return [prop["x-source-task-id"]];
|
|
440
|
+
}
|
|
441
|
+
return [];
|
|
442
|
+
}
|
|
443
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
444
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
445
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
446
|
+
if (hasInputTask && hasOutputTask) {
|
|
447
|
+
return json;
|
|
448
|
+
}
|
|
449
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
450
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const prependTasks = [];
|
|
452
|
+
const appendTasks = [];
|
|
453
|
+
const inputDataflows = [];
|
|
454
|
+
const outputDataflows = [];
|
|
455
|
+
if (!hasInputTask && inputSchema) {
|
|
456
|
+
const inputTaskId = uuid4();
|
|
457
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
458
|
+
prependTasks.push({
|
|
459
|
+
id: inputTaskId,
|
|
460
|
+
type: "InputTask",
|
|
461
|
+
config: {
|
|
462
|
+
inputSchema: strippedInputSchema,
|
|
463
|
+
outputSchema: strippedInputSchema
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
467
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
468
|
+
if (!prop || typeof prop === "boolean")
|
|
469
|
+
continue;
|
|
470
|
+
const origins = getOriginTaskIds(prop);
|
|
471
|
+
for (const originId of origins) {
|
|
472
|
+
inputDataflows.push({
|
|
473
|
+
sourceTaskId: inputTaskId,
|
|
474
|
+
sourceTaskPortId: propName,
|
|
475
|
+
targetTaskId: originId,
|
|
476
|
+
targetTaskPortId: propName
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
if (!hasOutputTask && outputSchema) {
|
|
483
|
+
const outputTaskId = uuid4();
|
|
484
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
485
|
+
appendTasks.push({
|
|
486
|
+
id: outputTaskId,
|
|
487
|
+
type: "OutputTask",
|
|
488
|
+
config: {
|
|
489
|
+
inputSchema: strippedOutputSchema,
|
|
490
|
+
outputSchema: strippedOutputSchema
|
|
491
|
+
}
|
|
492
|
+
});
|
|
493
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
494
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
495
|
+
if (!prop || typeof prop === "boolean")
|
|
496
|
+
continue;
|
|
497
|
+
const origins = getOriginTaskIds(prop);
|
|
498
|
+
for (const originId of origins) {
|
|
499
|
+
outputDataflows.push({
|
|
500
|
+
sourceTaskId: originId,
|
|
501
|
+
sourceTaskPortId: propName,
|
|
502
|
+
targetTaskId: outputTaskId,
|
|
503
|
+
targetTaskPortId: propName
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return {
|
|
510
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
511
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
515
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
516
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
517
|
+
if (hasInputTask && hasOutputTask) {
|
|
518
|
+
return items;
|
|
519
|
+
}
|
|
520
|
+
const prependItems = [];
|
|
521
|
+
const appendItems = [];
|
|
522
|
+
if (!hasInputTask) {
|
|
523
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
524
|
+
const inputTaskId = uuid4();
|
|
525
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
526
|
+
prependItems.push({
|
|
527
|
+
id: inputTaskId,
|
|
528
|
+
type: "InputTask",
|
|
529
|
+
config: {
|
|
530
|
+
inputSchema: strippedInputSchema,
|
|
531
|
+
outputSchema: strippedInputSchema
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
535
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
536
|
+
if (!prop || typeof prop === "boolean")
|
|
537
|
+
continue;
|
|
538
|
+
const origins = getOriginTaskIds(prop);
|
|
539
|
+
for (const originId of origins) {
|
|
540
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
541
|
+
if (!targetItem)
|
|
542
|
+
continue;
|
|
543
|
+
if (!targetItem.dependencies) {
|
|
544
|
+
targetItem.dependencies = {};
|
|
545
|
+
}
|
|
546
|
+
const existing = targetItem.dependencies[propName];
|
|
547
|
+
const dep = { id: inputTaskId, output: propName };
|
|
548
|
+
if (!existing) {
|
|
549
|
+
targetItem.dependencies[propName] = dep;
|
|
550
|
+
} else if (Array.isArray(existing)) {
|
|
551
|
+
existing.push(dep);
|
|
552
|
+
} else {
|
|
553
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (!hasOutputTask) {
|
|
560
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
561
|
+
const outputTaskId = uuid4();
|
|
562
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
563
|
+
const outputDependencies = {};
|
|
564
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
565
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
566
|
+
if (!prop || typeof prop === "boolean")
|
|
567
|
+
continue;
|
|
568
|
+
const origins = getOriginTaskIds(prop);
|
|
569
|
+
if (origins.length === 1) {
|
|
570
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
571
|
+
} else if (origins.length > 1) {
|
|
572
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
appendItems.push({
|
|
577
|
+
id: outputTaskId,
|
|
578
|
+
type: "OutputTask",
|
|
579
|
+
config: {
|
|
580
|
+
inputSchema: strippedOutputSchema,
|
|
581
|
+
outputSchema: strippedOutputSchema
|
|
582
|
+
},
|
|
583
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
return [...prependItems, ...items, ...appendItems];
|
|
587
|
+
}
|
|
243
588
|
// src/task-graph/TaskGraph.ts
|
|
244
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
589
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
245
590
|
|
|
246
591
|
// src/task/GraphAsTask.ts
|
|
247
592
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -249,9 +594,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
249
594
|
// src/task-graph/TaskGraphRunner.ts
|
|
250
595
|
import {
|
|
251
596
|
collectPropertyValues,
|
|
597
|
+
getLogger as getLogger3,
|
|
252
598
|
globalServiceRegistry as globalServiceRegistry2,
|
|
253
599
|
ServiceRegistry as ServiceRegistry2,
|
|
254
|
-
uuid4 as
|
|
600
|
+
uuid4 as uuid43
|
|
255
601
|
} from "@workglow/util";
|
|
256
602
|
|
|
257
603
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -284,6 +630,9 @@ class TaskOutputRepository {
|
|
|
284
630
|
}
|
|
285
631
|
}
|
|
286
632
|
|
|
633
|
+
// src/task/ConditionalTask.ts
|
|
634
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
635
|
+
|
|
287
636
|
// src/task/ConditionUtils.ts
|
|
288
637
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
289
638
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -356,7 +705,7 @@ import {
|
|
|
356
705
|
compileSchema,
|
|
357
706
|
deepEqual,
|
|
358
707
|
EventEmitter as EventEmitter3,
|
|
359
|
-
uuid4
|
|
708
|
+
uuid4 as uuid42
|
|
360
709
|
} from "@workglow/util";
|
|
361
710
|
|
|
362
711
|
// src/task/TaskError.ts
|
|
@@ -390,6 +739,13 @@ class TaskAbortedError extends TaskError {
|
|
|
390
739
|
}
|
|
391
740
|
}
|
|
392
741
|
|
|
742
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
743
|
+
static type = "TaskTimeoutError";
|
|
744
|
+
constructor(timeoutMs) {
|
|
745
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
393
749
|
class TaskFailedError extends TaskError {
|
|
394
750
|
static type = "TaskFailedError";
|
|
395
751
|
constructor(message = "Task failed") {
|
|
@@ -421,7 +777,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
421
777
|
}
|
|
422
778
|
|
|
423
779
|
// src/task/TaskRunner.ts
|
|
424
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
780
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
425
781
|
|
|
426
782
|
// src/task/InputResolver.ts
|
|
427
783
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -485,7 +841,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
485
841
|
if (!prop || typeof prop === "boolean")
|
|
486
842
|
return "none";
|
|
487
843
|
const xStream = prop["x-stream"];
|
|
488
|
-
if (xStream === "append" || xStream === "replace")
|
|
844
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
489
845
|
return xStream;
|
|
490
846
|
return "none";
|
|
491
847
|
}
|
|
@@ -500,7 +856,7 @@ function getStreamingPorts(schema) {
|
|
|
500
856
|
if (!prop || typeof prop === "boolean")
|
|
501
857
|
continue;
|
|
502
858
|
const xStream = prop["x-stream"];
|
|
503
|
-
if (xStream === "append" || xStream === "replace") {
|
|
859
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
504
860
|
result.push({ port: name, mode: xStream });
|
|
505
861
|
}
|
|
506
862
|
}
|
|
@@ -544,6 +900,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
544
900
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
545
901
|
return sourceMode !== targetMode;
|
|
546
902
|
}
|
|
903
|
+
function getObjectPortId(schema) {
|
|
904
|
+
if (typeof schema === "boolean")
|
|
905
|
+
return;
|
|
906
|
+
const props = schema.properties;
|
|
907
|
+
if (!props)
|
|
908
|
+
return;
|
|
909
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
910
|
+
if (!prop || typeof prop === "boolean")
|
|
911
|
+
continue;
|
|
912
|
+
if (prop["x-stream"] === "object")
|
|
913
|
+
return name;
|
|
914
|
+
}
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
function getStructuredOutputSchemas(schema) {
|
|
918
|
+
const result = new Map;
|
|
919
|
+
if (typeof schema === "boolean")
|
|
920
|
+
return result;
|
|
921
|
+
const props = schema.properties;
|
|
922
|
+
if (!props)
|
|
923
|
+
return result;
|
|
924
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
925
|
+
if (!prop || typeof prop === "boolean")
|
|
926
|
+
continue;
|
|
927
|
+
if (prop["x-structured-output"] === true) {
|
|
928
|
+
result.set(name, prop);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
return result;
|
|
932
|
+
}
|
|
933
|
+
function hasStructuredOutput(schema) {
|
|
934
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
935
|
+
}
|
|
547
936
|
|
|
548
937
|
// src/task/TaskRunner.ts
|
|
549
938
|
class TaskRunner {
|
|
@@ -554,12 +943,17 @@ class TaskRunner {
|
|
|
554
943
|
outputCache;
|
|
555
944
|
registry = globalServiceRegistry;
|
|
556
945
|
inputStreams;
|
|
946
|
+
timeoutTimer;
|
|
947
|
+
pendingTimeoutError;
|
|
557
948
|
shouldAccumulate = true;
|
|
558
949
|
constructor(task) {
|
|
559
950
|
this.task = task;
|
|
560
951
|
this.own = this.own.bind(this);
|
|
561
952
|
this.handleProgress = this.handleProgress.bind(this);
|
|
562
953
|
}
|
|
954
|
+
get timerLabel() {
|
|
955
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
956
|
+
}
|
|
563
957
|
async run(overrides = {}, config = {}) {
|
|
564
958
|
await this.handleStart(config);
|
|
565
959
|
try {
|
|
@@ -606,7 +1000,7 @@ class TaskRunner {
|
|
|
606
1000
|
return this.task.runOutputData;
|
|
607
1001
|
} catch (err) {
|
|
608
1002
|
await this.handleError(err);
|
|
609
|
-
throw err;
|
|
1003
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
610
1004
|
}
|
|
611
1005
|
}
|
|
612
1006
|
async runReactive(overrides = {}) {
|
|
@@ -663,7 +1057,14 @@ class TaskRunner {
|
|
|
663
1057
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
664
1058
|
}
|
|
665
1059
|
}
|
|
1060
|
+
if (streamMode === "object") {
|
|
1061
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1062
|
+
if (ports.length === 0) {
|
|
1063
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
666
1066
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1067
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
667
1068
|
let chunkCount = 0;
|
|
668
1069
|
let finalOutput;
|
|
669
1070
|
this.task.emit("stream_start");
|
|
@@ -694,6 +1095,13 @@ class TaskRunner {
|
|
|
694
1095
|
break;
|
|
695
1096
|
}
|
|
696
1097
|
case "object-delta": {
|
|
1098
|
+
if (accumulatedObjects) {
|
|
1099
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1100
|
+
}
|
|
1101
|
+
this.task.runOutputData = {
|
|
1102
|
+
...this.task.runOutputData,
|
|
1103
|
+
[event.port]: event.objectDelta
|
|
1104
|
+
};
|
|
697
1105
|
this.task.emit("stream_chunk", event);
|
|
698
1106
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
699
1107
|
await this.handleProgress(progress);
|
|
@@ -706,11 +1114,18 @@ class TaskRunner {
|
|
|
706
1114
|
break;
|
|
707
1115
|
}
|
|
708
1116
|
case "finish": {
|
|
709
|
-
if (accumulated) {
|
|
1117
|
+
if (accumulated || accumulatedObjects) {
|
|
710
1118
|
const merged = { ...event.data || {} };
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1119
|
+
if (accumulated) {
|
|
1120
|
+
for (const [port, text] of accumulated) {
|
|
1121
|
+
if (text.length > 0)
|
|
1122
|
+
merged[port] = text;
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
if (accumulatedObjects) {
|
|
1126
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1127
|
+
merged[port] = obj;
|
|
1128
|
+
}
|
|
714
1129
|
}
|
|
715
1130
|
finalOutput = merged;
|
|
716
1131
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -756,12 +1171,20 @@ class TaskRunner {
|
|
|
756
1171
|
this.outputCache = cache;
|
|
757
1172
|
}
|
|
758
1173
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1174
|
+
const timeout = this.task.config.timeout;
|
|
1175
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1176
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1177
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1178
|
+
this.abort();
|
|
1179
|
+
}, timeout);
|
|
1180
|
+
}
|
|
759
1181
|
if (config.updateProgress) {
|
|
760
1182
|
this.updateProgress = config.updateProgress;
|
|
761
1183
|
}
|
|
762
1184
|
if (config.registry) {
|
|
763
1185
|
this.registry = config.registry;
|
|
764
1186
|
}
|
|
1187
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
765
1188
|
this.task.emit("start");
|
|
766
1189
|
this.task.emit("status", this.task.status);
|
|
767
1190
|
}
|
|
@@ -769,12 +1192,21 @@ class TaskRunner {
|
|
|
769
1192
|
async handleStartReactive() {
|
|
770
1193
|
this.reactiveRunning = true;
|
|
771
1194
|
}
|
|
1195
|
+
clearTimeoutTimer() {
|
|
1196
|
+
if (this.timeoutTimer !== undefined) {
|
|
1197
|
+
clearTimeout(this.timeoutTimer);
|
|
1198
|
+
this.timeoutTimer = undefined;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
772
1201
|
async handleAbort() {
|
|
773
1202
|
if (this.task.status === TaskStatus.ABORTING)
|
|
774
1203
|
return;
|
|
1204
|
+
this.clearTimeoutTimer();
|
|
1205
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
775
1206
|
this.task.status = TaskStatus.ABORTING;
|
|
776
1207
|
this.task.progress = 100;
|
|
777
|
-
this.task.error = new TaskAbortedError;
|
|
1208
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1209
|
+
this.pendingTimeoutError = undefined;
|
|
778
1210
|
this.task.emit("abort", this.task.error);
|
|
779
1211
|
this.task.emit("status", this.task.status);
|
|
780
1212
|
}
|
|
@@ -784,6 +1216,9 @@ class TaskRunner {
|
|
|
784
1216
|
async handleComplete() {
|
|
785
1217
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
786
1218
|
return;
|
|
1219
|
+
this.clearTimeoutTimer();
|
|
1220
|
+
this.pendingTimeoutError = undefined;
|
|
1221
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
787
1222
|
this.task.completedAt = new Date;
|
|
788
1223
|
this.task.progress = 100;
|
|
789
1224
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -812,6 +1247,9 @@ class TaskRunner {
|
|
|
812
1247
|
return this.handleAbort();
|
|
813
1248
|
if (this.task.status === TaskStatus.FAILED)
|
|
814
1249
|
return;
|
|
1250
|
+
this.clearTimeoutTimer();
|
|
1251
|
+
this.pendingTimeoutError = undefined;
|
|
1252
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
815
1253
|
if (this.task.hasChildren()) {
|
|
816
1254
|
this.task.subGraph.abort();
|
|
817
1255
|
}
|
|
@@ -916,6 +1354,9 @@ class Task {
|
|
|
916
1354
|
runInputData = {};
|
|
917
1355
|
runOutputData = {};
|
|
918
1356
|
config;
|
|
1357
|
+
get id() {
|
|
1358
|
+
return this.config.id;
|
|
1359
|
+
}
|
|
919
1360
|
runConfig = {};
|
|
920
1361
|
status = TaskStatus.PENDING;
|
|
921
1362
|
progress = 0;
|
|
@@ -937,9 +1378,11 @@ class Task {
|
|
|
937
1378
|
this.resetInputData();
|
|
938
1379
|
const title = this.constructor.title || undefined;
|
|
939
1380
|
const baseConfig = Object.assign({
|
|
940
|
-
id: uuid4(),
|
|
941
1381
|
...title ? { title } : {}
|
|
942
1382
|
}, config);
|
|
1383
|
+
if (baseConfig.id === undefined) {
|
|
1384
|
+
baseConfig.id = uuid42();
|
|
1385
|
+
}
|
|
943
1386
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
944
1387
|
this.runConfig = runConfig;
|
|
945
1388
|
}
|
|
@@ -949,7 +1392,7 @@ class Task {
|
|
|
949
1392
|
return {};
|
|
950
1393
|
}
|
|
951
1394
|
try {
|
|
952
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1395
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
953
1396
|
const defaultData = compiledSchema.getData(undefined, {
|
|
954
1397
|
addOptionalProps: true,
|
|
955
1398
|
removeInvalidData: false,
|
|
@@ -1067,7 +1510,7 @@ class Task {
|
|
|
1067
1510
|
continue;
|
|
1068
1511
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1069
1512
|
if (isArray) {
|
|
1070
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1513
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1071
1514
|
const newitems = [...existingItems];
|
|
1072
1515
|
const overrideItem = overrides[inputId];
|
|
1073
1516
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1123,25 +1566,29 @@ class Task {
|
|
|
1123
1566
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1124
1567
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1125
1568
|
}
|
|
1126
|
-
static
|
|
1127
|
-
static getConfigSchemaNode(type) {
|
|
1569
|
+
static getConfigSchemaNode() {
|
|
1128
1570
|
const schema = this.configSchema();
|
|
1129
1571
|
if (!schema)
|
|
1130
1572
|
return;
|
|
1131
|
-
if (!
|
|
1573
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1132
1574
|
try {
|
|
1133
1575
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1134
|
-
|
|
1576
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1577
|
+
value: schemaNode,
|
|
1578
|
+
writable: true,
|
|
1579
|
+
configurable: true,
|
|
1580
|
+
enumerable: false
|
|
1581
|
+
});
|
|
1135
1582
|
} catch (error) {
|
|
1136
1583
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1137
1584
|
return;
|
|
1138
1585
|
}
|
|
1139
1586
|
}
|
|
1140
|
-
return this.
|
|
1587
|
+
return this.__compiledConfigSchema;
|
|
1141
1588
|
}
|
|
1142
1589
|
validateAndApplyConfigDefaults(config) {
|
|
1143
1590
|
const ctor = this.constructor;
|
|
1144
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1591
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1145
1592
|
if (!schemaNode)
|
|
1146
1593
|
return config;
|
|
1147
1594
|
const result = schemaNode.validate(config);
|
|
@@ -1154,7 +1601,6 @@ class Task {
|
|
|
1154
1601
|
}
|
|
1155
1602
|
return config;
|
|
1156
1603
|
}
|
|
1157
|
-
static _inputSchemaNode = new Map;
|
|
1158
1604
|
static generateInputSchemaNode(schema) {
|
|
1159
1605
|
if (typeof schema === "boolean") {
|
|
1160
1606
|
if (schema === false) {
|
|
@@ -1164,21 +1610,31 @@ class Task {
|
|
|
1164
1610
|
}
|
|
1165
1611
|
return compileSchema(schema);
|
|
1166
1612
|
}
|
|
1167
|
-
static getInputSchemaNode(
|
|
1168
|
-
if (!
|
|
1613
|
+
static getInputSchemaNode() {
|
|
1614
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1169
1615
|
const dataPortSchema = this.inputSchema();
|
|
1170
1616
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1171
1617
|
try {
|
|
1172
|
-
|
|
1618
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1619
|
+
value: schemaNode,
|
|
1620
|
+
writable: true,
|
|
1621
|
+
configurable: true,
|
|
1622
|
+
enumerable: false
|
|
1623
|
+
});
|
|
1173
1624
|
} catch (error) {
|
|
1174
1625
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1175
|
-
|
|
1626
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1627
|
+
value: compileSchema({}),
|
|
1628
|
+
writable: true,
|
|
1629
|
+
configurable: true,
|
|
1630
|
+
enumerable: false
|
|
1631
|
+
});
|
|
1176
1632
|
}
|
|
1177
1633
|
}
|
|
1178
|
-
return this.
|
|
1634
|
+
return this.__compiledInputSchema;
|
|
1179
1635
|
}
|
|
1180
|
-
getInputSchemaNode(
|
|
1181
|
-
return this.constructor.getInputSchemaNode(
|
|
1636
|
+
getInputSchemaNode() {
|
|
1637
|
+
return this.constructor.getInputSchemaNode();
|
|
1182
1638
|
}
|
|
1183
1639
|
async validateInput(input) {
|
|
1184
1640
|
const ctor = this.constructor;
|
|
@@ -1187,7 +1643,7 @@ class Task {
|
|
|
1187
1643
|
const instanceSchema = this.inputSchema();
|
|
1188
1644
|
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1189
1645
|
} else {
|
|
1190
|
-
schemaNode = this.getInputSchemaNode(
|
|
1646
|
+
schemaNode = this.getInputSchemaNode();
|
|
1191
1647
|
}
|
|
1192
1648
|
const result = schemaNode.validate(input);
|
|
1193
1649
|
if (!result.valid) {
|
|
@@ -1199,9 +1655,6 @@ class Task {
|
|
|
1199
1655
|
}
|
|
1200
1656
|
return true;
|
|
1201
1657
|
}
|
|
1202
|
-
id() {
|
|
1203
|
-
return this.config.id;
|
|
1204
|
-
}
|
|
1205
1658
|
stripSymbols(obj) {
|
|
1206
1659
|
if (obj === null || obj === undefined) {
|
|
1207
1660
|
return obj;
|
|
@@ -1223,14 +1676,15 @@ class Task {
|
|
|
1223
1676
|
}
|
|
1224
1677
|
return obj;
|
|
1225
1678
|
}
|
|
1226
|
-
toJSON() {
|
|
1679
|
+
toJSON(_options) {
|
|
1227
1680
|
const extras = this.config.extras;
|
|
1228
1681
|
const json = this.stripSymbols({
|
|
1229
|
-
id: this.
|
|
1682
|
+
id: this.id,
|
|
1230
1683
|
type: this.type,
|
|
1231
1684
|
defaults: this.defaults,
|
|
1232
1685
|
config: {
|
|
1233
1686
|
...this.config.title ? { title: this.config.title } : {},
|
|
1687
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1234
1688
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1235
1689
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1236
1690
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1238,8 +1692,8 @@ class Task {
|
|
|
1238
1692
|
});
|
|
1239
1693
|
return json;
|
|
1240
1694
|
}
|
|
1241
|
-
toDependencyJSON() {
|
|
1242
|
-
const json = this.toJSON();
|
|
1695
|
+
toDependencyJSON(options) {
|
|
1696
|
+
const json = this.toJSON(options);
|
|
1243
1697
|
return json;
|
|
1244
1698
|
}
|
|
1245
1699
|
hasChildren() {
|
|
@@ -1269,7 +1723,7 @@ class Task {
|
|
|
1269
1723
|
this.subGraph.removeDataflow(dataflow);
|
|
1270
1724
|
}
|
|
1271
1725
|
for (const child of this.subGraph.getTasks()) {
|
|
1272
|
-
this.subGraph.removeTask(child.
|
|
1726
|
+
this.subGraph.removeTask(child.id);
|
|
1273
1727
|
}
|
|
1274
1728
|
}
|
|
1275
1729
|
this.events.emit("regenerate");
|
|
@@ -1360,7 +1814,7 @@ class ConditionalTask extends Task {
|
|
|
1360
1814
|
}
|
|
1361
1815
|
}
|
|
1362
1816
|
} catch (error) {
|
|
1363
|
-
|
|
1817
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1364
1818
|
}
|
|
1365
1819
|
}
|
|
1366
1820
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1525,7 +1979,7 @@ class DependencyBasedScheduler {
|
|
|
1525
1979
|
if (task.status === TaskStatus.DISABLED) {
|
|
1526
1980
|
return false;
|
|
1527
1981
|
}
|
|
1528
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
1982
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1529
1983
|
if (sourceDataflows.length > 0) {
|
|
1530
1984
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1531
1985
|
if (allIncomingDisabled) {
|
|
@@ -1652,6 +2106,10 @@ class TaskGraphRunner {
|
|
|
1652
2106
|
graph.outputCache = outputCache;
|
|
1653
2107
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1654
2108
|
}
|
|
2109
|
+
runId = "";
|
|
2110
|
+
get timerLabel() {
|
|
2111
|
+
return `graph:${this.runId}`;
|
|
2112
|
+
}
|
|
1655
2113
|
async runGraph(input = {}, config) {
|
|
1656
2114
|
await this.handleStart(config);
|
|
1657
2115
|
const results = [];
|
|
@@ -1664,25 +2122,33 @@ class TaskGraphRunner {
|
|
|
1664
2122
|
if (this.failedTaskErrors.size > 0) {
|
|
1665
2123
|
break;
|
|
1666
2124
|
}
|
|
1667
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2125
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1668
2126
|
const runAsync = async () => {
|
|
2127
|
+
let errorRouted = false;
|
|
1669
2128
|
try {
|
|
1670
2129
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1671
2130
|
const taskPromise = this.runTask(task, taskInput);
|
|
1672
|
-
this.inProgressTasks.set(task.
|
|
2131
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1673
2132
|
const taskResult = await taskPromise;
|
|
1674
|
-
if (this.graph.getTargetDataflows(task.
|
|
2133
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1675
2134
|
results.push(taskResult);
|
|
1676
2135
|
}
|
|
1677
2136
|
} catch (error2) {
|
|
1678
|
-
this.
|
|
2137
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2138
|
+
errorRouted = true;
|
|
2139
|
+
this.pushErrorOutputToEdges(task);
|
|
2140
|
+
} else {
|
|
2141
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2142
|
+
}
|
|
1679
2143
|
} finally {
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
2144
|
+
if (!errorRouted) {
|
|
2145
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2146
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2147
|
+
}
|
|
2148
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1683
2149
|
}
|
|
1684
2150
|
};
|
|
1685
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2151
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1686
2152
|
}
|
|
1687
2153
|
} catch (err) {
|
|
1688
2154
|
error = err;
|
|
@@ -1706,7 +2172,7 @@ class TaskGraphRunner {
|
|
|
1706
2172
|
const results = [];
|
|
1707
2173
|
try {
|
|
1708
2174
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1709
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2175
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1710
2176
|
if (task.status === TaskStatus.PENDING) {
|
|
1711
2177
|
task.resetInputData();
|
|
1712
2178
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1714,9 +2180,9 @@ class TaskGraphRunner {
|
|
|
1714
2180
|
const taskInput = isRootTask ? input : {};
|
|
1715
2181
|
const taskResult = await task.runReactive(taskInput);
|
|
1716
2182
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1717
|
-
if (this.graph.getTargetDataflows(task.
|
|
2183
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1718
2184
|
results.push({
|
|
1719
|
-
id: task.
|
|
2185
|
+
id: task.id,
|
|
1720
2186
|
type: task.constructor.runtype || task.constructor.type,
|
|
1721
2187
|
data: taskResult
|
|
1722
2188
|
});
|
|
@@ -1736,7 +2202,7 @@ class TaskGraphRunner {
|
|
|
1736
2202
|
await this.handleDisable();
|
|
1737
2203
|
}
|
|
1738
2204
|
filterInputForTask(task, input) {
|
|
1739
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2205
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1740
2206
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1741
2207
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1742
2208
|
const filteredInput = {};
|
|
@@ -1775,13 +2241,13 @@ class TaskGraphRunner {
|
|
|
1775
2241
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1776
2242
|
}
|
|
1777
2243
|
copyInputFromEdgesToNode(task) {
|
|
1778
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2244
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1779
2245
|
for (const dataflow of dataflows) {
|
|
1780
2246
|
this.addInputData(task, dataflow.getPortData());
|
|
1781
2247
|
}
|
|
1782
2248
|
}
|
|
1783
2249
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1784
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2250
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1785
2251
|
for (const dataflow of dataflows) {
|
|
1786
2252
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1787
2253
|
if (compatibility === "static") {
|
|
@@ -1796,7 +2262,7 @@ class TaskGraphRunner {
|
|
|
1796
2262
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1797
2263
|
if (!node?.config?.id)
|
|
1798
2264
|
return;
|
|
1799
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2265
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1800
2266
|
const effectiveStatus = status ?? node.status;
|
|
1801
2267
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1802
2268
|
const branches = node.config.branches ?? [];
|
|
@@ -1827,10 +2293,31 @@ class TaskGraphRunner {
|
|
|
1827
2293
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1828
2294
|
if (!node?.config?.id)
|
|
1829
2295
|
return;
|
|
1830
|
-
graph.getTargetDataflows(node.
|
|
2296
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1831
2297
|
dataflow.error = node.error;
|
|
1832
2298
|
});
|
|
1833
2299
|
}
|
|
2300
|
+
hasErrorOutputEdges(task) {
|
|
2301
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2302
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2303
|
+
}
|
|
2304
|
+
pushErrorOutputToEdges(task) {
|
|
2305
|
+
const taskError = task.error;
|
|
2306
|
+
const errorData = {
|
|
2307
|
+
error: taskError?.message ?? "Unknown error",
|
|
2308
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2309
|
+
};
|
|
2310
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2311
|
+
for (const df of dataflows) {
|
|
2312
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2313
|
+
df.value = errorData;
|
|
2314
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2315
|
+
} else {
|
|
2316
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
this.propagateDisabledStatus(this.graph);
|
|
2320
|
+
}
|
|
1834
2321
|
propagateDisabledStatus(graph) {
|
|
1835
2322
|
let changed = true;
|
|
1836
2323
|
while (changed) {
|
|
@@ -1839,7 +2326,7 @@ class TaskGraphRunner {
|
|
|
1839
2326
|
if (task.status !== TaskStatus.PENDING) {
|
|
1840
2327
|
continue;
|
|
1841
2328
|
}
|
|
1842
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2329
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1843
2330
|
if (incomingDataflows.length === 0) {
|
|
1844
2331
|
continue;
|
|
1845
2332
|
}
|
|
@@ -1850,10 +2337,10 @@ class TaskGraphRunner {
|
|
|
1850
2337
|
task.completedAt = new Date;
|
|
1851
2338
|
task.emit("disabled");
|
|
1852
2339
|
task.emit("status", task.status);
|
|
1853
|
-
graph.getTargetDataflows(task.
|
|
2340
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1854
2341
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1855
2342
|
});
|
|
1856
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2343
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1857
2344
|
changed = true;
|
|
1858
2345
|
}
|
|
1859
2346
|
}
|
|
@@ -1862,7 +2349,7 @@ class TaskGraphRunner {
|
|
|
1862
2349
|
taskNeedsAccumulation(task) {
|
|
1863
2350
|
if (this.outputCache)
|
|
1864
2351
|
return true;
|
|
1865
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2352
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1866
2353
|
if (outEdges.length === 0)
|
|
1867
2354
|
return this.accumulateLeafOutputs;
|
|
1868
2355
|
const outSchema = task.outputSchema();
|
|
@@ -1885,7 +2372,7 @@ class TaskGraphRunner {
|
|
|
1885
2372
|
async runTask(task, input) {
|
|
1886
2373
|
const isStreamable = isTaskStreamable(task);
|
|
1887
2374
|
if (isStreamable) {
|
|
1888
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2375
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1889
2376
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1890
2377
|
if (streamingEdges.length > 0) {
|
|
1891
2378
|
const inputStreams = new Map;
|
|
@@ -1910,13 +2397,13 @@ class TaskGraphRunner {
|
|
|
1910
2397
|
});
|
|
1911
2398
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1912
2399
|
return {
|
|
1913
|
-
id: task.
|
|
2400
|
+
id: task.id,
|
|
1914
2401
|
type: task.constructor.runtype || task.constructor.type,
|
|
1915
2402
|
data: results
|
|
1916
2403
|
};
|
|
1917
2404
|
}
|
|
1918
2405
|
async awaitStreamInputs(task) {
|
|
1919
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2406
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1920
2407
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1921
2408
|
if (streamPromises.length > 0) {
|
|
1922
2409
|
await Promise.all(streamPromises);
|
|
@@ -1931,17 +2418,17 @@ class TaskGraphRunner {
|
|
|
1931
2418
|
streamingNotified = true;
|
|
1932
2419
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1933
2420
|
this.pushStreamToEdges(task, streamMode);
|
|
1934
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2421
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1935
2422
|
}
|
|
1936
2423
|
};
|
|
1937
2424
|
const onStreamStart = () => {
|
|
1938
|
-
this.graph.emit("task_stream_start", task.
|
|
2425
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1939
2426
|
};
|
|
1940
2427
|
const onStreamChunk = (event) => {
|
|
1941
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2428
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1942
2429
|
};
|
|
1943
2430
|
const onStreamEnd = (output) => {
|
|
1944
|
-
this.graph.emit("task_stream_end", task.
|
|
2431
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1945
2432
|
};
|
|
1946
2433
|
task.on("status", onStatus);
|
|
1947
2434
|
task.on("stream_start", onStreamStart);
|
|
@@ -1956,7 +2443,7 @@ class TaskGraphRunner {
|
|
|
1956
2443
|
});
|
|
1957
2444
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1958
2445
|
return {
|
|
1959
|
-
id: task.
|
|
2446
|
+
id: task.id,
|
|
1960
2447
|
type: task.constructor.runtype || task.constructor.type,
|
|
1961
2448
|
data: results
|
|
1962
2449
|
};
|
|
@@ -1994,7 +2481,7 @@ class TaskGraphRunner {
|
|
|
1994
2481
|
});
|
|
1995
2482
|
}
|
|
1996
2483
|
pushStreamToEdges(task, streamMode) {
|
|
1997
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2484
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1998
2485
|
if (targetDataflows.length === 0)
|
|
1999
2486
|
return;
|
|
2000
2487
|
const groups = new Map;
|
|
@@ -2085,11 +2572,15 @@ class TaskGraphRunner {
|
|
|
2085
2572
|
this.abortController?.abort();
|
|
2086
2573
|
}, { once: true });
|
|
2087
2574
|
}
|
|
2088
|
-
this.
|
|
2575
|
+
this.runId = uuid43();
|
|
2576
|
+
this.resetGraph(this.graph, this.runId);
|
|
2089
2577
|
this.processScheduler.reset();
|
|
2090
2578
|
this.inProgressTasks.clear();
|
|
2091
2579
|
this.inProgressFunctions.clear();
|
|
2092
2580
|
this.failedTaskErrors.clear();
|
|
2581
|
+
const logger = getLogger3();
|
|
2582
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2583
|
+
logger.time(this.timerLabel);
|
|
2093
2584
|
this.graph.emit("start");
|
|
2094
2585
|
}
|
|
2095
2586
|
async handleStartReactive() {
|
|
@@ -2101,6 +2592,9 @@ class TaskGraphRunner {
|
|
|
2101
2592
|
}
|
|
2102
2593
|
async handleComplete() {
|
|
2103
2594
|
this.running = false;
|
|
2595
|
+
const logger = getLogger3();
|
|
2596
|
+
logger.timeEnd(this.timerLabel);
|
|
2597
|
+
logger.groupEnd();
|
|
2104
2598
|
this.graph.emit("complete");
|
|
2105
2599
|
}
|
|
2106
2600
|
async handleCompleteReactive() {
|
|
@@ -2113,6 +2607,9 @@ class TaskGraphRunner {
|
|
|
2113
2607
|
}
|
|
2114
2608
|
}));
|
|
2115
2609
|
this.running = false;
|
|
2610
|
+
const logger = getLogger3();
|
|
2611
|
+
logger.timeEnd(this.timerLabel);
|
|
2612
|
+
logger.groupEnd();
|
|
2116
2613
|
this.graph.emit("error", error);
|
|
2117
2614
|
}
|
|
2118
2615
|
async handleErrorReactive() {
|
|
@@ -2125,6 +2622,9 @@ class TaskGraphRunner {
|
|
|
2125
2622
|
}
|
|
2126
2623
|
});
|
|
2127
2624
|
this.running = false;
|
|
2625
|
+
const logger = getLogger3();
|
|
2626
|
+
logger.timeEnd(this.timerLabel);
|
|
2627
|
+
logger.groupEnd();
|
|
2128
2628
|
this.graph.emit("abort");
|
|
2129
2629
|
}
|
|
2130
2630
|
async handleAbortReactive() {
|
|
@@ -2239,118 +2739,27 @@ class GraphAsTask extends Task {
|
|
|
2239
2739
|
if (!this.hasChildren()) {
|
|
2240
2740
|
return this.constructor.inputSchema();
|
|
2241
2741
|
}
|
|
2242
|
-
|
|
2243
|
-
const required = [];
|
|
2244
|
-
const tasks = this.subGraph.getTasks();
|
|
2245
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2246
|
-
for (const task of startingNodes) {
|
|
2247
|
-
const taskInputSchema = task.inputSchema();
|
|
2248
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2249
|
-
if (taskInputSchema === false) {
|
|
2250
|
-
continue;
|
|
2251
|
-
}
|
|
2252
|
-
if (taskInputSchema === true) {
|
|
2253
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2254
|
-
continue;
|
|
2255
|
-
}
|
|
2256
|
-
}
|
|
2257
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2258
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2259
|
-
if (!properties[inputName]) {
|
|
2260
|
-
properties[inputName] = inputProp;
|
|
2261
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2262
|
-
required.push(inputName);
|
|
2263
|
-
}
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
return {
|
|
2268
|
-
type: "object",
|
|
2269
|
-
properties,
|
|
2270
|
-
...required.length > 0 ? { required } : {},
|
|
2271
|
-
additionalProperties: false
|
|
2272
|
-
};
|
|
2742
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2273
2743
|
}
|
|
2274
2744
|
_inputSchemaNode;
|
|
2275
|
-
getInputSchemaNode(
|
|
2745
|
+
getInputSchemaNode() {
|
|
2276
2746
|
if (!this._inputSchemaNode) {
|
|
2277
|
-
const dataPortSchema = this.inputSchema();
|
|
2278
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2279
2747
|
try {
|
|
2748
|
+
const dataPortSchema = this.inputSchema();
|
|
2749
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2280
2750
|
this._inputSchemaNode = schemaNode;
|
|
2281
2751
|
} catch (error) {
|
|
2282
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2752
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2283
2753
|
this._inputSchemaNode = compileSchema2({});
|
|
2284
2754
|
}
|
|
2285
2755
|
}
|
|
2286
2756
|
return this._inputSchemaNode;
|
|
2287
2757
|
}
|
|
2288
|
-
calculateNodeDepths() {
|
|
2289
|
-
const depths = new Map;
|
|
2290
|
-
const tasks = this.subGraph.getTasks();
|
|
2291
|
-
for (const task of tasks) {
|
|
2292
|
-
depths.set(task.config.id, 0);
|
|
2293
|
-
}
|
|
2294
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2295
|
-
for (const task of sortedTasks) {
|
|
2296
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2297
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2298
|
-
for (const targetTask of targetTasks) {
|
|
2299
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2300
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2301
|
-
}
|
|
2302
|
-
}
|
|
2303
|
-
return depths;
|
|
2304
|
-
}
|
|
2305
2758
|
outputSchema() {
|
|
2306
2759
|
if (!this.hasChildren()) {
|
|
2307
2760
|
return this.constructor.outputSchema();
|
|
2308
2761
|
}
|
|
2309
|
-
|
|
2310
|
-
const required = [];
|
|
2311
|
-
const tasks = this.subGraph.getTasks();
|
|
2312
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2313
|
-
const depths = this.calculateNodeDepths();
|
|
2314
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2315
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2316
|
-
const propertyCount = {};
|
|
2317
|
-
const propertySchema = {};
|
|
2318
|
-
for (const task of lastLevelNodes) {
|
|
2319
|
-
const taskOutputSchema = task.outputSchema();
|
|
2320
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2321
|
-
if (taskOutputSchema === false) {
|
|
2322
|
-
continue;
|
|
2323
|
-
}
|
|
2324
|
-
if (taskOutputSchema === true) {
|
|
2325
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2326
|
-
continue;
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2330
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2331
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2332
|
-
if (!propertySchema[outputName]) {
|
|
2333
|
-
propertySchema[outputName] = outputProp;
|
|
2334
|
-
}
|
|
2335
|
-
}
|
|
2336
|
-
}
|
|
2337
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2338
|
-
const outputProp = propertySchema[outputName];
|
|
2339
|
-
if (lastLevelNodes.length === 1) {
|
|
2340
|
-
properties[outputName] = outputProp;
|
|
2341
|
-
} else {
|
|
2342
|
-
properties[outputName] = {
|
|
2343
|
-
type: "array",
|
|
2344
|
-
items: outputProp
|
|
2345
|
-
};
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
return {
|
|
2349
|
-
type: "object",
|
|
2350
|
-
properties,
|
|
2351
|
-
...required.length > 0 ? { required } : {},
|
|
2352
|
-
additionalProperties: false
|
|
2353
|
-
};
|
|
2762
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2354
2763
|
}
|
|
2355
2764
|
resetInputData() {
|
|
2356
2765
|
super.resetInputData();
|
|
@@ -2385,8 +2794,8 @@ class GraphAsTask extends Task {
|
|
|
2385
2794
|
const endingNodeIds = new Set;
|
|
2386
2795
|
const tasks = this.subGraph.getTasks();
|
|
2387
2796
|
for (const task of tasks) {
|
|
2388
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2389
|
-
endingNodeIds.add(task.
|
|
2797
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2798
|
+
endingNodeIds.add(task.id);
|
|
2390
2799
|
}
|
|
2391
2800
|
}
|
|
2392
2801
|
const eventQueue = [];
|
|
@@ -2430,32 +2839,36 @@ class GraphAsTask extends Task {
|
|
|
2430
2839
|
this._inputSchemaNode = undefined;
|
|
2431
2840
|
this.events.emit("regenerate");
|
|
2432
2841
|
}
|
|
2433
|
-
toJSON() {
|
|
2434
|
-
let json = super.toJSON();
|
|
2842
|
+
toJSON(options) {
|
|
2843
|
+
let json = super.toJSON(options);
|
|
2435
2844
|
const hasChildren = this.hasChildren();
|
|
2436
2845
|
if (hasChildren) {
|
|
2437
2846
|
json = {
|
|
2438
2847
|
...json,
|
|
2439
2848
|
merge: this.compoundMerge,
|
|
2440
|
-
subgraph: this.subGraph.toJSON()
|
|
2849
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2441
2850
|
};
|
|
2442
2851
|
}
|
|
2443
2852
|
return json;
|
|
2444
2853
|
}
|
|
2445
|
-
toDependencyJSON() {
|
|
2446
|
-
const json = this.toJSON();
|
|
2854
|
+
toDependencyJSON(options) {
|
|
2855
|
+
const json = this.toJSON(options);
|
|
2447
2856
|
if (this.hasChildren()) {
|
|
2448
2857
|
if ("subgraph" in json) {
|
|
2449
2858
|
delete json.subgraph;
|
|
2450
2859
|
}
|
|
2451
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2860
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2452
2861
|
}
|
|
2453
2862
|
return json;
|
|
2454
2863
|
}
|
|
2455
2864
|
}
|
|
2456
2865
|
|
|
2457
2866
|
// src/task-graph/Workflow.ts
|
|
2458
|
-
import {
|
|
2867
|
+
import {
|
|
2868
|
+
EventEmitter as EventEmitter4,
|
|
2869
|
+
getLogger as getLogger4,
|
|
2870
|
+
uuid4 as uuid44
|
|
2871
|
+
} from "@workglow/util";
|
|
2459
2872
|
function CreateWorkflow(taskClass) {
|
|
2460
2873
|
return Workflow.createWorkflow(taskClass);
|
|
2461
2874
|
}
|
|
@@ -2556,43 +2969,48 @@ class Workflow {
|
|
|
2556
2969
|
const helper = function(input = {}, config = {}) {
|
|
2557
2970
|
this._error = "";
|
|
2558
2971
|
const parent = getLastTask(this);
|
|
2559
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
2972
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2560
2973
|
if (this._dataFlows.length > 0) {
|
|
2561
2974
|
this._dataFlows.forEach((dataflow) => {
|
|
2562
2975
|
const taskSchema = task.inputSchema();
|
|
2563
2976
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2564
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2565
|
-
|
|
2977
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
2978
|
+
getLogger4().error(this._error);
|
|
2566
2979
|
return;
|
|
2567
2980
|
}
|
|
2568
|
-
dataflow.targetTaskId = task.
|
|
2981
|
+
dataflow.targetTaskId = task.id;
|
|
2569
2982
|
this.graph.addDataflow(dataflow);
|
|
2570
2983
|
});
|
|
2571
2984
|
this._dataFlows = [];
|
|
2572
2985
|
}
|
|
2573
|
-
if (parent
|
|
2986
|
+
if (parent) {
|
|
2574
2987
|
const nodes = this._graph.getTasks();
|
|
2575
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
2988
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2576
2989
|
const earlierTasks = [];
|
|
2577
2990
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2578
2991
|
earlierTasks.push(nodes[i]);
|
|
2579
2992
|
}
|
|
2580
2993
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
2994
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2581
2995
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2582
2996
|
providedInputKeys,
|
|
2997
|
+
connectedInputKeys,
|
|
2583
2998
|
earlierTasks
|
|
2584
2999
|
});
|
|
2585
3000
|
if (result.error) {
|
|
2586
3001
|
if (this.isLoopBuilder) {
|
|
2587
3002
|
this._error = result.error;
|
|
2588
|
-
|
|
3003
|
+
getLogger4().warn(this._error);
|
|
2589
3004
|
} else {
|
|
2590
3005
|
this._error = result.error + " Task not added.";
|
|
2591
|
-
|
|
2592
|
-
this.graph.removeTask(task.
|
|
3006
|
+
getLogger4().error(this._error);
|
|
3007
|
+
this.graph.removeTask(task.id);
|
|
2593
3008
|
}
|
|
2594
3009
|
}
|
|
2595
3010
|
}
|
|
3011
|
+
if (!this._error) {
|
|
3012
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3013
|
+
}
|
|
2596
3014
|
return this;
|
|
2597
3015
|
};
|
|
2598
3016
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2672,18 +3090,18 @@ class Workflow {
|
|
|
2672
3090
|
const nodes = this._graph.getTasks();
|
|
2673
3091
|
if (nodes.length === 0) {
|
|
2674
3092
|
this._error = "No tasks to remove";
|
|
2675
|
-
|
|
3093
|
+
getLogger4().error(this._error);
|
|
2676
3094
|
return this;
|
|
2677
3095
|
}
|
|
2678
3096
|
const lastNode = nodes[nodes.length - 1];
|
|
2679
|
-
this._graph.removeTask(lastNode.
|
|
3097
|
+
this._graph.removeTask(lastNode.id);
|
|
2680
3098
|
return this;
|
|
2681
3099
|
}
|
|
2682
|
-
toJSON() {
|
|
2683
|
-
return this._graph.toJSON();
|
|
3100
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3101
|
+
return this._graph.toJSON(options);
|
|
2684
3102
|
}
|
|
2685
|
-
toDependencyJSON() {
|
|
2686
|
-
return this._graph.toDependencyJSON();
|
|
3103
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3104
|
+
return this._graph.toDependencyJSON(options);
|
|
2687
3105
|
}
|
|
2688
3106
|
pipe(...args) {
|
|
2689
3107
|
return pipe(args, this);
|
|
@@ -2703,25 +3121,40 @@ class Workflow {
|
|
|
2703
3121
|
if (-index > nodes.length) {
|
|
2704
3122
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2705
3123
|
this._error = errorMsg;
|
|
2706
|
-
|
|
3124
|
+
getLogger4().error(this._error);
|
|
2707
3125
|
throw new WorkflowError(errorMsg);
|
|
2708
3126
|
}
|
|
2709
3127
|
const lastNode = nodes[nodes.length + index];
|
|
2710
3128
|
const outputSchema = lastNode.outputSchema();
|
|
2711
3129
|
if (typeof outputSchema === "boolean") {
|
|
2712
3130
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2713
|
-
const errorMsg = `Task ${lastNode.
|
|
3131
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2714
3132
|
this._error = errorMsg;
|
|
2715
|
-
|
|
3133
|
+
getLogger4().error(this._error);
|
|
2716
3134
|
throw new WorkflowError(errorMsg);
|
|
2717
3135
|
}
|
|
2718
3136
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2719
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3137
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2720
3138
|
this._error = errorMsg;
|
|
2721
|
-
|
|
3139
|
+
getLogger4().error(this._error);
|
|
2722
3140
|
throw new WorkflowError(errorMsg);
|
|
2723
3141
|
}
|
|
2724
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3142
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3143
|
+
return this;
|
|
3144
|
+
}
|
|
3145
|
+
onError(handler) {
|
|
3146
|
+
this._error = "";
|
|
3147
|
+
const parent = getLastTask(this);
|
|
3148
|
+
if (!parent) {
|
|
3149
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3150
|
+
getLogger4().error(this._error);
|
|
3151
|
+
throw new WorkflowError(this._error);
|
|
3152
|
+
}
|
|
3153
|
+
const handlerTask = ensureTask(handler);
|
|
3154
|
+
this.graph.addTask(handlerTask);
|
|
3155
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3156
|
+
this.graph.addDataflow(dataflow);
|
|
3157
|
+
this.events.emit("changed", handlerTask.id);
|
|
2725
3158
|
return this;
|
|
2726
3159
|
}
|
|
2727
3160
|
toTaskGraph() {
|
|
@@ -2806,16 +3239,16 @@ class Workflow {
|
|
|
2806
3239
|
addLoopTask(taskClass, config = {}) {
|
|
2807
3240
|
this._error = "";
|
|
2808
3241
|
const parent = getLastTask(this);
|
|
2809
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3242
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2810
3243
|
if (this._dataFlows.length > 0) {
|
|
2811
3244
|
this._dataFlows.forEach((dataflow) => {
|
|
2812
3245
|
const taskSchema = task.inputSchema();
|
|
2813
3246
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2814
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2815
|
-
|
|
3247
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3248
|
+
getLogger4().error(this._error);
|
|
2816
3249
|
return;
|
|
2817
3250
|
}
|
|
2818
|
-
dataflow.targetTaskId = task.
|
|
3251
|
+
dataflow.targetTaskId = task.id;
|
|
2819
3252
|
this.graph.addDataflow(dataflow);
|
|
2820
3253
|
});
|
|
2821
3254
|
this._dataFlows = [];
|
|
@@ -2830,9 +3263,9 @@ class Workflow {
|
|
|
2830
3263
|
if (!pending)
|
|
2831
3264
|
return;
|
|
2832
3265
|
const { parent, iteratorTask } = pending;
|
|
2833
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3266
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2834
3267
|
const nodes = this._graph.getTasks();
|
|
2835
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3268
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2836
3269
|
const earlierTasks = [];
|
|
2837
3270
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2838
3271
|
earlierTasks.push(nodes[i]);
|
|
@@ -2842,8 +3275,81 @@ class Workflow {
|
|
|
2842
3275
|
});
|
|
2843
3276
|
if (result.error) {
|
|
2844
3277
|
this._error = result.error + " Task not added.";
|
|
2845
|
-
|
|
2846
|
-
this.graph.removeTask(iteratorTask.
|
|
3278
|
+
getLogger4().error(this._error);
|
|
3279
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3284
|
+
const tasks = graph.getTasks();
|
|
3285
|
+
for (const task of tasks) {
|
|
3286
|
+
if (task.type === "InputTask") {
|
|
3287
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3288
|
+
if (outgoing.length === 0)
|
|
3289
|
+
continue;
|
|
3290
|
+
const properties = {};
|
|
3291
|
+
const required = [];
|
|
3292
|
+
for (const df of outgoing) {
|
|
3293
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3294
|
+
if (!targetTask)
|
|
3295
|
+
continue;
|
|
3296
|
+
const targetSchema = targetTask.inputSchema();
|
|
3297
|
+
if (typeof targetSchema === "boolean")
|
|
3298
|
+
continue;
|
|
3299
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3300
|
+
if (prop && typeof prop !== "boolean") {
|
|
3301
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3302
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3303
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3304
|
+
required.push(df.sourceTaskPortId);
|
|
3305
|
+
}
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
const schema = {
|
|
3310
|
+
type: "object",
|
|
3311
|
+
properties,
|
|
3312
|
+
...required.length > 0 ? { required } : {},
|
|
3313
|
+
additionalProperties: false
|
|
3314
|
+
};
|
|
3315
|
+
task.config = {
|
|
3316
|
+
...task.config,
|
|
3317
|
+
inputSchema: schema,
|
|
3318
|
+
outputSchema: schema
|
|
3319
|
+
};
|
|
3320
|
+
}
|
|
3321
|
+
if (task.type === "OutputTask") {
|
|
3322
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3323
|
+
if (incoming.length === 0)
|
|
3324
|
+
continue;
|
|
3325
|
+
const properties = {};
|
|
3326
|
+
const required = [];
|
|
3327
|
+
for (const df of incoming) {
|
|
3328
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3329
|
+
if (!sourceTask)
|
|
3330
|
+
continue;
|
|
3331
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3332
|
+
if (typeof sourceSchema === "boolean")
|
|
3333
|
+
continue;
|
|
3334
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3335
|
+
if (prop && typeof prop !== "boolean") {
|
|
3336
|
+
properties[df.targetTaskPortId] = prop;
|
|
3337
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3338
|
+
required.push(df.targetTaskPortId);
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
const schema = {
|
|
3343
|
+
type: "object",
|
|
3344
|
+
properties,
|
|
3345
|
+
...required.length > 0 ? { required } : {},
|
|
3346
|
+
additionalProperties: false
|
|
3347
|
+
};
|
|
3348
|
+
task.config = {
|
|
3349
|
+
...task.config,
|
|
3350
|
+
inputSchema: schema,
|
|
3351
|
+
outputSchema: schema
|
|
3352
|
+
};
|
|
2847
3353
|
}
|
|
2848
3354
|
}
|
|
2849
3355
|
}
|
|
@@ -2853,6 +3359,7 @@ class Workflow {
|
|
|
2853
3359
|
const sourceSchema = sourceTask.outputSchema();
|
|
2854
3360
|
const targetSchema = targetTask.inputSchema();
|
|
2855
3361
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3362
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2856
3363
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2857
3364
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2858
3365
|
const formats = new Set;
|
|
@@ -2928,18 +3435,33 @@ class Workflow {
|
|
|
2928
3435
|
if (typeof fromSchema === "object") {
|
|
2929
3436
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2930
3437
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3438
|
+
if (matches.has(fromOutputPortId))
|
|
3439
|
+
continue;
|
|
2931
3440
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2932
3441
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2933
3442
|
}
|
|
2934
3443
|
return;
|
|
2935
3444
|
}
|
|
2936
3445
|
}
|
|
3446
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3447
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3448
|
+
if (matches.has(toInputPortId))
|
|
3449
|
+
continue;
|
|
3450
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3451
|
+
continue;
|
|
3452
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3453
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3454
|
+
}
|
|
3455
|
+
return;
|
|
3456
|
+
}
|
|
2937
3457
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2938
3458
|
return;
|
|
2939
3459
|
}
|
|
2940
3460
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2941
3461
|
if (matches.has(toInputPortId))
|
|
2942
3462
|
continue;
|
|
3463
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3464
|
+
continue;
|
|
2943
3465
|
const candidates = [];
|
|
2944
3466
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2945
3467
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2959,22 +3481,32 @@ class Workflow {
|
|
|
2959
3481
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2960
3482
|
}
|
|
2961
3483
|
};
|
|
2962
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3484
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2963
3485
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2964
3486
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2965
3487
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2966
3488
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2967
3489
|
});
|
|
2968
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3490
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2969
3491
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2970
3492
|
});
|
|
2971
3493
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2972
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3494
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2973
3495
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2974
3496
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2975
3497
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2976
3498
|
const earlierTask = earlierTasks[i];
|
|
2977
3499
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3500
|
+
if (earlierTask.type === "InputTask") {
|
|
3501
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3502
|
+
if (matches.has(requiredInputId))
|
|
3503
|
+
continue;
|
|
3504
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3505
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3506
|
+
}
|
|
3507
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3508
|
+
continue;
|
|
3509
|
+
}
|
|
2978
3510
|
const makeMatchFromEarlier = (comparator) => {
|
|
2979
3511
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2980
3512
|
return;
|
|
@@ -2984,7 +3516,7 @@ class Workflow {
|
|
|
2984
3516
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2985
3517
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2986
3518
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2987
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3519
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2988
3520
|
}
|
|
2989
3521
|
}
|
|
2990
3522
|
}
|
|
@@ -3010,6 +3542,10 @@ class Workflow {
|
|
|
3010
3542
|
};
|
|
3011
3543
|
}
|
|
3012
3544
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3545
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3546
|
+
if (existingTargetConnections.length > 0) {
|
|
3547
|
+
return { matches, unmatchedRequired: [] };
|
|
3548
|
+
}
|
|
3013
3549
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
3014
3550
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
3015
3551
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3132,7 +3668,7 @@ function getLastTask(workflow) {
|
|
|
3132
3668
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3133
3669
|
}
|
|
3134
3670
|
function connect(source, target, workflow) {
|
|
3135
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3671
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3136
3672
|
}
|
|
3137
3673
|
function pipe(args, workflow = new Workflow) {
|
|
3138
3674
|
let previousTask = getLastTask(workflow);
|
|
@@ -3188,7 +3724,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3188
3724
|
// src/task-graph/TaskGraph.ts
|
|
3189
3725
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3190
3726
|
constructor() {
|
|
3191
|
-
super((task) => task.
|
|
3727
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3192
3728
|
}
|
|
3193
3729
|
}
|
|
3194
3730
|
|
|
@@ -3285,18 +3821,22 @@ class TaskGraph {
|
|
|
3285
3821
|
return this._dag.removeNode(taskId);
|
|
3286
3822
|
}
|
|
3287
3823
|
resetGraph() {
|
|
3288
|
-
this.runner.resetGraph(this,
|
|
3824
|
+
this.runner.resetGraph(this, uuid45());
|
|
3289
3825
|
}
|
|
3290
|
-
toJSON() {
|
|
3291
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3826
|
+
toJSON(options) {
|
|
3827
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3292
3828
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3293
|
-
|
|
3829
|
+
let json = {
|
|
3294
3830
|
tasks,
|
|
3295
3831
|
dataflows
|
|
3296
3832
|
};
|
|
3833
|
+
if (options?.withBoundaryNodes) {
|
|
3834
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3835
|
+
}
|
|
3836
|
+
return json;
|
|
3297
3837
|
}
|
|
3298
|
-
toDependencyJSON() {
|
|
3299
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3838
|
+
toDependencyJSON(options) {
|
|
3839
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3300
3840
|
this.getDataflows().forEach((df) => {
|
|
3301
3841
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3302
3842
|
if (!target.dependencies) {
|
|
@@ -3322,6 +3862,9 @@ class TaskGraph {
|
|
|
3322
3862
|
}
|
|
3323
3863
|
}
|
|
3324
3864
|
});
|
|
3865
|
+
if (options?.withBoundaryNodes) {
|
|
3866
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3867
|
+
}
|
|
3325
3868
|
return tasks;
|
|
3326
3869
|
}
|
|
3327
3870
|
get events() {
|
|
@@ -3340,7 +3883,7 @@ class TaskGraph {
|
|
|
3340
3883
|
const tasks = this.getTasks();
|
|
3341
3884
|
tasks.forEach((task) => {
|
|
3342
3885
|
const unsub = task.subscribe("status", (status) => {
|
|
3343
|
-
callback(task.
|
|
3886
|
+
callback(task.id, status);
|
|
3344
3887
|
});
|
|
3345
3888
|
unsubscribes.push(unsub);
|
|
3346
3889
|
});
|
|
@@ -3349,7 +3892,7 @@ class TaskGraph {
|
|
|
3349
3892
|
if (!task || typeof task.subscribe !== "function")
|
|
3350
3893
|
return;
|
|
3351
3894
|
const unsub = task.subscribe("status", (status) => {
|
|
3352
|
-
callback(task.
|
|
3895
|
+
callback(task.id, status);
|
|
3353
3896
|
});
|
|
3354
3897
|
unsubscribes.push(unsub);
|
|
3355
3898
|
};
|
|
@@ -3364,7 +3907,7 @@ class TaskGraph {
|
|
|
3364
3907
|
const tasks = this.getTasks();
|
|
3365
3908
|
tasks.forEach((task) => {
|
|
3366
3909
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3367
|
-
callback(task.
|
|
3910
|
+
callback(task.id, progress, message, ...args);
|
|
3368
3911
|
});
|
|
3369
3912
|
unsubscribes.push(unsub);
|
|
3370
3913
|
});
|
|
@@ -3373,7 +3916,7 @@ class TaskGraph {
|
|
|
3373
3916
|
if (!task || typeof task.subscribe !== "function")
|
|
3374
3917
|
return;
|
|
3375
3918
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3376
|
-
callback(task.
|
|
3919
|
+
callback(task.id, progress, message, ...args);
|
|
3377
3920
|
});
|
|
3378
3921
|
unsubscribes.push(unsub);
|
|
3379
3922
|
};
|
|
@@ -3458,7 +4001,7 @@ class TaskGraph {
|
|
|
3458
4001
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3459
4002
|
const edges = [];
|
|
3460
4003
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3461
|
-
edges.push(new Dataflow(tasks[i].
|
|
4004
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3462
4005
|
}
|
|
3463
4006
|
return edges;
|
|
3464
4007
|
}
|
|
@@ -3468,9 +4011,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3468
4011
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3469
4012
|
return graph;
|
|
3470
4013
|
}
|
|
4014
|
+
// src/task/FallbackTaskRunner.ts
|
|
4015
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4016
|
+
async executeTask(input) {
|
|
4017
|
+
if (this.task.fallbackMode === "data") {
|
|
4018
|
+
return this.executeDataFallback(input);
|
|
4019
|
+
}
|
|
4020
|
+
return this.executeTaskFallback(input);
|
|
4021
|
+
}
|
|
4022
|
+
async executeTaskReactive(input, output) {
|
|
4023
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4024
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4025
|
+
}
|
|
4026
|
+
async executeTaskFallback(input) {
|
|
4027
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4028
|
+
if (tasks.length === 0) {
|
|
4029
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4030
|
+
}
|
|
4031
|
+
const errors = [];
|
|
4032
|
+
const totalAttempts = tasks.length;
|
|
4033
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4034
|
+
if (this.abortController?.signal.aborted) {
|
|
4035
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4036
|
+
}
|
|
4037
|
+
const alternativeTask = tasks[i];
|
|
4038
|
+
const attemptNumber = i + 1;
|
|
4039
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4040
|
+
try {
|
|
4041
|
+
this.resetTask(alternativeTask);
|
|
4042
|
+
const result = await alternativeTask.run(input);
|
|
4043
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4044
|
+
return await this.executeTaskReactive(input, result);
|
|
4045
|
+
} catch (error) {
|
|
4046
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4047
|
+
throw error;
|
|
4048
|
+
}
|
|
4049
|
+
errors.push({ task: alternativeTask, error });
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
4052
|
+
throw this.buildAggregateError(errors, "task");
|
|
4053
|
+
}
|
|
4054
|
+
async executeDataFallback(input) {
|
|
4055
|
+
const alternatives = this.task.alternatives;
|
|
4056
|
+
if (alternatives.length === 0) {
|
|
4057
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4058
|
+
}
|
|
4059
|
+
const errors = [];
|
|
4060
|
+
const totalAttempts = alternatives.length;
|
|
4061
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4062
|
+
if (this.abortController?.signal.aborted) {
|
|
4063
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4064
|
+
}
|
|
4065
|
+
const alternative = alternatives[i];
|
|
4066
|
+
const attemptNumber = i + 1;
|
|
4067
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4068
|
+
try {
|
|
4069
|
+
this.resetSubgraph();
|
|
4070
|
+
const mergedInput = { ...input, ...alternative };
|
|
4071
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4072
|
+
parentSignal: this.abortController?.signal,
|
|
4073
|
+
outputCache: this.outputCache
|
|
4074
|
+
});
|
|
4075
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4076
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4077
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4078
|
+
} catch (error) {
|
|
4079
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4080
|
+
throw error;
|
|
4081
|
+
}
|
|
4082
|
+
errors.push({ alternative, error });
|
|
4083
|
+
}
|
|
4084
|
+
}
|
|
4085
|
+
throw this.buildAggregateError(errors, "data");
|
|
4086
|
+
}
|
|
4087
|
+
resetTask(task) {
|
|
4088
|
+
task.status = TaskStatus.PENDING;
|
|
4089
|
+
task.progress = 0;
|
|
4090
|
+
task.error = undefined;
|
|
4091
|
+
task.completedAt = undefined;
|
|
4092
|
+
task.startedAt = undefined;
|
|
4093
|
+
task.resetInputData();
|
|
4094
|
+
}
|
|
4095
|
+
resetSubgraph() {
|
|
4096
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4097
|
+
this.resetTask(task);
|
|
4098
|
+
}
|
|
4099
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4100
|
+
dataflow.reset();
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
buildAggregateError(errors, mode) {
|
|
4104
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4105
|
+
const details = errors.map((e, i) => {
|
|
4106
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4107
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4108
|
+
}).join(`
|
|
4109
|
+
`);
|
|
4110
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4111
|
+
${details}`);
|
|
4112
|
+
}
|
|
4113
|
+
}
|
|
4114
|
+
|
|
4115
|
+
// src/task/FallbackTask.ts
|
|
4116
|
+
var fallbackTaskConfigSchema = {
|
|
4117
|
+
type: "object",
|
|
4118
|
+
properties: {
|
|
4119
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4120
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4121
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4122
|
+
},
|
|
4123
|
+
additionalProperties: false
|
|
4124
|
+
};
|
|
4125
|
+
|
|
4126
|
+
class FallbackTask extends GraphAsTask {
|
|
4127
|
+
static type = "FallbackTask";
|
|
4128
|
+
static category = "Flow Control";
|
|
4129
|
+
static title = "Fallback";
|
|
4130
|
+
static description = "Try alternatives until one succeeds";
|
|
4131
|
+
static hasDynamicSchemas = true;
|
|
4132
|
+
static configSchema() {
|
|
4133
|
+
return fallbackTaskConfigSchema;
|
|
4134
|
+
}
|
|
4135
|
+
get runner() {
|
|
4136
|
+
if (!this._runner) {
|
|
4137
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4138
|
+
}
|
|
4139
|
+
return this._runner;
|
|
4140
|
+
}
|
|
4141
|
+
get fallbackMode() {
|
|
4142
|
+
return this.config?.fallbackMode ?? "task";
|
|
4143
|
+
}
|
|
4144
|
+
get alternatives() {
|
|
4145
|
+
return this.config?.alternatives ?? [];
|
|
4146
|
+
}
|
|
4147
|
+
inputSchema() {
|
|
4148
|
+
if (!this.hasChildren()) {
|
|
4149
|
+
return this.constructor.inputSchema();
|
|
4150
|
+
}
|
|
4151
|
+
if (this.fallbackMode === "data") {
|
|
4152
|
+
return super.inputSchema();
|
|
4153
|
+
}
|
|
4154
|
+
const properties = {};
|
|
4155
|
+
const tasks = this.subGraph.getTasks();
|
|
4156
|
+
for (const task of tasks) {
|
|
4157
|
+
const taskInputSchema = task.inputSchema();
|
|
4158
|
+
if (typeof taskInputSchema === "boolean")
|
|
4159
|
+
continue;
|
|
4160
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4161
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4162
|
+
if (!properties[inputName]) {
|
|
4163
|
+
properties[inputName] = inputProp;
|
|
4164
|
+
}
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
return {
|
|
4168
|
+
type: "object",
|
|
4169
|
+
properties,
|
|
4170
|
+
additionalProperties: true
|
|
4171
|
+
};
|
|
4172
|
+
}
|
|
4173
|
+
outputSchema() {
|
|
4174
|
+
if (!this.hasChildren()) {
|
|
4175
|
+
return this.constructor.outputSchema();
|
|
4176
|
+
}
|
|
4177
|
+
const tasks = this.subGraph.getTasks();
|
|
4178
|
+
if (tasks.length === 0) {
|
|
4179
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4180
|
+
}
|
|
4181
|
+
if (this.fallbackMode === "task") {
|
|
4182
|
+
const firstTask = tasks[0];
|
|
4183
|
+
return firstTask.outputSchema();
|
|
4184
|
+
}
|
|
4185
|
+
return super.outputSchema();
|
|
4186
|
+
}
|
|
4187
|
+
toJSON() {
|
|
4188
|
+
const json = super.toJSON();
|
|
4189
|
+
return {
|
|
4190
|
+
...json,
|
|
4191
|
+
config: {
|
|
4192
|
+
..."config" in json ? json.config : {},
|
|
4193
|
+
fallbackMode: this.fallbackMode,
|
|
4194
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4195
|
+
}
|
|
4196
|
+
};
|
|
4197
|
+
}
|
|
4198
|
+
}
|
|
4199
|
+
queueMicrotask(() => {
|
|
4200
|
+
Workflow.prototype.fallback = function() {
|
|
4201
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4202
|
+
};
|
|
4203
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4204
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4205
|
+
return this.addLoopTask(FallbackTask, {
|
|
4206
|
+
fallbackMode: "data",
|
|
4207
|
+
alternatives
|
|
4208
|
+
});
|
|
4209
|
+
};
|
|
4210
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4211
|
+
});
|
|
3471
4212
|
// src/task/IteratorTaskRunner.ts
|
|
3472
4213
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3473
|
-
subGraphRunChain = Promise.resolve();
|
|
3474
4214
|
async executeTask(input) {
|
|
3475
4215
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3476
4216
|
if (analysis.iterationCount === 0) {
|
|
@@ -3492,13 +4232,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3492
4232
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3493
4233
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3494
4234
|
const completionOrderResults = [];
|
|
4235
|
+
let completedCount = 0;
|
|
3495
4236
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3496
4237
|
if (this.abortController?.signal.aborted) {
|
|
3497
4238
|
break;
|
|
3498
4239
|
}
|
|
3499
4240
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3500
4241
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3501
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4242
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4243
|
+
completedCount++;
|
|
4244
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4245
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4246
|
+
});
|
|
3502
4247
|
for (const { index, result } of batchResults) {
|
|
3503
4248
|
if (result === undefined)
|
|
3504
4249
|
continue;
|
|
@@ -3508,8 +4253,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3508
4253
|
completionOrderResults.push(result);
|
|
3509
4254
|
}
|
|
3510
4255
|
}
|
|
3511
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3512
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3513
4256
|
}
|
|
3514
4257
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3515
4258
|
return this.task.collectResults(collected);
|
|
@@ -3531,7 +4274,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3531
4274
|
}
|
|
3532
4275
|
return accumulator;
|
|
3533
4276
|
}
|
|
3534
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4277
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3535
4278
|
const results = [];
|
|
3536
4279
|
let cursor = 0;
|
|
3537
4280
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3549,33 +4292,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3549
4292
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3550
4293
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3551
4294
|
results.push({ index, result });
|
|
4295
|
+
await onItemComplete?.();
|
|
3552
4296
|
}
|
|
3553
4297
|
});
|
|
3554
4298
|
await Promise.all(workers);
|
|
3555
4299
|
return results;
|
|
3556
4300
|
}
|
|
4301
|
+
cloneGraph(graph) {
|
|
4302
|
+
const clone = new TaskGraph;
|
|
4303
|
+
for (const task of graph.getTasks()) {
|
|
4304
|
+
const ctor = task.constructor;
|
|
4305
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4306
|
+
if (task.hasChildren()) {
|
|
4307
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4308
|
+
}
|
|
4309
|
+
clone.addTask(newTask);
|
|
4310
|
+
}
|
|
4311
|
+
for (const df of graph.getDataflows()) {
|
|
4312
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4313
|
+
}
|
|
4314
|
+
return clone;
|
|
4315
|
+
}
|
|
3557
4316
|
async executeSubgraphIteration(input) {
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
4317
|
+
if (this.abortController?.signal.aborted) {
|
|
4318
|
+
return;
|
|
4319
|
+
}
|
|
4320
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4321
|
+
const results = await graphClone.run(input, {
|
|
4322
|
+
parentSignal: this.abortController?.signal,
|
|
4323
|
+
outputCache: this.outputCache
|
|
3562
4324
|
});
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
if (this.abortController?.signal.aborted) {
|
|
3566
|
-
return;
|
|
3567
|
-
}
|
|
3568
|
-
const results = await this.task.subGraph.run(input, {
|
|
3569
|
-
parentSignal: this.abortController?.signal,
|
|
3570
|
-
outputCache: this.outputCache
|
|
3571
|
-
});
|
|
3572
|
-
if (results.length === 0) {
|
|
3573
|
-
return;
|
|
3574
|
-
}
|
|
3575
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3576
|
-
} finally {
|
|
3577
|
-
releaseTurn?.();
|
|
4325
|
+
if (results.length === 0) {
|
|
4326
|
+
return;
|
|
3578
4327
|
}
|
|
4328
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3579
4329
|
}
|
|
3580
4330
|
}
|
|
3581
4331
|
|
|
@@ -3859,7 +4609,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3859
4609
|
const tasks = this.subGraph.getTasks();
|
|
3860
4610
|
if (tasks.length === 0)
|
|
3861
4611
|
return;
|
|
3862
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4612
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3863
4613
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3864
4614
|
const properties = {};
|
|
3865
4615
|
const required = [];
|
|
@@ -3886,6 +4636,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3886
4636
|
}
|
|
3887
4637
|
}
|
|
3888
4638
|
}
|
|
4639
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4640
|
+
for (const task of tasks) {
|
|
4641
|
+
if (sourceIds.has(task.id))
|
|
4642
|
+
continue;
|
|
4643
|
+
const inputSchema = task.inputSchema();
|
|
4644
|
+
if (typeof inputSchema === "boolean")
|
|
4645
|
+
continue;
|
|
4646
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4647
|
+
if (requiredKeys.size === 0)
|
|
4648
|
+
continue;
|
|
4649
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4650
|
+
for (const key of requiredKeys) {
|
|
4651
|
+
if (connectedPorts.has(key))
|
|
4652
|
+
continue;
|
|
4653
|
+
if (properties[key])
|
|
4654
|
+
continue;
|
|
4655
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4656
|
+
continue;
|
|
4657
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4658
|
+
if (!prop || typeof prop === "boolean")
|
|
4659
|
+
continue;
|
|
4660
|
+
properties[key] = prop;
|
|
4661
|
+
if (!required.includes(key)) {
|
|
4662
|
+
required.push(key);
|
|
4663
|
+
}
|
|
4664
|
+
}
|
|
4665
|
+
}
|
|
3889
4666
|
return {
|
|
3890
4667
|
type: "object",
|
|
3891
4668
|
properties,
|
|
@@ -4020,7 +4797,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4020
4797
|
if (!this.hasChildren()) {
|
|
4021
4798
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4022
4799
|
}
|
|
4023
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4800
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4024
4801
|
if (endingNodes.length === 0) {
|
|
4025
4802
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4026
4803
|
}
|
|
@@ -4230,8 +5007,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4230
5007
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4231
5008
|
}
|
|
4232
5009
|
this._currentIteration++;
|
|
4233
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4234
|
-
await context.updateProgress(progress, `
|
|
5010
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5011
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4235
5012
|
}
|
|
4236
5013
|
return currentOutput;
|
|
4237
5014
|
}
|
|
@@ -4274,8 +5051,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4274
5051
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4275
5052
|
}
|
|
4276
5053
|
this._currentIteration++;
|
|
4277
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4278
|
-
await context.updateProgress(progress, `
|
|
5054
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5055
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4279
5056
|
}
|
|
4280
5057
|
yield { type: "finish", data: currentOutput };
|
|
4281
5058
|
}
|
|
@@ -4351,7 +5128,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4351
5128
|
return this.constructor.outputSchema();
|
|
4352
5129
|
}
|
|
4353
5130
|
const tasks = this.subGraph.getTasks();
|
|
4354
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5131
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4355
5132
|
if (endingNodes.length === 0) {
|
|
4356
5133
|
return this.constructor.outputSchema();
|
|
4357
5134
|
}
|
|
@@ -5018,7 +5795,7 @@ class ReduceTask extends IteratorTask {
|
|
|
5018
5795
|
if (!this.hasChildren()) {
|
|
5019
5796
|
return this.constructor.outputSchema();
|
|
5020
5797
|
}
|
|
5021
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5798
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5022
5799
|
if (endingNodes.length === 0) {
|
|
5023
5800
|
return this.constructor.outputSchema();
|
|
5024
5801
|
}
|
|
@@ -5056,14 +5833,14 @@ var TaskRegistry = {
|
|
|
5056
5833
|
};
|
|
5057
5834
|
|
|
5058
5835
|
// src/task/TaskJSON.ts
|
|
5059
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5836
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5060
5837
|
if (!item.id)
|
|
5061
5838
|
throw new TaskJSONError("Task id required");
|
|
5062
5839
|
if (!item.type)
|
|
5063
5840
|
throw new TaskJSONError("Task type required");
|
|
5064
5841
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5065
5842
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5066
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5843
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5067
5844
|
if (!taskClass)
|
|
5068
5845
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5069
5846
|
const taskConfig = {
|
|
@@ -5090,20 +5867,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5090
5867
|
}
|
|
5091
5868
|
return subGraph;
|
|
5092
5869
|
};
|
|
5093
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5094
|
-
const task = createSingleTaskFromJSON(item);
|
|
5870
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5871
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5095
5872
|
if (item.subgraph) {
|
|
5096
5873
|
if (!(task instanceof GraphAsTask)) {
|
|
5097
5874
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5098
5875
|
}
|
|
5099
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5876
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5100
5877
|
}
|
|
5101
5878
|
return task;
|
|
5102
5879
|
};
|
|
5103
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5880
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5104
5881
|
const subGraph = new TaskGraph;
|
|
5105
5882
|
for (const subitem of graphJsonObj.tasks) {
|
|
5106
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5883
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5107
5884
|
}
|
|
5108
5885
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5109
5886
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5112,7 +5889,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5112
5889
|
};
|
|
5113
5890
|
// src/task/index.ts
|
|
5114
5891
|
var registerBaseTasks = () => {
|
|
5115
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5892
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5116
5893
|
tasks.map(TaskRegistry.registerTask);
|
|
5117
5894
|
return tasks;
|
|
5118
5895
|
};
|
|
@@ -5295,11 +6072,14 @@ export {
|
|
|
5295
6072
|
isFlexibleSchema,
|
|
5296
6073
|
hasVectorOutput,
|
|
5297
6074
|
hasVectorLikeInput,
|
|
6075
|
+
hasStructuredOutput,
|
|
5298
6076
|
graphAsTaskConfigSchema,
|
|
5299
6077
|
getTaskQueueRegistry,
|
|
6078
|
+
getStructuredOutputSchemas,
|
|
5300
6079
|
getStreamingPorts,
|
|
5301
6080
|
getPortStreamMode,
|
|
5302
6081
|
getOutputStreamMode,
|
|
6082
|
+
getObjectPortId,
|
|
5303
6083
|
getNestedValue,
|
|
5304
6084
|
getLastTask,
|
|
5305
6085
|
getJobQueueFactory,
|
|
@@ -5308,6 +6088,7 @@ export {
|
|
|
5308
6088
|
getAppendPortId,
|
|
5309
6089
|
findArrayPorts,
|
|
5310
6090
|
filterIterationProperties,
|
|
6091
|
+
fallbackTaskConfigSchema,
|
|
5311
6092
|
extractIterationProperties,
|
|
5312
6093
|
extractBaseSchema,
|
|
5313
6094
|
evaluateCondition,
|
|
@@ -5322,13 +6103,19 @@ export {
|
|
|
5322
6103
|
createArraySchema,
|
|
5323
6104
|
connect,
|
|
5324
6105
|
conditionalTaskConfigSchema,
|
|
6106
|
+
computeGraphOutputSchema,
|
|
6107
|
+
computeGraphInputSchema,
|
|
6108
|
+
calculateNodeDepths,
|
|
5325
6109
|
buildIterationInputSchema,
|
|
5326
6110
|
addIterationContextToSchema,
|
|
6111
|
+
addBoundaryNodesToGraphJson,
|
|
6112
|
+
addBoundaryNodesToDependencyJson,
|
|
5327
6113
|
WorkflowError,
|
|
5328
6114
|
Workflow,
|
|
5329
6115
|
WhileTaskRunner,
|
|
5330
6116
|
WhileTask,
|
|
5331
6117
|
WHILE_CONTEXT_SCHEMA,
|
|
6118
|
+
TaskTimeoutError,
|
|
5332
6119
|
TaskStatus,
|
|
5333
6120
|
TaskRegistry,
|
|
5334
6121
|
TaskQueueRegistry,
|
|
@@ -5364,6 +6151,8 @@ export {
|
|
|
5364
6151
|
GraphAsTaskRunner,
|
|
5365
6152
|
GraphAsTask,
|
|
5366
6153
|
GRAPH_RESULT_ARRAY,
|
|
6154
|
+
FallbackTaskRunner,
|
|
6155
|
+
FallbackTask,
|
|
5367
6156
|
EventTaskGraphToDagMapping,
|
|
5368
6157
|
EventDagToTaskGraphMapping,
|
|
5369
6158
|
DataflowArrow,
|
|
@@ -5377,4 +6166,4 @@ export {
|
|
|
5377
6166
|
ConditionalTask
|
|
5378
6167
|
};
|
|
5379
6168
|
|
|
5380
|
-
//# debugId=
|
|
6169
|
+
//# debugId=2417CE87582BE09664756E2164756E21
|