@workglow/task-graph 0.0.102 → 0.0.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1096 -279
- package/dist/browser.js.map +25 -22
- package/dist/bun.js +1096 -279
- package/dist/bun.js.map +25 -22
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1096 -279
- package/dist/node.js.map +25 -22
- package/dist/task/ConditionalTask.d.ts +5 -1
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +15 -12
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +5 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +9 -1
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +4 -0
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +20 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +7 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +5 -0
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/bun.js
CHANGED
|
@@ -21,6 +21,7 @@ var TaskConfigSchema = {
|
|
|
21
21
|
title: { type: "string" },
|
|
22
22
|
description: { type: "string" },
|
|
23
23
|
cacheable: { type: "boolean" },
|
|
24
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
24
25
|
inputSchema: {
|
|
25
26
|
type: "object",
|
|
26
27
|
properties: {},
|
|
@@ -241,8 +242,352 @@ class DataflowArrow extends Dataflow {
|
|
|
241
242
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
242
243
|
}
|
|
243
244
|
}
|
|
245
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
246
|
+
import { uuid4 } from "@workglow/util";
|
|
247
|
+
function calculateNodeDepths(graph) {
|
|
248
|
+
const depths = new Map;
|
|
249
|
+
const tasks = graph.getTasks();
|
|
250
|
+
for (const task of tasks) {
|
|
251
|
+
depths.set(task.id, 0);
|
|
252
|
+
}
|
|
253
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
254
|
+
for (const task of sortedTasks) {
|
|
255
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
256
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
257
|
+
for (const targetTask of targetTasks) {
|
|
258
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
259
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return depths;
|
|
263
|
+
}
|
|
264
|
+
function computeGraphInputSchema(graph, options) {
|
|
265
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
266
|
+
const properties = {};
|
|
267
|
+
const required = [];
|
|
268
|
+
const propertyOrigins = {};
|
|
269
|
+
const tasks = graph.getTasks();
|
|
270
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
271
|
+
for (const task of startingNodes) {
|
|
272
|
+
const taskInputSchema = task.inputSchema();
|
|
273
|
+
if (typeof taskInputSchema === "boolean") {
|
|
274
|
+
if (taskInputSchema === false) {
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (taskInputSchema === true) {
|
|
278
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
283
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
284
|
+
if (!properties[inputName]) {
|
|
285
|
+
properties[inputName] = inputProp;
|
|
286
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
287
|
+
required.push(inputName);
|
|
288
|
+
}
|
|
289
|
+
if (trackOrigins) {
|
|
290
|
+
propertyOrigins[inputName] = [task.id];
|
|
291
|
+
}
|
|
292
|
+
} else if (trackOrigins) {
|
|
293
|
+
propertyOrigins[inputName].push(task.id);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
298
|
+
for (const task of tasks) {
|
|
299
|
+
if (sourceIds.has(task.id))
|
|
300
|
+
continue;
|
|
301
|
+
const taskInputSchema = task.inputSchema();
|
|
302
|
+
if (typeof taskInputSchema === "boolean")
|
|
303
|
+
continue;
|
|
304
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
305
|
+
if (requiredKeys.size === 0)
|
|
306
|
+
continue;
|
|
307
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
308
|
+
for (const key of requiredKeys) {
|
|
309
|
+
if (connectedPorts.has(key))
|
|
310
|
+
continue;
|
|
311
|
+
if (properties[key]) {
|
|
312
|
+
if (trackOrigins) {
|
|
313
|
+
propertyOrigins[key].push(task.id);
|
|
314
|
+
}
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
318
|
+
continue;
|
|
319
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
320
|
+
if (!prop || typeof prop === "boolean")
|
|
321
|
+
continue;
|
|
322
|
+
properties[key] = prop;
|
|
323
|
+
if (!required.includes(key)) {
|
|
324
|
+
required.push(key);
|
|
325
|
+
}
|
|
326
|
+
if (trackOrigins) {
|
|
327
|
+
propertyOrigins[key] = [task.id];
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (trackOrigins) {
|
|
332
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
333
|
+
const prop = properties[propName];
|
|
334
|
+
if (!prop || typeof prop === "boolean")
|
|
335
|
+
continue;
|
|
336
|
+
if (origins.length === 1) {
|
|
337
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
338
|
+
} else {
|
|
339
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return {
|
|
344
|
+
type: "object",
|
|
345
|
+
properties,
|
|
346
|
+
...required.length > 0 ? { required } : {},
|
|
347
|
+
additionalProperties: false
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
function computeGraphOutputSchema(graph, options) {
|
|
351
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
352
|
+
const properties = {};
|
|
353
|
+
const required = [];
|
|
354
|
+
const propertyOrigins = {};
|
|
355
|
+
const tasks = graph.getTasks();
|
|
356
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
357
|
+
const depths = calculateNodeDepths(graph);
|
|
358
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
359
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
360
|
+
const propertyCount = {};
|
|
361
|
+
const propertySchema = {};
|
|
362
|
+
for (const task of lastLevelNodes) {
|
|
363
|
+
const taskOutputSchema = task.outputSchema();
|
|
364
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
365
|
+
if (taskOutputSchema === false) {
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
if (taskOutputSchema === true) {
|
|
369
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
374
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
375
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
376
|
+
if (!propertySchema[outputName]) {
|
|
377
|
+
propertySchema[outputName] = outputProp;
|
|
378
|
+
}
|
|
379
|
+
if (trackOrigins) {
|
|
380
|
+
if (!propertyOrigins[outputName]) {
|
|
381
|
+
propertyOrigins[outputName] = [task.id];
|
|
382
|
+
} else {
|
|
383
|
+
propertyOrigins[outputName].push(task.id);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
389
|
+
const outputProp = propertySchema[outputName];
|
|
390
|
+
if (lastLevelNodes.length === 1) {
|
|
391
|
+
properties[outputName] = outputProp;
|
|
392
|
+
} else {
|
|
393
|
+
properties[outputName] = {
|
|
394
|
+
type: "array",
|
|
395
|
+
items: outputProp
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
if (trackOrigins) {
|
|
400
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
401
|
+
const prop = properties[propName];
|
|
402
|
+
if (!prop || typeof prop === "boolean")
|
|
403
|
+
continue;
|
|
404
|
+
if (origins.length === 1) {
|
|
405
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
406
|
+
} else {
|
|
407
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return {
|
|
412
|
+
type: "object",
|
|
413
|
+
properties,
|
|
414
|
+
...required.length > 0 ? { required } : {},
|
|
415
|
+
additionalProperties: false
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
function stripOriginAnnotations(schema) {
|
|
419
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
420
|
+
return schema;
|
|
421
|
+
const properties = schema.properties;
|
|
422
|
+
if (!properties)
|
|
423
|
+
return schema;
|
|
424
|
+
const strippedProperties = {};
|
|
425
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
426
|
+
if (!prop || typeof prop !== "object") {
|
|
427
|
+
strippedProperties[key] = prop;
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
431
|
+
strippedProperties[key] = rest;
|
|
432
|
+
}
|
|
433
|
+
return { ...schema, properties: strippedProperties };
|
|
434
|
+
}
|
|
435
|
+
function getOriginTaskIds(prop) {
|
|
436
|
+
if (prop["x-source-task-ids"]) {
|
|
437
|
+
return prop["x-source-task-ids"];
|
|
438
|
+
}
|
|
439
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
440
|
+
return [prop["x-source-task-id"]];
|
|
441
|
+
}
|
|
442
|
+
return [];
|
|
443
|
+
}
|
|
444
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
445
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
446
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
447
|
+
if (hasInputTask && hasOutputTask) {
|
|
448
|
+
return json;
|
|
449
|
+
}
|
|
450
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
452
|
+
const prependTasks = [];
|
|
453
|
+
const appendTasks = [];
|
|
454
|
+
const inputDataflows = [];
|
|
455
|
+
const outputDataflows = [];
|
|
456
|
+
if (!hasInputTask && inputSchema) {
|
|
457
|
+
const inputTaskId = uuid4();
|
|
458
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
459
|
+
prependTasks.push({
|
|
460
|
+
id: inputTaskId,
|
|
461
|
+
type: "InputTask",
|
|
462
|
+
config: {
|
|
463
|
+
inputSchema: strippedInputSchema,
|
|
464
|
+
outputSchema: strippedInputSchema
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
468
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
469
|
+
if (!prop || typeof prop === "boolean")
|
|
470
|
+
continue;
|
|
471
|
+
const origins = getOriginTaskIds(prop);
|
|
472
|
+
for (const originId of origins) {
|
|
473
|
+
inputDataflows.push({
|
|
474
|
+
sourceTaskId: inputTaskId,
|
|
475
|
+
sourceTaskPortId: propName,
|
|
476
|
+
targetTaskId: originId,
|
|
477
|
+
targetTaskPortId: propName
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
if (!hasOutputTask && outputSchema) {
|
|
484
|
+
const outputTaskId = uuid4();
|
|
485
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
486
|
+
appendTasks.push({
|
|
487
|
+
id: outputTaskId,
|
|
488
|
+
type: "OutputTask",
|
|
489
|
+
config: {
|
|
490
|
+
inputSchema: strippedOutputSchema,
|
|
491
|
+
outputSchema: strippedOutputSchema
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
495
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
496
|
+
if (!prop || typeof prop === "boolean")
|
|
497
|
+
continue;
|
|
498
|
+
const origins = getOriginTaskIds(prop);
|
|
499
|
+
for (const originId of origins) {
|
|
500
|
+
outputDataflows.push({
|
|
501
|
+
sourceTaskId: originId,
|
|
502
|
+
sourceTaskPortId: propName,
|
|
503
|
+
targetTaskId: outputTaskId,
|
|
504
|
+
targetTaskPortId: propName
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return {
|
|
511
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
512
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
516
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
517
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
518
|
+
if (hasInputTask && hasOutputTask) {
|
|
519
|
+
return items;
|
|
520
|
+
}
|
|
521
|
+
const prependItems = [];
|
|
522
|
+
const appendItems = [];
|
|
523
|
+
if (!hasInputTask) {
|
|
524
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
525
|
+
const inputTaskId = uuid4();
|
|
526
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
527
|
+
prependItems.push({
|
|
528
|
+
id: inputTaskId,
|
|
529
|
+
type: "InputTask",
|
|
530
|
+
config: {
|
|
531
|
+
inputSchema: strippedInputSchema,
|
|
532
|
+
outputSchema: strippedInputSchema
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
536
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
537
|
+
if (!prop || typeof prop === "boolean")
|
|
538
|
+
continue;
|
|
539
|
+
const origins = getOriginTaskIds(prop);
|
|
540
|
+
for (const originId of origins) {
|
|
541
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
542
|
+
if (!targetItem)
|
|
543
|
+
continue;
|
|
544
|
+
if (!targetItem.dependencies) {
|
|
545
|
+
targetItem.dependencies = {};
|
|
546
|
+
}
|
|
547
|
+
const existing = targetItem.dependencies[propName];
|
|
548
|
+
const dep = { id: inputTaskId, output: propName };
|
|
549
|
+
if (!existing) {
|
|
550
|
+
targetItem.dependencies[propName] = dep;
|
|
551
|
+
} else if (Array.isArray(existing)) {
|
|
552
|
+
existing.push(dep);
|
|
553
|
+
} else {
|
|
554
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
if (!hasOutputTask) {
|
|
561
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
562
|
+
const outputTaskId = uuid4();
|
|
563
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
564
|
+
const outputDependencies = {};
|
|
565
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
566
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
567
|
+
if (!prop || typeof prop === "boolean")
|
|
568
|
+
continue;
|
|
569
|
+
const origins = getOriginTaskIds(prop);
|
|
570
|
+
if (origins.length === 1) {
|
|
571
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
572
|
+
} else if (origins.length > 1) {
|
|
573
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
appendItems.push({
|
|
578
|
+
id: outputTaskId,
|
|
579
|
+
type: "OutputTask",
|
|
580
|
+
config: {
|
|
581
|
+
inputSchema: strippedOutputSchema,
|
|
582
|
+
outputSchema: strippedOutputSchema
|
|
583
|
+
},
|
|
584
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
return [...prependItems, ...items, ...appendItems];
|
|
588
|
+
}
|
|
244
589
|
// src/task-graph/TaskGraph.ts
|
|
245
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
590
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
246
591
|
|
|
247
592
|
// src/task/GraphAsTask.ts
|
|
248
593
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -250,9 +595,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
250
595
|
// src/task-graph/TaskGraphRunner.ts
|
|
251
596
|
import {
|
|
252
597
|
collectPropertyValues,
|
|
598
|
+
getLogger as getLogger3,
|
|
253
599
|
globalServiceRegistry as globalServiceRegistry2,
|
|
254
600
|
ServiceRegistry as ServiceRegistry2,
|
|
255
|
-
uuid4 as
|
|
601
|
+
uuid4 as uuid43
|
|
256
602
|
} from "@workglow/util";
|
|
257
603
|
|
|
258
604
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -285,6 +631,9 @@ class TaskOutputRepository {
|
|
|
285
631
|
}
|
|
286
632
|
}
|
|
287
633
|
|
|
634
|
+
// src/task/ConditionalTask.ts
|
|
635
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
636
|
+
|
|
288
637
|
// src/task/ConditionUtils.ts
|
|
289
638
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
290
639
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -357,7 +706,7 @@ import {
|
|
|
357
706
|
compileSchema,
|
|
358
707
|
deepEqual,
|
|
359
708
|
EventEmitter as EventEmitter3,
|
|
360
|
-
uuid4
|
|
709
|
+
uuid4 as uuid42
|
|
361
710
|
} from "@workglow/util";
|
|
362
711
|
|
|
363
712
|
// src/task/TaskError.ts
|
|
@@ -391,6 +740,13 @@ class TaskAbortedError extends TaskError {
|
|
|
391
740
|
}
|
|
392
741
|
}
|
|
393
742
|
|
|
743
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
744
|
+
static type = "TaskTimeoutError";
|
|
745
|
+
constructor(timeoutMs) {
|
|
746
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
394
750
|
class TaskFailedError extends TaskError {
|
|
395
751
|
static type = "TaskFailedError";
|
|
396
752
|
constructor(message = "Task failed") {
|
|
@@ -422,7 +778,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
422
778
|
}
|
|
423
779
|
|
|
424
780
|
// src/task/TaskRunner.ts
|
|
425
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
781
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
426
782
|
|
|
427
783
|
// src/task/InputResolver.ts
|
|
428
784
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -444,6 +800,26 @@ function getSchemaFormat(schema) {
|
|
|
444
800
|
}
|
|
445
801
|
return;
|
|
446
802
|
}
|
|
803
|
+
function getObjectSchema(schema) {
|
|
804
|
+
if (typeof schema !== "object" || schema === null)
|
|
805
|
+
return;
|
|
806
|
+
const s = schema;
|
|
807
|
+
if (s.type === "object" && s.properties && typeof s.properties === "object") {
|
|
808
|
+
return s;
|
|
809
|
+
}
|
|
810
|
+
const variants = s.oneOf ?? s.anyOf;
|
|
811
|
+
if (Array.isArray(variants)) {
|
|
812
|
+
for (const variant of variants) {
|
|
813
|
+
if (typeof variant === "object" && variant !== null) {
|
|
814
|
+
const v = variant;
|
|
815
|
+
if (v.type === "object" && v.properties && typeof v.properties === "object") {
|
|
816
|
+
return v;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
447
823
|
function getFormatPrefix(format) {
|
|
448
824
|
const colonIndex = format.indexOf(":");
|
|
449
825
|
return colonIndex >= 0 ? format.substring(0, colonIndex) : format;
|
|
@@ -457,22 +833,30 @@ async function resolveSchemaInputs(input, schema, config) {
|
|
|
457
833
|
const resolvers = getInputResolvers();
|
|
458
834
|
const resolved = { ...input };
|
|
459
835
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
460
|
-
|
|
836
|
+
let value = resolved[key];
|
|
461
837
|
const format = getSchemaFormat(propSchema);
|
|
462
|
-
if (
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
838
|
+
if (format) {
|
|
839
|
+
let resolver = resolvers.get(format);
|
|
840
|
+
if (!resolver) {
|
|
841
|
+
const prefix = getFormatPrefix(format);
|
|
842
|
+
resolver = resolvers.get(prefix);
|
|
843
|
+
}
|
|
844
|
+
if (resolver) {
|
|
845
|
+
if (typeof value === "string") {
|
|
846
|
+
value = await resolver(value, format, config.registry);
|
|
847
|
+
resolved[key] = value;
|
|
848
|
+
} else if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
|
|
849
|
+
const results = await Promise.all(value.map((item) => resolver(item, format, config.registry)));
|
|
850
|
+
value = results.filter((result) => result !== undefined);
|
|
851
|
+
resolved[key] = value;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
468
854
|
}
|
|
469
|
-
if (!
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
const results = await Promise.all(value.map((item) => resolver(item, format, config.registry)));
|
|
475
|
-
resolved[key] = results.filter((result) => result !== undefined);
|
|
855
|
+
if (value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value)) {
|
|
856
|
+
const objectSchema = getObjectSchema(propSchema);
|
|
857
|
+
if (objectSchema) {
|
|
858
|
+
resolved[key] = await resolveSchemaInputs(value, objectSchema, config);
|
|
859
|
+
}
|
|
476
860
|
}
|
|
477
861
|
}
|
|
478
862
|
return resolved;
|
|
@@ -486,7 +870,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
486
870
|
if (!prop || typeof prop === "boolean")
|
|
487
871
|
return "none";
|
|
488
872
|
const xStream = prop["x-stream"];
|
|
489
|
-
if (xStream === "append" || xStream === "replace")
|
|
873
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
490
874
|
return xStream;
|
|
491
875
|
return "none";
|
|
492
876
|
}
|
|
@@ -501,7 +885,7 @@ function getStreamingPorts(schema) {
|
|
|
501
885
|
if (!prop || typeof prop === "boolean")
|
|
502
886
|
continue;
|
|
503
887
|
const xStream = prop["x-stream"];
|
|
504
|
-
if (xStream === "append" || xStream === "replace") {
|
|
888
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
505
889
|
result.push({ port: name, mode: xStream });
|
|
506
890
|
}
|
|
507
891
|
}
|
|
@@ -545,6 +929,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
545
929
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
546
930
|
return sourceMode !== targetMode;
|
|
547
931
|
}
|
|
932
|
+
function getObjectPortId(schema) {
|
|
933
|
+
if (typeof schema === "boolean")
|
|
934
|
+
return;
|
|
935
|
+
const props = schema.properties;
|
|
936
|
+
if (!props)
|
|
937
|
+
return;
|
|
938
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
939
|
+
if (!prop || typeof prop === "boolean")
|
|
940
|
+
continue;
|
|
941
|
+
if (prop["x-stream"] === "object")
|
|
942
|
+
return name;
|
|
943
|
+
}
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
function getStructuredOutputSchemas(schema) {
|
|
947
|
+
const result = new Map;
|
|
948
|
+
if (typeof schema === "boolean")
|
|
949
|
+
return result;
|
|
950
|
+
const props = schema.properties;
|
|
951
|
+
if (!props)
|
|
952
|
+
return result;
|
|
953
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
954
|
+
if (!prop || typeof prop === "boolean")
|
|
955
|
+
continue;
|
|
956
|
+
if (prop["x-structured-output"] === true) {
|
|
957
|
+
result.set(name, prop);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
return result;
|
|
961
|
+
}
|
|
962
|
+
function hasStructuredOutput(schema) {
|
|
963
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
964
|
+
}
|
|
548
965
|
|
|
549
966
|
// src/task/TaskRunner.ts
|
|
550
967
|
class TaskRunner {
|
|
@@ -555,12 +972,17 @@ class TaskRunner {
|
|
|
555
972
|
outputCache;
|
|
556
973
|
registry = globalServiceRegistry;
|
|
557
974
|
inputStreams;
|
|
975
|
+
timeoutTimer;
|
|
976
|
+
pendingTimeoutError;
|
|
558
977
|
shouldAccumulate = true;
|
|
559
978
|
constructor(task) {
|
|
560
979
|
this.task = task;
|
|
561
980
|
this.own = this.own.bind(this);
|
|
562
981
|
this.handleProgress = this.handleProgress.bind(this);
|
|
563
982
|
}
|
|
983
|
+
get timerLabel() {
|
|
984
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
985
|
+
}
|
|
564
986
|
async run(overrides = {}, config = {}) {
|
|
565
987
|
await this.handleStart(config);
|
|
566
988
|
try {
|
|
@@ -607,7 +1029,7 @@ class TaskRunner {
|
|
|
607
1029
|
return this.task.runOutputData;
|
|
608
1030
|
} catch (err) {
|
|
609
1031
|
await this.handleError(err);
|
|
610
|
-
throw err;
|
|
1032
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
611
1033
|
}
|
|
612
1034
|
}
|
|
613
1035
|
async runReactive(overrides = {}) {
|
|
@@ -664,7 +1086,14 @@ class TaskRunner {
|
|
|
664
1086
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
665
1087
|
}
|
|
666
1088
|
}
|
|
1089
|
+
if (streamMode === "object") {
|
|
1090
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1091
|
+
if (ports.length === 0) {
|
|
1092
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
667
1095
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1096
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
668
1097
|
let chunkCount = 0;
|
|
669
1098
|
let finalOutput;
|
|
670
1099
|
this.task.emit("stream_start");
|
|
@@ -695,6 +1124,13 @@ class TaskRunner {
|
|
|
695
1124
|
break;
|
|
696
1125
|
}
|
|
697
1126
|
case "object-delta": {
|
|
1127
|
+
if (accumulatedObjects) {
|
|
1128
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1129
|
+
}
|
|
1130
|
+
this.task.runOutputData = {
|
|
1131
|
+
...this.task.runOutputData,
|
|
1132
|
+
[event.port]: event.objectDelta
|
|
1133
|
+
};
|
|
698
1134
|
this.task.emit("stream_chunk", event);
|
|
699
1135
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
700
1136
|
await this.handleProgress(progress);
|
|
@@ -707,11 +1143,18 @@ class TaskRunner {
|
|
|
707
1143
|
break;
|
|
708
1144
|
}
|
|
709
1145
|
case "finish": {
|
|
710
|
-
if (accumulated) {
|
|
1146
|
+
if (accumulated || accumulatedObjects) {
|
|
711
1147
|
const merged = { ...event.data || {} };
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
1148
|
+
if (accumulated) {
|
|
1149
|
+
for (const [port, text] of accumulated) {
|
|
1150
|
+
if (text.length > 0)
|
|
1151
|
+
merged[port] = text;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
if (accumulatedObjects) {
|
|
1155
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1156
|
+
merged[port] = obj;
|
|
1157
|
+
}
|
|
715
1158
|
}
|
|
716
1159
|
finalOutput = merged;
|
|
717
1160
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -757,12 +1200,20 @@ class TaskRunner {
|
|
|
757
1200
|
this.outputCache = cache;
|
|
758
1201
|
}
|
|
759
1202
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1203
|
+
const timeout = this.task.config.timeout;
|
|
1204
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1205
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1206
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1207
|
+
this.abort();
|
|
1208
|
+
}, timeout);
|
|
1209
|
+
}
|
|
760
1210
|
if (config.updateProgress) {
|
|
761
1211
|
this.updateProgress = config.updateProgress;
|
|
762
1212
|
}
|
|
763
1213
|
if (config.registry) {
|
|
764
1214
|
this.registry = config.registry;
|
|
765
1215
|
}
|
|
1216
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
766
1217
|
this.task.emit("start");
|
|
767
1218
|
this.task.emit("status", this.task.status);
|
|
768
1219
|
}
|
|
@@ -770,12 +1221,21 @@ class TaskRunner {
|
|
|
770
1221
|
async handleStartReactive() {
|
|
771
1222
|
this.reactiveRunning = true;
|
|
772
1223
|
}
|
|
1224
|
+
clearTimeoutTimer() {
|
|
1225
|
+
if (this.timeoutTimer !== undefined) {
|
|
1226
|
+
clearTimeout(this.timeoutTimer);
|
|
1227
|
+
this.timeoutTimer = undefined;
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
773
1230
|
async handleAbort() {
|
|
774
1231
|
if (this.task.status === TaskStatus.ABORTING)
|
|
775
1232
|
return;
|
|
1233
|
+
this.clearTimeoutTimer();
|
|
1234
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
776
1235
|
this.task.status = TaskStatus.ABORTING;
|
|
777
1236
|
this.task.progress = 100;
|
|
778
|
-
this.task.error = new TaskAbortedError;
|
|
1237
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1238
|
+
this.pendingTimeoutError = undefined;
|
|
779
1239
|
this.task.emit("abort", this.task.error);
|
|
780
1240
|
this.task.emit("status", this.task.status);
|
|
781
1241
|
}
|
|
@@ -785,6 +1245,9 @@ class TaskRunner {
|
|
|
785
1245
|
async handleComplete() {
|
|
786
1246
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
787
1247
|
return;
|
|
1248
|
+
this.clearTimeoutTimer();
|
|
1249
|
+
this.pendingTimeoutError = undefined;
|
|
1250
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
788
1251
|
this.task.completedAt = new Date;
|
|
789
1252
|
this.task.progress = 100;
|
|
790
1253
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -813,6 +1276,9 @@ class TaskRunner {
|
|
|
813
1276
|
return this.handleAbort();
|
|
814
1277
|
if (this.task.status === TaskStatus.FAILED)
|
|
815
1278
|
return;
|
|
1279
|
+
this.clearTimeoutTimer();
|
|
1280
|
+
this.pendingTimeoutError = undefined;
|
|
1281
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
816
1282
|
if (this.task.hasChildren()) {
|
|
817
1283
|
this.task.subGraph.abort();
|
|
818
1284
|
}
|
|
@@ -917,6 +1383,9 @@ class Task {
|
|
|
917
1383
|
runInputData = {};
|
|
918
1384
|
runOutputData = {};
|
|
919
1385
|
config;
|
|
1386
|
+
get id() {
|
|
1387
|
+
return this.config.id;
|
|
1388
|
+
}
|
|
920
1389
|
runConfig = {};
|
|
921
1390
|
status = TaskStatus.PENDING;
|
|
922
1391
|
progress = 0;
|
|
@@ -938,9 +1407,11 @@ class Task {
|
|
|
938
1407
|
this.resetInputData();
|
|
939
1408
|
const title = this.constructor.title || undefined;
|
|
940
1409
|
const baseConfig = Object.assign({
|
|
941
|
-
id: uuid4(),
|
|
942
1410
|
...title ? { title } : {}
|
|
943
1411
|
}, config);
|
|
1412
|
+
if (baseConfig.id === undefined) {
|
|
1413
|
+
baseConfig.id = uuid42();
|
|
1414
|
+
}
|
|
944
1415
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
945
1416
|
this.runConfig = runConfig;
|
|
946
1417
|
}
|
|
@@ -950,7 +1421,7 @@ class Task {
|
|
|
950
1421
|
return {};
|
|
951
1422
|
}
|
|
952
1423
|
try {
|
|
953
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1424
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
954
1425
|
const defaultData = compiledSchema.getData(undefined, {
|
|
955
1426
|
addOptionalProps: true,
|
|
956
1427
|
removeInvalidData: false,
|
|
@@ -1033,7 +1504,7 @@ class Task {
|
|
|
1033
1504
|
this.runInputData[inputId] = prop.default;
|
|
1034
1505
|
}
|
|
1035
1506
|
}
|
|
1036
|
-
if (schema.additionalProperties
|
|
1507
|
+
if (schema.additionalProperties) {
|
|
1037
1508
|
for (const [inputId, value] of Object.entries(input)) {
|
|
1038
1509
|
if (!(inputId in properties)) {
|
|
1039
1510
|
this.runInputData[inputId] = value;
|
|
@@ -1068,7 +1539,7 @@ class Task {
|
|
|
1068
1539
|
continue;
|
|
1069
1540
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1070
1541
|
if (isArray) {
|
|
1071
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1542
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1072
1543
|
const newitems = [...existingItems];
|
|
1073
1544
|
const overrideItem = overrides[inputId];
|
|
1074
1545
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1086,7 +1557,7 @@ class Task {
|
|
|
1086
1557
|
}
|
|
1087
1558
|
}
|
|
1088
1559
|
}
|
|
1089
|
-
if (inputSchema.additionalProperties
|
|
1560
|
+
if (inputSchema.additionalProperties) {
|
|
1090
1561
|
for (const [inputId, value] of Object.entries(overrides)) {
|
|
1091
1562
|
if (!(inputId in properties)) {
|
|
1092
1563
|
if (!deepEqual(this.runInputData[inputId], value)) {
|
|
@@ -1124,25 +1595,29 @@ class Task {
|
|
|
1124
1595
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1125
1596
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1126
1597
|
}
|
|
1127
|
-
static
|
|
1128
|
-
static getConfigSchemaNode(type) {
|
|
1598
|
+
static getConfigSchemaNode() {
|
|
1129
1599
|
const schema = this.configSchema();
|
|
1130
1600
|
if (!schema)
|
|
1131
1601
|
return;
|
|
1132
|
-
if (!
|
|
1602
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1133
1603
|
try {
|
|
1134
1604
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1135
|
-
|
|
1605
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1606
|
+
value: schemaNode,
|
|
1607
|
+
writable: true,
|
|
1608
|
+
configurable: true,
|
|
1609
|
+
enumerable: false
|
|
1610
|
+
});
|
|
1136
1611
|
} catch (error) {
|
|
1137
1612
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1138
1613
|
return;
|
|
1139
1614
|
}
|
|
1140
1615
|
}
|
|
1141
|
-
return this.
|
|
1616
|
+
return this.__compiledConfigSchema;
|
|
1142
1617
|
}
|
|
1143
1618
|
validateAndApplyConfigDefaults(config) {
|
|
1144
1619
|
const ctor = this.constructor;
|
|
1145
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1620
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1146
1621
|
if (!schemaNode)
|
|
1147
1622
|
return config;
|
|
1148
1623
|
const result = schemaNode.validate(config);
|
|
@@ -1155,7 +1630,6 @@ class Task {
|
|
|
1155
1630
|
}
|
|
1156
1631
|
return config;
|
|
1157
1632
|
}
|
|
1158
|
-
static _inputSchemaNode = new Map;
|
|
1159
1633
|
static generateInputSchemaNode(schema) {
|
|
1160
1634
|
if (typeof schema === "boolean") {
|
|
1161
1635
|
if (schema === false) {
|
|
@@ -1165,21 +1639,31 @@ class Task {
|
|
|
1165
1639
|
}
|
|
1166
1640
|
return compileSchema(schema);
|
|
1167
1641
|
}
|
|
1168
|
-
static getInputSchemaNode(
|
|
1169
|
-
if (!
|
|
1642
|
+
static getInputSchemaNode() {
|
|
1643
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1170
1644
|
const dataPortSchema = this.inputSchema();
|
|
1171
1645
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1172
1646
|
try {
|
|
1173
|
-
|
|
1647
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1648
|
+
value: schemaNode,
|
|
1649
|
+
writable: true,
|
|
1650
|
+
configurable: true,
|
|
1651
|
+
enumerable: false
|
|
1652
|
+
});
|
|
1174
1653
|
} catch (error) {
|
|
1175
1654
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1176
|
-
|
|
1655
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1656
|
+
value: compileSchema({}),
|
|
1657
|
+
writable: true,
|
|
1658
|
+
configurable: true,
|
|
1659
|
+
enumerable: false
|
|
1660
|
+
});
|
|
1177
1661
|
}
|
|
1178
1662
|
}
|
|
1179
|
-
return this.
|
|
1663
|
+
return this.__compiledInputSchema;
|
|
1180
1664
|
}
|
|
1181
|
-
getInputSchemaNode(
|
|
1182
|
-
return this.constructor.getInputSchemaNode(
|
|
1665
|
+
getInputSchemaNode() {
|
|
1666
|
+
return this.constructor.getInputSchemaNode();
|
|
1183
1667
|
}
|
|
1184
1668
|
async validateInput(input) {
|
|
1185
1669
|
const ctor = this.constructor;
|
|
@@ -1188,7 +1672,7 @@ class Task {
|
|
|
1188
1672
|
const instanceSchema = this.inputSchema();
|
|
1189
1673
|
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1190
1674
|
} else {
|
|
1191
|
-
schemaNode = this.getInputSchemaNode(
|
|
1675
|
+
schemaNode = this.getInputSchemaNode();
|
|
1192
1676
|
}
|
|
1193
1677
|
const result = schemaNode.validate(input);
|
|
1194
1678
|
if (!result.valid) {
|
|
@@ -1200,9 +1684,6 @@ class Task {
|
|
|
1200
1684
|
}
|
|
1201
1685
|
return true;
|
|
1202
1686
|
}
|
|
1203
|
-
id() {
|
|
1204
|
-
return this.config.id;
|
|
1205
|
-
}
|
|
1206
1687
|
stripSymbols(obj) {
|
|
1207
1688
|
if (obj === null || obj === undefined) {
|
|
1208
1689
|
return obj;
|
|
@@ -1224,14 +1705,15 @@ class Task {
|
|
|
1224
1705
|
}
|
|
1225
1706
|
return obj;
|
|
1226
1707
|
}
|
|
1227
|
-
toJSON() {
|
|
1708
|
+
toJSON(_options) {
|
|
1228
1709
|
const extras = this.config.extras;
|
|
1229
1710
|
const json = this.stripSymbols({
|
|
1230
|
-
id: this.
|
|
1711
|
+
id: this.id,
|
|
1231
1712
|
type: this.type,
|
|
1232
1713
|
defaults: this.defaults,
|
|
1233
1714
|
config: {
|
|
1234
1715
|
...this.config.title ? { title: this.config.title } : {},
|
|
1716
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1235
1717
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1236
1718
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1237
1719
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1239,8 +1721,8 @@ class Task {
|
|
|
1239
1721
|
});
|
|
1240
1722
|
return json;
|
|
1241
1723
|
}
|
|
1242
|
-
toDependencyJSON() {
|
|
1243
|
-
const json = this.toJSON();
|
|
1724
|
+
toDependencyJSON(options) {
|
|
1725
|
+
const json = this.toJSON(options);
|
|
1244
1726
|
return json;
|
|
1245
1727
|
}
|
|
1246
1728
|
hasChildren() {
|
|
@@ -1270,7 +1752,7 @@ class Task {
|
|
|
1270
1752
|
this.subGraph.removeDataflow(dataflow);
|
|
1271
1753
|
}
|
|
1272
1754
|
for (const child of this.subGraph.getTasks()) {
|
|
1273
|
-
this.subGraph.removeTask(child.
|
|
1755
|
+
this.subGraph.removeTask(child.id);
|
|
1274
1756
|
}
|
|
1275
1757
|
}
|
|
1276
1758
|
this.events.emit("regenerate");
|
|
@@ -1361,7 +1843,7 @@ class ConditionalTask extends Task {
|
|
|
1361
1843
|
}
|
|
1362
1844
|
}
|
|
1363
1845
|
} catch (error) {
|
|
1364
|
-
|
|
1846
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1365
1847
|
}
|
|
1366
1848
|
}
|
|
1367
1849
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1526,7 +2008,7 @@ class DependencyBasedScheduler {
|
|
|
1526
2008
|
if (task.status === TaskStatus.DISABLED) {
|
|
1527
2009
|
return false;
|
|
1528
2010
|
}
|
|
1529
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
2011
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1530
2012
|
if (sourceDataflows.length > 0) {
|
|
1531
2013
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1532
2014
|
if (allIncomingDisabled) {
|
|
@@ -1653,6 +2135,10 @@ class TaskGraphRunner {
|
|
|
1653
2135
|
graph.outputCache = outputCache;
|
|
1654
2136
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1655
2137
|
}
|
|
2138
|
+
runId = "";
|
|
2139
|
+
get timerLabel() {
|
|
2140
|
+
return `graph:${this.runId}`;
|
|
2141
|
+
}
|
|
1656
2142
|
async runGraph(input = {}, config) {
|
|
1657
2143
|
await this.handleStart(config);
|
|
1658
2144
|
const results = [];
|
|
@@ -1665,25 +2151,33 @@ class TaskGraphRunner {
|
|
|
1665
2151
|
if (this.failedTaskErrors.size > 0) {
|
|
1666
2152
|
break;
|
|
1667
2153
|
}
|
|
1668
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2154
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1669
2155
|
const runAsync = async () => {
|
|
2156
|
+
let errorRouted = false;
|
|
1670
2157
|
try {
|
|
1671
2158
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1672
2159
|
const taskPromise = this.runTask(task, taskInput);
|
|
1673
|
-
this.inProgressTasks.set(task.
|
|
2160
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1674
2161
|
const taskResult = await taskPromise;
|
|
1675
|
-
if (this.graph.getTargetDataflows(task.
|
|
2162
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1676
2163
|
results.push(taskResult);
|
|
1677
2164
|
}
|
|
1678
2165
|
} catch (error2) {
|
|
1679
|
-
this.
|
|
2166
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2167
|
+
errorRouted = true;
|
|
2168
|
+
this.pushErrorOutputToEdges(task);
|
|
2169
|
+
} else {
|
|
2170
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2171
|
+
}
|
|
1680
2172
|
} finally {
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
2173
|
+
if (!errorRouted) {
|
|
2174
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2175
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2176
|
+
}
|
|
2177
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1684
2178
|
}
|
|
1685
2179
|
};
|
|
1686
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2180
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1687
2181
|
}
|
|
1688
2182
|
} catch (err) {
|
|
1689
2183
|
error = err;
|
|
@@ -1707,7 +2201,7 @@ class TaskGraphRunner {
|
|
|
1707
2201
|
const results = [];
|
|
1708
2202
|
try {
|
|
1709
2203
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1710
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2204
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1711
2205
|
if (task.status === TaskStatus.PENDING) {
|
|
1712
2206
|
task.resetInputData();
|
|
1713
2207
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1715,9 +2209,9 @@ class TaskGraphRunner {
|
|
|
1715
2209
|
const taskInput = isRootTask ? input : {};
|
|
1716
2210
|
const taskResult = await task.runReactive(taskInput);
|
|
1717
2211
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1718
|
-
if (this.graph.getTargetDataflows(task.
|
|
2212
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1719
2213
|
results.push({
|
|
1720
|
-
id: task.
|
|
2214
|
+
id: task.id,
|
|
1721
2215
|
type: task.constructor.runtype || task.constructor.type,
|
|
1722
2216
|
data: taskResult
|
|
1723
2217
|
});
|
|
@@ -1737,7 +2231,7 @@ class TaskGraphRunner {
|
|
|
1737
2231
|
await this.handleDisable();
|
|
1738
2232
|
}
|
|
1739
2233
|
filterInputForTask(task, input) {
|
|
1740
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2234
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1741
2235
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1742
2236
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1743
2237
|
const filteredInput = {};
|
|
@@ -1776,13 +2270,13 @@ class TaskGraphRunner {
|
|
|
1776
2270
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1777
2271
|
}
|
|
1778
2272
|
copyInputFromEdgesToNode(task) {
|
|
1779
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2273
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1780
2274
|
for (const dataflow of dataflows) {
|
|
1781
2275
|
this.addInputData(task, dataflow.getPortData());
|
|
1782
2276
|
}
|
|
1783
2277
|
}
|
|
1784
2278
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1785
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2279
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1786
2280
|
for (const dataflow of dataflows) {
|
|
1787
2281
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1788
2282
|
if (compatibility === "static") {
|
|
@@ -1797,7 +2291,7 @@ class TaskGraphRunner {
|
|
|
1797
2291
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1798
2292
|
if (!node?.config?.id)
|
|
1799
2293
|
return;
|
|
1800
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2294
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1801
2295
|
const effectiveStatus = status ?? node.status;
|
|
1802
2296
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1803
2297
|
const branches = node.config.branches ?? [];
|
|
@@ -1828,10 +2322,31 @@ class TaskGraphRunner {
|
|
|
1828
2322
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1829
2323
|
if (!node?.config?.id)
|
|
1830
2324
|
return;
|
|
1831
|
-
graph.getTargetDataflows(node.
|
|
2325
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1832
2326
|
dataflow.error = node.error;
|
|
1833
2327
|
});
|
|
1834
2328
|
}
|
|
2329
|
+
hasErrorOutputEdges(task) {
|
|
2330
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2331
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2332
|
+
}
|
|
2333
|
+
pushErrorOutputToEdges(task) {
|
|
2334
|
+
const taskError = task.error;
|
|
2335
|
+
const errorData = {
|
|
2336
|
+
error: taskError?.message ?? "Unknown error",
|
|
2337
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2338
|
+
};
|
|
2339
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2340
|
+
for (const df of dataflows) {
|
|
2341
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2342
|
+
df.value = errorData;
|
|
2343
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2344
|
+
} else {
|
|
2345
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
this.propagateDisabledStatus(this.graph);
|
|
2349
|
+
}
|
|
1835
2350
|
propagateDisabledStatus(graph) {
|
|
1836
2351
|
let changed = true;
|
|
1837
2352
|
while (changed) {
|
|
@@ -1840,7 +2355,7 @@ class TaskGraphRunner {
|
|
|
1840
2355
|
if (task.status !== TaskStatus.PENDING) {
|
|
1841
2356
|
continue;
|
|
1842
2357
|
}
|
|
1843
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2358
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1844
2359
|
if (incomingDataflows.length === 0) {
|
|
1845
2360
|
continue;
|
|
1846
2361
|
}
|
|
@@ -1851,10 +2366,10 @@ class TaskGraphRunner {
|
|
|
1851
2366
|
task.completedAt = new Date;
|
|
1852
2367
|
task.emit("disabled");
|
|
1853
2368
|
task.emit("status", task.status);
|
|
1854
|
-
graph.getTargetDataflows(task.
|
|
2369
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1855
2370
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1856
2371
|
});
|
|
1857
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2372
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1858
2373
|
changed = true;
|
|
1859
2374
|
}
|
|
1860
2375
|
}
|
|
@@ -1863,7 +2378,7 @@ class TaskGraphRunner {
|
|
|
1863
2378
|
taskNeedsAccumulation(task) {
|
|
1864
2379
|
if (this.outputCache)
|
|
1865
2380
|
return true;
|
|
1866
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2381
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1867
2382
|
if (outEdges.length === 0)
|
|
1868
2383
|
return this.accumulateLeafOutputs;
|
|
1869
2384
|
const outSchema = task.outputSchema();
|
|
@@ -1886,7 +2401,7 @@ class TaskGraphRunner {
|
|
|
1886
2401
|
async runTask(task, input) {
|
|
1887
2402
|
const isStreamable = isTaskStreamable(task);
|
|
1888
2403
|
if (isStreamable) {
|
|
1889
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2404
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1890
2405
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1891
2406
|
if (streamingEdges.length > 0) {
|
|
1892
2407
|
const inputStreams = new Map;
|
|
@@ -1911,13 +2426,13 @@ class TaskGraphRunner {
|
|
|
1911
2426
|
});
|
|
1912
2427
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1913
2428
|
return {
|
|
1914
|
-
id: task.
|
|
2429
|
+
id: task.id,
|
|
1915
2430
|
type: task.constructor.runtype || task.constructor.type,
|
|
1916
2431
|
data: results
|
|
1917
2432
|
};
|
|
1918
2433
|
}
|
|
1919
2434
|
async awaitStreamInputs(task) {
|
|
1920
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2435
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1921
2436
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1922
2437
|
if (streamPromises.length > 0) {
|
|
1923
2438
|
await Promise.all(streamPromises);
|
|
@@ -1932,17 +2447,17 @@ class TaskGraphRunner {
|
|
|
1932
2447
|
streamingNotified = true;
|
|
1933
2448
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1934
2449
|
this.pushStreamToEdges(task, streamMode);
|
|
1935
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2450
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1936
2451
|
}
|
|
1937
2452
|
};
|
|
1938
2453
|
const onStreamStart = () => {
|
|
1939
|
-
this.graph.emit("task_stream_start", task.
|
|
2454
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1940
2455
|
};
|
|
1941
2456
|
const onStreamChunk = (event) => {
|
|
1942
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2457
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1943
2458
|
};
|
|
1944
2459
|
const onStreamEnd = (output) => {
|
|
1945
|
-
this.graph.emit("task_stream_end", task.
|
|
2460
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1946
2461
|
};
|
|
1947
2462
|
task.on("status", onStatus);
|
|
1948
2463
|
task.on("stream_start", onStreamStart);
|
|
@@ -1957,7 +2472,7 @@ class TaskGraphRunner {
|
|
|
1957
2472
|
});
|
|
1958
2473
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1959
2474
|
return {
|
|
1960
|
-
id: task.
|
|
2475
|
+
id: task.id,
|
|
1961
2476
|
type: task.constructor.runtype || task.constructor.type,
|
|
1962
2477
|
data: results
|
|
1963
2478
|
};
|
|
@@ -1995,7 +2510,7 @@ class TaskGraphRunner {
|
|
|
1995
2510
|
});
|
|
1996
2511
|
}
|
|
1997
2512
|
pushStreamToEdges(task, streamMode) {
|
|
1998
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2513
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1999
2514
|
if (targetDataflows.length === 0)
|
|
2000
2515
|
return;
|
|
2001
2516
|
const groups = new Map;
|
|
@@ -2086,11 +2601,15 @@ class TaskGraphRunner {
|
|
|
2086
2601
|
this.abortController?.abort();
|
|
2087
2602
|
}, { once: true });
|
|
2088
2603
|
}
|
|
2089
|
-
this.
|
|
2604
|
+
this.runId = uuid43();
|
|
2605
|
+
this.resetGraph(this.graph, this.runId);
|
|
2090
2606
|
this.processScheduler.reset();
|
|
2091
2607
|
this.inProgressTasks.clear();
|
|
2092
2608
|
this.inProgressFunctions.clear();
|
|
2093
2609
|
this.failedTaskErrors.clear();
|
|
2610
|
+
const logger = getLogger3();
|
|
2611
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2612
|
+
logger.time(this.timerLabel);
|
|
2094
2613
|
this.graph.emit("start");
|
|
2095
2614
|
}
|
|
2096
2615
|
async handleStartReactive() {
|
|
@@ -2102,6 +2621,9 @@ class TaskGraphRunner {
|
|
|
2102
2621
|
}
|
|
2103
2622
|
async handleComplete() {
|
|
2104
2623
|
this.running = false;
|
|
2624
|
+
const logger = getLogger3();
|
|
2625
|
+
logger.timeEnd(this.timerLabel);
|
|
2626
|
+
logger.groupEnd();
|
|
2105
2627
|
this.graph.emit("complete");
|
|
2106
2628
|
}
|
|
2107
2629
|
async handleCompleteReactive() {
|
|
@@ -2114,6 +2636,9 @@ class TaskGraphRunner {
|
|
|
2114
2636
|
}
|
|
2115
2637
|
}));
|
|
2116
2638
|
this.running = false;
|
|
2639
|
+
const logger = getLogger3();
|
|
2640
|
+
logger.timeEnd(this.timerLabel);
|
|
2641
|
+
logger.groupEnd();
|
|
2117
2642
|
this.graph.emit("error", error);
|
|
2118
2643
|
}
|
|
2119
2644
|
async handleErrorReactive() {
|
|
@@ -2126,6 +2651,9 @@ class TaskGraphRunner {
|
|
|
2126
2651
|
}
|
|
2127
2652
|
});
|
|
2128
2653
|
this.running = false;
|
|
2654
|
+
const logger = getLogger3();
|
|
2655
|
+
logger.timeEnd(this.timerLabel);
|
|
2656
|
+
logger.groupEnd();
|
|
2129
2657
|
this.graph.emit("abort");
|
|
2130
2658
|
}
|
|
2131
2659
|
async handleAbortReactive() {
|
|
@@ -2240,118 +2768,27 @@ class GraphAsTask extends Task {
|
|
|
2240
2768
|
if (!this.hasChildren()) {
|
|
2241
2769
|
return this.constructor.inputSchema();
|
|
2242
2770
|
}
|
|
2243
|
-
|
|
2244
|
-
const required = [];
|
|
2245
|
-
const tasks = this.subGraph.getTasks();
|
|
2246
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2247
|
-
for (const task of startingNodes) {
|
|
2248
|
-
const taskInputSchema = task.inputSchema();
|
|
2249
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2250
|
-
if (taskInputSchema === false) {
|
|
2251
|
-
continue;
|
|
2252
|
-
}
|
|
2253
|
-
if (taskInputSchema === true) {
|
|
2254
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2255
|
-
continue;
|
|
2256
|
-
}
|
|
2257
|
-
}
|
|
2258
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2259
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2260
|
-
if (!properties[inputName]) {
|
|
2261
|
-
properties[inputName] = inputProp;
|
|
2262
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2263
|
-
required.push(inputName);
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
}
|
|
2268
|
-
return {
|
|
2269
|
-
type: "object",
|
|
2270
|
-
properties,
|
|
2271
|
-
...required.length > 0 ? { required } : {},
|
|
2272
|
-
additionalProperties: false
|
|
2273
|
-
};
|
|
2771
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2274
2772
|
}
|
|
2275
2773
|
_inputSchemaNode;
|
|
2276
|
-
getInputSchemaNode(
|
|
2774
|
+
getInputSchemaNode() {
|
|
2277
2775
|
if (!this._inputSchemaNode) {
|
|
2278
|
-
const dataPortSchema = this.inputSchema();
|
|
2279
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2280
2776
|
try {
|
|
2777
|
+
const dataPortSchema = this.inputSchema();
|
|
2778
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2281
2779
|
this._inputSchemaNode = schemaNode;
|
|
2282
2780
|
} catch (error) {
|
|
2283
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2781
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2284
2782
|
this._inputSchemaNode = compileSchema2({});
|
|
2285
2783
|
}
|
|
2286
2784
|
}
|
|
2287
2785
|
return this._inputSchemaNode;
|
|
2288
2786
|
}
|
|
2289
|
-
calculateNodeDepths() {
|
|
2290
|
-
const depths = new Map;
|
|
2291
|
-
const tasks = this.subGraph.getTasks();
|
|
2292
|
-
for (const task of tasks) {
|
|
2293
|
-
depths.set(task.config.id, 0);
|
|
2294
|
-
}
|
|
2295
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2296
|
-
for (const task of sortedTasks) {
|
|
2297
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2298
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2299
|
-
for (const targetTask of targetTasks) {
|
|
2300
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2301
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2302
|
-
}
|
|
2303
|
-
}
|
|
2304
|
-
return depths;
|
|
2305
|
-
}
|
|
2306
2787
|
outputSchema() {
|
|
2307
2788
|
if (!this.hasChildren()) {
|
|
2308
2789
|
return this.constructor.outputSchema();
|
|
2309
2790
|
}
|
|
2310
|
-
|
|
2311
|
-
const required = [];
|
|
2312
|
-
const tasks = this.subGraph.getTasks();
|
|
2313
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2314
|
-
const depths = this.calculateNodeDepths();
|
|
2315
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2316
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2317
|
-
const propertyCount = {};
|
|
2318
|
-
const propertySchema = {};
|
|
2319
|
-
for (const task of lastLevelNodes) {
|
|
2320
|
-
const taskOutputSchema = task.outputSchema();
|
|
2321
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2322
|
-
if (taskOutputSchema === false) {
|
|
2323
|
-
continue;
|
|
2324
|
-
}
|
|
2325
|
-
if (taskOutputSchema === true) {
|
|
2326
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2327
|
-
continue;
|
|
2328
|
-
}
|
|
2329
|
-
}
|
|
2330
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2331
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2332
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2333
|
-
if (!propertySchema[outputName]) {
|
|
2334
|
-
propertySchema[outputName] = outputProp;
|
|
2335
|
-
}
|
|
2336
|
-
}
|
|
2337
|
-
}
|
|
2338
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2339
|
-
const outputProp = propertySchema[outputName];
|
|
2340
|
-
if (lastLevelNodes.length === 1) {
|
|
2341
|
-
properties[outputName] = outputProp;
|
|
2342
|
-
} else {
|
|
2343
|
-
properties[outputName] = {
|
|
2344
|
-
type: "array",
|
|
2345
|
-
items: outputProp
|
|
2346
|
-
};
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
return {
|
|
2350
|
-
type: "object",
|
|
2351
|
-
properties,
|
|
2352
|
-
...required.length > 0 ? { required } : {},
|
|
2353
|
-
additionalProperties: false
|
|
2354
|
-
};
|
|
2791
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2355
2792
|
}
|
|
2356
2793
|
resetInputData() {
|
|
2357
2794
|
super.resetInputData();
|
|
@@ -2386,8 +2823,8 @@ class GraphAsTask extends Task {
|
|
|
2386
2823
|
const endingNodeIds = new Set;
|
|
2387
2824
|
const tasks = this.subGraph.getTasks();
|
|
2388
2825
|
for (const task of tasks) {
|
|
2389
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2390
|
-
endingNodeIds.add(task.
|
|
2826
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2827
|
+
endingNodeIds.add(task.id);
|
|
2391
2828
|
}
|
|
2392
2829
|
}
|
|
2393
2830
|
const eventQueue = [];
|
|
@@ -2431,32 +2868,36 @@ class GraphAsTask extends Task {
|
|
|
2431
2868
|
this._inputSchemaNode = undefined;
|
|
2432
2869
|
this.events.emit("regenerate");
|
|
2433
2870
|
}
|
|
2434
|
-
toJSON() {
|
|
2435
|
-
let json = super.toJSON();
|
|
2871
|
+
toJSON(options) {
|
|
2872
|
+
let json = super.toJSON(options);
|
|
2436
2873
|
const hasChildren = this.hasChildren();
|
|
2437
2874
|
if (hasChildren) {
|
|
2438
2875
|
json = {
|
|
2439
2876
|
...json,
|
|
2440
2877
|
merge: this.compoundMerge,
|
|
2441
|
-
subgraph: this.subGraph.toJSON()
|
|
2878
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2442
2879
|
};
|
|
2443
2880
|
}
|
|
2444
2881
|
return json;
|
|
2445
2882
|
}
|
|
2446
|
-
toDependencyJSON() {
|
|
2447
|
-
const json = this.toJSON();
|
|
2883
|
+
toDependencyJSON(options) {
|
|
2884
|
+
const json = this.toJSON(options);
|
|
2448
2885
|
if (this.hasChildren()) {
|
|
2449
2886
|
if ("subgraph" in json) {
|
|
2450
2887
|
delete json.subgraph;
|
|
2451
2888
|
}
|
|
2452
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2889
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2453
2890
|
}
|
|
2454
2891
|
return json;
|
|
2455
2892
|
}
|
|
2456
2893
|
}
|
|
2457
2894
|
|
|
2458
2895
|
// src/task-graph/Workflow.ts
|
|
2459
|
-
import {
|
|
2896
|
+
import {
|
|
2897
|
+
EventEmitter as EventEmitter4,
|
|
2898
|
+
getLogger as getLogger4,
|
|
2899
|
+
uuid4 as uuid44
|
|
2900
|
+
} from "@workglow/util";
|
|
2460
2901
|
function CreateWorkflow(taskClass) {
|
|
2461
2902
|
return Workflow.createWorkflow(taskClass);
|
|
2462
2903
|
}
|
|
@@ -2557,43 +2998,48 @@ class Workflow {
|
|
|
2557
2998
|
const helper = function(input = {}, config = {}) {
|
|
2558
2999
|
this._error = "";
|
|
2559
3000
|
const parent = getLastTask(this);
|
|
2560
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
3001
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2561
3002
|
if (this._dataFlows.length > 0) {
|
|
2562
3003
|
this._dataFlows.forEach((dataflow) => {
|
|
2563
3004
|
const taskSchema = task.inputSchema();
|
|
2564
3005
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2565
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2566
|
-
|
|
3006
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3007
|
+
getLogger4().error(this._error);
|
|
2567
3008
|
return;
|
|
2568
3009
|
}
|
|
2569
|
-
dataflow.targetTaskId = task.
|
|
3010
|
+
dataflow.targetTaskId = task.id;
|
|
2570
3011
|
this.graph.addDataflow(dataflow);
|
|
2571
3012
|
});
|
|
2572
3013
|
this._dataFlows = [];
|
|
2573
3014
|
}
|
|
2574
|
-
if (parent
|
|
3015
|
+
if (parent) {
|
|
2575
3016
|
const nodes = this._graph.getTasks();
|
|
2576
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3017
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2577
3018
|
const earlierTasks = [];
|
|
2578
3019
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2579
3020
|
earlierTasks.push(nodes[i]);
|
|
2580
3021
|
}
|
|
2581
3022
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
3023
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2582
3024
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2583
3025
|
providedInputKeys,
|
|
3026
|
+
connectedInputKeys,
|
|
2584
3027
|
earlierTasks
|
|
2585
3028
|
});
|
|
2586
3029
|
if (result.error) {
|
|
2587
3030
|
if (this.isLoopBuilder) {
|
|
2588
3031
|
this._error = result.error;
|
|
2589
|
-
|
|
3032
|
+
getLogger4().warn(this._error);
|
|
2590
3033
|
} else {
|
|
2591
3034
|
this._error = result.error + " Task not added.";
|
|
2592
|
-
|
|
2593
|
-
this.graph.removeTask(task.
|
|
3035
|
+
getLogger4().error(this._error);
|
|
3036
|
+
this.graph.removeTask(task.id);
|
|
2594
3037
|
}
|
|
2595
3038
|
}
|
|
2596
3039
|
}
|
|
3040
|
+
if (!this._error) {
|
|
3041
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3042
|
+
}
|
|
2597
3043
|
return this;
|
|
2598
3044
|
};
|
|
2599
3045
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2673,18 +3119,18 @@ class Workflow {
|
|
|
2673
3119
|
const nodes = this._graph.getTasks();
|
|
2674
3120
|
if (nodes.length === 0) {
|
|
2675
3121
|
this._error = "No tasks to remove";
|
|
2676
|
-
|
|
3122
|
+
getLogger4().error(this._error);
|
|
2677
3123
|
return this;
|
|
2678
3124
|
}
|
|
2679
3125
|
const lastNode = nodes[nodes.length - 1];
|
|
2680
|
-
this._graph.removeTask(lastNode.
|
|
3126
|
+
this._graph.removeTask(lastNode.id);
|
|
2681
3127
|
return this;
|
|
2682
3128
|
}
|
|
2683
|
-
toJSON() {
|
|
2684
|
-
return this._graph.toJSON();
|
|
3129
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3130
|
+
return this._graph.toJSON(options);
|
|
2685
3131
|
}
|
|
2686
|
-
toDependencyJSON() {
|
|
2687
|
-
return this._graph.toDependencyJSON();
|
|
3132
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3133
|
+
return this._graph.toDependencyJSON(options);
|
|
2688
3134
|
}
|
|
2689
3135
|
pipe(...args) {
|
|
2690
3136
|
return pipe(args, this);
|
|
@@ -2704,25 +3150,40 @@ class Workflow {
|
|
|
2704
3150
|
if (-index > nodes.length) {
|
|
2705
3151
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2706
3152
|
this._error = errorMsg;
|
|
2707
|
-
|
|
3153
|
+
getLogger4().error(this._error);
|
|
2708
3154
|
throw new WorkflowError(errorMsg);
|
|
2709
3155
|
}
|
|
2710
3156
|
const lastNode = nodes[nodes.length + index];
|
|
2711
3157
|
const outputSchema = lastNode.outputSchema();
|
|
2712
3158
|
if (typeof outputSchema === "boolean") {
|
|
2713
3159
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2714
|
-
const errorMsg = `Task ${lastNode.
|
|
3160
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2715
3161
|
this._error = errorMsg;
|
|
2716
|
-
|
|
3162
|
+
getLogger4().error(this._error);
|
|
2717
3163
|
throw new WorkflowError(errorMsg);
|
|
2718
3164
|
}
|
|
2719
3165
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2720
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3166
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2721
3167
|
this._error = errorMsg;
|
|
2722
|
-
|
|
3168
|
+
getLogger4().error(this._error);
|
|
2723
3169
|
throw new WorkflowError(errorMsg);
|
|
2724
3170
|
}
|
|
2725
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3171
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3172
|
+
return this;
|
|
3173
|
+
}
|
|
3174
|
+
onError(handler) {
|
|
3175
|
+
this._error = "";
|
|
3176
|
+
const parent = getLastTask(this);
|
|
3177
|
+
if (!parent) {
|
|
3178
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3179
|
+
getLogger4().error(this._error);
|
|
3180
|
+
throw new WorkflowError(this._error);
|
|
3181
|
+
}
|
|
3182
|
+
const handlerTask = ensureTask(handler);
|
|
3183
|
+
this.graph.addTask(handlerTask);
|
|
3184
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3185
|
+
this.graph.addDataflow(dataflow);
|
|
3186
|
+
this.events.emit("changed", handlerTask.id);
|
|
2726
3187
|
return this;
|
|
2727
3188
|
}
|
|
2728
3189
|
toTaskGraph() {
|
|
@@ -2807,16 +3268,16 @@ class Workflow {
|
|
|
2807
3268
|
addLoopTask(taskClass, config = {}) {
|
|
2808
3269
|
this._error = "";
|
|
2809
3270
|
const parent = getLastTask(this);
|
|
2810
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3271
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2811
3272
|
if (this._dataFlows.length > 0) {
|
|
2812
3273
|
this._dataFlows.forEach((dataflow) => {
|
|
2813
3274
|
const taskSchema = task.inputSchema();
|
|
2814
3275
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2815
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2816
|
-
|
|
3276
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3277
|
+
getLogger4().error(this._error);
|
|
2817
3278
|
return;
|
|
2818
3279
|
}
|
|
2819
|
-
dataflow.targetTaskId = task.
|
|
3280
|
+
dataflow.targetTaskId = task.id;
|
|
2820
3281
|
this.graph.addDataflow(dataflow);
|
|
2821
3282
|
});
|
|
2822
3283
|
this._dataFlows = [];
|
|
@@ -2831,9 +3292,9 @@ class Workflow {
|
|
|
2831
3292
|
if (!pending)
|
|
2832
3293
|
return;
|
|
2833
3294
|
const { parent, iteratorTask } = pending;
|
|
2834
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3295
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2835
3296
|
const nodes = this._graph.getTasks();
|
|
2836
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3297
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2837
3298
|
const earlierTasks = [];
|
|
2838
3299
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2839
3300
|
earlierTasks.push(nodes[i]);
|
|
@@ -2843,8 +3304,81 @@ class Workflow {
|
|
|
2843
3304
|
});
|
|
2844
3305
|
if (result.error) {
|
|
2845
3306
|
this._error = result.error + " Task not added.";
|
|
2846
|
-
|
|
2847
|
-
this.graph.removeTask(iteratorTask.
|
|
3307
|
+
getLogger4().error(this._error);
|
|
3308
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3309
|
+
}
|
|
3310
|
+
}
|
|
3311
|
+
}
|
|
3312
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3313
|
+
const tasks = graph.getTasks();
|
|
3314
|
+
for (const task of tasks) {
|
|
3315
|
+
if (task.type === "InputTask") {
|
|
3316
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3317
|
+
if (outgoing.length === 0)
|
|
3318
|
+
continue;
|
|
3319
|
+
const properties = {};
|
|
3320
|
+
const required = [];
|
|
3321
|
+
for (const df of outgoing) {
|
|
3322
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3323
|
+
if (!targetTask)
|
|
3324
|
+
continue;
|
|
3325
|
+
const targetSchema = targetTask.inputSchema();
|
|
3326
|
+
if (typeof targetSchema === "boolean")
|
|
3327
|
+
continue;
|
|
3328
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3329
|
+
if (prop && typeof prop !== "boolean") {
|
|
3330
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3331
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3332
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3333
|
+
required.push(df.sourceTaskPortId);
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
}
|
|
3338
|
+
const schema = {
|
|
3339
|
+
type: "object",
|
|
3340
|
+
properties,
|
|
3341
|
+
...required.length > 0 ? { required } : {},
|
|
3342
|
+
additionalProperties: false
|
|
3343
|
+
};
|
|
3344
|
+
task.config = {
|
|
3345
|
+
...task.config,
|
|
3346
|
+
inputSchema: schema,
|
|
3347
|
+
outputSchema: schema
|
|
3348
|
+
};
|
|
3349
|
+
}
|
|
3350
|
+
if (task.type === "OutputTask") {
|
|
3351
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3352
|
+
if (incoming.length === 0)
|
|
3353
|
+
continue;
|
|
3354
|
+
const properties = {};
|
|
3355
|
+
const required = [];
|
|
3356
|
+
for (const df of incoming) {
|
|
3357
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3358
|
+
if (!sourceTask)
|
|
3359
|
+
continue;
|
|
3360
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3361
|
+
if (typeof sourceSchema === "boolean")
|
|
3362
|
+
continue;
|
|
3363
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3364
|
+
if (prop && typeof prop !== "boolean") {
|
|
3365
|
+
properties[df.targetTaskPortId] = prop;
|
|
3366
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3367
|
+
required.push(df.targetTaskPortId);
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
const schema = {
|
|
3372
|
+
type: "object",
|
|
3373
|
+
properties,
|
|
3374
|
+
...required.length > 0 ? { required } : {},
|
|
3375
|
+
additionalProperties: false
|
|
3376
|
+
};
|
|
3377
|
+
task.config = {
|
|
3378
|
+
...task.config,
|
|
3379
|
+
inputSchema: schema,
|
|
3380
|
+
outputSchema: schema
|
|
3381
|
+
};
|
|
2848
3382
|
}
|
|
2849
3383
|
}
|
|
2850
3384
|
}
|
|
@@ -2854,6 +3388,7 @@ class Workflow {
|
|
|
2854
3388
|
const sourceSchema = sourceTask.outputSchema();
|
|
2855
3389
|
const targetSchema = targetTask.inputSchema();
|
|
2856
3390
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3391
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2857
3392
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2858
3393
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2859
3394
|
const formats = new Set;
|
|
@@ -2929,18 +3464,33 @@ class Workflow {
|
|
|
2929
3464
|
if (typeof fromSchema === "object") {
|
|
2930
3465
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2931
3466
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3467
|
+
if (matches.has(fromOutputPortId))
|
|
3468
|
+
continue;
|
|
2932
3469
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2933
3470
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2934
3471
|
}
|
|
2935
3472
|
return;
|
|
2936
3473
|
}
|
|
2937
3474
|
}
|
|
3475
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3476
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3477
|
+
if (matches.has(toInputPortId))
|
|
3478
|
+
continue;
|
|
3479
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3480
|
+
continue;
|
|
3481
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3482
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3483
|
+
}
|
|
3484
|
+
return;
|
|
3485
|
+
}
|
|
2938
3486
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2939
3487
|
return;
|
|
2940
3488
|
}
|
|
2941
3489
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2942
3490
|
if (matches.has(toInputPortId))
|
|
2943
3491
|
continue;
|
|
3492
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3493
|
+
continue;
|
|
2944
3494
|
const candidates = [];
|
|
2945
3495
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2946
3496
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2960,22 +3510,32 @@ class Workflow {
|
|
|
2960
3510
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2961
3511
|
}
|
|
2962
3512
|
};
|
|
2963
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3513
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2964
3514
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2965
3515
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2966
3516
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2967
3517
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2968
3518
|
});
|
|
2969
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3519
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2970
3520
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2971
3521
|
});
|
|
2972
3522
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2973
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3523
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2974
3524
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2975
3525
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2976
3526
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2977
3527
|
const earlierTask = earlierTasks[i];
|
|
2978
3528
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3529
|
+
if (earlierTask.type === "InputTask") {
|
|
3530
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3531
|
+
if (matches.has(requiredInputId))
|
|
3532
|
+
continue;
|
|
3533
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3534
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3535
|
+
}
|
|
3536
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3537
|
+
continue;
|
|
3538
|
+
}
|
|
2979
3539
|
const makeMatchFromEarlier = (comparator) => {
|
|
2980
3540
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2981
3541
|
return;
|
|
@@ -2985,7 +3545,7 @@ class Workflow {
|
|
|
2985
3545
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2986
3546
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2987
3547
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2988
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3548
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2989
3549
|
}
|
|
2990
3550
|
}
|
|
2991
3551
|
}
|
|
@@ -3011,6 +3571,10 @@ class Workflow {
|
|
|
3011
3571
|
};
|
|
3012
3572
|
}
|
|
3013
3573
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3574
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3575
|
+
if (existingTargetConnections.length > 0) {
|
|
3576
|
+
return { matches, unmatchedRequired: [] };
|
|
3577
|
+
}
|
|
3014
3578
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
3015
3579
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
3016
3580
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3133,7 +3697,7 @@ function getLastTask(workflow) {
|
|
|
3133
3697
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3134
3698
|
}
|
|
3135
3699
|
function connect(source, target, workflow) {
|
|
3136
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3700
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3137
3701
|
}
|
|
3138
3702
|
function pipe(args, workflow = new Workflow) {
|
|
3139
3703
|
let previousTask = getLastTask(workflow);
|
|
@@ -3189,7 +3753,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3189
3753
|
// src/task-graph/TaskGraph.ts
|
|
3190
3754
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3191
3755
|
constructor() {
|
|
3192
|
-
super((task) => task.
|
|
3756
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3193
3757
|
}
|
|
3194
3758
|
}
|
|
3195
3759
|
|
|
@@ -3286,18 +3850,22 @@ class TaskGraph {
|
|
|
3286
3850
|
return this._dag.removeNode(taskId);
|
|
3287
3851
|
}
|
|
3288
3852
|
resetGraph() {
|
|
3289
|
-
this.runner.resetGraph(this,
|
|
3853
|
+
this.runner.resetGraph(this, uuid45());
|
|
3290
3854
|
}
|
|
3291
|
-
toJSON() {
|
|
3292
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3855
|
+
toJSON(options) {
|
|
3856
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3293
3857
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3294
|
-
|
|
3858
|
+
let json = {
|
|
3295
3859
|
tasks,
|
|
3296
3860
|
dataflows
|
|
3297
3861
|
};
|
|
3862
|
+
if (options?.withBoundaryNodes) {
|
|
3863
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3864
|
+
}
|
|
3865
|
+
return json;
|
|
3298
3866
|
}
|
|
3299
|
-
toDependencyJSON() {
|
|
3300
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3867
|
+
toDependencyJSON(options) {
|
|
3868
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3301
3869
|
this.getDataflows().forEach((df) => {
|
|
3302
3870
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3303
3871
|
if (!target.dependencies) {
|
|
@@ -3323,6 +3891,9 @@ class TaskGraph {
|
|
|
3323
3891
|
}
|
|
3324
3892
|
}
|
|
3325
3893
|
});
|
|
3894
|
+
if (options?.withBoundaryNodes) {
|
|
3895
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3896
|
+
}
|
|
3326
3897
|
return tasks;
|
|
3327
3898
|
}
|
|
3328
3899
|
get events() {
|
|
@@ -3341,7 +3912,7 @@ class TaskGraph {
|
|
|
3341
3912
|
const tasks = this.getTasks();
|
|
3342
3913
|
tasks.forEach((task) => {
|
|
3343
3914
|
const unsub = task.subscribe("status", (status) => {
|
|
3344
|
-
callback(task.
|
|
3915
|
+
callback(task.id, status);
|
|
3345
3916
|
});
|
|
3346
3917
|
unsubscribes.push(unsub);
|
|
3347
3918
|
});
|
|
@@ -3350,7 +3921,7 @@ class TaskGraph {
|
|
|
3350
3921
|
if (!task || typeof task.subscribe !== "function")
|
|
3351
3922
|
return;
|
|
3352
3923
|
const unsub = task.subscribe("status", (status) => {
|
|
3353
|
-
callback(task.
|
|
3924
|
+
callback(task.id, status);
|
|
3354
3925
|
});
|
|
3355
3926
|
unsubscribes.push(unsub);
|
|
3356
3927
|
};
|
|
@@ -3365,7 +3936,7 @@ class TaskGraph {
|
|
|
3365
3936
|
const tasks = this.getTasks();
|
|
3366
3937
|
tasks.forEach((task) => {
|
|
3367
3938
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3368
|
-
callback(task.
|
|
3939
|
+
callback(task.id, progress, message, ...args);
|
|
3369
3940
|
});
|
|
3370
3941
|
unsubscribes.push(unsub);
|
|
3371
3942
|
});
|
|
@@ -3374,7 +3945,7 @@ class TaskGraph {
|
|
|
3374
3945
|
if (!task || typeof task.subscribe !== "function")
|
|
3375
3946
|
return;
|
|
3376
3947
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3377
|
-
callback(task.
|
|
3948
|
+
callback(task.id, progress, message, ...args);
|
|
3378
3949
|
});
|
|
3379
3950
|
unsubscribes.push(unsub);
|
|
3380
3951
|
};
|
|
@@ -3459,7 +4030,7 @@ class TaskGraph {
|
|
|
3459
4030
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3460
4031
|
const edges = [];
|
|
3461
4032
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3462
|
-
edges.push(new Dataflow(tasks[i].
|
|
4033
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3463
4034
|
}
|
|
3464
4035
|
return edges;
|
|
3465
4036
|
}
|
|
@@ -3469,9 +4040,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3469
4040
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3470
4041
|
return graph;
|
|
3471
4042
|
}
|
|
4043
|
+
// src/task/FallbackTaskRunner.ts
|
|
4044
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4045
|
+
async executeTask(input) {
|
|
4046
|
+
if (this.task.fallbackMode === "data") {
|
|
4047
|
+
return this.executeDataFallback(input);
|
|
4048
|
+
}
|
|
4049
|
+
return this.executeTaskFallback(input);
|
|
4050
|
+
}
|
|
4051
|
+
async executeTaskReactive(input, output) {
|
|
4052
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4053
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4054
|
+
}
|
|
4055
|
+
async executeTaskFallback(input) {
|
|
4056
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4057
|
+
if (tasks.length === 0) {
|
|
4058
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4059
|
+
}
|
|
4060
|
+
const errors = [];
|
|
4061
|
+
const totalAttempts = tasks.length;
|
|
4062
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4063
|
+
if (this.abortController?.signal.aborted) {
|
|
4064
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4065
|
+
}
|
|
4066
|
+
const alternativeTask = tasks[i];
|
|
4067
|
+
const attemptNumber = i + 1;
|
|
4068
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4069
|
+
try {
|
|
4070
|
+
this.resetTask(alternativeTask);
|
|
4071
|
+
const result = await alternativeTask.run(input);
|
|
4072
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4073
|
+
return await this.executeTaskReactive(input, result);
|
|
4074
|
+
} catch (error) {
|
|
4075
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4076
|
+
throw error;
|
|
4077
|
+
}
|
|
4078
|
+
errors.push({ task: alternativeTask, error });
|
|
4079
|
+
}
|
|
4080
|
+
}
|
|
4081
|
+
throw this.buildAggregateError(errors, "task");
|
|
4082
|
+
}
|
|
4083
|
+
async executeDataFallback(input) {
|
|
4084
|
+
const alternatives = this.task.alternatives;
|
|
4085
|
+
if (alternatives.length === 0) {
|
|
4086
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4087
|
+
}
|
|
4088
|
+
const errors = [];
|
|
4089
|
+
const totalAttempts = alternatives.length;
|
|
4090
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4091
|
+
if (this.abortController?.signal.aborted) {
|
|
4092
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4093
|
+
}
|
|
4094
|
+
const alternative = alternatives[i];
|
|
4095
|
+
const attemptNumber = i + 1;
|
|
4096
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4097
|
+
try {
|
|
4098
|
+
this.resetSubgraph();
|
|
4099
|
+
const mergedInput = { ...input, ...alternative };
|
|
4100
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4101
|
+
parentSignal: this.abortController?.signal,
|
|
4102
|
+
outputCache: this.outputCache
|
|
4103
|
+
});
|
|
4104
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4105
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4106
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4107
|
+
} catch (error) {
|
|
4108
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4109
|
+
throw error;
|
|
4110
|
+
}
|
|
4111
|
+
errors.push({ alternative, error });
|
|
4112
|
+
}
|
|
4113
|
+
}
|
|
4114
|
+
throw this.buildAggregateError(errors, "data");
|
|
4115
|
+
}
|
|
4116
|
+
resetTask(task) {
|
|
4117
|
+
task.status = TaskStatus.PENDING;
|
|
4118
|
+
task.progress = 0;
|
|
4119
|
+
task.error = undefined;
|
|
4120
|
+
task.completedAt = undefined;
|
|
4121
|
+
task.startedAt = undefined;
|
|
4122
|
+
task.resetInputData();
|
|
4123
|
+
}
|
|
4124
|
+
resetSubgraph() {
|
|
4125
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4126
|
+
this.resetTask(task);
|
|
4127
|
+
}
|
|
4128
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4129
|
+
dataflow.reset();
|
|
4130
|
+
}
|
|
4131
|
+
}
|
|
4132
|
+
buildAggregateError(errors, mode) {
|
|
4133
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4134
|
+
const details = errors.map((e, i) => {
|
|
4135
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4136
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4137
|
+
}).join(`
|
|
4138
|
+
`);
|
|
4139
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4140
|
+
${details}`);
|
|
4141
|
+
}
|
|
4142
|
+
}
|
|
4143
|
+
|
|
4144
|
+
// src/task/FallbackTask.ts
|
|
4145
|
+
var fallbackTaskConfigSchema = {
|
|
4146
|
+
type: "object",
|
|
4147
|
+
properties: {
|
|
4148
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4149
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4150
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4151
|
+
},
|
|
4152
|
+
additionalProperties: false
|
|
4153
|
+
};
|
|
4154
|
+
|
|
4155
|
+
class FallbackTask extends GraphAsTask {
|
|
4156
|
+
static type = "FallbackTask";
|
|
4157
|
+
static category = "Flow Control";
|
|
4158
|
+
static title = "Fallback";
|
|
4159
|
+
static description = "Try alternatives until one succeeds";
|
|
4160
|
+
static hasDynamicSchemas = true;
|
|
4161
|
+
static configSchema() {
|
|
4162
|
+
return fallbackTaskConfigSchema;
|
|
4163
|
+
}
|
|
4164
|
+
get runner() {
|
|
4165
|
+
if (!this._runner) {
|
|
4166
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4167
|
+
}
|
|
4168
|
+
return this._runner;
|
|
4169
|
+
}
|
|
4170
|
+
get fallbackMode() {
|
|
4171
|
+
return this.config?.fallbackMode ?? "task";
|
|
4172
|
+
}
|
|
4173
|
+
get alternatives() {
|
|
4174
|
+
return this.config?.alternatives ?? [];
|
|
4175
|
+
}
|
|
4176
|
+
inputSchema() {
|
|
4177
|
+
if (!this.hasChildren()) {
|
|
4178
|
+
return this.constructor.inputSchema();
|
|
4179
|
+
}
|
|
4180
|
+
if (this.fallbackMode === "data") {
|
|
4181
|
+
return super.inputSchema();
|
|
4182
|
+
}
|
|
4183
|
+
const properties = {};
|
|
4184
|
+
const tasks = this.subGraph.getTasks();
|
|
4185
|
+
for (const task of tasks) {
|
|
4186
|
+
const taskInputSchema = task.inputSchema();
|
|
4187
|
+
if (typeof taskInputSchema === "boolean")
|
|
4188
|
+
continue;
|
|
4189
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4190
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4191
|
+
if (!properties[inputName]) {
|
|
4192
|
+
properties[inputName] = inputProp;
|
|
4193
|
+
}
|
|
4194
|
+
}
|
|
4195
|
+
}
|
|
4196
|
+
return {
|
|
4197
|
+
type: "object",
|
|
4198
|
+
properties,
|
|
4199
|
+
additionalProperties: true
|
|
4200
|
+
};
|
|
4201
|
+
}
|
|
4202
|
+
outputSchema() {
|
|
4203
|
+
if (!this.hasChildren()) {
|
|
4204
|
+
return this.constructor.outputSchema();
|
|
4205
|
+
}
|
|
4206
|
+
const tasks = this.subGraph.getTasks();
|
|
4207
|
+
if (tasks.length === 0) {
|
|
4208
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4209
|
+
}
|
|
4210
|
+
if (this.fallbackMode === "task") {
|
|
4211
|
+
const firstTask = tasks[0];
|
|
4212
|
+
return firstTask.outputSchema();
|
|
4213
|
+
}
|
|
4214
|
+
return super.outputSchema();
|
|
4215
|
+
}
|
|
4216
|
+
toJSON() {
|
|
4217
|
+
const json = super.toJSON();
|
|
4218
|
+
return {
|
|
4219
|
+
...json,
|
|
4220
|
+
config: {
|
|
4221
|
+
..."config" in json ? json.config : {},
|
|
4222
|
+
fallbackMode: this.fallbackMode,
|
|
4223
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4224
|
+
}
|
|
4225
|
+
};
|
|
4226
|
+
}
|
|
4227
|
+
}
|
|
4228
|
+
queueMicrotask(() => {
|
|
4229
|
+
Workflow.prototype.fallback = function() {
|
|
4230
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4231
|
+
};
|
|
4232
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4233
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4234
|
+
return this.addLoopTask(FallbackTask, {
|
|
4235
|
+
fallbackMode: "data",
|
|
4236
|
+
alternatives
|
|
4237
|
+
});
|
|
4238
|
+
};
|
|
4239
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4240
|
+
});
|
|
3472
4241
|
// src/task/IteratorTaskRunner.ts
|
|
3473
4242
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3474
|
-
subGraphRunChain = Promise.resolve();
|
|
3475
4243
|
async executeTask(input) {
|
|
3476
4244
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3477
4245
|
if (analysis.iterationCount === 0) {
|
|
@@ -3493,13 +4261,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3493
4261
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3494
4262
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3495
4263
|
const completionOrderResults = [];
|
|
4264
|
+
let completedCount = 0;
|
|
3496
4265
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3497
4266
|
if (this.abortController?.signal.aborted) {
|
|
3498
4267
|
break;
|
|
3499
4268
|
}
|
|
3500
4269
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3501
4270
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3502
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4271
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4272
|
+
completedCount++;
|
|
4273
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4274
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4275
|
+
});
|
|
3503
4276
|
for (const { index, result } of batchResults) {
|
|
3504
4277
|
if (result === undefined)
|
|
3505
4278
|
continue;
|
|
@@ -3509,8 +4282,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3509
4282
|
completionOrderResults.push(result);
|
|
3510
4283
|
}
|
|
3511
4284
|
}
|
|
3512
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3513
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3514
4285
|
}
|
|
3515
4286
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3516
4287
|
return this.task.collectResults(collected);
|
|
@@ -3532,7 +4303,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3532
4303
|
}
|
|
3533
4304
|
return accumulator;
|
|
3534
4305
|
}
|
|
3535
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4306
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3536
4307
|
const results = [];
|
|
3537
4308
|
let cursor = 0;
|
|
3538
4309
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3550,33 +4321,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3550
4321
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3551
4322
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3552
4323
|
results.push({ index, result });
|
|
4324
|
+
await onItemComplete?.();
|
|
3553
4325
|
}
|
|
3554
4326
|
});
|
|
3555
4327
|
await Promise.all(workers);
|
|
3556
4328
|
return results;
|
|
3557
4329
|
}
|
|
4330
|
+
cloneGraph(graph) {
|
|
4331
|
+
const clone = new TaskGraph;
|
|
4332
|
+
for (const task of graph.getTasks()) {
|
|
4333
|
+
const ctor = task.constructor;
|
|
4334
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4335
|
+
if (task.hasChildren()) {
|
|
4336
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4337
|
+
}
|
|
4338
|
+
clone.addTask(newTask);
|
|
4339
|
+
}
|
|
4340
|
+
for (const df of graph.getDataflows()) {
|
|
4341
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4342
|
+
}
|
|
4343
|
+
return clone;
|
|
4344
|
+
}
|
|
3558
4345
|
async executeSubgraphIteration(input) {
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
4346
|
+
if (this.abortController?.signal.aborted) {
|
|
4347
|
+
return;
|
|
4348
|
+
}
|
|
4349
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4350
|
+
const results = await graphClone.run(input, {
|
|
4351
|
+
parentSignal: this.abortController?.signal,
|
|
4352
|
+
outputCache: this.outputCache
|
|
3563
4353
|
});
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
if (this.abortController?.signal.aborted) {
|
|
3567
|
-
return;
|
|
3568
|
-
}
|
|
3569
|
-
const results = await this.task.subGraph.run(input, {
|
|
3570
|
-
parentSignal: this.abortController?.signal,
|
|
3571
|
-
outputCache: this.outputCache
|
|
3572
|
-
});
|
|
3573
|
-
if (results.length === 0) {
|
|
3574
|
-
return;
|
|
3575
|
-
}
|
|
3576
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3577
|
-
} finally {
|
|
3578
|
-
releaseTurn?.();
|
|
4354
|
+
if (results.length === 0) {
|
|
4355
|
+
return;
|
|
3579
4356
|
}
|
|
4357
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3580
4358
|
}
|
|
3581
4359
|
}
|
|
3582
4360
|
|
|
@@ -3860,7 +4638,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3860
4638
|
const tasks = this.subGraph.getTasks();
|
|
3861
4639
|
if (tasks.length === 0)
|
|
3862
4640
|
return;
|
|
3863
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4641
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3864
4642
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3865
4643
|
const properties = {};
|
|
3866
4644
|
const required = [];
|
|
@@ -3887,6 +4665,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3887
4665
|
}
|
|
3888
4666
|
}
|
|
3889
4667
|
}
|
|
4668
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4669
|
+
for (const task of tasks) {
|
|
4670
|
+
if (sourceIds.has(task.id))
|
|
4671
|
+
continue;
|
|
4672
|
+
const inputSchema = task.inputSchema();
|
|
4673
|
+
if (typeof inputSchema === "boolean")
|
|
4674
|
+
continue;
|
|
4675
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4676
|
+
if (requiredKeys.size === 0)
|
|
4677
|
+
continue;
|
|
4678
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4679
|
+
for (const key of requiredKeys) {
|
|
4680
|
+
if (connectedPorts.has(key))
|
|
4681
|
+
continue;
|
|
4682
|
+
if (properties[key])
|
|
4683
|
+
continue;
|
|
4684
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4685
|
+
continue;
|
|
4686
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4687
|
+
if (!prop || typeof prop === "boolean")
|
|
4688
|
+
continue;
|
|
4689
|
+
properties[key] = prop;
|
|
4690
|
+
if (!required.includes(key)) {
|
|
4691
|
+
required.push(key);
|
|
4692
|
+
}
|
|
4693
|
+
}
|
|
4694
|
+
}
|
|
3890
4695
|
return {
|
|
3891
4696
|
type: "object",
|
|
3892
4697
|
properties,
|
|
@@ -4021,7 +4826,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4021
4826
|
if (!this.hasChildren()) {
|
|
4022
4827
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4023
4828
|
}
|
|
4024
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4829
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4025
4830
|
if (endingNodes.length === 0) {
|
|
4026
4831
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4027
4832
|
}
|
|
@@ -4231,8 +5036,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4231
5036
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4232
5037
|
}
|
|
4233
5038
|
this._currentIteration++;
|
|
4234
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4235
|
-
await context.updateProgress(progress, `
|
|
5039
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5040
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4236
5041
|
}
|
|
4237
5042
|
return currentOutput;
|
|
4238
5043
|
}
|
|
@@ -4275,8 +5080,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4275
5080
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4276
5081
|
}
|
|
4277
5082
|
this._currentIteration++;
|
|
4278
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4279
|
-
await context.updateProgress(progress, `
|
|
5083
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5084
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4280
5085
|
}
|
|
4281
5086
|
yield { type: "finish", data: currentOutput };
|
|
4282
5087
|
}
|
|
@@ -4352,7 +5157,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4352
5157
|
return this.constructor.outputSchema();
|
|
4353
5158
|
}
|
|
4354
5159
|
const tasks = this.subGraph.getTasks();
|
|
4355
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5160
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4356
5161
|
if (endingNodes.length === 0) {
|
|
4357
5162
|
return this.constructor.outputSchema();
|
|
4358
5163
|
}
|
|
@@ -5019,7 +5824,7 @@ class ReduceTask extends IteratorTask {
|
|
|
5019
5824
|
if (!this.hasChildren()) {
|
|
5020
5825
|
return this.constructor.outputSchema();
|
|
5021
5826
|
}
|
|
5022
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5827
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5023
5828
|
if (endingNodes.length === 0) {
|
|
5024
5829
|
return this.constructor.outputSchema();
|
|
5025
5830
|
}
|
|
@@ -5057,14 +5862,14 @@ var TaskRegistry = {
|
|
|
5057
5862
|
};
|
|
5058
5863
|
|
|
5059
5864
|
// src/task/TaskJSON.ts
|
|
5060
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5865
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5061
5866
|
if (!item.id)
|
|
5062
5867
|
throw new TaskJSONError("Task id required");
|
|
5063
5868
|
if (!item.type)
|
|
5064
5869
|
throw new TaskJSONError("Task type required");
|
|
5065
5870
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5066
5871
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5067
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5872
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5068
5873
|
if (!taskClass)
|
|
5069
5874
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5070
5875
|
const taskConfig = {
|
|
@@ -5091,20 +5896,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5091
5896
|
}
|
|
5092
5897
|
return subGraph;
|
|
5093
5898
|
};
|
|
5094
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5095
|
-
const task = createSingleTaskFromJSON(item);
|
|
5899
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5900
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5096
5901
|
if (item.subgraph) {
|
|
5097
5902
|
if (!(task instanceof GraphAsTask)) {
|
|
5098
5903
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5099
5904
|
}
|
|
5100
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5905
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5101
5906
|
}
|
|
5102
5907
|
return task;
|
|
5103
5908
|
};
|
|
5104
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5909
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5105
5910
|
const subGraph = new TaskGraph;
|
|
5106
5911
|
for (const subitem of graphJsonObj.tasks) {
|
|
5107
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5912
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5108
5913
|
}
|
|
5109
5914
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5110
5915
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5113,7 +5918,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5113
5918
|
};
|
|
5114
5919
|
// src/task/index.ts
|
|
5115
5920
|
var registerBaseTasks = () => {
|
|
5116
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5921
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5117
5922
|
tasks.map(TaskRegistry.registerTask);
|
|
5118
5923
|
return tasks;
|
|
5119
5924
|
};
|
|
@@ -5296,11 +6101,14 @@ export {
|
|
|
5296
6101
|
isFlexibleSchema,
|
|
5297
6102
|
hasVectorOutput,
|
|
5298
6103
|
hasVectorLikeInput,
|
|
6104
|
+
hasStructuredOutput,
|
|
5299
6105
|
graphAsTaskConfigSchema,
|
|
5300
6106
|
getTaskQueueRegistry,
|
|
6107
|
+
getStructuredOutputSchemas,
|
|
5301
6108
|
getStreamingPorts,
|
|
5302
6109
|
getPortStreamMode,
|
|
5303
6110
|
getOutputStreamMode,
|
|
6111
|
+
getObjectPortId,
|
|
5304
6112
|
getNestedValue,
|
|
5305
6113
|
getLastTask,
|
|
5306
6114
|
getJobQueueFactory,
|
|
@@ -5309,6 +6117,7 @@ export {
|
|
|
5309
6117
|
getAppendPortId,
|
|
5310
6118
|
findArrayPorts,
|
|
5311
6119
|
filterIterationProperties,
|
|
6120
|
+
fallbackTaskConfigSchema,
|
|
5312
6121
|
extractIterationProperties,
|
|
5313
6122
|
extractBaseSchema,
|
|
5314
6123
|
evaluateCondition,
|
|
@@ -5323,13 +6132,19 @@ export {
|
|
|
5323
6132
|
createArraySchema,
|
|
5324
6133
|
connect,
|
|
5325
6134
|
conditionalTaskConfigSchema,
|
|
6135
|
+
computeGraphOutputSchema,
|
|
6136
|
+
computeGraphInputSchema,
|
|
6137
|
+
calculateNodeDepths,
|
|
5326
6138
|
buildIterationInputSchema,
|
|
5327
6139
|
addIterationContextToSchema,
|
|
6140
|
+
addBoundaryNodesToGraphJson,
|
|
6141
|
+
addBoundaryNodesToDependencyJson,
|
|
5328
6142
|
WorkflowError,
|
|
5329
6143
|
Workflow,
|
|
5330
6144
|
WhileTaskRunner,
|
|
5331
6145
|
WhileTask,
|
|
5332
6146
|
WHILE_CONTEXT_SCHEMA,
|
|
6147
|
+
TaskTimeoutError,
|
|
5333
6148
|
TaskStatus,
|
|
5334
6149
|
TaskRegistry,
|
|
5335
6150
|
TaskQueueRegistry,
|
|
@@ -5365,6 +6180,8 @@ export {
|
|
|
5365
6180
|
GraphAsTaskRunner,
|
|
5366
6181
|
GraphAsTask,
|
|
5367
6182
|
GRAPH_RESULT_ARRAY,
|
|
6183
|
+
FallbackTaskRunner,
|
|
6184
|
+
FallbackTask,
|
|
5368
6185
|
EventTaskGraphToDagMapping,
|
|
5369
6186
|
EventDagToTaskGraphMapping,
|
|
5370
6187
|
DataflowArrow,
|
|
@@ -5378,4 +6195,4 @@ export {
|
|
|
5378
6195
|
ConditionalTask
|
|
5379
6196
|
};
|
|
5380
6197
|
|
|
5381
|
-
//# debugId=
|
|
6198
|
+
//# debugId=AA716C41A75EE12C64756E2164756E21
|