@workglow/task-graph 0.0.102 → 0.0.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/browser.js +1096 -279
- package/dist/browser.js.map +25 -22
- package/dist/bun.js +1096 -279
- package/dist/bun.js.map +25 -22
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1096 -279
- package/dist/node.js.map +25 -22
- package/dist/task/ConditionalTask.d.ts +5 -1
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +244 -0
- package/dist/task/FallbackTask.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +49 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -0
- package/dist/task/GraphAsTask.d.ts +15 -12
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +5 -4
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +9 -1
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +7 -5
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueTask.d.ts +4 -0
- package/dist/task/JobQueueTask.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/StreamTypes.d.ts +26 -4
- package/dist/task/StreamTypes.d.ts.map +1 -1
- package/dist/task/Task.d.ts +20 -18
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +9 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +7 -2
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +20 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +5 -0
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +4 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts +51 -0
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -0
- package/dist/task-graph/ITaskGraph.d.ts +3 -3
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +6 -4
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +26 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +26 -4
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +1 -1
- package/src/task/ConditionalTask.README.md +4 -4
package/dist/node.js
CHANGED
|
@@ -20,6 +20,7 @@ var TaskConfigSchema = {
|
|
|
20
20
|
title: { type: "string" },
|
|
21
21
|
description: { type: "string" },
|
|
22
22
|
cacheable: { type: "boolean" },
|
|
23
|
+
timeout: { type: "number", description: "Max execution time in milliseconds" },
|
|
23
24
|
inputSchema: {
|
|
24
25
|
type: "object",
|
|
25
26
|
properties: {},
|
|
@@ -240,8 +241,352 @@ class DataflowArrow extends Dataflow {
|
|
|
240
241
|
super(sourceTaskId, sourceTaskPortId, targetTaskId, targetTaskPortId);
|
|
241
242
|
}
|
|
242
243
|
}
|
|
244
|
+
// src/task-graph/GraphSchemaUtils.ts
|
|
245
|
+
import { uuid4 } from "@workglow/util";
|
|
246
|
+
function calculateNodeDepths(graph) {
|
|
247
|
+
const depths = new Map;
|
|
248
|
+
const tasks = graph.getTasks();
|
|
249
|
+
for (const task of tasks) {
|
|
250
|
+
depths.set(task.id, 0);
|
|
251
|
+
}
|
|
252
|
+
const sortedTasks = graph.topologicallySortedNodes();
|
|
253
|
+
for (const task of sortedTasks) {
|
|
254
|
+
const currentDepth = depths.get(task.id) || 0;
|
|
255
|
+
const targetTasks = graph.getTargetTasks(task.id);
|
|
256
|
+
for (const targetTask of targetTasks) {
|
|
257
|
+
const targetDepth = depths.get(targetTask.id) || 0;
|
|
258
|
+
depths.set(targetTask.id, Math.max(targetDepth, currentDepth + 1));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return depths;
|
|
262
|
+
}
|
|
263
|
+
function computeGraphInputSchema(graph, options) {
|
|
264
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
265
|
+
const properties = {};
|
|
266
|
+
const required = [];
|
|
267
|
+
const propertyOrigins = {};
|
|
268
|
+
const tasks = graph.getTasks();
|
|
269
|
+
const startingNodes = tasks.filter((task) => graph.getSourceDataflows(task.id).length === 0);
|
|
270
|
+
for (const task of startingNodes) {
|
|
271
|
+
const taskInputSchema = task.inputSchema();
|
|
272
|
+
if (typeof taskInputSchema === "boolean") {
|
|
273
|
+
if (taskInputSchema === false) {
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
if (taskInputSchema === true) {
|
|
277
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
282
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
283
|
+
if (!properties[inputName]) {
|
|
284
|
+
properties[inputName] = inputProp;
|
|
285
|
+
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
286
|
+
required.push(inputName);
|
|
287
|
+
}
|
|
288
|
+
if (trackOrigins) {
|
|
289
|
+
propertyOrigins[inputName] = [task.id];
|
|
290
|
+
}
|
|
291
|
+
} else if (trackOrigins) {
|
|
292
|
+
propertyOrigins[inputName].push(task.id);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const sourceIds = new Set(startingNodes.map((t) => t.id));
|
|
297
|
+
for (const task of tasks) {
|
|
298
|
+
if (sourceIds.has(task.id))
|
|
299
|
+
continue;
|
|
300
|
+
const taskInputSchema = task.inputSchema();
|
|
301
|
+
if (typeof taskInputSchema === "boolean")
|
|
302
|
+
continue;
|
|
303
|
+
const requiredKeys = new Set(taskInputSchema.required || []);
|
|
304
|
+
if (requiredKeys.size === 0)
|
|
305
|
+
continue;
|
|
306
|
+
const connectedPorts = new Set(graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
307
|
+
for (const key of requiredKeys) {
|
|
308
|
+
if (connectedPorts.has(key))
|
|
309
|
+
continue;
|
|
310
|
+
if (properties[key]) {
|
|
311
|
+
if (trackOrigins) {
|
|
312
|
+
propertyOrigins[key].push(task.id);
|
|
313
|
+
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
317
|
+
continue;
|
|
318
|
+
const prop = (taskInputSchema.properties || {})[key];
|
|
319
|
+
if (!prop || typeof prop === "boolean")
|
|
320
|
+
continue;
|
|
321
|
+
properties[key] = prop;
|
|
322
|
+
if (!required.includes(key)) {
|
|
323
|
+
required.push(key);
|
|
324
|
+
}
|
|
325
|
+
if (trackOrigins) {
|
|
326
|
+
propertyOrigins[key] = [task.id];
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (trackOrigins) {
|
|
331
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
332
|
+
const prop = properties[propName];
|
|
333
|
+
if (!prop || typeof prop === "boolean")
|
|
334
|
+
continue;
|
|
335
|
+
if (origins.length === 1) {
|
|
336
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
337
|
+
} else {
|
|
338
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
type: "object",
|
|
344
|
+
properties,
|
|
345
|
+
...required.length > 0 ? { required } : {},
|
|
346
|
+
additionalProperties: false
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
function computeGraphOutputSchema(graph, options) {
|
|
350
|
+
const trackOrigins = options?.trackOrigins ?? false;
|
|
351
|
+
const properties = {};
|
|
352
|
+
const required = [];
|
|
353
|
+
const propertyOrigins = {};
|
|
354
|
+
const tasks = graph.getTasks();
|
|
355
|
+
const endingNodes = tasks.filter((task) => graph.getTargetDataflows(task.id).length === 0);
|
|
356
|
+
const depths = calculateNodeDepths(graph);
|
|
357
|
+
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.id) || 0));
|
|
358
|
+
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.id) === maxDepth);
|
|
359
|
+
const propertyCount = {};
|
|
360
|
+
const propertySchema = {};
|
|
361
|
+
for (const task of lastLevelNodes) {
|
|
362
|
+
const taskOutputSchema = task.outputSchema();
|
|
363
|
+
if (typeof taskOutputSchema === "boolean") {
|
|
364
|
+
if (taskOutputSchema === false) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (taskOutputSchema === true) {
|
|
368
|
+
properties[DATAFLOW_ALL_PORTS] = {};
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const taskProperties = taskOutputSchema.properties || {};
|
|
373
|
+
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
374
|
+
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
375
|
+
if (!propertySchema[outputName]) {
|
|
376
|
+
propertySchema[outputName] = outputProp;
|
|
377
|
+
}
|
|
378
|
+
if (trackOrigins) {
|
|
379
|
+
if (!propertyOrigins[outputName]) {
|
|
380
|
+
propertyOrigins[outputName] = [task.id];
|
|
381
|
+
} else {
|
|
382
|
+
propertyOrigins[outputName].push(task.id);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
for (const [outputName] of Object.entries(propertyCount)) {
|
|
388
|
+
const outputProp = propertySchema[outputName];
|
|
389
|
+
if (lastLevelNodes.length === 1) {
|
|
390
|
+
properties[outputName] = outputProp;
|
|
391
|
+
} else {
|
|
392
|
+
properties[outputName] = {
|
|
393
|
+
type: "array",
|
|
394
|
+
items: outputProp
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
if (trackOrigins) {
|
|
399
|
+
for (const [propName, origins] of Object.entries(propertyOrigins)) {
|
|
400
|
+
const prop = properties[propName];
|
|
401
|
+
if (!prop || typeof prop === "boolean")
|
|
402
|
+
continue;
|
|
403
|
+
if (origins.length === 1) {
|
|
404
|
+
properties[propName] = { ...prop, "x-source-task-id": origins[0] };
|
|
405
|
+
} else {
|
|
406
|
+
properties[propName] = { ...prop, "x-source-task-ids": origins };
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
return {
|
|
411
|
+
type: "object",
|
|
412
|
+
properties,
|
|
413
|
+
...required.length > 0 ? { required } : {},
|
|
414
|
+
additionalProperties: false
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
function stripOriginAnnotations(schema) {
|
|
418
|
+
if (typeof schema === "boolean" || !schema || typeof schema !== "object")
|
|
419
|
+
return schema;
|
|
420
|
+
const properties = schema.properties;
|
|
421
|
+
if (!properties)
|
|
422
|
+
return schema;
|
|
423
|
+
const strippedProperties = {};
|
|
424
|
+
for (const [key, prop] of Object.entries(properties)) {
|
|
425
|
+
if (!prop || typeof prop !== "object") {
|
|
426
|
+
strippedProperties[key] = prop;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
const { "x-source-task-id": _id, "x-source-task-ids": _ids, ...rest } = prop;
|
|
430
|
+
strippedProperties[key] = rest;
|
|
431
|
+
}
|
|
432
|
+
return { ...schema, properties: strippedProperties };
|
|
433
|
+
}
|
|
434
|
+
function getOriginTaskIds(prop) {
|
|
435
|
+
if (prop["x-source-task-ids"]) {
|
|
436
|
+
return prop["x-source-task-ids"];
|
|
437
|
+
}
|
|
438
|
+
if (prop["x-source-task-id"] !== undefined) {
|
|
439
|
+
return [prop["x-source-task-id"]];
|
|
440
|
+
}
|
|
441
|
+
return [];
|
|
442
|
+
}
|
|
443
|
+
function addBoundaryNodesToGraphJson(json, graph) {
|
|
444
|
+
const hasInputTask = json.tasks.some((t) => t.type === "InputTask");
|
|
445
|
+
const hasOutputTask = json.tasks.some((t) => t.type === "OutputTask");
|
|
446
|
+
if (hasInputTask && hasOutputTask) {
|
|
447
|
+
return json;
|
|
448
|
+
}
|
|
449
|
+
const inputSchema = !hasInputTask ? computeGraphInputSchema(graph, { trackOrigins: true }) : undefined;
|
|
450
|
+
const outputSchema = !hasOutputTask ? computeGraphOutputSchema(graph, { trackOrigins: true }) : undefined;
|
|
451
|
+
const prependTasks = [];
|
|
452
|
+
const appendTasks = [];
|
|
453
|
+
const inputDataflows = [];
|
|
454
|
+
const outputDataflows = [];
|
|
455
|
+
if (!hasInputTask && inputSchema) {
|
|
456
|
+
const inputTaskId = uuid4();
|
|
457
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
458
|
+
prependTasks.push({
|
|
459
|
+
id: inputTaskId,
|
|
460
|
+
type: "InputTask",
|
|
461
|
+
config: {
|
|
462
|
+
inputSchema: strippedInputSchema,
|
|
463
|
+
outputSchema: strippedInputSchema
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
467
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
468
|
+
if (!prop || typeof prop === "boolean")
|
|
469
|
+
continue;
|
|
470
|
+
const origins = getOriginTaskIds(prop);
|
|
471
|
+
for (const originId of origins) {
|
|
472
|
+
inputDataflows.push({
|
|
473
|
+
sourceTaskId: inputTaskId,
|
|
474
|
+
sourceTaskPortId: propName,
|
|
475
|
+
targetTaskId: originId,
|
|
476
|
+
targetTaskPortId: propName
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
if (!hasOutputTask && outputSchema) {
|
|
483
|
+
const outputTaskId = uuid4();
|
|
484
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
485
|
+
appendTasks.push({
|
|
486
|
+
id: outputTaskId,
|
|
487
|
+
type: "OutputTask",
|
|
488
|
+
config: {
|
|
489
|
+
inputSchema: strippedOutputSchema,
|
|
490
|
+
outputSchema: strippedOutputSchema
|
|
491
|
+
}
|
|
492
|
+
});
|
|
493
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
494
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
495
|
+
if (!prop || typeof prop === "boolean")
|
|
496
|
+
continue;
|
|
497
|
+
const origins = getOriginTaskIds(prop);
|
|
498
|
+
for (const originId of origins) {
|
|
499
|
+
outputDataflows.push({
|
|
500
|
+
sourceTaskId: originId,
|
|
501
|
+
sourceTaskPortId: propName,
|
|
502
|
+
targetTaskId: outputTaskId,
|
|
503
|
+
targetTaskPortId: propName
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return {
|
|
510
|
+
tasks: [...prependTasks, ...json.tasks, ...appendTasks],
|
|
511
|
+
dataflows: [...inputDataflows, ...json.dataflows, ...outputDataflows]
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
function addBoundaryNodesToDependencyJson(items, graph) {
|
|
515
|
+
const hasInputTask = items.some((t) => t.type === "InputTask");
|
|
516
|
+
const hasOutputTask = items.some((t) => t.type === "OutputTask");
|
|
517
|
+
if (hasInputTask && hasOutputTask) {
|
|
518
|
+
return items;
|
|
519
|
+
}
|
|
520
|
+
const prependItems = [];
|
|
521
|
+
const appendItems = [];
|
|
522
|
+
if (!hasInputTask) {
|
|
523
|
+
const inputSchema = computeGraphInputSchema(graph, { trackOrigins: true });
|
|
524
|
+
const inputTaskId = uuid4();
|
|
525
|
+
const strippedInputSchema = stripOriginAnnotations(inputSchema);
|
|
526
|
+
prependItems.push({
|
|
527
|
+
id: inputTaskId,
|
|
528
|
+
type: "InputTask",
|
|
529
|
+
config: {
|
|
530
|
+
inputSchema: strippedInputSchema,
|
|
531
|
+
outputSchema: strippedInputSchema
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
if (typeof inputSchema !== "boolean" && inputSchema.properties) {
|
|
535
|
+
for (const [propName, prop] of Object.entries(inputSchema.properties)) {
|
|
536
|
+
if (!prop || typeof prop === "boolean")
|
|
537
|
+
continue;
|
|
538
|
+
const origins = getOriginTaskIds(prop);
|
|
539
|
+
for (const originId of origins) {
|
|
540
|
+
const targetItem = items.find((item) => item.id === originId);
|
|
541
|
+
if (!targetItem)
|
|
542
|
+
continue;
|
|
543
|
+
if (!targetItem.dependencies) {
|
|
544
|
+
targetItem.dependencies = {};
|
|
545
|
+
}
|
|
546
|
+
const existing = targetItem.dependencies[propName];
|
|
547
|
+
const dep = { id: inputTaskId, output: propName };
|
|
548
|
+
if (!existing) {
|
|
549
|
+
targetItem.dependencies[propName] = dep;
|
|
550
|
+
} else if (Array.isArray(existing)) {
|
|
551
|
+
existing.push(dep);
|
|
552
|
+
} else {
|
|
553
|
+
targetItem.dependencies[propName] = [existing, dep];
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (!hasOutputTask) {
|
|
560
|
+
const outputSchema = computeGraphOutputSchema(graph, { trackOrigins: true });
|
|
561
|
+
const outputTaskId = uuid4();
|
|
562
|
+
const strippedOutputSchema = stripOriginAnnotations(outputSchema);
|
|
563
|
+
const outputDependencies = {};
|
|
564
|
+
if (typeof outputSchema !== "boolean" && outputSchema.properties) {
|
|
565
|
+
for (const [propName, prop] of Object.entries(outputSchema.properties)) {
|
|
566
|
+
if (!prop || typeof prop === "boolean")
|
|
567
|
+
continue;
|
|
568
|
+
const origins = getOriginTaskIds(prop);
|
|
569
|
+
if (origins.length === 1) {
|
|
570
|
+
outputDependencies[propName] = { id: origins[0], output: propName };
|
|
571
|
+
} else if (origins.length > 1) {
|
|
572
|
+
outputDependencies[propName] = origins.map((id) => ({ id, output: propName }));
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
appendItems.push({
|
|
577
|
+
id: outputTaskId,
|
|
578
|
+
type: "OutputTask",
|
|
579
|
+
config: {
|
|
580
|
+
inputSchema: strippedOutputSchema,
|
|
581
|
+
outputSchema: strippedOutputSchema
|
|
582
|
+
},
|
|
583
|
+
...Object.keys(outputDependencies).length > 0 ? { dependencies: outputDependencies } : {}
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
return [...prependItems, ...items, ...appendItems];
|
|
587
|
+
}
|
|
243
588
|
// src/task-graph/TaskGraph.ts
|
|
244
|
-
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as
|
|
589
|
+
import { DirectedAcyclicGraph, EventEmitter as EventEmitter5, uuid4 as uuid45 } from "@workglow/util";
|
|
245
590
|
|
|
246
591
|
// src/task/GraphAsTask.ts
|
|
247
592
|
import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
@@ -249,9 +594,10 @@ import { compileSchema as compileSchema2 } from "@workglow/util";
|
|
|
249
594
|
// src/task-graph/TaskGraphRunner.ts
|
|
250
595
|
import {
|
|
251
596
|
collectPropertyValues,
|
|
597
|
+
getLogger as getLogger3,
|
|
252
598
|
globalServiceRegistry as globalServiceRegistry2,
|
|
253
599
|
ServiceRegistry as ServiceRegistry2,
|
|
254
|
-
uuid4 as
|
|
600
|
+
uuid4 as uuid43
|
|
255
601
|
} from "@workglow/util";
|
|
256
602
|
|
|
257
603
|
// src/storage/TaskOutputRepository.ts
|
|
@@ -284,6 +630,9 @@ class TaskOutputRepository {
|
|
|
284
630
|
}
|
|
285
631
|
}
|
|
286
632
|
|
|
633
|
+
// src/task/ConditionalTask.ts
|
|
634
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
635
|
+
|
|
287
636
|
// src/task/ConditionUtils.ts
|
|
288
637
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
289
638
|
if (fieldValue === null || fieldValue === undefined) {
|
|
@@ -356,7 +705,7 @@ import {
|
|
|
356
705
|
compileSchema,
|
|
357
706
|
deepEqual,
|
|
358
707
|
EventEmitter as EventEmitter3,
|
|
359
|
-
uuid4
|
|
708
|
+
uuid4 as uuid42
|
|
360
709
|
} from "@workglow/util";
|
|
361
710
|
|
|
362
711
|
// src/task/TaskError.ts
|
|
@@ -390,6 +739,13 @@ class TaskAbortedError extends TaskError {
|
|
|
390
739
|
}
|
|
391
740
|
}
|
|
392
741
|
|
|
742
|
+
class TaskTimeoutError extends TaskAbortedError {
|
|
743
|
+
static type = "TaskTimeoutError";
|
|
744
|
+
constructor(timeoutMs) {
|
|
745
|
+
super(timeoutMs ? `Task timed out after ${timeoutMs}ms` : "Task timed out");
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
393
749
|
class TaskFailedError extends TaskError {
|
|
394
750
|
static type = "TaskFailedError";
|
|
395
751
|
constructor(message = "Task failed") {
|
|
@@ -421,7 +777,7 @@ class TaskInvalidInputError extends TaskError {
|
|
|
421
777
|
}
|
|
422
778
|
|
|
423
779
|
// src/task/TaskRunner.ts
|
|
424
|
-
import { globalServiceRegistry } from "@workglow/util";
|
|
780
|
+
import { getLogger, globalServiceRegistry } from "@workglow/util";
|
|
425
781
|
|
|
426
782
|
// src/task/InputResolver.ts
|
|
427
783
|
import { getInputResolvers } from "@workglow/util";
|
|
@@ -443,6 +799,26 @@ function getSchemaFormat(schema) {
|
|
|
443
799
|
}
|
|
444
800
|
return;
|
|
445
801
|
}
|
|
802
|
+
function getObjectSchema(schema) {
|
|
803
|
+
if (typeof schema !== "object" || schema === null)
|
|
804
|
+
return;
|
|
805
|
+
const s = schema;
|
|
806
|
+
if (s.type === "object" && s.properties && typeof s.properties === "object") {
|
|
807
|
+
return s;
|
|
808
|
+
}
|
|
809
|
+
const variants = s.oneOf ?? s.anyOf;
|
|
810
|
+
if (Array.isArray(variants)) {
|
|
811
|
+
for (const variant of variants) {
|
|
812
|
+
if (typeof variant === "object" && variant !== null) {
|
|
813
|
+
const v = variant;
|
|
814
|
+
if (v.type === "object" && v.properties && typeof v.properties === "object") {
|
|
815
|
+
return v;
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
446
822
|
function getFormatPrefix(format) {
|
|
447
823
|
const colonIndex = format.indexOf(":");
|
|
448
824
|
return colonIndex >= 0 ? format.substring(0, colonIndex) : format;
|
|
@@ -456,22 +832,30 @@ async function resolveSchemaInputs(input, schema, config) {
|
|
|
456
832
|
const resolvers = getInputResolvers();
|
|
457
833
|
const resolved = { ...input };
|
|
458
834
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
459
|
-
|
|
835
|
+
let value = resolved[key];
|
|
460
836
|
const format = getSchemaFormat(propSchema);
|
|
461
|
-
if (
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
837
|
+
if (format) {
|
|
838
|
+
let resolver = resolvers.get(format);
|
|
839
|
+
if (!resolver) {
|
|
840
|
+
const prefix = getFormatPrefix(format);
|
|
841
|
+
resolver = resolvers.get(prefix);
|
|
842
|
+
}
|
|
843
|
+
if (resolver) {
|
|
844
|
+
if (typeof value === "string") {
|
|
845
|
+
value = await resolver(value, format, config.registry);
|
|
846
|
+
resolved[key] = value;
|
|
847
|
+
} else if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
|
|
848
|
+
const results = await Promise.all(value.map((item) => resolver(item, format, config.registry)));
|
|
849
|
+
value = results.filter((result) => result !== undefined);
|
|
850
|
+
resolved[key] = value;
|
|
851
|
+
}
|
|
852
|
+
}
|
|
467
853
|
}
|
|
468
|
-
if (!
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const results = await Promise.all(value.map((item) => resolver(item, format, config.registry)));
|
|
474
|
-
resolved[key] = results.filter((result) => result !== undefined);
|
|
854
|
+
if (value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value)) {
|
|
855
|
+
const objectSchema = getObjectSchema(propSchema);
|
|
856
|
+
if (objectSchema) {
|
|
857
|
+
resolved[key] = await resolveSchemaInputs(value, objectSchema, config);
|
|
858
|
+
}
|
|
475
859
|
}
|
|
476
860
|
}
|
|
477
861
|
return resolved;
|
|
@@ -485,7 +869,7 @@ function getPortStreamMode(schema, portId) {
|
|
|
485
869
|
if (!prop || typeof prop === "boolean")
|
|
486
870
|
return "none";
|
|
487
871
|
const xStream = prop["x-stream"];
|
|
488
|
-
if (xStream === "append" || xStream === "replace")
|
|
872
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object")
|
|
489
873
|
return xStream;
|
|
490
874
|
return "none";
|
|
491
875
|
}
|
|
@@ -500,7 +884,7 @@ function getStreamingPorts(schema) {
|
|
|
500
884
|
if (!prop || typeof prop === "boolean")
|
|
501
885
|
continue;
|
|
502
886
|
const xStream = prop["x-stream"];
|
|
503
|
-
if (xStream === "append" || xStream === "replace") {
|
|
887
|
+
if (xStream === "append" || xStream === "replace" || xStream === "object") {
|
|
504
888
|
result.push({ port: name, mode: xStream });
|
|
505
889
|
}
|
|
506
890
|
}
|
|
@@ -544,6 +928,39 @@ function edgeNeedsAccumulation(sourceSchema, sourcePort, targetSchema, targetPor
|
|
|
544
928
|
const targetMode = getPortStreamMode(targetSchema, targetPort);
|
|
545
929
|
return sourceMode !== targetMode;
|
|
546
930
|
}
|
|
931
|
+
function getObjectPortId(schema) {
|
|
932
|
+
if (typeof schema === "boolean")
|
|
933
|
+
return;
|
|
934
|
+
const props = schema.properties;
|
|
935
|
+
if (!props)
|
|
936
|
+
return;
|
|
937
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
938
|
+
if (!prop || typeof prop === "boolean")
|
|
939
|
+
continue;
|
|
940
|
+
if (prop["x-stream"] === "object")
|
|
941
|
+
return name;
|
|
942
|
+
}
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
function getStructuredOutputSchemas(schema) {
|
|
946
|
+
const result = new Map;
|
|
947
|
+
if (typeof schema === "boolean")
|
|
948
|
+
return result;
|
|
949
|
+
const props = schema.properties;
|
|
950
|
+
if (!props)
|
|
951
|
+
return result;
|
|
952
|
+
for (const [name, prop] of Object.entries(props)) {
|
|
953
|
+
if (!prop || typeof prop === "boolean")
|
|
954
|
+
continue;
|
|
955
|
+
if (prop["x-structured-output"] === true) {
|
|
956
|
+
result.set(name, prop);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
return result;
|
|
960
|
+
}
|
|
961
|
+
function hasStructuredOutput(schema) {
|
|
962
|
+
return getStructuredOutputSchemas(schema).size > 0;
|
|
963
|
+
}
|
|
547
964
|
|
|
548
965
|
// src/task/TaskRunner.ts
|
|
549
966
|
class TaskRunner {
|
|
@@ -554,12 +971,17 @@ class TaskRunner {
|
|
|
554
971
|
outputCache;
|
|
555
972
|
registry = globalServiceRegistry;
|
|
556
973
|
inputStreams;
|
|
974
|
+
timeoutTimer;
|
|
975
|
+
pendingTimeoutError;
|
|
557
976
|
shouldAccumulate = true;
|
|
558
977
|
constructor(task) {
|
|
559
978
|
this.task = task;
|
|
560
979
|
this.own = this.own.bind(this);
|
|
561
980
|
this.handleProgress = this.handleProgress.bind(this);
|
|
562
981
|
}
|
|
982
|
+
get timerLabel() {
|
|
983
|
+
return `task:${this.task.type}:${this.task.config.id}`;
|
|
984
|
+
}
|
|
563
985
|
async run(overrides = {}, config = {}) {
|
|
564
986
|
await this.handleStart(config);
|
|
565
987
|
try {
|
|
@@ -606,7 +1028,7 @@ class TaskRunner {
|
|
|
606
1028
|
return this.task.runOutputData;
|
|
607
1029
|
} catch (err) {
|
|
608
1030
|
await this.handleError(err);
|
|
609
|
-
throw err;
|
|
1031
|
+
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
610
1032
|
}
|
|
611
1033
|
}
|
|
612
1034
|
async runReactive(overrides = {}) {
|
|
@@ -663,7 +1085,14 @@ class TaskRunner {
|
|
|
663
1085
|
throw new TaskError(`Task ${this.task.type} declares append streaming but no output port has x-stream: "append"`);
|
|
664
1086
|
}
|
|
665
1087
|
}
|
|
1088
|
+
if (streamMode === "object") {
|
|
1089
|
+
const ports = getStreamingPorts(this.task.outputSchema());
|
|
1090
|
+
if (ports.length === 0) {
|
|
1091
|
+
throw new TaskError(`Task ${this.task.type} declares object streaming but no output port has x-stream: "object"`);
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
666
1094
|
const accumulated = this.shouldAccumulate ? new Map : undefined;
|
|
1095
|
+
const accumulatedObjects = this.shouldAccumulate ? new Map : undefined;
|
|
667
1096
|
let chunkCount = 0;
|
|
668
1097
|
let finalOutput;
|
|
669
1098
|
this.task.emit("stream_start");
|
|
@@ -694,6 +1123,13 @@ class TaskRunner {
|
|
|
694
1123
|
break;
|
|
695
1124
|
}
|
|
696
1125
|
case "object-delta": {
|
|
1126
|
+
if (accumulatedObjects) {
|
|
1127
|
+
accumulatedObjects.set(event.port, event.objectDelta);
|
|
1128
|
+
}
|
|
1129
|
+
this.task.runOutputData = {
|
|
1130
|
+
...this.task.runOutputData,
|
|
1131
|
+
[event.port]: event.objectDelta
|
|
1132
|
+
};
|
|
697
1133
|
this.task.emit("stream_chunk", event);
|
|
698
1134
|
const progress = Math.min(99, Math.round(100 * (1 - Math.exp(-0.05 * chunkCount))));
|
|
699
1135
|
await this.handleProgress(progress);
|
|
@@ -706,11 +1142,18 @@ class TaskRunner {
|
|
|
706
1142
|
break;
|
|
707
1143
|
}
|
|
708
1144
|
case "finish": {
|
|
709
|
-
if (accumulated) {
|
|
1145
|
+
if (accumulated || accumulatedObjects) {
|
|
710
1146
|
const merged = { ...event.data || {} };
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1147
|
+
if (accumulated) {
|
|
1148
|
+
for (const [port, text] of accumulated) {
|
|
1149
|
+
if (text.length > 0)
|
|
1150
|
+
merged[port] = text;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
if (accumulatedObjects) {
|
|
1154
|
+
for (const [port, obj] of accumulatedObjects) {
|
|
1155
|
+
merged[port] = obj;
|
|
1156
|
+
}
|
|
714
1157
|
}
|
|
715
1158
|
finalOutput = merged;
|
|
716
1159
|
this.task.emit("stream_chunk", { type: "finish", data: merged });
|
|
@@ -756,12 +1199,20 @@ class TaskRunner {
|
|
|
756
1199
|
this.outputCache = cache;
|
|
757
1200
|
}
|
|
758
1201
|
this.shouldAccumulate = config.shouldAccumulate !== false;
|
|
1202
|
+
const timeout = this.task.config.timeout;
|
|
1203
|
+
if (timeout !== undefined && timeout > 0) {
|
|
1204
|
+
this.pendingTimeoutError = new TaskTimeoutError(timeout);
|
|
1205
|
+
this.timeoutTimer = setTimeout(() => {
|
|
1206
|
+
this.abort();
|
|
1207
|
+
}, timeout);
|
|
1208
|
+
}
|
|
759
1209
|
if (config.updateProgress) {
|
|
760
1210
|
this.updateProgress = config.updateProgress;
|
|
761
1211
|
}
|
|
762
1212
|
if (config.registry) {
|
|
763
1213
|
this.registry = config.registry;
|
|
764
1214
|
}
|
|
1215
|
+
getLogger().time(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
765
1216
|
this.task.emit("start");
|
|
766
1217
|
this.task.emit("status", this.task.status);
|
|
767
1218
|
}
|
|
@@ -769,12 +1220,21 @@ class TaskRunner {
|
|
|
769
1220
|
async handleStartReactive() {
|
|
770
1221
|
this.reactiveRunning = true;
|
|
771
1222
|
}
|
|
1223
|
+
clearTimeoutTimer() {
|
|
1224
|
+
if (this.timeoutTimer !== undefined) {
|
|
1225
|
+
clearTimeout(this.timeoutTimer);
|
|
1226
|
+
this.timeoutTimer = undefined;
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
772
1229
|
async handleAbort() {
|
|
773
1230
|
if (this.task.status === TaskStatus.ABORTING)
|
|
774
1231
|
return;
|
|
1232
|
+
this.clearTimeoutTimer();
|
|
1233
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
775
1234
|
this.task.status = TaskStatus.ABORTING;
|
|
776
1235
|
this.task.progress = 100;
|
|
777
|
-
this.task.error = new TaskAbortedError;
|
|
1236
|
+
this.task.error = this.pendingTimeoutError ?? new TaskAbortedError;
|
|
1237
|
+
this.pendingTimeoutError = undefined;
|
|
778
1238
|
this.task.emit("abort", this.task.error);
|
|
779
1239
|
this.task.emit("status", this.task.status);
|
|
780
1240
|
}
|
|
@@ -784,6 +1244,9 @@ class TaskRunner {
|
|
|
784
1244
|
async handleComplete() {
|
|
785
1245
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
786
1246
|
return;
|
|
1247
|
+
this.clearTimeoutTimer();
|
|
1248
|
+
this.pendingTimeoutError = undefined;
|
|
1249
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
787
1250
|
this.task.completedAt = new Date;
|
|
788
1251
|
this.task.progress = 100;
|
|
789
1252
|
this.task.status = TaskStatus.COMPLETED;
|
|
@@ -812,6 +1275,9 @@ class TaskRunner {
|
|
|
812
1275
|
return this.handleAbort();
|
|
813
1276
|
if (this.task.status === TaskStatus.FAILED)
|
|
814
1277
|
return;
|
|
1278
|
+
this.clearTimeoutTimer();
|
|
1279
|
+
this.pendingTimeoutError = undefined;
|
|
1280
|
+
getLogger().timeEnd(this.timerLabel, { taskType: this.task.type, taskId: this.task.config.id });
|
|
815
1281
|
if (this.task.hasChildren()) {
|
|
816
1282
|
this.task.subGraph.abort();
|
|
817
1283
|
}
|
|
@@ -916,6 +1382,9 @@ class Task {
|
|
|
916
1382
|
runInputData = {};
|
|
917
1383
|
runOutputData = {};
|
|
918
1384
|
config;
|
|
1385
|
+
get id() {
|
|
1386
|
+
return this.config.id;
|
|
1387
|
+
}
|
|
919
1388
|
runConfig = {};
|
|
920
1389
|
status = TaskStatus.PENDING;
|
|
921
1390
|
progress = 0;
|
|
@@ -937,9 +1406,11 @@ class Task {
|
|
|
937
1406
|
this.resetInputData();
|
|
938
1407
|
const title = this.constructor.title || undefined;
|
|
939
1408
|
const baseConfig = Object.assign({
|
|
940
|
-
id: uuid4(),
|
|
941
1409
|
...title ? { title } : {}
|
|
942
1410
|
}, config);
|
|
1411
|
+
if (baseConfig.id === undefined) {
|
|
1412
|
+
baseConfig.id = uuid42();
|
|
1413
|
+
}
|
|
943
1414
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
944
1415
|
this.runConfig = runConfig;
|
|
945
1416
|
}
|
|
@@ -949,7 +1420,7 @@ class Task {
|
|
|
949
1420
|
return {};
|
|
950
1421
|
}
|
|
951
1422
|
try {
|
|
952
|
-
const compiledSchema = this.getInputSchemaNode(
|
|
1423
|
+
const compiledSchema = this.getInputSchemaNode();
|
|
953
1424
|
const defaultData = compiledSchema.getData(undefined, {
|
|
954
1425
|
addOptionalProps: true,
|
|
955
1426
|
removeInvalidData: false,
|
|
@@ -1032,7 +1503,7 @@ class Task {
|
|
|
1032
1503
|
this.runInputData[inputId] = prop.default;
|
|
1033
1504
|
}
|
|
1034
1505
|
}
|
|
1035
|
-
if (schema.additionalProperties
|
|
1506
|
+
if (schema.additionalProperties) {
|
|
1036
1507
|
for (const [inputId, value] of Object.entries(input)) {
|
|
1037
1508
|
if (!(inputId in properties)) {
|
|
1038
1509
|
this.runInputData[inputId] = value;
|
|
@@ -1067,7 +1538,7 @@ class Task {
|
|
|
1067
1538
|
continue;
|
|
1068
1539
|
const isArray = prop?.type === "array" || prop?.type === "any" && (Array.isArray(overrides[inputId]) || Array.isArray(this.runInputData[inputId]));
|
|
1069
1540
|
if (isArray) {
|
|
1070
|
-
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : [this.runInputData[inputId]];
|
|
1541
|
+
const existingItems = Array.isArray(this.runInputData[inputId]) ? this.runInputData[inputId] : this.runInputData[inputId] !== undefined ? [this.runInputData[inputId]] : [];
|
|
1071
1542
|
const newitems = [...existingItems];
|
|
1072
1543
|
const overrideItem = overrides[inputId];
|
|
1073
1544
|
if (Array.isArray(overrideItem)) {
|
|
@@ -1085,7 +1556,7 @@ class Task {
|
|
|
1085
1556
|
}
|
|
1086
1557
|
}
|
|
1087
1558
|
}
|
|
1088
|
-
if (inputSchema.additionalProperties
|
|
1559
|
+
if (inputSchema.additionalProperties) {
|
|
1089
1560
|
for (const [inputId, value] of Object.entries(overrides)) {
|
|
1090
1561
|
if (!(inputId in properties)) {
|
|
1091
1562
|
if (!deepEqual(this.runInputData[inputId], value)) {
|
|
@@ -1123,25 +1594,29 @@ class Task {
|
|
|
1123
1594
|
const finalOutputSchema = outputSchema ?? this.outputSchema();
|
|
1124
1595
|
this.emit("schemaChange", finalInputSchema, finalOutputSchema);
|
|
1125
1596
|
}
|
|
1126
|
-
static
|
|
1127
|
-
static getConfigSchemaNode(type) {
|
|
1597
|
+
static getConfigSchemaNode() {
|
|
1128
1598
|
const schema = this.configSchema();
|
|
1129
1599
|
if (!schema)
|
|
1130
1600
|
return;
|
|
1131
|
-
if (!
|
|
1601
|
+
if (!Object.hasOwn(this, "__compiledConfigSchema")) {
|
|
1132
1602
|
try {
|
|
1133
1603
|
const schemaNode = typeof schema === "boolean" ? compileSchema(schema ? {} : { not: {} }) : compileSchema(schema);
|
|
1134
|
-
|
|
1604
|
+
Object.defineProperty(this, "__compiledConfigSchema", {
|
|
1605
|
+
value: schemaNode,
|
|
1606
|
+
writable: true,
|
|
1607
|
+
configurable: true,
|
|
1608
|
+
enumerable: false
|
|
1609
|
+
});
|
|
1135
1610
|
} catch (error) {
|
|
1136
1611
|
console.warn(`Failed to compile config schema for ${this.type}:`, error);
|
|
1137
1612
|
return;
|
|
1138
1613
|
}
|
|
1139
1614
|
}
|
|
1140
|
-
return this.
|
|
1615
|
+
return this.__compiledConfigSchema;
|
|
1141
1616
|
}
|
|
1142
1617
|
validateAndApplyConfigDefaults(config) {
|
|
1143
1618
|
const ctor = this.constructor;
|
|
1144
|
-
const schemaNode = ctor.getConfigSchemaNode(
|
|
1619
|
+
const schemaNode = ctor.getConfigSchemaNode();
|
|
1145
1620
|
if (!schemaNode)
|
|
1146
1621
|
return config;
|
|
1147
1622
|
const result = schemaNode.validate(config);
|
|
@@ -1154,7 +1629,6 @@ class Task {
|
|
|
1154
1629
|
}
|
|
1155
1630
|
return config;
|
|
1156
1631
|
}
|
|
1157
|
-
static _inputSchemaNode = new Map;
|
|
1158
1632
|
static generateInputSchemaNode(schema) {
|
|
1159
1633
|
if (typeof schema === "boolean") {
|
|
1160
1634
|
if (schema === false) {
|
|
@@ -1164,21 +1638,31 @@ class Task {
|
|
|
1164
1638
|
}
|
|
1165
1639
|
return compileSchema(schema);
|
|
1166
1640
|
}
|
|
1167
|
-
static getInputSchemaNode(
|
|
1168
|
-
if (!
|
|
1641
|
+
static getInputSchemaNode() {
|
|
1642
|
+
if (!Object.hasOwn(this, "__compiledInputSchema")) {
|
|
1169
1643
|
const dataPortSchema = this.inputSchema();
|
|
1170
1644
|
const schemaNode = this.generateInputSchemaNode(dataPortSchema);
|
|
1171
1645
|
try {
|
|
1172
|
-
|
|
1646
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1647
|
+
value: schemaNode,
|
|
1648
|
+
writable: true,
|
|
1649
|
+
configurable: true,
|
|
1650
|
+
enumerable: false
|
|
1651
|
+
});
|
|
1173
1652
|
} catch (error) {
|
|
1174
1653
|
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
1175
|
-
|
|
1654
|
+
Object.defineProperty(this, "__compiledInputSchema", {
|
|
1655
|
+
value: compileSchema({}),
|
|
1656
|
+
writable: true,
|
|
1657
|
+
configurable: true,
|
|
1658
|
+
enumerable: false
|
|
1659
|
+
});
|
|
1176
1660
|
}
|
|
1177
1661
|
}
|
|
1178
|
-
return this.
|
|
1662
|
+
return this.__compiledInputSchema;
|
|
1179
1663
|
}
|
|
1180
|
-
getInputSchemaNode(
|
|
1181
|
-
return this.constructor.getInputSchemaNode(
|
|
1664
|
+
getInputSchemaNode() {
|
|
1665
|
+
return this.constructor.getInputSchemaNode();
|
|
1182
1666
|
}
|
|
1183
1667
|
async validateInput(input) {
|
|
1184
1668
|
const ctor = this.constructor;
|
|
@@ -1187,7 +1671,7 @@ class Task {
|
|
|
1187
1671
|
const instanceSchema = this.inputSchema();
|
|
1188
1672
|
schemaNode = ctor.generateInputSchemaNode(instanceSchema);
|
|
1189
1673
|
} else {
|
|
1190
|
-
schemaNode = this.getInputSchemaNode(
|
|
1674
|
+
schemaNode = this.getInputSchemaNode();
|
|
1191
1675
|
}
|
|
1192
1676
|
const result = schemaNode.validate(input);
|
|
1193
1677
|
if (!result.valid) {
|
|
@@ -1199,9 +1683,6 @@ class Task {
|
|
|
1199
1683
|
}
|
|
1200
1684
|
return true;
|
|
1201
1685
|
}
|
|
1202
|
-
id() {
|
|
1203
|
-
return this.config.id;
|
|
1204
|
-
}
|
|
1205
1686
|
stripSymbols(obj) {
|
|
1206
1687
|
if (obj === null || obj === undefined) {
|
|
1207
1688
|
return obj;
|
|
@@ -1223,14 +1704,15 @@ class Task {
|
|
|
1223
1704
|
}
|
|
1224
1705
|
return obj;
|
|
1225
1706
|
}
|
|
1226
|
-
toJSON() {
|
|
1707
|
+
toJSON(_options) {
|
|
1227
1708
|
const extras = this.config.extras;
|
|
1228
1709
|
const json = this.stripSymbols({
|
|
1229
|
-
id: this.
|
|
1710
|
+
id: this.id,
|
|
1230
1711
|
type: this.type,
|
|
1231
1712
|
defaults: this.defaults,
|
|
1232
1713
|
config: {
|
|
1233
1714
|
...this.config.title ? { title: this.config.title } : {},
|
|
1715
|
+
...this.config.description ? { description: this.config.description } : {},
|
|
1234
1716
|
...this.config.inputSchema ? { inputSchema: this.config.inputSchema } : {},
|
|
1235
1717
|
...this.config.outputSchema ? { outputSchema: this.config.outputSchema } : {},
|
|
1236
1718
|
...extras && Object.keys(extras).length ? { extras } : {}
|
|
@@ -1238,8 +1720,8 @@ class Task {
|
|
|
1238
1720
|
});
|
|
1239
1721
|
return json;
|
|
1240
1722
|
}
|
|
1241
|
-
toDependencyJSON() {
|
|
1242
|
-
const json = this.toJSON();
|
|
1723
|
+
toDependencyJSON(options) {
|
|
1724
|
+
const json = this.toJSON(options);
|
|
1243
1725
|
return json;
|
|
1244
1726
|
}
|
|
1245
1727
|
hasChildren() {
|
|
@@ -1269,7 +1751,7 @@ class Task {
|
|
|
1269
1751
|
this.subGraph.removeDataflow(dataflow);
|
|
1270
1752
|
}
|
|
1271
1753
|
for (const child of this.subGraph.getTasks()) {
|
|
1272
|
-
this.subGraph.removeTask(child.
|
|
1754
|
+
this.subGraph.removeTask(child.id);
|
|
1273
1755
|
}
|
|
1274
1756
|
}
|
|
1275
1757
|
this.events.emit("regenerate");
|
|
@@ -1360,7 +1842,7 @@ class ConditionalTask extends Task {
|
|
|
1360
1842
|
}
|
|
1361
1843
|
}
|
|
1362
1844
|
} catch (error) {
|
|
1363
|
-
|
|
1845
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1364
1846
|
}
|
|
1365
1847
|
}
|
|
1366
1848
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -1525,7 +2007,7 @@ class DependencyBasedScheduler {
|
|
|
1525
2007
|
if (task.status === TaskStatus.DISABLED) {
|
|
1526
2008
|
return false;
|
|
1527
2009
|
}
|
|
1528
|
-
const sourceDataflows = this.dag.getSourceDataflows(task.
|
|
2010
|
+
const sourceDataflows = this.dag.getSourceDataflows(task.id);
|
|
1529
2011
|
if (sourceDataflows.length > 0) {
|
|
1530
2012
|
const allIncomingDisabled = sourceDataflows.every((df) => df.status === TaskStatus.DISABLED);
|
|
1531
2013
|
if (allIncomingDisabled) {
|
|
@@ -1652,6 +2134,10 @@ class TaskGraphRunner {
|
|
|
1652
2134
|
graph.outputCache = outputCache;
|
|
1653
2135
|
this.handleProgress = this.handleProgress.bind(this);
|
|
1654
2136
|
}
|
|
2137
|
+
runId = "";
|
|
2138
|
+
get timerLabel() {
|
|
2139
|
+
return `graph:${this.runId}`;
|
|
2140
|
+
}
|
|
1655
2141
|
async runGraph(input = {}, config) {
|
|
1656
2142
|
await this.handleStart(config);
|
|
1657
2143
|
const results = [];
|
|
@@ -1664,25 +2150,33 @@ class TaskGraphRunner {
|
|
|
1664
2150
|
if (this.failedTaskErrors.size > 0) {
|
|
1665
2151
|
break;
|
|
1666
2152
|
}
|
|
1667
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2153
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1668
2154
|
const runAsync = async () => {
|
|
2155
|
+
let errorRouted = false;
|
|
1669
2156
|
try {
|
|
1670
2157
|
const taskInput = isRootTask ? input : this.filterInputForTask(task, input);
|
|
1671
2158
|
const taskPromise = this.runTask(task, taskInput);
|
|
1672
|
-
this.inProgressTasks.set(task.
|
|
2159
|
+
this.inProgressTasks.set(task.id, taskPromise);
|
|
1673
2160
|
const taskResult = await taskPromise;
|
|
1674
|
-
if (this.graph.getTargetDataflows(task.
|
|
2161
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1675
2162
|
results.push(taskResult);
|
|
1676
2163
|
}
|
|
1677
2164
|
} catch (error2) {
|
|
1678
|
-
this.
|
|
2165
|
+
if (this.hasErrorOutputEdges(task)) {
|
|
2166
|
+
errorRouted = true;
|
|
2167
|
+
this.pushErrorOutputToEdges(task);
|
|
2168
|
+
} else {
|
|
2169
|
+
this.failedTaskErrors.set(task.id, error2);
|
|
2170
|
+
}
|
|
1679
2171
|
} finally {
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
2172
|
+
if (!errorRouted) {
|
|
2173
|
+
this.pushStatusFromNodeToEdges(this.graph, task);
|
|
2174
|
+
this.pushErrorFromNodeToEdges(this.graph, task);
|
|
2175
|
+
}
|
|
2176
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1683
2177
|
}
|
|
1684
2178
|
};
|
|
1685
|
-
this.inProgressFunctions.set(Symbol(task.
|
|
2179
|
+
this.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
1686
2180
|
}
|
|
1687
2181
|
} catch (err) {
|
|
1688
2182
|
error = err;
|
|
@@ -1706,7 +2200,7 @@ class TaskGraphRunner {
|
|
|
1706
2200
|
const results = [];
|
|
1707
2201
|
try {
|
|
1708
2202
|
for await (const task of this.reactiveScheduler.tasks()) {
|
|
1709
|
-
const isRootTask = this.graph.getSourceDataflows(task.
|
|
2203
|
+
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
1710
2204
|
if (task.status === TaskStatus.PENDING) {
|
|
1711
2205
|
task.resetInputData();
|
|
1712
2206
|
this.copyInputFromEdgesToNode(task);
|
|
@@ -1714,9 +2208,9 @@ class TaskGraphRunner {
|
|
|
1714
2208
|
const taskInput = isRootTask ? input : {};
|
|
1715
2209
|
const taskResult = await task.runReactive(taskInput);
|
|
1716
2210
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
1717
|
-
if (this.graph.getTargetDataflows(task.
|
|
2211
|
+
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
1718
2212
|
results.push({
|
|
1719
|
-
id: task.
|
|
2213
|
+
id: task.id,
|
|
1720
2214
|
type: task.constructor.runtype || task.constructor.type,
|
|
1721
2215
|
data: taskResult
|
|
1722
2216
|
});
|
|
@@ -1736,7 +2230,7 @@ class TaskGraphRunner {
|
|
|
1736
2230
|
await this.handleDisable();
|
|
1737
2231
|
}
|
|
1738
2232
|
filterInputForTask(task, input) {
|
|
1739
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.
|
|
2233
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1740
2234
|
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1741
2235
|
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1742
2236
|
const filteredInput = {};
|
|
@@ -1775,13 +2269,13 @@ class TaskGraphRunner {
|
|
|
1775
2269
|
throw new TaskConfigurationError(`Unknown compound merge strategy: ${compoundMerge}`);
|
|
1776
2270
|
}
|
|
1777
2271
|
copyInputFromEdgesToNode(task) {
|
|
1778
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2272
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1779
2273
|
for (const dataflow of dataflows) {
|
|
1780
2274
|
this.addInputData(task, dataflow.getPortData());
|
|
1781
2275
|
}
|
|
1782
2276
|
}
|
|
1783
2277
|
async pushOutputFromNodeToEdges(node, results) {
|
|
1784
|
-
const dataflows = this.graph.getTargetDataflows(node.
|
|
2278
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1785
2279
|
for (const dataflow of dataflows) {
|
|
1786
2280
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
1787
2281
|
if (compatibility === "static") {
|
|
@@ -1796,7 +2290,7 @@ class TaskGraphRunner {
|
|
|
1796
2290
|
pushStatusFromNodeToEdges(graph, node, status) {
|
|
1797
2291
|
if (!node?.config?.id)
|
|
1798
2292
|
return;
|
|
1799
|
-
const dataflows = graph.getTargetDataflows(node.
|
|
2293
|
+
const dataflows = graph.getTargetDataflows(node.id);
|
|
1800
2294
|
const effectiveStatus = status ?? node.status;
|
|
1801
2295
|
if (node instanceof ConditionalTask && effectiveStatus === TaskStatus.COMPLETED) {
|
|
1802
2296
|
const branches = node.config.branches ?? [];
|
|
@@ -1827,10 +2321,31 @@ class TaskGraphRunner {
|
|
|
1827
2321
|
pushErrorFromNodeToEdges(graph, node) {
|
|
1828
2322
|
if (!node?.config?.id)
|
|
1829
2323
|
return;
|
|
1830
|
-
graph.getTargetDataflows(node.
|
|
2324
|
+
graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1831
2325
|
dataflow.error = node.error;
|
|
1832
2326
|
});
|
|
1833
2327
|
}
|
|
2328
|
+
hasErrorOutputEdges(task) {
|
|
2329
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2330
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
2331
|
+
}
|
|
2332
|
+
pushErrorOutputToEdges(task) {
|
|
2333
|
+
const taskError = task.error;
|
|
2334
|
+
const errorData = {
|
|
2335
|
+
error: taskError?.message ?? "Unknown error",
|
|
2336
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2337
|
+
};
|
|
2338
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
2339
|
+
for (const df of dataflows) {
|
|
2340
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
2341
|
+
df.value = errorData;
|
|
2342
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
2343
|
+
} else {
|
|
2344
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
this.propagateDisabledStatus(this.graph);
|
|
2348
|
+
}
|
|
1834
2349
|
propagateDisabledStatus(graph) {
|
|
1835
2350
|
let changed = true;
|
|
1836
2351
|
while (changed) {
|
|
@@ -1839,7 +2354,7 @@ class TaskGraphRunner {
|
|
|
1839
2354
|
if (task.status !== TaskStatus.PENDING) {
|
|
1840
2355
|
continue;
|
|
1841
2356
|
}
|
|
1842
|
-
const incomingDataflows = graph.getSourceDataflows(task.
|
|
2357
|
+
const incomingDataflows = graph.getSourceDataflows(task.id);
|
|
1843
2358
|
if (incomingDataflows.length === 0) {
|
|
1844
2359
|
continue;
|
|
1845
2360
|
}
|
|
@@ -1850,10 +2365,10 @@ class TaskGraphRunner {
|
|
|
1850
2365
|
task.completedAt = new Date;
|
|
1851
2366
|
task.emit("disabled");
|
|
1852
2367
|
task.emit("status", task.status);
|
|
1853
|
-
graph.getTargetDataflows(task.
|
|
2368
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1854
2369
|
dataflow.setStatus(TaskStatus.DISABLED);
|
|
1855
2370
|
});
|
|
1856
|
-
this.processScheduler.onTaskCompleted(task.
|
|
2371
|
+
this.processScheduler.onTaskCompleted(task.id);
|
|
1857
2372
|
changed = true;
|
|
1858
2373
|
}
|
|
1859
2374
|
}
|
|
@@ -1862,7 +2377,7 @@ class TaskGraphRunner {
|
|
|
1862
2377
|
taskNeedsAccumulation(task) {
|
|
1863
2378
|
if (this.outputCache)
|
|
1864
2379
|
return true;
|
|
1865
|
-
const outEdges = this.graph.getTargetDataflows(task.
|
|
2380
|
+
const outEdges = this.graph.getTargetDataflows(task.id);
|
|
1866
2381
|
if (outEdges.length === 0)
|
|
1867
2382
|
return this.accumulateLeafOutputs;
|
|
1868
2383
|
const outSchema = task.outputSchema();
|
|
@@ -1885,7 +2400,7 @@ class TaskGraphRunner {
|
|
|
1885
2400
|
async runTask(task, input) {
|
|
1886
2401
|
const isStreamable = isTaskStreamable(task);
|
|
1887
2402
|
if (isStreamable) {
|
|
1888
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2403
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1889
2404
|
const streamingEdges = dataflows.filter((df) => df.stream !== undefined);
|
|
1890
2405
|
if (streamingEdges.length > 0) {
|
|
1891
2406
|
const inputStreams = new Map;
|
|
@@ -1910,13 +2425,13 @@ class TaskGraphRunner {
|
|
|
1910
2425
|
});
|
|
1911
2426
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1912
2427
|
return {
|
|
1913
|
-
id: task.
|
|
2428
|
+
id: task.id,
|
|
1914
2429
|
type: task.constructor.runtype || task.constructor.type,
|
|
1915
2430
|
data: results
|
|
1916
2431
|
};
|
|
1917
2432
|
}
|
|
1918
2433
|
async awaitStreamInputs(task) {
|
|
1919
|
-
const dataflows = this.graph.getSourceDataflows(task.
|
|
2434
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1920
2435
|
const streamPromises = dataflows.filter((df) => df.stream !== undefined).map((df) => df.awaitStreamValue());
|
|
1921
2436
|
if (streamPromises.length > 0) {
|
|
1922
2437
|
await Promise.all(streamPromises);
|
|
@@ -1931,17 +2446,17 @@ class TaskGraphRunner {
|
|
|
1931
2446
|
streamingNotified = true;
|
|
1932
2447
|
this.pushStatusFromNodeToEdges(this.graph, task, TaskStatus.STREAMING);
|
|
1933
2448
|
this.pushStreamToEdges(task, streamMode);
|
|
1934
|
-
this.processScheduler.onTaskStreaming(task.
|
|
2449
|
+
this.processScheduler.onTaskStreaming(task.id);
|
|
1935
2450
|
}
|
|
1936
2451
|
};
|
|
1937
2452
|
const onStreamStart = () => {
|
|
1938
|
-
this.graph.emit("task_stream_start", task.
|
|
2453
|
+
this.graph.emit("task_stream_start", task.id);
|
|
1939
2454
|
};
|
|
1940
2455
|
const onStreamChunk = (event) => {
|
|
1941
|
-
this.graph.emit("task_stream_chunk", task.
|
|
2456
|
+
this.graph.emit("task_stream_chunk", task.id, event);
|
|
1942
2457
|
};
|
|
1943
2458
|
const onStreamEnd = (output) => {
|
|
1944
|
-
this.graph.emit("task_stream_end", task.
|
|
2459
|
+
this.graph.emit("task_stream_end", task.id, output);
|
|
1945
2460
|
};
|
|
1946
2461
|
task.on("status", onStatus);
|
|
1947
2462
|
task.on("stream_start", onStreamStart);
|
|
@@ -1956,7 +2471,7 @@ class TaskGraphRunner {
|
|
|
1956
2471
|
});
|
|
1957
2472
|
await this.pushOutputFromNodeToEdges(task, results);
|
|
1958
2473
|
return {
|
|
1959
|
-
id: task.
|
|
2474
|
+
id: task.id,
|
|
1960
2475
|
type: task.constructor.runtype || task.constructor.type,
|
|
1961
2476
|
data: results
|
|
1962
2477
|
};
|
|
@@ -1994,7 +2509,7 @@ class TaskGraphRunner {
|
|
|
1994
2509
|
});
|
|
1995
2510
|
}
|
|
1996
2511
|
pushStreamToEdges(task, streamMode) {
|
|
1997
|
-
const targetDataflows = this.graph.getTargetDataflows(task.
|
|
2512
|
+
const targetDataflows = this.graph.getTargetDataflows(task.id);
|
|
1998
2513
|
if (targetDataflows.length === 0)
|
|
1999
2514
|
return;
|
|
2000
2515
|
const groups = new Map;
|
|
@@ -2085,11 +2600,15 @@ class TaskGraphRunner {
|
|
|
2085
2600
|
this.abortController?.abort();
|
|
2086
2601
|
}, { once: true });
|
|
2087
2602
|
}
|
|
2088
|
-
this.
|
|
2603
|
+
this.runId = uuid43();
|
|
2604
|
+
this.resetGraph(this.graph, this.runId);
|
|
2089
2605
|
this.processScheduler.reset();
|
|
2090
2606
|
this.inProgressTasks.clear();
|
|
2091
2607
|
this.inProgressFunctions.clear();
|
|
2092
2608
|
this.failedTaskErrors.clear();
|
|
2609
|
+
const logger = getLogger3();
|
|
2610
|
+
logger.group(this.timerLabel, { graph: this.graph });
|
|
2611
|
+
logger.time(this.timerLabel);
|
|
2093
2612
|
this.graph.emit("start");
|
|
2094
2613
|
}
|
|
2095
2614
|
async handleStartReactive() {
|
|
@@ -2101,6 +2620,9 @@ class TaskGraphRunner {
|
|
|
2101
2620
|
}
|
|
2102
2621
|
async handleComplete() {
|
|
2103
2622
|
this.running = false;
|
|
2623
|
+
const logger = getLogger3();
|
|
2624
|
+
logger.timeEnd(this.timerLabel);
|
|
2625
|
+
logger.groupEnd();
|
|
2104
2626
|
this.graph.emit("complete");
|
|
2105
2627
|
}
|
|
2106
2628
|
async handleCompleteReactive() {
|
|
@@ -2113,6 +2635,9 @@ class TaskGraphRunner {
|
|
|
2113
2635
|
}
|
|
2114
2636
|
}));
|
|
2115
2637
|
this.running = false;
|
|
2638
|
+
const logger = getLogger3();
|
|
2639
|
+
logger.timeEnd(this.timerLabel);
|
|
2640
|
+
logger.groupEnd();
|
|
2116
2641
|
this.graph.emit("error", error);
|
|
2117
2642
|
}
|
|
2118
2643
|
async handleErrorReactive() {
|
|
@@ -2125,6 +2650,9 @@ class TaskGraphRunner {
|
|
|
2125
2650
|
}
|
|
2126
2651
|
});
|
|
2127
2652
|
this.running = false;
|
|
2653
|
+
const logger = getLogger3();
|
|
2654
|
+
logger.timeEnd(this.timerLabel);
|
|
2655
|
+
logger.groupEnd();
|
|
2128
2656
|
this.graph.emit("abort");
|
|
2129
2657
|
}
|
|
2130
2658
|
async handleAbortReactive() {
|
|
@@ -2239,118 +2767,27 @@ class GraphAsTask extends Task {
|
|
|
2239
2767
|
if (!this.hasChildren()) {
|
|
2240
2768
|
return this.constructor.inputSchema();
|
|
2241
2769
|
}
|
|
2242
|
-
|
|
2243
|
-
const required = [];
|
|
2244
|
-
const tasks = this.subGraph.getTasks();
|
|
2245
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.config.id).length === 0);
|
|
2246
|
-
for (const task of startingNodes) {
|
|
2247
|
-
const taskInputSchema = task.inputSchema();
|
|
2248
|
-
if (typeof taskInputSchema === "boolean") {
|
|
2249
|
-
if (taskInputSchema === false) {
|
|
2250
|
-
continue;
|
|
2251
|
-
}
|
|
2252
|
-
if (taskInputSchema === true) {
|
|
2253
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2254
|
-
continue;
|
|
2255
|
-
}
|
|
2256
|
-
}
|
|
2257
|
-
const taskProperties = taskInputSchema.properties || {};
|
|
2258
|
-
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
2259
|
-
if (!properties[inputName]) {
|
|
2260
|
-
properties[inputName] = inputProp;
|
|
2261
|
-
if (taskInputSchema.required && taskInputSchema.required.includes(inputName)) {
|
|
2262
|
-
required.push(inputName);
|
|
2263
|
-
}
|
|
2264
|
-
}
|
|
2265
|
-
}
|
|
2266
|
-
}
|
|
2267
|
-
return {
|
|
2268
|
-
type: "object",
|
|
2269
|
-
properties,
|
|
2270
|
-
...required.length > 0 ? { required } : {},
|
|
2271
|
-
additionalProperties: false
|
|
2272
|
-
};
|
|
2770
|
+
return computeGraphInputSchema(this.subGraph);
|
|
2273
2771
|
}
|
|
2274
2772
|
_inputSchemaNode;
|
|
2275
|
-
getInputSchemaNode(
|
|
2773
|
+
getInputSchemaNode() {
|
|
2276
2774
|
if (!this._inputSchemaNode) {
|
|
2277
|
-
const dataPortSchema = this.inputSchema();
|
|
2278
|
-
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2279
2775
|
try {
|
|
2776
|
+
const dataPortSchema = this.inputSchema();
|
|
2777
|
+
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2280
2778
|
this._inputSchemaNode = schemaNode;
|
|
2281
2779
|
} catch (error) {
|
|
2282
|
-
console.warn(`Failed to compile input schema for ${type}, falling back to permissive validation:`, error);
|
|
2780
|
+
console.warn(`Failed to compile input schema for ${this.type}, falling back to permissive validation:`, error);
|
|
2283
2781
|
this._inputSchemaNode = compileSchema2({});
|
|
2284
2782
|
}
|
|
2285
2783
|
}
|
|
2286
2784
|
return this._inputSchemaNode;
|
|
2287
2785
|
}
|
|
2288
|
-
calculateNodeDepths() {
|
|
2289
|
-
const depths = new Map;
|
|
2290
|
-
const tasks = this.subGraph.getTasks();
|
|
2291
|
-
for (const task of tasks) {
|
|
2292
|
-
depths.set(task.config.id, 0);
|
|
2293
|
-
}
|
|
2294
|
-
const sortedTasks = this.subGraph.topologicallySortedNodes();
|
|
2295
|
-
for (const task of sortedTasks) {
|
|
2296
|
-
const currentDepth = depths.get(task.config.id) || 0;
|
|
2297
|
-
const targetTasks = this.subGraph.getTargetTasks(task.config.id);
|
|
2298
|
-
for (const targetTask of targetTasks) {
|
|
2299
|
-
const targetDepth = depths.get(targetTask.config.id) || 0;
|
|
2300
|
-
depths.set(targetTask.config.id, Math.max(targetDepth, currentDepth + 1));
|
|
2301
|
-
}
|
|
2302
|
-
}
|
|
2303
|
-
return depths;
|
|
2304
|
-
}
|
|
2305
2786
|
outputSchema() {
|
|
2306
2787
|
if (!this.hasChildren()) {
|
|
2307
2788
|
return this.constructor.outputSchema();
|
|
2308
2789
|
}
|
|
2309
|
-
|
|
2310
|
-
const required = [];
|
|
2311
|
-
const tasks = this.subGraph.getTasks();
|
|
2312
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.config.id).length === 0);
|
|
2313
|
-
const depths = this.calculateNodeDepths();
|
|
2314
|
-
const maxDepth = Math.max(...endingNodes.map((task) => depths.get(task.config.id) || 0));
|
|
2315
|
-
const lastLevelNodes = endingNodes.filter((task) => depths.get(task.config.id) === maxDepth);
|
|
2316
|
-
const propertyCount = {};
|
|
2317
|
-
const propertySchema = {};
|
|
2318
|
-
for (const task of lastLevelNodes) {
|
|
2319
|
-
const taskOutputSchema = task.outputSchema();
|
|
2320
|
-
if (typeof taskOutputSchema === "boolean") {
|
|
2321
|
-
if (taskOutputSchema === false) {
|
|
2322
|
-
continue;
|
|
2323
|
-
}
|
|
2324
|
-
if (taskOutputSchema === true) {
|
|
2325
|
-
properties[DATAFLOW_ALL_PORTS] = {};
|
|
2326
|
-
continue;
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
const taskProperties = taskOutputSchema.properties || {};
|
|
2330
|
-
for (const [outputName, outputProp] of Object.entries(taskProperties)) {
|
|
2331
|
-
propertyCount[outputName] = (propertyCount[outputName] || 0) + 1;
|
|
2332
|
-
if (!propertySchema[outputName]) {
|
|
2333
|
-
propertySchema[outputName] = outputProp;
|
|
2334
|
-
}
|
|
2335
|
-
}
|
|
2336
|
-
}
|
|
2337
|
-
for (const [outputName, count] of Object.entries(propertyCount)) {
|
|
2338
|
-
const outputProp = propertySchema[outputName];
|
|
2339
|
-
if (lastLevelNodes.length === 1) {
|
|
2340
|
-
properties[outputName] = outputProp;
|
|
2341
|
-
} else {
|
|
2342
|
-
properties[outputName] = {
|
|
2343
|
-
type: "array",
|
|
2344
|
-
items: outputProp
|
|
2345
|
-
};
|
|
2346
|
-
}
|
|
2347
|
-
}
|
|
2348
|
-
return {
|
|
2349
|
-
type: "object",
|
|
2350
|
-
properties,
|
|
2351
|
-
...required.length > 0 ? { required } : {},
|
|
2352
|
-
additionalProperties: false
|
|
2353
|
-
};
|
|
2790
|
+
return computeGraphOutputSchema(this.subGraph);
|
|
2354
2791
|
}
|
|
2355
2792
|
resetInputData() {
|
|
2356
2793
|
super.resetInputData();
|
|
@@ -2385,8 +2822,8 @@ class GraphAsTask extends Task {
|
|
|
2385
2822
|
const endingNodeIds = new Set;
|
|
2386
2823
|
const tasks = this.subGraph.getTasks();
|
|
2387
2824
|
for (const task of tasks) {
|
|
2388
|
-
if (this.subGraph.getTargetDataflows(task.
|
|
2389
|
-
endingNodeIds.add(task.
|
|
2825
|
+
if (this.subGraph.getTargetDataflows(task.id).length === 0) {
|
|
2826
|
+
endingNodeIds.add(task.id);
|
|
2390
2827
|
}
|
|
2391
2828
|
}
|
|
2392
2829
|
const eventQueue = [];
|
|
@@ -2430,32 +2867,36 @@ class GraphAsTask extends Task {
|
|
|
2430
2867
|
this._inputSchemaNode = undefined;
|
|
2431
2868
|
this.events.emit("regenerate");
|
|
2432
2869
|
}
|
|
2433
|
-
toJSON() {
|
|
2434
|
-
let json = super.toJSON();
|
|
2870
|
+
toJSON(options) {
|
|
2871
|
+
let json = super.toJSON(options);
|
|
2435
2872
|
const hasChildren = this.hasChildren();
|
|
2436
2873
|
if (hasChildren) {
|
|
2437
2874
|
json = {
|
|
2438
2875
|
...json,
|
|
2439
2876
|
merge: this.compoundMerge,
|
|
2440
|
-
subgraph: this.subGraph.toJSON()
|
|
2877
|
+
subgraph: this.subGraph.toJSON(options)
|
|
2441
2878
|
};
|
|
2442
2879
|
}
|
|
2443
2880
|
return json;
|
|
2444
2881
|
}
|
|
2445
|
-
toDependencyJSON() {
|
|
2446
|
-
const json = this.toJSON();
|
|
2882
|
+
toDependencyJSON(options) {
|
|
2883
|
+
const json = this.toJSON(options);
|
|
2447
2884
|
if (this.hasChildren()) {
|
|
2448
2885
|
if ("subgraph" in json) {
|
|
2449
2886
|
delete json.subgraph;
|
|
2450
2887
|
}
|
|
2451
|
-
return { ...json, subtasks: this.subGraph.toDependencyJSON() };
|
|
2888
|
+
return { ...json, subtasks: this.subGraph.toDependencyJSON(options) };
|
|
2452
2889
|
}
|
|
2453
2890
|
return json;
|
|
2454
2891
|
}
|
|
2455
2892
|
}
|
|
2456
2893
|
|
|
2457
2894
|
// src/task-graph/Workflow.ts
|
|
2458
|
-
import {
|
|
2895
|
+
import {
|
|
2896
|
+
EventEmitter as EventEmitter4,
|
|
2897
|
+
getLogger as getLogger4,
|
|
2898
|
+
uuid4 as uuid44
|
|
2899
|
+
} from "@workglow/util";
|
|
2459
2900
|
function CreateWorkflow(taskClass) {
|
|
2460
2901
|
return Workflow.createWorkflow(taskClass);
|
|
2461
2902
|
}
|
|
@@ -2556,43 +2997,48 @@ class Workflow {
|
|
|
2556
2997
|
const helper = function(input = {}, config = {}) {
|
|
2557
2998
|
this._error = "";
|
|
2558
2999
|
const parent = getLastTask(this);
|
|
2559
|
-
const task = this.addTaskToGraph(taskClass, input, { id:
|
|
3000
|
+
const task = this.addTaskToGraph(taskClass, input, { id: uuid44(), ...config });
|
|
2560
3001
|
if (this._dataFlows.length > 0) {
|
|
2561
3002
|
this._dataFlows.forEach((dataflow) => {
|
|
2562
3003
|
const taskSchema = task.inputSchema();
|
|
2563
3004
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2564
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2565
|
-
|
|
3005
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3006
|
+
getLogger4().error(this._error);
|
|
2566
3007
|
return;
|
|
2567
3008
|
}
|
|
2568
|
-
dataflow.targetTaskId = task.
|
|
3009
|
+
dataflow.targetTaskId = task.id;
|
|
2569
3010
|
this.graph.addDataflow(dataflow);
|
|
2570
3011
|
});
|
|
2571
3012
|
this._dataFlows = [];
|
|
2572
3013
|
}
|
|
2573
|
-
if (parent
|
|
3014
|
+
if (parent) {
|
|
2574
3015
|
const nodes = this._graph.getTasks();
|
|
2575
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3016
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2576
3017
|
const earlierTasks = [];
|
|
2577
3018
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2578
3019
|
earlierTasks.push(nodes[i]);
|
|
2579
3020
|
}
|
|
2580
3021
|
const providedInputKeys = new Set(Object.keys(input || {}));
|
|
3022
|
+
const connectedInputKeys = new Set(this.graph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
2581
3023
|
const result = Workflow.autoConnect(this.graph, parent, task, {
|
|
2582
3024
|
providedInputKeys,
|
|
3025
|
+
connectedInputKeys,
|
|
2583
3026
|
earlierTasks
|
|
2584
3027
|
});
|
|
2585
3028
|
if (result.error) {
|
|
2586
3029
|
if (this.isLoopBuilder) {
|
|
2587
3030
|
this._error = result.error;
|
|
2588
|
-
|
|
3031
|
+
getLogger4().warn(this._error);
|
|
2589
3032
|
} else {
|
|
2590
3033
|
this._error = result.error + " Task not added.";
|
|
2591
|
-
|
|
2592
|
-
this.graph.removeTask(task.
|
|
3034
|
+
getLogger4().error(this._error);
|
|
3035
|
+
this.graph.removeTask(task.id);
|
|
2593
3036
|
}
|
|
2594
3037
|
}
|
|
2595
3038
|
}
|
|
3039
|
+
if (!this._error) {
|
|
3040
|
+
Workflow.updateBoundaryTaskSchemas(this._graph);
|
|
3041
|
+
}
|
|
2596
3042
|
return this;
|
|
2597
3043
|
};
|
|
2598
3044
|
helper.type = taskClass.runtype ?? taskClass.type;
|
|
@@ -2672,18 +3118,18 @@ class Workflow {
|
|
|
2672
3118
|
const nodes = this._graph.getTasks();
|
|
2673
3119
|
if (nodes.length === 0) {
|
|
2674
3120
|
this._error = "No tasks to remove";
|
|
2675
|
-
|
|
3121
|
+
getLogger4().error(this._error);
|
|
2676
3122
|
return this;
|
|
2677
3123
|
}
|
|
2678
3124
|
const lastNode = nodes[nodes.length - 1];
|
|
2679
|
-
this._graph.removeTask(lastNode.
|
|
3125
|
+
this._graph.removeTask(lastNode.id);
|
|
2680
3126
|
return this;
|
|
2681
3127
|
}
|
|
2682
|
-
toJSON() {
|
|
2683
|
-
return this._graph.toJSON();
|
|
3128
|
+
toJSON(options = { withBoundaryNodes: true }) {
|
|
3129
|
+
return this._graph.toJSON(options);
|
|
2684
3130
|
}
|
|
2685
|
-
toDependencyJSON() {
|
|
2686
|
-
return this._graph.toDependencyJSON();
|
|
3131
|
+
toDependencyJSON(options = { withBoundaryNodes: true }) {
|
|
3132
|
+
return this._graph.toDependencyJSON(options);
|
|
2687
3133
|
}
|
|
2688
3134
|
pipe(...args) {
|
|
2689
3135
|
return pipe(args, this);
|
|
@@ -2703,25 +3149,40 @@ class Workflow {
|
|
|
2703
3149
|
if (-index > nodes.length) {
|
|
2704
3150
|
const errorMsg = `Back index greater than number of tasks`;
|
|
2705
3151
|
this._error = errorMsg;
|
|
2706
|
-
|
|
3152
|
+
getLogger4().error(this._error);
|
|
2707
3153
|
throw new WorkflowError(errorMsg);
|
|
2708
3154
|
}
|
|
2709
3155
|
const lastNode = nodes[nodes.length + index];
|
|
2710
3156
|
const outputSchema = lastNode.outputSchema();
|
|
2711
3157
|
if (typeof outputSchema === "boolean") {
|
|
2712
3158
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
2713
|
-
const errorMsg = `Task ${lastNode.
|
|
3159
|
+
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
2714
3160
|
this._error = errorMsg;
|
|
2715
|
-
|
|
3161
|
+
getLogger4().error(this._error);
|
|
2716
3162
|
throw new WorkflowError(errorMsg);
|
|
2717
3163
|
}
|
|
2718
3164
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
2719
|
-
const errorMsg = `Output ${source} not found on task ${lastNode.
|
|
3165
|
+
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
2720
3166
|
this._error = errorMsg;
|
|
2721
|
-
|
|
3167
|
+
getLogger4().error(this._error);
|
|
2722
3168
|
throw new WorkflowError(errorMsg);
|
|
2723
3169
|
}
|
|
2724
|
-
this._dataFlows.push(new Dataflow(lastNode.
|
|
3170
|
+
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
3171
|
+
return this;
|
|
3172
|
+
}
|
|
3173
|
+
onError(handler) {
|
|
3174
|
+
this._error = "";
|
|
3175
|
+
const parent = getLastTask(this);
|
|
3176
|
+
if (!parent) {
|
|
3177
|
+
this._error = "onError() requires a preceding task in the workflow";
|
|
3178
|
+
getLogger4().error(this._error);
|
|
3179
|
+
throw new WorkflowError(this._error);
|
|
3180
|
+
}
|
|
3181
|
+
const handlerTask = ensureTask(handler);
|
|
3182
|
+
this.graph.addTask(handlerTask);
|
|
3183
|
+
const dataflow = new Dataflow(parent.id, DATAFLOW_ERROR_PORT, handlerTask.id, DATAFLOW_ALL_PORTS);
|
|
3184
|
+
this.graph.addDataflow(dataflow);
|
|
3185
|
+
this.events.emit("changed", handlerTask.id);
|
|
2725
3186
|
return this;
|
|
2726
3187
|
}
|
|
2727
3188
|
toTaskGraph() {
|
|
@@ -2806,16 +3267,16 @@ class Workflow {
|
|
|
2806
3267
|
addLoopTask(taskClass, config = {}) {
|
|
2807
3268
|
this._error = "";
|
|
2808
3269
|
const parent = getLastTask(this);
|
|
2809
|
-
const task = this.addTaskToGraph(taskClass, {}, { id:
|
|
3270
|
+
const task = this.addTaskToGraph(taskClass, {}, { id: uuid44(), ...config });
|
|
2810
3271
|
if (this._dataFlows.length > 0) {
|
|
2811
3272
|
this._dataFlows.forEach((dataflow) => {
|
|
2812
3273
|
const taskSchema = task.inputSchema();
|
|
2813
3274
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
2814
|
-
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.
|
|
2815
|
-
|
|
3275
|
+
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3276
|
+
getLogger4().error(this._error);
|
|
2816
3277
|
return;
|
|
2817
3278
|
}
|
|
2818
|
-
dataflow.targetTaskId = task.
|
|
3279
|
+
dataflow.targetTaskId = task.id;
|
|
2819
3280
|
this.graph.addDataflow(dataflow);
|
|
2820
3281
|
});
|
|
2821
3282
|
this._dataFlows = [];
|
|
@@ -2830,9 +3291,9 @@ class Workflow {
|
|
|
2830
3291
|
if (!pending)
|
|
2831
3292
|
return;
|
|
2832
3293
|
const { parent, iteratorTask } = pending;
|
|
2833
|
-
if (this.graph.getTargetDataflows(parent.
|
|
3294
|
+
if (this.graph.getTargetDataflows(parent.id).length === 0) {
|
|
2834
3295
|
const nodes = this._graph.getTasks();
|
|
2835
|
-
const parentIndex = nodes.findIndex((n) => n.
|
|
3296
|
+
const parentIndex = nodes.findIndex((n) => n.id === parent.id);
|
|
2836
3297
|
const earlierTasks = [];
|
|
2837
3298
|
for (let i = parentIndex - 1;i >= 0; i--) {
|
|
2838
3299
|
earlierTasks.push(nodes[i]);
|
|
@@ -2842,8 +3303,81 @@ class Workflow {
|
|
|
2842
3303
|
});
|
|
2843
3304
|
if (result.error) {
|
|
2844
3305
|
this._error = result.error + " Task not added.";
|
|
2845
|
-
|
|
2846
|
-
this.graph.removeTask(iteratorTask.
|
|
3306
|
+
getLogger4().error(this._error);
|
|
3307
|
+
this.graph.removeTask(iteratorTask.id);
|
|
3308
|
+
}
|
|
3309
|
+
}
|
|
3310
|
+
}
|
|
3311
|
+
static updateBoundaryTaskSchemas(graph) {
|
|
3312
|
+
const tasks = graph.getTasks();
|
|
3313
|
+
for (const task of tasks) {
|
|
3314
|
+
if (task.type === "InputTask") {
|
|
3315
|
+
const outgoing = graph.getTargetDataflows(task.id);
|
|
3316
|
+
if (outgoing.length === 0)
|
|
3317
|
+
continue;
|
|
3318
|
+
const properties = {};
|
|
3319
|
+
const required = [];
|
|
3320
|
+
for (const df of outgoing) {
|
|
3321
|
+
const targetTask = graph.getTask(df.targetTaskId);
|
|
3322
|
+
if (!targetTask)
|
|
3323
|
+
continue;
|
|
3324
|
+
const targetSchema = targetTask.inputSchema();
|
|
3325
|
+
if (typeof targetSchema === "boolean")
|
|
3326
|
+
continue;
|
|
3327
|
+
const prop = targetSchema.properties?.[df.targetTaskPortId];
|
|
3328
|
+
if (prop && typeof prop !== "boolean") {
|
|
3329
|
+
properties[df.sourceTaskPortId] = prop;
|
|
3330
|
+
if (targetSchema.required?.includes(df.targetTaskPortId)) {
|
|
3331
|
+
if (!required.includes(df.sourceTaskPortId)) {
|
|
3332
|
+
required.push(df.sourceTaskPortId);
|
|
3333
|
+
}
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
const schema = {
|
|
3338
|
+
type: "object",
|
|
3339
|
+
properties,
|
|
3340
|
+
...required.length > 0 ? { required } : {},
|
|
3341
|
+
additionalProperties: false
|
|
3342
|
+
};
|
|
3343
|
+
task.config = {
|
|
3344
|
+
...task.config,
|
|
3345
|
+
inputSchema: schema,
|
|
3346
|
+
outputSchema: schema
|
|
3347
|
+
};
|
|
3348
|
+
}
|
|
3349
|
+
if (task.type === "OutputTask") {
|
|
3350
|
+
const incoming = graph.getSourceDataflows(task.id);
|
|
3351
|
+
if (incoming.length === 0)
|
|
3352
|
+
continue;
|
|
3353
|
+
const properties = {};
|
|
3354
|
+
const required = [];
|
|
3355
|
+
for (const df of incoming) {
|
|
3356
|
+
const sourceTask = graph.getTask(df.sourceTaskId);
|
|
3357
|
+
if (!sourceTask)
|
|
3358
|
+
continue;
|
|
3359
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
3360
|
+
if (typeof sourceSchema === "boolean")
|
|
3361
|
+
continue;
|
|
3362
|
+
const prop = sourceSchema.properties?.[df.sourceTaskPortId];
|
|
3363
|
+
if (prop && typeof prop !== "boolean") {
|
|
3364
|
+
properties[df.targetTaskPortId] = prop;
|
|
3365
|
+
if (sourceSchema.required?.includes(df.sourceTaskPortId) && !required.includes(df.targetTaskPortId)) {
|
|
3366
|
+
required.push(df.targetTaskPortId);
|
|
3367
|
+
}
|
|
3368
|
+
}
|
|
3369
|
+
}
|
|
3370
|
+
const schema = {
|
|
3371
|
+
type: "object",
|
|
3372
|
+
properties,
|
|
3373
|
+
...required.length > 0 ? { required } : {},
|
|
3374
|
+
additionalProperties: false
|
|
3375
|
+
};
|
|
3376
|
+
task.config = {
|
|
3377
|
+
...task.config,
|
|
3378
|
+
inputSchema: schema,
|
|
3379
|
+
outputSchema: schema
|
|
3380
|
+
};
|
|
2847
3381
|
}
|
|
2848
3382
|
}
|
|
2849
3383
|
}
|
|
@@ -2853,6 +3387,7 @@ class Workflow {
|
|
|
2853
3387
|
const sourceSchema = sourceTask.outputSchema();
|
|
2854
3388
|
const targetSchema = targetTask.inputSchema();
|
|
2855
3389
|
const providedInputKeys = options?.providedInputKeys ?? new Set;
|
|
3390
|
+
const connectedInputKeys = options?.connectedInputKeys ?? new Set;
|
|
2856
3391
|
const earlierTasks = options?.earlierTasks ?? [];
|
|
2857
3392
|
const getSpecificTypeIdentifiers = (schema) => {
|
|
2858
3393
|
const formats = new Set;
|
|
@@ -2928,18 +3463,33 @@ class Workflow {
|
|
|
2928
3463
|
if (typeof fromSchema === "object") {
|
|
2929
3464
|
if (toSchema === true || typeof toSchema === "object" && toSchema.additionalProperties === true) {
|
|
2930
3465
|
for (const fromOutputPortId of Object.keys(fromSchema.properties || {})) {
|
|
3466
|
+
if (matches.has(fromOutputPortId))
|
|
3467
|
+
continue;
|
|
2931
3468
|
matches.set(fromOutputPortId, fromOutputPortId);
|
|
2932
3469
|
graph.addDataflow(new Dataflow(fromTaskId, fromOutputPortId, toTaskId, fromOutputPortId));
|
|
2933
3470
|
}
|
|
2934
3471
|
return;
|
|
2935
3472
|
}
|
|
2936
3473
|
}
|
|
3474
|
+
if (typeof fromSchema === "object" && fromSchema.additionalProperties === true && typeof toSchema === "object" && (sourceTask.type === "InputTask" || sourceTask.type === "OutputTask")) {
|
|
3475
|
+
for (const toInputPortId of Object.keys(toSchema.properties || {})) {
|
|
3476
|
+
if (matches.has(toInputPortId))
|
|
3477
|
+
continue;
|
|
3478
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3479
|
+
continue;
|
|
3480
|
+
matches.set(toInputPortId, toInputPortId);
|
|
3481
|
+
graph.addDataflow(new Dataflow(fromTaskId, toInputPortId, toTaskId, toInputPortId));
|
|
3482
|
+
}
|
|
3483
|
+
return;
|
|
3484
|
+
}
|
|
2937
3485
|
if (typeof fromSchema === "boolean" || typeof toSchema === "boolean") {
|
|
2938
3486
|
return;
|
|
2939
3487
|
}
|
|
2940
3488
|
for (const [toInputPortId, toPortInputSchema] of Object.entries(toSchema.properties || {})) {
|
|
2941
3489
|
if (matches.has(toInputPortId))
|
|
2942
3490
|
continue;
|
|
3491
|
+
if (connectedInputKeys.has(toInputPortId))
|
|
3492
|
+
continue;
|
|
2943
3493
|
const candidates = [];
|
|
2944
3494
|
for (const [fromOutputPortId, fromPortOutputSchema] of Object.entries(fromSchema.properties || {})) {
|
|
2945
3495
|
if (comparator([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema])) {
|
|
@@ -2959,22 +3509,32 @@ class Workflow {
|
|
|
2959
3509
|
graph.addDataflow(new Dataflow(fromTaskId, winner, toTaskId, toInputPortId));
|
|
2960
3510
|
}
|
|
2961
3511
|
};
|
|
2962
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3512
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([fromOutputPortId, fromPortOutputSchema], [toInputPortId, toPortInputSchema]) => {
|
|
2963
3513
|
const outputPortIdMatch = fromOutputPortId === toInputPortId;
|
|
2964
3514
|
const outputPortIdOutputInput = fromOutputPortId === "output" && toInputPortId === "input";
|
|
2965
3515
|
const portIdsCompatible = outputPortIdMatch || outputPortIdOutputInput;
|
|
2966
3516
|
return portIdsCompatible && isTypeCompatible(fromPortOutputSchema, toPortInputSchema, false);
|
|
2967
3517
|
});
|
|
2968
|
-
makeMatch(sourceSchema, targetSchema, sourceTask.
|
|
3518
|
+
makeMatch(sourceSchema, targetSchema, sourceTask.id, targetTask.id, ([_fromOutputPortId, fromPortOutputSchema], [_toInputPortId, toPortInputSchema]) => {
|
|
2969
3519
|
return isTypeCompatible(fromPortOutputSchema, toPortInputSchema, true);
|
|
2970
3520
|
});
|
|
2971
3521
|
const requiredInputs = new Set(typeof targetSchema === "object" ? targetSchema.required || [] : []);
|
|
2972
|
-
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r));
|
|
3522
|
+
const requiredInputsNeedingConnection = [...requiredInputs].filter((r) => !providedInputKeys.has(r) && !connectedInputKeys.has(r));
|
|
2973
3523
|
let unmatchedRequired = requiredInputsNeedingConnection.filter((r) => !matches.has(r));
|
|
2974
3524
|
if (unmatchedRequired.length > 0 && earlierTasks.length > 0) {
|
|
2975
3525
|
for (let i = 0;i < earlierTasks.length && unmatchedRequired.length > 0; i++) {
|
|
2976
3526
|
const earlierTask = earlierTasks[i];
|
|
2977
3527
|
const earlierOutputSchema = earlierTask.outputSchema();
|
|
3528
|
+
if (earlierTask.type === "InputTask") {
|
|
3529
|
+
for (const requiredInputId of [...unmatchedRequired]) {
|
|
3530
|
+
if (matches.has(requiredInputId))
|
|
3531
|
+
continue;
|
|
3532
|
+
matches.set(requiredInputId, requiredInputId);
|
|
3533
|
+
graph.addDataflow(new Dataflow(earlierTask.id, requiredInputId, targetTask.id, requiredInputId));
|
|
3534
|
+
}
|
|
3535
|
+
unmatchedRequired = unmatchedRequired.filter((r) => !matches.has(r));
|
|
3536
|
+
continue;
|
|
3537
|
+
}
|
|
2978
3538
|
const makeMatchFromEarlier = (comparator) => {
|
|
2979
3539
|
if (typeof earlierOutputSchema === "boolean" || typeof targetSchema === "boolean") {
|
|
2980
3540
|
return;
|
|
@@ -2984,7 +3544,7 @@ class Workflow {
|
|
|
2984
3544
|
const toPortInputSchema = targetSchema.properties?.[requiredInputId];
|
|
2985
3545
|
if (!matches.has(requiredInputId) && toPortInputSchema && comparator([fromOutputPortId, fromPortOutputSchema], [requiredInputId, toPortInputSchema])) {
|
|
2986
3546
|
matches.set(requiredInputId, fromOutputPortId);
|
|
2987
|
-
graph.addDataflow(new Dataflow(earlierTask.
|
|
3547
|
+
graph.addDataflow(new Dataflow(earlierTask.id, fromOutputPortId, targetTask.id, requiredInputId));
|
|
2988
3548
|
}
|
|
2989
3549
|
}
|
|
2990
3550
|
}
|
|
@@ -3010,6 +3570,10 @@ class Workflow {
|
|
|
3010
3570
|
};
|
|
3011
3571
|
}
|
|
3012
3572
|
if (matches.size === 0 && requiredInputsNeedingConnection.length === 0) {
|
|
3573
|
+
const existingTargetConnections = graph.getSourceDataflows(targetTask.id);
|
|
3574
|
+
if (existingTargetConnections.length > 0) {
|
|
3575
|
+
return { matches, unmatchedRequired: [] };
|
|
3576
|
+
}
|
|
3013
3577
|
const hasRequiredInputs = requiredInputs.size > 0;
|
|
3014
3578
|
const allRequiredInputsProvided = hasRequiredInputs && [...requiredInputs].every((r) => providedInputKeys.has(r));
|
|
3015
3579
|
const hasInputsWithDefaults = typeof targetSchema === "object" && targetSchema.properties && Object.values(targetSchema.properties).some((prop) => prop && typeof prop === "object" && ("default" in prop));
|
|
@@ -3132,7 +3696,7 @@ function getLastTask(workflow) {
|
|
|
3132
3696
|
return tasks.length > 0 ? tasks[tasks.length - 1] : undefined;
|
|
3133
3697
|
}
|
|
3134
3698
|
function connect(source, target, workflow) {
|
|
3135
|
-
workflow.graph.addDataflow(new Dataflow(source.
|
|
3699
|
+
workflow.graph.addDataflow(new Dataflow(source.id, "*", target.id, "*"));
|
|
3136
3700
|
}
|
|
3137
3701
|
function pipe(args, workflow = new Workflow) {
|
|
3138
3702
|
let previousTask = getLastTask(workflow);
|
|
@@ -3188,7 +3752,7 @@ var EventTaskGraphToDagMapping = {
|
|
|
3188
3752
|
// src/task-graph/TaskGraph.ts
|
|
3189
3753
|
class TaskGraphDAG extends DirectedAcyclicGraph {
|
|
3190
3754
|
constructor() {
|
|
3191
|
-
super((task) => task.
|
|
3755
|
+
super((task) => task.id, (dataflow) => dataflow.id);
|
|
3192
3756
|
}
|
|
3193
3757
|
}
|
|
3194
3758
|
|
|
@@ -3285,18 +3849,22 @@ class TaskGraph {
|
|
|
3285
3849
|
return this._dag.removeNode(taskId);
|
|
3286
3850
|
}
|
|
3287
3851
|
resetGraph() {
|
|
3288
|
-
this.runner.resetGraph(this,
|
|
3852
|
+
this.runner.resetGraph(this, uuid45());
|
|
3289
3853
|
}
|
|
3290
|
-
toJSON() {
|
|
3291
|
-
const tasks = this.getTasks().map((node) => node.toJSON());
|
|
3854
|
+
toJSON(options) {
|
|
3855
|
+
const tasks = this.getTasks().map((node) => node.toJSON(options));
|
|
3292
3856
|
const dataflows = this.getDataflows().map((df) => df.toJSON());
|
|
3293
|
-
|
|
3857
|
+
let json = {
|
|
3294
3858
|
tasks,
|
|
3295
3859
|
dataflows
|
|
3296
3860
|
};
|
|
3861
|
+
if (options?.withBoundaryNodes) {
|
|
3862
|
+
json = addBoundaryNodesToGraphJson(json, this);
|
|
3863
|
+
}
|
|
3864
|
+
return json;
|
|
3297
3865
|
}
|
|
3298
|
-
toDependencyJSON() {
|
|
3299
|
-
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON());
|
|
3866
|
+
toDependencyJSON(options) {
|
|
3867
|
+
const tasks = this.getTasks().flatMap((node) => node.toDependencyJSON(options));
|
|
3300
3868
|
this.getDataflows().forEach((df) => {
|
|
3301
3869
|
const target = tasks.find((node) => node.id === df.targetTaskId);
|
|
3302
3870
|
if (!target.dependencies) {
|
|
@@ -3322,6 +3890,9 @@ class TaskGraph {
|
|
|
3322
3890
|
}
|
|
3323
3891
|
}
|
|
3324
3892
|
});
|
|
3893
|
+
if (options?.withBoundaryNodes) {
|
|
3894
|
+
return addBoundaryNodesToDependencyJson(tasks, this);
|
|
3895
|
+
}
|
|
3325
3896
|
return tasks;
|
|
3326
3897
|
}
|
|
3327
3898
|
get events() {
|
|
@@ -3340,7 +3911,7 @@ class TaskGraph {
|
|
|
3340
3911
|
const tasks = this.getTasks();
|
|
3341
3912
|
tasks.forEach((task) => {
|
|
3342
3913
|
const unsub = task.subscribe("status", (status) => {
|
|
3343
|
-
callback(task.
|
|
3914
|
+
callback(task.id, status);
|
|
3344
3915
|
});
|
|
3345
3916
|
unsubscribes.push(unsub);
|
|
3346
3917
|
});
|
|
@@ -3349,7 +3920,7 @@ class TaskGraph {
|
|
|
3349
3920
|
if (!task || typeof task.subscribe !== "function")
|
|
3350
3921
|
return;
|
|
3351
3922
|
const unsub = task.subscribe("status", (status) => {
|
|
3352
|
-
callback(task.
|
|
3923
|
+
callback(task.id, status);
|
|
3353
3924
|
});
|
|
3354
3925
|
unsubscribes.push(unsub);
|
|
3355
3926
|
};
|
|
@@ -3364,7 +3935,7 @@ class TaskGraph {
|
|
|
3364
3935
|
const tasks = this.getTasks();
|
|
3365
3936
|
tasks.forEach((task) => {
|
|
3366
3937
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3367
|
-
callback(task.
|
|
3938
|
+
callback(task.id, progress, message, ...args);
|
|
3368
3939
|
});
|
|
3369
3940
|
unsubscribes.push(unsub);
|
|
3370
3941
|
});
|
|
@@ -3373,7 +3944,7 @@ class TaskGraph {
|
|
|
3373
3944
|
if (!task || typeof task.subscribe !== "function")
|
|
3374
3945
|
return;
|
|
3375
3946
|
const unsub = task.subscribe("progress", (progress, message, ...args) => {
|
|
3376
|
-
callback(task.
|
|
3947
|
+
callback(task.id, progress, message, ...args);
|
|
3377
3948
|
});
|
|
3378
3949
|
unsubscribes.push(unsub);
|
|
3379
3950
|
};
|
|
@@ -3458,7 +4029,7 @@ class TaskGraph {
|
|
|
3458
4029
|
function serialGraphEdges(tasks, inputHandle, outputHandle) {
|
|
3459
4030
|
const edges = [];
|
|
3460
4031
|
for (let i = 0;i < tasks.length - 1; i++) {
|
|
3461
|
-
edges.push(new Dataflow(tasks[i].
|
|
4032
|
+
edges.push(new Dataflow(tasks[i].id, inputHandle, tasks[i + 1].id, outputHandle));
|
|
3462
4033
|
}
|
|
3463
4034
|
return edges;
|
|
3464
4035
|
}
|
|
@@ -3468,9 +4039,206 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3468
4039
|
graph.addDataflows(serialGraphEdges(tasks, inputHandle, outputHandle));
|
|
3469
4040
|
return graph;
|
|
3470
4041
|
}
|
|
4042
|
+
// src/task/FallbackTaskRunner.ts
|
|
4043
|
+
class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
4044
|
+
async executeTask(input) {
|
|
4045
|
+
if (this.task.fallbackMode === "data") {
|
|
4046
|
+
return this.executeDataFallback(input);
|
|
4047
|
+
}
|
|
4048
|
+
return this.executeTaskFallback(input);
|
|
4049
|
+
}
|
|
4050
|
+
async executeTaskReactive(input, output) {
|
|
4051
|
+
const reactiveResult = await this.task.executeReactive(input, output, { own: this.own });
|
|
4052
|
+
return Object.assign({}, output, reactiveResult ?? {});
|
|
4053
|
+
}
|
|
4054
|
+
async executeTaskFallback(input) {
|
|
4055
|
+
const tasks = this.task.subGraph.getTasks();
|
|
4056
|
+
if (tasks.length === 0) {
|
|
4057
|
+
throw new TaskFailedError("FallbackTask has no alternatives to try");
|
|
4058
|
+
}
|
|
4059
|
+
const errors = [];
|
|
4060
|
+
const totalAttempts = tasks.length;
|
|
4061
|
+
for (let i = 0;i < tasks.length; i++) {
|
|
4062
|
+
if (this.abortController?.signal.aborted) {
|
|
4063
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4064
|
+
}
|
|
4065
|
+
const alternativeTask = tasks[i];
|
|
4066
|
+
const attemptNumber = i + 1;
|
|
4067
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying alternative ${attemptNumber}/${totalAttempts}: ${alternativeTask.type}`);
|
|
4068
|
+
try {
|
|
4069
|
+
this.resetTask(alternativeTask);
|
|
4070
|
+
const result = await alternativeTask.run(input);
|
|
4071
|
+
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
4072
|
+
return await this.executeTaskReactive(input, result);
|
|
4073
|
+
} catch (error) {
|
|
4074
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4075
|
+
throw error;
|
|
4076
|
+
}
|
|
4077
|
+
errors.push({ task: alternativeTask, error });
|
|
4078
|
+
}
|
|
4079
|
+
}
|
|
4080
|
+
throw this.buildAggregateError(errors, "task");
|
|
4081
|
+
}
|
|
4082
|
+
async executeDataFallback(input) {
|
|
4083
|
+
const alternatives = this.task.alternatives;
|
|
4084
|
+
if (alternatives.length === 0) {
|
|
4085
|
+
throw new TaskFailedError("FallbackTask has no data alternatives to try");
|
|
4086
|
+
}
|
|
4087
|
+
const errors = [];
|
|
4088
|
+
const totalAttempts = alternatives.length;
|
|
4089
|
+
for (let i = 0;i < alternatives.length; i++) {
|
|
4090
|
+
if (this.abortController?.signal.aborted) {
|
|
4091
|
+
throw new TaskAbortedError("Fallback aborted");
|
|
4092
|
+
}
|
|
4093
|
+
const alternative = alternatives[i];
|
|
4094
|
+
const attemptNumber = i + 1;
|
|
4095
|
+
await this.handleProgress(Math.round((i + 0.5) / totalAttempts * 100), `Trying data alternative ${attemptNumber}/${totalAttempts}`);
|
|
4096
|
+
try {
|
|
4097
|
+
this.resetSubgraph();
|
|
4098
|
+
const mergedInput = { ...input, ...alternative };
|
|
4099
|
+
const results = await this.task.subGraph.run(mergedInput, {
|
|
4100
|
+
parentSignal: this.abortController?.signal,
|
|
4101
|
+
outputCache: this.outputCache
|
|
4102
|
+
});
|
|
4103
|
+
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
4104
|
+
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
4105
|
+
return await this.executeTaskReactive(input, mergedOutput);
|
|
4106
|
+
} catch (error) {
|
|
4107
|
+
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
4108
|
+
throw error;
|
|
4109
|
+
}
|
|
4110
|
+
errors.push({ alternative, error });
|
|
4111
|
+
}
|
|
4112
|
+
}
|
|
4113
|
+
throw this.buildAggregateError(errors, "data");
|
|
4114
|
+
}
|
|
4115
|
+
resetTask(task) {
|
|
4116
|
+
task.status = TaskStatus.PENDING;
|
|
4117
|
+
task.progress = 0;
|
|
4118
|
+
task.error = undefined;
|
|
4119
|
+
task.completedAt = undefined;
|
|
4120
|
+
task.startedAt = undefined;
|
|
4121
|
+
task.resetInputData();
|
|
4122
|
+
}
|
|
4123
|
+
resetSubgraph() {
|
|
4124
|
+
for (const task of this.task.subGraph.getTasks()) {
|
|
4125
|
+
this.resetTask(task);
|
|
4126
|
+
}
|
|
4127
|
+
for (const dataflow of this.task.subGraph.getDataflows()) {
|
|
4128
|
+
dataflow.reset();
|
|
4129
|
+
}
|
|
4130
|
+
}
|
|
4131
|
+
buildAggregateError(errors, mode) {
|
|
4132
|
+
const label = mode === "task" ? "alternative" : "data alternative";
|
|
4133
|
+
const details = errors.map((e, i) => {
|
|
4134
|
+
const prefix = e.error instanceof TaskTimeoutError ? "[timeout] " : "";
|
|
4135
|
+
return ` ${label} ${i + 1}: ${prefix}${e.error.message}`;
|
|
4136
|
+
}).join(`
|
|
4137
|
+
`);
|
|
4138
|
+
return new TaskFailedError(`All ${errors.length} ${label}s failed:
|
|
4139
|
+
${details}`);
|
|
4140
|
+
}
|
|
4141
|
+
}
|
|
4142
|
+
|
|
4143
|
+
// src/task/FallbackTask.ts
|
|
4144
|
+
var fallbackTaskConfigSchema = {
|
|
4145
|
+
type: "object",
|
|
4146
|
+
properties: {
|
|
4147
|
+
...graphAsTaskConfigSchema["properties"],
|
|
4148
|
+
fallbackMode: { type: "string", enum: ["task", "data"] },
|
|
4149
|
+
alternatives: { type: "array", items: { type: "object", additionalProperties: true } }
|
|
4150
|
+
},
|
|
4151
|
+
additionalProperties: false
|
|
4152
|
+
};
|
|
4153
|
+
|
|
4154
|
+
class FallbackTask extends GraphAsTask {
|
|
4155
|
+
static type = "FallbackTask";
|
|
4156
|
+
static category = "Flow Control";
|
|
4157
|
+
static title = "Fallback";
|
|
4158
|
+
static description = "Try alternatives until one succeeds";
|
|
4159
|
+
static hasDynamicSchemas = true;
|
|
4160
|
+
static configSchema() {
|
|
4161
|
+
return fallbackTaskConfigSchema;
|
|
4162
|
+
}
|
|
4163
|
+
get runner() {
|
|
4164
|
+
if (!this._runner) {
|
|
4165
|
+
this._runner = new FallbackTaskRunner(this);
|
|
4166
|
+
}
|
|
4167
|
+
return this._runner;
|
|
4168
|
+
}
|
|
4169
|
+
get fallbackMode() {
|
|
4170
|
+
return this.config?.fallbackMode ?? "task";
|
|
4171
|
+
}
|
|
4172
|
+
get alternatives() {
|
|
4173
|
+
return this.config?.alternatives ?? [];
|
|
4174
|
+
}
|
|
4175
|
+
inputSchema() {
|
|
4176
|
+
if (!this.hasChildren()) {
|
|
4177
|
+
return this.constructor.inputSchema();
|
|
4178
|
+
}
|
|
4179
|
+
if (this.fallbackMode === "data") {
|
|
4180
|
+
return super.inputSchema();
|
|
4181
|
+
}
|
|
4182
|
+
const properties = {};
|
|
4183
|
+
const tasks = this.subGraph.getTasks();
|
|
4184
|
+
for (const task of tasks) {
|
|
4185
|
+
const taskInputSchema = task.inputSchema();
|
|
4186
|
+
if (typeof taskInputSchema === "boolean")
|
|
4187
|
+
continue;
|
|
4188
|
+
const taskProperties = taskInputSchema.properties || {};
|
|
4189
|
+
for (const [inputName, inputProp] of Object.entries(taskProperties)) {
|
|
4190
|
+
if (!properties[inputName]) {
|
|
4191
|
+
properties[inputName] = inputProp;
|
|
4192
|
+
}
|
|
4193
|
+
}
|
|
4194
|
+
}
|
|
4195
|
+
return {
|
|
4196
|
+
type: "object",
|
|
4197
|
+
properties,
|
|
4198
|
+
additionalProperties: true
|
|
4199
|
+
};
|
|
4200
|
+
}
|
|
4201
|
+
outputSchema() {
|
|
4202
|
+
if (!this.hasChildren()) {
|
|
4203
|
+
return this.constructor.outputSchema();
|
|
4204
|
+
}
|
|
4205
|
+
const tasks = this.subGraph.getTasks();
|
|
4206
|
+
if (tasks.length === 0) {
|
|
4207
|
+
return { type: "object", properties: {}, additionalProperties: false };
|
|
4208
|
+
}
|
|
4209
|
+
if (this.fallbackMode === "task") {
|
|
4210
|
+
const firstTask = tasks[0];
|
|
4211
|
+
return firstTask.outputSchema();
|
|
4212
|
+
}
|
|
4213
|
+
return super.outputSchema();
|
|
4214
|
+
}
|
|
4215
|
+
toJSON() {
|
|
4216
|
+
const json = super.toJSON();
|
|
4217
|
+
return {
|
|
4218
|
+
...json,
|
|
4219
|
+
config: {
|
|
4220
|
+
..."config" in json ? json.config : {},
|
|
4221
|
+
fallbackMode: this.fallbackMode,
|
|
4222
|
+
...this.alternatives.length > 0 ? { alternatives: this.alternatives } : {}
|
|
4223
|
+
}
|
|
4224
|
+
};
|
|
4225
|
+
}
|
|
4226
|
+
}
|
|
4227
|
+
queueMicrotask(() => {
|
|
4228
|
+
Workflow.prototype.fallback = function() {
|
|
4229
|
+
return this.addLoopTask(FallbackTask, { fallbackMode: "task" });
|
|
4230
|
+
};
|
|
4231
|
+
Workflow.prototype.endFallback = CreateEndLoopWorkflow("endFallback");
|
|
4232
|
+
Workflow.prototype.fallbackWith = function(alternatives) {
|
|
4233
|
+
return this.addLoopTask(FallbackTask, {
|
|
4234
|
+
fallbackMode: "data",
|
|
4235
|
+
alternatives
|
|
4236
|
+
});
|
|
4237
|
+
};
|
|
4238
|
+
Workflow.prototype.endFallbackWith = CreateEndLoopWorkflow("endFallbackWith");
|
|
4239
|
+
});
|
|
3471
4240
|
// src/task/IteratorTaskRunner.ts
|
|
3472
4241
|
class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
3473
|
-
subGraphRunChain = Promise.resolve();
|
|
3474
4242
|
async executeTask(input) {
|
|
3475
4243
|
const analysis = this.task.analyzeIterationInput(input);
|
|
3476
4244
|
if (analysis.iterationCount === 0) {
|
|
@@ -3492,13 +4260,18 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3492
4260
|
const concurrency = Math.max(1, Math.min(requestedConcurrency, iterationCount));
|
|
3493
4261
|
const orderedResults = preserveOrder ? new Array(iterationCount) : [];
|
|
3494
4262
|
const completionOrderResults = [];
|
|
4263
|
+
let completedCount = 0;
|
|
3495
4264
|
for (let batchStart = 0;batchStart < iterationCount; batchStart += batchSize) {
|
|
3496
4265
|
if (this.abortController?.signal.aborted) {
|
|
3497
4266
|
break;
|
|
3498
4267
|
}
|
|
3499
4268
|
const batchEnd = Math.min(batchStart + batchSize, iterationCount);
|
|
3500
4269
|
const batchIndices = Array.from({ length: batchEnd - batchStart }, (_, i) => batchStart + i);
|
|
3501
|
-
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency)
|
|
4270
|
+
const batchResults = await this.executeBatch(batchIndices, analysis, iterationCount, concurrency, async () => {
|
|
4271
|
+
completedCount++;
|
|
4272
|
+
const progress = Math.round(completedCount / iterationCount * 100);
|
|
4273
|
+
await this.handleProgress(progress, `Completed ${completedCount}/${iterationCount} iterations`);
|
|
4274
|
+
});
|
|
3502
4275
|
for (const { index, result } of batchResults) {
|
|
3503
4276
|
if (result === undefined)
|
|
3504
4277
|
continue;
|
|
@@ -3508,8 +4281,6 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3508
4281
|
completionOrderResults.push(result);
|
|
3509
4282
|
}
|
|
3510
4283
|
}
|
|
3511
|
-
const progress = Math.round(batchEnd / iterationCount * 100);
|
|
3512
|
-
await this.handleProgress(progress, `Completed ${batchEnd}/${iterationCount} iterations`);
|
|
3513
4284
|
}
|
|
3514
4285
|
const collected = preserveOrder ? orderedResults.filter((result) => result !== undefined) : completionOrderResults;
|
|
3515
4286
|
return this.task.collectResults(collected);
|
|
@@ -3531,7 +4302,7 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3531
4302
|
}
|
|
3532
4303
|
return accumulator;
|
|
3533
4304
|
}
|
|
3534
|
-
async executeBatch(indices, analysis, iterationCount, concurrency) {
|
|
4305
|
+
async executeBatch(indices, analysis, iterationCount, concurrency, onItemComplete) {
|
|
3535
4306
|
const results = [];
|
|
3536
4307
|
let cursor = 0;
|
|
3537
4308
|
const workerCount = Math.max(1, Math.min(concurrency, indices.length));
|
|
@@ -3549,33 +4320,40 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
3549
4320
|
const iterationInput = this.task.buildIterationRunInput(analysis, index, iterationCount);
|
|
3550
4321
|
const result = await this.executeSubgraphIteration(iterationInput);
|
|
3551
4322
|
results.push({ index, result });
|
|
4323
|
+
await onItemComplete?.();
|
|
3552
4324
|
}
|
|
3553
4325
|
});
|
|
3554
4326
|
await Promise.all(workers);
|
|
3555
4327
|
return results;
|
|
3556
4328
|
}
|
|
4329
|
+
cloneGraph(graph) {
|
|
4330
|
+
const clone = new TaskGraph;
|
|
4331
|
+
for (const task of graph.getTasks()) {
|
|
4332
|
+
const ctor = task.constructor;
|
|
4333
|
+
const newTask = new ctor(task.defaults, task.config);
|
|
4334
|
+
if (task.hasChildren()) {
|
|
4335
|
+
newTask.subGraph = this.cloneGraph(task.subGraph);
|
|
4336
|
+
}
|
|
4337
|
+
clone.addTask(newTask);
|
|
4338
|
+
}
|
|
4339
|
+
for (const df of graph.getDataflows()) {
|
|
4340
|
+
clone.addDataflow(new Dataflow(df.sourceTaskId, df.sourceTaskPortId, df.targetTaskId, df.targetTaskPortId));
|
|
4341
|
+
}
|
|
4342
|
+
return clone;
|
|
4343
|
+
}
|
|
3557
4344
|
async executeSubgraphIteration(input) {
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
4345
|
+
if (this.abortController?.signal.aborted) {
|
|
4346
|
+
return;
|
|
4347
|
+
}
|
|
4348
|
+
const graphClone = this.cloneGraph(this.task.subGraph);
|
|
4349
|
+
const results = await graphClone.run(input, {
|
|
4350
|
+
parentSignal: this.abortController?.signal,
|
|
4351
|
+
outputCache: this.outputCache
|
|
3562
4352
|
});
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
if (this.abortController?.signal.aborted) {
|
|
3566
|
-
return;
|
|
3567
|
-
}
|
|
3568
|
-
const results = await this.task.subGraph.run(input, {
|
|
3569
|
-
parentSignal: this.abortController?.signal,
|
|
3570
|
-
outputCache: this.outputCache
|
|
3571
|
-
});
|
|
3572
|
-
if (results.length === 0) {
|
|
3573
|
-
return;
|
|
3574
|
-
}
|
|
3575
|
-
return this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3576
|
-
} finally {
|
|
3577
|
-
releaseTurn?.();
|
|
4353
|
+
if (results.length === 0) {
|
|
4354
|
+
return;
|
|
3578
4355
|
}
|
|
4356
|
+
return graphClone.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
3579
4357
|
}
|
|
3580
4358
|
}
|
|
3581
4359
|
|
|
@@ -3859,7 +4637,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
3859
4637
|
const tasks = this.subGraph.getTasks();
|
|
3860
4638
|
if (tasks.length === 0)
|
|
3861
4639
|
return;
|
|
3862
|
-
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.
|
|
4640
|
+
const startingNodes = tasks.filter((task) => this.subGraph.getSourceDataflows(task.id).length === 0);
|
|
3863
4641
|
const sources = startingNodes.length > 0 ? startingNodes : tasks;
|
|
3864
4642
|
const properties = {};
|
|
3865
4643
|
const required = [];
|
|
@@ -3886,6 +4664,33 @@ class IteratorTask extends GraphAsTask {
|
|
|
3886
4664
|
}
|
|
3887
4665
|
}
|
|
3888
4666
|
}
|
|
4667
|
+
const sourceIds = new Set(sources.map((t) => t.id));
|
|
4668
|
+
for (const task of tasks) {
|
|
4669
|
+
if (sourceIds.has(task.id))
|
|
4670
|
+
continue;
|
|
4671
|
+
const inputSchema = task.inputSchema();
|
|
4672
|
+
if (typeof inputSchema === "boolean")
|
|
4673
|
+
continue;
|
|
4674
|
+
const requiredKeys = new Set(inputSchema.required || []);
|
|
4675
|
+
if (requiredKeys.size === 0)
|
|
4676
|
+
continue;
|
|
4677
|
+
const connectedPorts = new Set(this.subGraph.getSourceDataflows(task.id).map((df) => df.targetTaskPortId));
|
|
4678
|
+
for (const key of requiredKeys) {
|
|
4679
|
+
if (connectedPorts.has(key))
|
|
4680
|
+
continue;
|
|
4681
|
+
if (properties[key])
|
|
4682
|
+
continue;
|
|
4683
|
+
if (task.defaults && task.defaults[key] !== undefined)
|
|
4684
|
+
continue;
|
|
4685
|
+
const prop = (inputSchema.properties || {})[key];
|
|
4686
|
+
if (!prop || typeof prop === "boolean")
|
|
4687
|
+
continue;
|
|
4688
|
+
properties[key] = prop;
|
|
4689
|
+
if (!required.includes(key)) {
|
|
4690
|
+
required.push(key);
|
|
4691
|
+
}
|
|
4692
|
+
}
|
|
4693
|
+
}
|
|
3889
4694
|
return {
|
|
3890
4695
|
type: "object",
|
|
3891
4696
|
properties,
|
|
@@ -4020,7 +4825,7 @@ class IteratorTask extends GraphAsTask {
|
|
|
4020
4825
|
if (!this.hasChildren()) {
|
|
4021
4826
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4022
4827
|
}
|
|
4023
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
4828
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4024
4829
|
if (endingNodes.length === 0) {
|
|
4025
4830
|
return { type: "object", properties: {}, additionalProperties: false };
|
|
4026
4831
|
}
|
|
@@ -4230,8 +5035,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4230
5035
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4231
5036
|
}
|
|
4232
5037
|
this._currentIteration++;
|
|
4233
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4234
|
-
await context.updateProgress(progress, `
|
|
5038
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5039
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4235
5040
|
}
|
|
4236
5041
|
return currentOutput;
|
|
4237
5042
|
}
|
|
@@ -4274,8 +5079,8 @@ class WhileTask extends GraphAsTask {
|
|
|
4274
5079
|
currentInput = { ...currentInput, ...currentOutput };
|
|
4275
5080
|
}
|
|
4276
5081
|
this._currentIteration++;
|
|
4277
|
-
const progress = Math.min(this._currentIteration / effectiveMax * 100, 99);
|
|
4278
|
-
await context.updateProgress(progress, `
|
|
5082
|
+
const progress = Math.min(Math.round(this._currentIteration / effectiveMax * 100), 99);
|
|
5083
|
+
await context.updateProgress(progress, `Completed ${this._currentIteration}/${effectiveMax} iterations`);
|
|
4279
5084
|
}
|
|
4280
5085
|
yield { type: "finish", data: currentOutput };
|
|
4281
5086
|
}
|
|
@@ -4351,7 +5156,7 @@ class WhileTask extends GraphAsTask {
|
|
|
4351
5156
|
return this.constructor.outputSchema();
|
|
4352
5157
|
}
|
|
4353
5158
|
const tasks = this.subGraph.getTasks();
|
|
4354
|
-
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5159
|
+
const endingNodes = tasks.filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
4355
5160
|
if (endingNodes.length === 0) {
|
|
4356
5161
|
return this.constructor.outputSchema();
|
|
4357
5162
|
}
|
|
@@ -5018,7 +5823,7 @@ class ReduceTask extends IteratorTask {
|
|
|
5018
5823
|
if (!this.hasChildren()) {
|
|
5019
5824
|
return this.constructor.outputSchema();
|
|
5020
5825
|
}
|
|
5021
|
-
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.
|
|
5826
|
+
const endingNodes = this.subGraph.getTasks().filter((task) => this.subGraph.getTargetDataflows(task.id).length === 0);
|
|
5022
5827
|
if (endingNodes.length === 0) {
|
|
5023
5828
|
return this.constructor.outputSchema();
|
|
5024
5829
|
}
|
|
@@ -5056,14 +5861,14 @@ var TaskRegistry = {
|
|
|
5056
5861
|
};
|
|
5057
5862
|
|
|
5058
5863
|
// src/task/TaskJSON.ts
|
|
5059
|
-
var createSingleTaskFromJSON = (item) => {
|
|
5864
|
+
var createSingleTaskFromJSON = (item, taskRegistry) => {
|
|
5060
5865
|
if (!item.id)
|
|
5061
5866
|
throw new TaskJSONError("Task id required");
|
|
5062
5867
|
if (!item.type)
|
|
5063
5868
|
throw new TaskJSONError("Task type required");
|
|
5064
5869
|
if (item.defaults && Array.isArray(item.defaults))
|
|
5065
5870
|
throw new TaskJSONError("Task defaults must be an object");
|
|
5066
|
-
const taskClass = TaskRegistry.all.get(item.type);
|
|
5871
|
+
const taskClass = taskRegistry?.get(item.type) ?? TaskRegistry.all.get(item.type);
|
|
5067
5872
|
if (!taskClass)
|
|
5068
5873
|
throw new TaskJSONError(`Task type ${item.type} not found, perhaps not registered?`);
|
|
5069
5874
|
const taskConfig = {
|
|
@@ -5090,20 +5895,20 @@ var createGraphFromDependencyJSON = (jsonItems) => {
|
|
|
5090
5895
|
}
|
|
5091
5896
|
return subGraph;
|
|
5092
5897
|
};
|
|
5093
|
-
var createTaskFromGraphJSON = (item) => {
|
|
5094
|
-
const task = createSingleTaskFromJSON(item);
|
|
5898
|
+
var createTaskFromGraphJSON = (item, taskRegistry) => {
|
|
5899
|
+
const task = createSingleTaskFromJSON(item, taskRegistry);
|
|
5095
5900
|
if (item.subgraph) {
|
|
5096
5901
|
if (!(task instanceof GraphAsTask)) {
|
|
5097
5902
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
5098
5903
|
}
|
|
5099
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph);
|
|
5904
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, taskRegistry);
|
|
5100
5905
|
}
|
|
5101
5906
|
return task;
|
|
5102
5907
|
};
|
|
5103
|
-
var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
5908
|
+
var createGraphFromGraphJSON = (graphJsonObj, taskRegistry) => {
|
|
5104
5909
|
const subGraph = new TaskGraph;
|
|
5105
5910
|
for (const subitem of graphJsonObj.tasks) {
|
|
5106
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem));
|
|
5911
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, taskRegistry));
|
|
5107
5912
|
}
|
|
5108
5913
|
for (const subitem of graphJsonObj.dataflows) {
|
|
5109
5914
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
@@ -5112,7 +5917,7 @@ var createGraphFromGraphJSON = (graphJsonObj) => {
|
|
|
5112
5917
|
};
|
|
5113
5918
|
// src/task/index.ts
|
|
5114
5919
|
var registerBaseTasks = () => {
|
|
5115
|
-
const tasks = [GraphAsTask, ConditionalTask, MapTask, WhileTask, ReduceTask];
|
|
5920
|
+
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
5116
5921
|
tasks.map(TaskRegistry.registerTask);
|
|
5117
5922
|
return tasks;
|
|
5118
5923
|
};
|
|
@@ -5295,11 +6100,14 @@ export {
|
|
|
5295
6100
|
isFlexibleSchema,
|
|
5296
6101
|
hasVectorOutput,
|
|
5297
6102
|
hasVectorLikeInput,
|
|
6103
|
+
hasStructuredOutput,
|
|
5298
6104
|
graphAsTaskConfigSchema,
|
|
5299
6105
|
getTaskQueueRegistry,
|
|
6106
|
+
getStructuredOutputSchemas,
|
|
5300
6107
|
getStreamingPorts,
|
|
5301
6108
|
getPortStreamMode,
|
|
5302
6109
|
getOutputStreamMode,
|
|
6110
|
+
getObjectPortId,
|
|
5303
6111
|
getNestedValue,
|
|
5304
6112
|
getLastTask,
|
|
5305
6113
|
getJobQueueFactory,
|
|
@@ -5308,6 +6116,7 @@ export {
|
|
|
5308
6116
|
getAppendPortId,
|
|
5309
6117
|
findArrayPorts,
|
|
5310
6118
|
filterIterationProperties,
|
|
6119
|
+
fallbackTaskConfigSchema,
|
|
5311
6120
|
extractIterationProperties,
|
|
5312
6121
|
extractBaseSchema,
|
|
5313
6122
|
evaluateCondition,
|
|
@@ -5322,13 +6131,19 @@ export {
|
|
|
5322
6131
|
createArraySchema,
|
|
5323
6132
|
connect,
|
|
5324
6133
|
conditionalTaskConfigSchema,
|
|
6134
|
+
computeGraphOutputSchema,
|
|
6135
|
+
computeGraphInputSchema,
|
|
6136
|
+
calculateNodeDepths,
|
|
5325
6137
|
buildIterationInputSchema,
|
|
5326
6138
|
addIterationContextToSchema,
|
|
6139
|
+
addBoundaryNodesToGraphJson,
|
|
6140
|
+
addBoundaryNodesToDependencyJson,
|
|
5327
6141
|
WorkflowError,
|
|
5328
6142
|
Workflow,
|
|
5329
6143
|
WhileTaskRunner,
|
|
5330
6144
|
WhileTask,
|
|
5331
6145
|
WHILE_CONTEXT_SCHEMA,
|
|
6146
|
+
TaskTimeoutError,
|
|
5332
6147
|
TaskStatus,
|
|
5333
6148
|
TaskRegistry,
|
|
5334
6149
|
TaskQueueRegistry,
|
|
@@ -5364,6 +6179,8 @@ export {
|
|
|
5364
6179
|
GraphAsTaskRunner,
|
|
5365
6180
|
GraphAsTask,
|
|
5366
6181
|
GRAPH_RESULT_ARRAY,
|
|
6182
|
+
FallbackTaskRunner,
|
|
6183
|
+
FallbackTask,
|
|
5367
6184
|
EventTaskGraphToDagMapping,
|
|
5368
6185
|
EventDagToTaskGraphMapping,
|
|
5369
6186
|
DataflowArrow,
|
|
@@ -5377,4 +6194,4 @@ export {
|
|
|
5377
6194
|
ConditionalTask
|
|
5378
6195
|
};
|
|
5379
6196
|
|
|
5380
|
-
//# debugId=
|
|
6197
|
+
//# debugId=CAEDE44FEB7E806964756E2164756E21
|