@workglow/task-graph 0.0.126 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -24
- package/dist/browser.js +289 -266
- package/dist/browser.js.map +23 -24
- package/dist/bun.js +289 -266
- package/dist/bun.js.map +22 -23
- package/dist/debug/console/ConsoleFormatters.d.ts.map +1 -1
- package/dist/node.js +289 -266
- package/dist/node.js.map +22 -23
- package/dist/task/ConditionalTask.d.ts +1 -0
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +0 -1
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +6 -0
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts +6 -0
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +11 -0
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueFactory.d.ts +3 -4
- package/dist/task/JobQueueFactory.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/Task.d.ts +13 -1
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +23 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +1 -1
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +17 -7
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskQueueRegistry.d.ts +8 -0
- package/dist/task/TaskQueueRegistry.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +1 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +0 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/Dataflow.d.ts +11 -0
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +10 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +15 -2
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/task/README.md +0 -17
- package/dist/task/JobQueueTask.d.ts +0 -130
- package/dist/task/JobQueueTask.d.ts.map +0 -1
package/dist/bun.js
CHANGED
|
@@ -119,6 +119,7 @@ class Dataflow {
|
|
|
119
119
|
this.error = undefined;
|
|
120
120
|
this.value = undefined;
|
|
121
121
|
this.stream = undefined;
|
|
122
|
+
this._compatibilityCache = undefined;
|
|
122
123
|
this.emit("reset");
|
|
123
124
|
this.emit("status", this.status);
|
|
124
125
|
}
|
|
@@ -179,9 +180,19 @@ class Dataflow {
|
|
|
179
180
|
targetTaskPortId: this.targetTaskPortId
|
|
180
181
|
};
|
|
181
182
|
}
|
|
183
|
+
_compatibilityCache;
|
|
184
|
+
invalidateCompatibilityCache() {
|
|
185
|
+
this._compatibilityCache = undefined;
|
|
186
|
+
}
|
|
182
187
|
semanticallyCompatible(graph, dataflow) {
|
|
183
|
-
const
|
|
184
|
-
const
|
|
188
|
+
const sourceTask = graph.getTask(dataflow.sourceTaskId);
|
|
189
|
+
const targetTask = graph.getTask(dataflow.targetTaskId);
|
|
190
|
+
const shouldCache = !(sourceTask.constructor.hasDynamicSchemas ?? true) && !(targetTask.constructor.hasDynamicSchemas ?? true);
|
|
191
|
+
if (shouldCache && this._compatibilityCache !== undefined) {
|
|
192
|
+
return this._compatibilityCache;
|
|
193
|
+
}
|
|
194
|
+
const targetSchema = targetTask.inputSchema();
|
|
195
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
185
196
|
if (typeof targetSchema === "boolean") {
|
|
186
197
|
if (targetSchema === false) {
|
|
187
198
|
return "incompatible";
|
|
@@ -202,8 +213,11 @@ class Dataflow {
|
|
|
202
213
|
if (sourceSchemaProperty === undefined && sourceSchema.additionalProperties === true) {
|
|
203
214
|
sourceSchemaProperty = true;
|
|
204
215
|
}
|
|
205
|
-
const
|
|
206
|
-
|
|
216
|
+
const result = areSemanticallyCompatible(sourceSchemaProperty, targetSchemaProperty);
|
|
217
|
+
if (shouldCache) {
|
|
218
|
+
this._compatibilityCache = result;
|
|
219
|
+
}
|
|
220
|
+
return result;
|
|
207
221
|
}
|
|
208
222
|
get events() {
|
|
209
223
|
if (!this._events) {
|
|
@@ -596,12 +610,13 @@ import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
|
596
610
|
import { EventEmitter as EventEmitter4, uuid4 as uuid44 } from "@workglow/util";
|
|
597
611
|
|
|
598
612
|
// src/task/GraphAsTask.ts
|
|
613
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
599
614
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
600
615
|
|
|
601
616
|
// src/task-graph/TaskGraphRunner.ts
|
|
602
617
|
import {
|
|
603
618
|
collectPropertyValues,
|
|
604
|
-
getLogger as
|
|
619
|
+
getLogger as getLogger3,
|
|
605
620
|
getTelemetryProvider as getTelemetryProvider2,
|
|
606
621
|
globalServiceRegistry as globalServiceRegistry2,
|
|
607
622
|
ServiceRegistry as ServiceRegistry2,
|
|
@@ -640,7 +655,7 @@ class TaskOutputRepository {
|
|
|
640
655
|
}
|
|
641
656
|
|
|
642
657
|
// src/task/ConditionalTask.ts
|
|
643
|
-
import { getLogger } from "@workglow/util";
|
|
658
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
644
659
|
|
|
645
660
|
// src/task/ConditionUtils.ts
|
|
646
661
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
@@ -718,6 +733,8 @@ import { BaseError } from "@workglow/util";
|
|
|
718
733
|
|
|
719
734
|
class TaskError extends BaseError {
|
|
720
735
|
static type = "TaskError";
|
|
736
|
+
taskType;
|
|
737
|
+
taskId;
|
|
721
738
|
constructor(message) {
|
|
722
739
|
super(message);
|
|
723
740
|
}
|
|
@@ -751,6 +768,14 @@ class TaskTimeoutError extends TaskAbortedError {
|
|
|
751
768
|
}
|
|
752
769
|
}
|
|
753
770
|
|
|
771
|
+
class TaskGraphTimeoutError extends TaskTimeoutError {
|
|
772
|
+
static type = "TaskGraphTimeoutError";
|
|
773
|
+
constructor(timeoutMs) {
|
|
774
|
+
super(timeoutMs);
|
|
775
|
+
this.message = timeoutMs ? `Graph execution timed out after ${timeoutMs}ms` : "Graph execution timed out";
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
754
779
|
class TaskFailedError extends TaskError {
|
|
755
780
|
static type = "TaskFailedError";
|
|
756
781
|
constructor(message = "Task failed") {
|
|
@@ -781,8 +806,16 @@ class TaskInvalidInputError extends TaskError {
|
|
|
781
806
|
}
|
|
782
807
|
}
|
|
783
808
|
|
|
809
|
+
class TaskSerializationError extends TaskError {
|
|
810
|
+
static type = "TaskSerializationError";
|
|
811
|
+
constructor(taskType) {
|
|
812
|
+
super(`Task "${taskType}" cannot be serialized: config contains non-serializable values. ` + `Use a declarative config alternative or remove function-valued config properties.`);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
784
816
|
// src/task/TaskRunner.ts
|
|
785
817
|
import {
|
|
818
|
+
getLogger,
|
|
786
819
|
getTelemetryProvider,
|
|
787
820
|
globalServiceRegistry,
|
|
788
821
|
SpanStatusCode
|
|
@@ -832,6 +865,18 @@ function getFormatPrefix(format) {
|
|
|
832
865
|
const colonIndex = format.indexOf(":");
|
|
833
866
|
return colonIndex >= 0 ? format.substring(0, colonIndex) : format;
|
|
834
867
|
}
|
|
868
|
+
function schemaHasFormatAnnotations(schema) {
|
|
869
|
+
if (typeof schema === "boolean")
|
|
870
|
+
return false;
|
|
871
|
+
const properties = schema.properties;
|
|
872
|
+
if (!properties || typeof properties !== "object")
|
|
873
|
+
return false;
|
|
874
|
+
for (const propSchema of Object.values(properties)) {
|
|
875
|
+
if (getSchemaFormat(propSchema) !== undefined)
|
|
876
|
+
return true;
|
|
877
|
+
}
|
|
878
|
+
return false;
|
|
879
|
+
}
|
|
835
880
|
async function resolveSchemaInputs(input, schema, config) {
|
|
836
881
|
if (typeof schema === "boolean")
|
|
837
882
|
return input;
|
|
@@ -997,6 +1042,12 @@ class TaskRunner {
|
|
|
997
1042
|
await this.handleStart(config);
|
|
998
1043
|
try {
|
|
999
1044
|
this.task.setInput(overrides);
|
|
1045
|
+
const configSchema = this.task.constructor.configSchema();
|
|
1046
|
+
if (schemaHasFormatAnnotations(configSchema)) {
|
|
1047
|
+
const source = this.task.originalConfig ?? this.task.config;
|
|
1048
|
+
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
1049
|
+
Object.assign(this.task.config, resolved);
|
|
1050
|
+
}
|
|
1000
1051
|
const schema = this.task.constructor.inputSchema();
|
|
1001
1052
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1002
1053
|
const inputs = this.task.runInputData;
|
|
@@ -1010,6 +1061,12 @@ class TaskRunner {
|
|
|
1010
1061
|
}
|
|
1011
1062
|
let outputs;
|
|
1012
1063
|
const isStreamable = isTaskStreamable(this.task);
|
|
1064
|
+
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
1065
|
+
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
1066
|
+
if (streamMode !== "none") {
|
|
1067
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1013
1070
|
if (this.task.cacheable) {
|
|
1014
1071
|
outputs = await this.outputCache?.getOutput(this.task.type, inputs);
|
|
1015
1072
|
if (outputs) {
|
|
@@ -1048,6 +1105,12 @@ class TaskRunner {
|
|
|
1048
1105
|
return this.task.runOutputData;
|
|
1049
1106
|
}
|
|
1050
1107
|
this.task.setInput(overrides);
|
|
1108
|
+
const configSchema = this.task.constructor.configSchema();
|
|
1109
|
+
if (schemaHasFormatAnnotations(configSchema)) {
|
|
1110
|
+
const source = this.task.originalConfig ?? this.task.config;
|
|
1111
|
+
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
1112
|
+
Object.assign(this.task.config, resolved);
|
|
1113
|
+
}
|
|
1051
1114
|
const schema = this.task.constructor.inputSchema();
|
|
1052
1115
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1053
1116
|
await this.handleStartReactive();
|
|
@@ -1276,6 +1339,11 @@ class TaskRunner {
|
|
|
1276
1339
|
this.telemetrySpan.end();
|
|
1277
1340
|
this.telemetrySpan = undefined;
|
|
1278
1341
|
}
|
|
1342
|
+
if (typeof this.task.cleanup === "function") {
|
|
1343
|
+
try {
|
|
1344
|
+
await this.task.cleanup();
|
|
1345
|
+
} catch {}
|
|
1346
|
+
}
|
|
1279
1347
|
this.task.emit("abort", this.task.error);
|
|
1280
1348
|
this.task.emit("status", this.task.status);
|
|
1281
1349
|
}
|
|
@@ -1328,7 +1396,15 @@ class TaskRunner {
|
|
|
1328
1396
|
this.task.completedAt = new Date;
|
|
1329
1397
|
this.task.progress = 100;
|
|
1330
1398
|
this.task.status = TaskStatus.FAILED;
|
|
1331
|
-
|
|
1399
|
+
if (err instanceof TaskError) {
|
|
1400
|
+
this.task.error = err;
|
|
1401
|
+
} else {
|
|
1402
|
+
this.task.error = new TaskFailedError(`Task "${this.task.type}" (${this.task.id}): ${err?.message || "Task failed"}`);
|
|
1403
|
+
}
|
|
1404
|
+
if (this.task.error instanceof TaskError) {
|
|
1405
|
+
this.task.error.taskType ??= this.task.type;
|
|
1406
|
+
this.task.error.taskId ??= this.task.id;
|
|
1407
|
+
}
|
|
1332
1408
|
this.abortController = undefined;
|
|
1333
1409
|
if (this.telemetrySpan) {
|
|
1334
1410
|
this.telemetrySpan.setStatus(SpanStatusCode.ERROR, this.task.error.message);
|
|
@@ -1432,6 +1508,7 @@ class Task {
|
|
|
1432
1508
|
runInputData = {};
|
|
1433
1509
|
runOutputData = {};
|
|
1434
1510
|
config;
|
|
1511
|
+
originalConfig;
|
|
1435
1512
|
get id() {
|
|
1436
1513
|
return this.config.id;
|
|
1437
1514
|
}
|
|
@@ -1462,6 +1539,11 @@ class Task {
|
|
|
1462
1539
|
baseConfig.id = uuid42();
|
|
1463
1540
|
}
|
|
1464
1541
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
1542
|
+
try {
|
|
1543
|
+
this.originalConfig = Object.freeze(structuredClone(this.config));
|
|
1544
|
+
} catch {
|
|
1545
|
+
this.originalConfig = undefined;
|
|
1546
|
+
}
|
|
1465
1547
|
this.runConfig = runConfig;
|
|
1466
1548
|
}
|
|
1467
1549
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
@@ -1732,7 +1814,10 @@ class Task {
|
|
|
1732
1814
|
const path = e.data.pointer || "";
|
|
1733
1815
|
return `${e.message}${path ? ` (${path})` : ""}`;
|
|
1734
1816
|
});
|
|
1735
|
-
|
|
1817
|
+
const err = new TaskInvalidInputError(`Task "${this.type}" (${this.id}): Input ${JSON.stringify(Object.keys(input))} does not match schema: ${errorMessages.join(", ")}`);
|
|
1818
|
+
err.taskType = this.type;
|
|
1819
|
+
err.taskId = this.id;
|
|
1820
|
+
throw err;
|
|
1736
1821
|
}
|
|
1737
1822
|
return true;
|
|
1738
1823
|
}
|
|
@@ -1757,8 +1842,14 @@ class Task {
|
|
|
1757
1842
|
}
|
|
1758
1843
|
return obj;
|
|
1759
1844
|
}
|
|
1845
|
+
canSerializeConfig() {
|
|
1846
|
+
return true;
|
|
1847
|
+
}
|
|
1760
1848
|
toJSON(_options) {
|
|
1761
1849
|
const ctor = this.constructor;
|
|
1850
|
+
if (!this.canSerializeConfig() || !this.originalConfig) {
|
|
1851
|
+
throw new TaskSerializationError(this.type);
|
|
1852
|
+
}
|
|
1762
1853
|
const schema = ctor.configSchema();
|
|
1763
1854
|
const schemaProperties = typeof schema !== "boolean" && schema?.properties ? schema.properties : {};
|
|
1764
1855
|
const config = {};
|
|
@@ -1768,7 +1859,7 @@ class Task {
|
|
|
1768
1859
|
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
1769
1860
|
continue;
|
|
1770
1861
|
}
|
|
1771
|
-
const value = this.
|
|
1862
|
+
const value = this.originalConfig[key];
|
|
1772
1863
|
if (value === undefined)
|
|
1773
1864
|
continue;
|
|
1774
1865
|
if (typeof value === "function" || typeof value === "symbol")
|
|
@@ -1852,6 +1943,11 @@ class ConditionalTask extends Task {
|
|
|
1852
1943
|
static configSchema() {
|
|
1853
1944
|
return conditionalTaskConfigSchema;
|
|
1854
1945
|
}
|
|
1946
|
+
canSerializeConfig() {
|
|
1947
|
+
if (!this.config.branches)
|
|
1948
|
+
return true;
|
|
1949
|
+
return !this.config.branches.some((b) => typeof b.condition === "function");
|
|
1950
|
+
}
|
|
1855
1951
|
activeBranches = new Set;
|
|
1856
1952
|
buildBranchesFromConditionConfig(conditionConfig) {
|
|
1857
1953
|
if (!conditionConfig?.branches || conditionConfig.branches.length === 0) {
|
|
@@ -1914,7 +2010,7 @@ class ConditionalTask extends Task {
|
|
|
1914
2010
|
}
|
|
1915
2011
|
}
|
|
1916
2012
|
} catch (error) {
|
|
1917
|
-
|
|
2013
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1918
2014
|
}
|
|
1919
2015
|
}
|
|
1920
2016
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -2204,6 +2300,8 @@ class TaskGraphRunner {
|
|
|
2204
2300
|
inProgressFunctions = new Map;
|
|
2205
2301
|
failedTaskErrors = new Map;
|
|
2206
2302
|
telemetrySpan;
|
|
2303
|
+
graphTimeoutTimer;
|
|
2304
|
+
pendingGraphTimeoutError;
|
|
2207
2305
|
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph), reactiveScheduler = new TopologicalScheduler(graph)) {
|
|
2208
2306
|
this.processScheduler = processScheduler;
|
|
2209
2307
|
this.reactiveScheduler = reactiveScheduler;
|
|
@@ -2254,10 +2352,14 @@ class TaskGraphRunner {
|
|
|
2254
2352
|
}
|
|
2255
2353
|
} catch (err) {
|
|
2256
2354
|
error = err;
|
|
2257
|
-
|
|
2355
|
+
getLogger3().error("Error running graph", { error });
|
|
2258
2356
|
}
|
|
2259
2357
|
await Promise.allSettled(Array.from(this.inProgressTasks.values()));
|
|
2260
2358
|
await Promise.allSettled(Array.from(this.inProgressFunctions.values()));
|
|
2359
|
+
if (this.pendingGraphTimeoutError) {
|
|
2360
|
+
await this.handleAbort();
|
|
2361
|
+
throw this.pendingGraphTimeoutError;
|
|
2362
|
+
}
|
|
2261
2363
|
if (this.failedTaskErrors.size > 0) {
|
|
2262
2364
|
const latestError = this.failedTaskErrors.values().next().value;
|
|
2263
2365
|
this.handleError(latestError);
|
|
@@ -2345,6 +2447,7 @@ class TaskGraphRunner {
|
|
|
2345
2447
|
}
|
|
2346
2448
|
copyInputFromEdgesToNode(task) {
|
|
2347
2449
|
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2450
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2348
2451
|
for (const dataflow of dataflows) {
|
|
2349
2452
|
this.addInputData(task, dataflow.getPortData());
|
|
2350
2453
|
}
|
|
@@ -2353,7 +2456,7 @@ class TaskGraphRunner {
|
|
|
2353
2456
|
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2354
2457
|
for (const dataflow of dataflows) {
|
|
2355
2458
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
2356
|
-
|
|
2459
|
+
getLogger3().debug("pushOutputFromNodeToEdges", {
|
|
2357
2460
|
dataflowId: dataflow.id,
|
|
2358
2461
|
compatibility,
|
|
2359
2462
|
resultsKeys: Object.keys(results)
|
|
@@ -2367,7 +2470,7 @@ class TaskGraphRunner {
|
|
|
2367
2470
|
} else {
|
|
2368
2471
|
const resultsKeys = Object.keys(results);
|
|
2369
2472
|
if (resultsKeys.length > 0) {
|
|
2370
|
-
|
|
2473
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2371
2474
|
dataflowId: dataflow.id,
|
|
2372
2475
|
compatibility,
|
|
2373
2476
|
resultsKeys
|
|
@@ -2673,6 +2776,12 @@ class TaskGraphRunner {
|
|
|
2673
2776
|
}
|
|
2674
2777
|
this.graph.outputCache = this.outputCache;
|
|
2675
2778
|
}
|
|
2779
|
+
if (config?.maxTasks !== undefined && config.maxTasks > 0) {
|
|
2780
|
+
const taskCount = this.graph.getTasks().length;
|
|
2781
|
+
if (taskCount > config.maxTasks) {
|
|
2782
|
+
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2676
2785
|
if (this.running || this.reactiveRunning) {
|
|
2677
2786
|
throw new TaskConfigurationError("Graph is already running");
|
|
2678
2787
|
}
|
|
@@ -2681,6 +2790,13 @@ class TaskGraphRunner {
|
|
|
2681
2790
|
this.abortController.signal.addEventListener("abort", () => {
|
|
2682
2791
|
this.handleAbort();
|
|
2683
2792
|
});
|
|
2793
|
+
if (config?.timeout !== undefined && config.timeout > 0) {
|
|
2794
|
+
this.pendingGraphTimeoutError = undefined;
|
|
2795
|
+
this.graphTimeoutTimer = setTimeout(() => {
|
|
2796
|
+
this.pendingGraphTimeoutError = new TaskGraphTimeoutError(config.timeout);
|
|
2797
|
+
this.abortController?.abort();
|
|
2798
|
+
}, config.timeout);
|
|
2799
|
+
}
|
|
2684
2800
|
if (config?.parentSignal?.aborted) {
|
|
2685
2801
|
this.abortController.abort();
|
|
2686
2802
|
return;
|
|
@@ -2714,10 +2830,23 @@ class TaskGraphRunner {
|
|
|
2714
2830
|
if (config?.registry !== undefined) {
|
|
2715
2831
|
this.registry = config.registry;
|
|
2716
2832
|
}
|
|
2833
|
+
if (config?.maxTasks !== undefined && config.maxTasks > 0) {
|
|
2834
|
+
const taskCount = this.graph.getTasks().length;
|
|
2835
|
+
if (taskCount > config.maxTasks) {
|
|
2836
|
+
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
2837
|
+
}
|
|
2838
|
+
}
|
|
2717
2839
|
this.reactiveScheduler.reset();
|
|
2718
2840
|
this.reactiveRunning = true;
|
|
2719
2841
|
}
|
|
2842
|
+
clearGraphTimeout() {
|
|
2843
|
+
if (this.graphTimeoutTimer !== undefined) {
|
|
2844
|
+
clearTimeout(this.graphTimeoutTimer);
|
|
2845
|
+
this.graphTimeoutTimer = undefined;
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2720
2848
|
async handleComplete() {
|
|
2849
|
+
this.clearGraphTimeout();
|
|
2721
2850
|
this.running = false;
|
|
2722
2851
|
if (this.telemetrySpan) {
|
|
2723
2852
|
this.telemetrySpan.setStatus(SpanStatusCode2.OK);
|
|
@@ -2730,6 +2859,7 @@ class TaskGraphRunner {
|
|
|
2730
2859
|
this.reactiveRunning = false;
|
|
2731
2860
|
}
|
|
2732
2861
|
async handleError(error) {
|
|
2862
|
+
this.clearGraphTimeout();
|
|
2733
2863
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
2734
2864
|
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
2735
2865
|
return task.abort();
|
|
@@ -2748,6 +2878,7 @@ class TaskGraphRunner {
|
|
|
2748
2878
|
this.reactiveRunning = false;
|
|
2749
2879
|
}
|
|
2750
2880
|
async handleAbort() {
|
|
2881
|
+
this.clearGraphTimeout();
|
|
2751
2882
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
2752
2883
|
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
2753
2884
|
return task.abort();
|
|
@@ -2892,7 +3023,7 @@ class GraphAsTask extends Task {
|
|
|
2892
3023
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2893
3024
|
this._inputSchemaNode = schemaNode;
|
|
2894
3025
|
} catch (error) {
|
|
2895
|
-
|
|
3026
|
+
getLogger4().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
2896
3027
|
this._inputSchemaNode = compileSchema2({});
|
|
2897
3028
|
}
|
|
2898
3029
|
}
|
|
@@ -3151,7 +3282,9 @@ class TaskGraph {
|
|
|
3151
3282
|
outputCache: config?.outputCache || this.outputCache,
|
|
3152
3283
|
parentSignal: config?.parentSignal || undefined,
|
|
3153
3284
|
accumulateLeafOutputs: config?.accumulateLeafOutputs,
|
|
3154
|
-
registry: config?.registry
|
|
3285
|
+
registry: config?.registry,
|
|
3286
|
+
timeout: config?.timeout,
|
|
3287
|
+
maxTasks: config?.maxTasks
|
|
3155
3288
|
});
|
|
3156
3289
|
}
|
|
3157
3290
|
runReactive(input = {}, config = {}) {
|
|
@@ -3191,18 +3324,12 @@ class TaskGraph {
|
|
|
3191
3324
|
return this._dag.addEdges(addedEdges);
|
|
3192
3325
|
}
|
|
3193
3326
|
getDataflow(id) {
|
|
3194
|
-
for (const
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
if (maybeEdges !== null) {
|
|
3198
|
-
for (const edge of maybeEdges) {
|
|
3199
|
-
if (this._dag.edgeIdentity(edge, "", "") == id) {
|
|
3200
|
-
return edge;
|
|
3201
|
-
}
|
|
3202
|
-
}
|
|
3203
|
-
}
|
|
3327
|
+
for (const [, , edge] of this._dag.getEdges()) {
|
|
3328
|
+
if (edge.id === id) {
|
|
3329
|
+
return edge;
|
|
3204
3330
|
}
|
|
3205
3331
|
}
|
|
3332
|
+
return;
|
|
3206
3333
|
}
|
|
3207
3334
|
getDataflows() {
|
|
3208
3335
|
return this._dag.getEdges().map((edge) => edge[2]);
|
|
@@ -3390,9 +3517,9 @@ class TaskGraph {
|
|
|
3390
3517
|
emit(name, ...args) {
|
|
3391
3518
|
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
3392
3519
|
if (dagEvent) {
|
|
3393
|
-
return this.emit_dag(name, ...args);
|
|
3520
|
+
return this.emit_dag.call(this, name, ...args);
|
|
3394
3521
|
} else {
|
|
3395
|
-
return this.emit_local(name, ...args);
|
|
3522
|
+
return this.emit_local.call(this, name, ...args);
|
|
3396
3523
|
}
|
|
3397
3524
|
}
|
|
3398
3525
|
emit_local(name, ...args) {
|
|
@@ -3419,7 +3546,7 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3419
3546
|
// src/task-graph/Workflow.ts
|
|
3420
3547
|
import {
|
|
3421
3548
|
EventEmitter as EventEmitter5,
|
|
3422
|
-
getLogger as
|
|
3549
|
+
getLogger as getLogger5,
|
|
3423
3550
|
uuid4 as uuid45
|
|
3424
3551
|
} from "@workglow/util";
|
|
3425
3552
|
function getLastTask(workflow) {
|
|
@@ -3567,7 +3694,7 @@ class Workflow {
|
|
|
3567
3694
|
const taskSchema = task.inputSchema();
|
|
3568
3695
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
3569
3696
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3570
|
-
|
|
3697
|
+
getLogger5().error(this._error);
|
|
3571
3698
|
return;
|
|
3572
3699
|
}
|
|
3573
3700
|
dataflow.targetTaskId = task.id;
|
|
@@ -3592,10 +3719,10 @@ class Workflow {
|
|
|
3592
3719
|
if (result.error) {
|
|
3593
3720
|
if (this.isLoopBuilder) {
|
|
3594
3721
|
this._error = result.error;
|
|
3595
|
-
|
|
3722
|
+
getLogger5().warn(this._error);
|
|
3596
3723
|
} else {
|
|
3597
3724
|
this._error = result.error + " Task not added.";
|
|
3598
|
-
|
|
3725
|
+
getLogger5().error(this._error);
|
|
3599
3726
|
this.graph.removeTask(task.id);
|
|
3600
3727
|
}
|
|
3601
3728
|
}
|
|
@@ -3683,7 +3810,7 @@ class Workflow {
|
|
|
3683
3810
|
const nodes = this._graph.getTasks();
|
|
3684
3811
|
if (nodes.length === 0) {
|
|
3685
3812
|
this._error = "No tasks to remove";
|
|
3686
|
-
|
|
3813
|
+
getLogger5().error(this._error);
|
|
3687
3814
|
return this;
|
|
3688
3815
|
}
|
|
3689
3816
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -3714,7 +3841,7 @@ class Workflow {
|
|
|
3714
3841
|
if (-index > nodes.length) {
|
|
3715
3842
|
const errorMsg = `Back index greater than number of tasks`;
|
|
3716
3843
|
this._error = errorMsg;
|
|
3717
|
-
|
|
3844
|
+
getLogger5().error(this._error);
|
|
3718
3845
|
throw new WorkflowError(errorMsg);
|
|
3719
3846
|
}
|
|
3720
3847
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -3723,13 +3850,13 @@ class Workflow {
|
|
|
3723
3850
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
3724
3851
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
3725
3852
|
this._error = errorMsg;
|
|
3726
|
-
|
|
3853
|
+
getLogger5().error(this._error);
|
|
3727
3854
|
throw new WorkflowError(errorMsg);
|
|
3728
3855
|
}
|
|
3729
3856
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
3730
3857
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
3731
3858
|
this._error = errorMsg;
|
|
3732
|
-
|
|
3859
|
+
getLogger5().error(this._error);
|
|
3733
3860
|
throw new WorkflowError(errorMsg);
|
|
3734
3861
|
}
|
|
3735
3862
|
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
@@ -3740,7 +3867,7 @@ class Workflow {
|
|
|
3740
3867
|
const parent = getLastTask(this);
|
|
3741
3868
|
if (!parent) {
|
|
3742
3869
|
this._error = "onError() requires a preceding task in the workflow";
|
|
3743
|
-
|
|
3870
|
+
getLogger5().error(this._error);
|
|
3744
3871
|
throw new WorkflowError(this._error);
|
|
3745
3872
|
}
|
|
3746
3873
|
const handlerTask = ensureTask(handler);
|
|
@@ -3838,7 +3965,7 @@ class Workflow {
|
|
|
3838
3965
|
const taskSchema = task.inputSchema();
|
|
3839
3966
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
3840
3967
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3841
|
-
|
|
3968
|
+
getLogger5().error(this._error);
|
|
3842
3969
|
return;
|
|
3843
3970
|
}
|
|
3844
3971
|
dataflow.targetTaskId = task.id;
|
|
@@ -3868,7 +3995,7 @@ class Workflow {
|
|
|
3868
3995
|
});
|
|
3869
3996
|
if (result.error) {
|
|
3870
3997
|
this._error = result.error + " Task not added.";
|
|
3871
|
-
|
|
3998
|
+
getLogger5().error(this._error);
|
|
3872
3999
|
this.graph.removeTask(iteratorTask.id);
|
|
3873
4000
|
}
|
|
3874
4001
|
}
|
|
@@ -4714,7 +4841,11 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
4714
4841
|
mapPartialProgress = [];
|
|
4715
4842
|
mapPartialIterationCount = 0;
|
|
4716
4843
|
async executeTask(input) {
|
|
4717
|
-
|
|
4844
|
+
let analysis = this.task.analyzeIterationInput(input);
|
|
4845
|
+
const maxIterations = this.task.config.maxIterations;
|
|
4846
|
+
if (maxIterations !== undefined && maxIterations > 0 && analysis.iterationCount > maxIterations) {
|
|
4847
|
+
analysis = { ...analysis, iterationCount: maxIterations };
|
|
4848
|
+
}
|
|
4718
4849
|
if (analysis.iterationCount === 0) {
|
|
4719
4850
|
const emptyResult = this.task.getEmptyResult();
|
|
4720
4851
|
return this.executeTaskReactive(input, emptyResult);
|
|
@@ -4899,6 +5030,7 @@ var iteratorTaskConfigSchema = {
|
|
|
4899
5030
|
...graphAsTaskConfigSchema["properties"],
|
|
4900
5031
|
concurrencyLimit: { type: "integer", minimum: 1 },
|
|
4901
5032
|
batchSize: { type: "integer", minimum: 1 },
|
|
5033
|
+
maxIterations: { type: "integer", minimum: 1 },
|
|
4902
5034
|
iterationInputConfig: { type: "object", additionalProperties: true }
|
|
4903
5035
|
},
|
|
4904
5036
|
additionalProperties: false
|
|
@@ -4967,22 +5099,29 @@ function extractBaseSchema(schema) {
|
|
|
4967
5099
|
}
|
|
4968
5100
|
const variants = schema.oneOf ?? schema.anyOf;
|
|
4969
5101
|
if (Array.isArray(variants)) {
|
|
5102
|
+
let hasScalar = false;
|
|
5103
|
+
let hasArray = false;
|
|
5104
|
+
let scalarVariant;
|
|
5105
|
+
let arrayVariant;
|
|
4970
5106
|
for (const variant of variants) {
|
|
4971
5107
|
if (typeof variant === "object") {
|
|
4972
|
-
const
|
|
4973
|
-
if (
|
|
4974
|
-
|
|
5108
|
+
const v = variant;
|
|
5109
|
+
if (v.type === "array" || "items" in v) {
|
|
5110
|
+
hasArray = true;
|
|
5111
|
+
arrayVariant = variant;
|
|
5112
|
+
} else {
|
|
5113
|
+
hasScalar = true;
|
|
5114
|
+
scalarVariant = variant;
|
|
4975
5115
|
}
|
|
4976
5116
|
}
|
|
4977
5117
|
}
|
|
4978
|
-
|
|
4979
|
-
|
|
4980
|
-
const variantType = variant.type;
|
|
4981
|
-
if (variantType === "array" && variant.items) {
|
|
4982
|
-
return variant.items;
|
|
4983
|
-
}
|
|
4984
|
-
}
|
|
5118
|
+
if (hasScalar && hasArray && variants.length === 2) {
|
|
5119
|
+
return scalarVariant;
|
|
4985
5120
|
}
|
|
5121
|
+
if (!hasScalar && hasArray && arrayVariant) {
|
|
5122
|
+
return arrayVariant.items;
|
|
5123
|
+
}
|
|
5124
|
+
return schema;
|
|
4986
5125
|
}
|
|
4987
5126
|
return schema;
|
|
4988
5127
|
}
|
|
@@ -5423,6 +5562,9 @@ class WhileTask extends GraphAsTask {
|
|
|
5423
5562
|
return WHILE_CONTEXT_SCHEMA;
|
|
5424
5563
|
}
|
|
5425
5564
|
_currentIteration = 0;
|
|
5565
|
+
canSerializeConfig() {
|
|
5566
|
+
return typeof this.config.condition !== "function";
|
|
5567
|
+
}
|
|
5426
5568
|
constructor(input = {}, config = {}) {
|
|
5427
5569
|
super(input, config);
|
|
5428
5570
|
}
|
|
@@ -5544,7 +5686,27 @@ class WhileTask extends GraphAsTask {
|
|
|
5544
5686
|
parentSignal: context.signal
|
|
5545
5687
|
});
|
|
5546
5688
|
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
5547
|
-
|
|
5689
|
+
let shouldContinue;
|
|
5690
|
+
try {
|
|
5691
|
+
shouldContinue = condition(currentOutput, this._currentIteration);
|
|
5692
|
+
} catch (err) {
|
|
5693
|
+
if (err instanceof TaskFailedError) {
|
|
5694
|
+
throw err;
|
|
5695
|
+
}
|
|
5696
|
+
const message = `${this.type}: Condition function threw at iteration ${this._currentIteration}: ${err instanceof Error ? err.message : String(err)}`;
|
|
5697
|
+
const wrappedError = new TaskFailedError(message);
|
|
5698
|
+
if (err instanceof Error && err.stack) {
|
|
5699
|
+
if (wrappedError.stack) {
|
|
5700
|
+
wrappedError.stack += `
|
|
5701
|
+
Caused by original error:
|
|
5702
|
+
${err.stack}`;
|
|
5703
|
+
} else {
|
|
5704
|
+
wrappedError.stack = err.stack;
|
|
5705
|
+
}
|
|
5706
|
+
}
|
|
5707
|
+
throw wrappedError;
|
|
5708
|
+
}
|
|
5709
|
+
if (!shouldContinue) {
|
|
5548
5710
|
break;
|
|
5549
5711
|
}
|
|
5550
5712
|
if (this.chainIterations) {
|
|
@@ -5588,7 +5750,13 @@ class WhileTask extends GraphAsTask {
|
|
|
5588
5750
|
parentSignal: context.signal
|
|
5589
5751
|
});
|
|
5590
5752
|
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
5591
|
-
|
|
5753
|
+
let shouldContinue;
|
|
5754
|
+
try {
|
|
5755
|
+
shouldContinue = condition(currentOutput, this._currentIteration);
|
|
5756
|
+
} catch (err) {
|
|
5757
|
+
throw new TaskFailedError(`${this.type}: Condition function threw at iteration ${this._currentIteration}: ${err instanceof Error ? err.message : String(err)}`);
|
|
5758
|
+
}
|
|
5759
|
+
if (!shouldContinue) {
|
|
5592
5760
|
break;
|
|
5593
5761
|
}
|
|
5594
5762
|
if (this.chainIterations) {
|
|
@@ -5991,205 +6159,6 @@ function getJobQueueFactory() {
|
|
|
5991
6159
|
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
5992
6160
|
registerJobQueueFactory(defaultJobQueueFactory);
|
|
5993
6161
|
}
|
|
5994
|
-
// src/task/JobQueueTask.ts
|
|
5995
|
-
import { Job as Job2 } from "@workglow/job-queue";
|
|
5996
|
-
|
|
5997
|
-
// src/task/TaskQueueRegistry.ts
|
|
5998
|
-
var taskQueueRegistry = null;
|
|
5999
|
-
|
|
6000
|
-
class TaskQueueRegistry {
|
|
6001
|
-
queues = new Map;
|
|
6002
|
-
registerQueue(queue) {
|
|
6003
|
-
const queueName = queue.server.queueName;
|
|
6004
|
-
if (this.queues.has(queueName)) {
|
|
6005
|
-
throw new Error(`Queue with name ${queueName} already exists`);
|
|
6006
|
-
}
|
|
6007
|
-
this.queues.set(queueName, queue);
|
|
6008
|
-
}
|
|
6009
|
-
getQueue(queueName) {
|
|
6010
|
-
return this.queues.get(queueName);
|
|
6011
|
-
}
|
|
6012
|
-
async startQueues() {
|
|
6013
|
-
for (const queue of this.queues.values()) {
|
|
6014
|
-
await queue.server.start();
|
|
6015
|
-
}
|
|
6016
|
-
}
|
|
6017
|
-
async stopQueues() {
|
|
6018
|
-
for (const queue of this.queues.values()) {
|
|
6019
|
-
await queue.server.stop();
|
|
6020
|
-
}
|
|
6021
|
-
}
|
|
6022
|
-
async clearQueues() {
|
|
6023
|
-
for (const queue of this.queues.values()) {
|
|
6024
|
-
await queue.storage.deleteAll();
|
|
6025
|
-
}
|
|
6026
|
-
}
|
|
6027
|
-
}
|
|
6028
|
-
function getTaskQueueRegistry() {
|
|
6029
|
-
if (!taskQueueRegistry) {
|
|
6030
|
-
taskQueueRegistry = new TaskQueueRegistry;
|
|
6031
|
-
}
|
|
6032
|
-
return taskQueueRegistry;
|
|
6033
|
-
}
|
|
6034
|
-
async function setTaskQueueRegistry(registry) {
|
|
6035
|
-
if (taskQueueRegistry) {
|
|
6036
|
-
await taskQueueRegistry.stopQueues();
|
|
6037
|
-
await taskQueueRegistry.clearQueues();
|
|
6038
|
-
}
|
|
6039
|
-
taskQueueRegistry = registry;
|
|
6040
|
-
}
|
|
6041
|
-
|
|
6042
|
-
// src/task/JobQueueTask.ts
|
|
6043
|
-
var jobQueueTaskConfigSchema = {
|
|
6044
|
-
type: "object",
|
|
6045
|
-
properties: {
|
|
6046
|
-
...graphAsTaskConfigSchema["properties"],
|
|
6047
|
-
queue: {
|
|
6048
|
-
oneOf: [{ type: "boolean" }, { type: "string" }],
|
|
6049
|
-
description: "Queue handling: false=run inline, true=use default, string=explicit queue name",
|
|
6050
|
-
"x-ui-hidden": true
|
|
6051
|
-
}
|
|
6052
|
-
},
|
|
6053
|
-
additionalProperties: false
|
|
6054
|
-
};
|
|
6055
|
-
|
|
6056
|
-
class JobQueueTask extends GraphAsTask {
|
|
6057
|
-
static type = "JobQueueTask";
|
|
6058
|
-
static canRunDirectly = true;
|
|
6059
|
-
static configSchema() {
|
|
6060
|
-
return jobQueueTaskConfigSchema;
|
|
6061
|
-
}
|
|
6062
|
-
currentQueueName;
|
|
6063
|
-
currentJobId;
|
|
6064
|
-
currentRunnerId;
|
|
6065
|
-
jobClass;
|
|
6066
|
-
constructor(input = {}, config = {}) {
|
|
6067
|
-
config.queue ??= true;
|
|
6068
|
-
super(input, config);
|
|
6069
|
-
this.jobClass = Job2;
|
|
6070
|
-
}
|
|
6071
|
-
async execute(input, executeContext) {
|
|
6072
|
-
let cleanup = () => {};
|
|
6073
|
-
try {
|
|
6074
|
-
if (this.config.queue === false && !this.constructor.canRunDirectly) {
|
|
6075
|
-
throw new TaskConfigurationError(`${this.type} cannot run directly without a queue`);
|
|
6076
|
-
}
|
|
6077
|
-
const registeredQueue = await this.resolveQueue(input);
|
|
6078
|
-
if (!registeredQueue) {
|
|
6079
|
-
if (!this.constructor.canRunDirectly) {
|
|
6080
|
-
const queueLabel = typeof this.config.queue === "string" ? this.config.queue : this.currentQueueName ?? this.type;
|
|
6081
|
-
throw new TaskConfigurationError(`Queue ${queueLabel} not found, and ${this.type} cannot run directly`);
|
|
6082
|
-
}
|
|
6083
|
-
this.currentJobId = undefined;
|
|
6084
|
-
const job = await this.createJob(input, this.currentQueueName);
|
|
6085
|
-
cleanup = job.onJobProgress((progress, message, details) => {
|
|
6086
|
-
executeContext.updateProgress(progress, message, details);
|
|
6087
|
-
});
|
|
6088
|
-
const output2 = await job.execute(job.input, {
|
|
6089
|
-
signal: executeContext.signal,
|
|
6090
|
-
updateProgress: executeContext.updateProgress.bind(this)
|
|
6091
|
-
});
|
|
6092
|
-
return output2;
|
|
6093
|
-
}
|
|
6094
|
-
const { client } = registeredQueue;
|
|
6095
|
-
const jobInput = await this.getJobInput(input);
|
|
6096
|
-
const handle = await client.submit(jobInput, {
|
|
6097
|
-
jobRunId: this.currentRunnerId,
|
|
6098
|
-
maxRetries: 10
|
|
6099
|
-
});
|
|
6100
|
-
this.currentJobId = handle.id;
|
|
6101
|
-
this.currentQueueName = client.queueName;
|
|
6102
|
-
cleanup = handle.onProgress((progress, message, details) => {
|
|
6103
|
-
executeContext.updateProgress(progress, message, details);
|
|
6104
|
-
});
|
|
6105
|
-
const output = await handle.waitFor();
|
|
6106
|
-
if (output === undefined) {
|
|
6107
|
-
throw new TaskConfigurationError("Job disabled, should not happen");
|
|
6108
|
-
}
|
|
6109
|
-
return output;
|
|
6110
|
-
} catch (err) {
|
|
6111
|
-
throw new JobTaskFailedError(err);
|
|
6112
|
-
} finally {
|
|
6113
|
-
cleanup();
|
|
6114
|
-
}
|
|
6115
|
-
}
|
|
6116
|
-
async getJobInput(input) {
|
|
6117
|
-
return input;
|
|
6118
|
-
}
|
|
6119
|
-
async createJob(input, queueName) {
|
|
6120
|
-
return new this.jobClass({
|
|
6121
|
-
queueName: queueName ?? this.currentQueueName,
|
|
6122
|
-
jobRunId: this.currentRunnerId,
|
|
6123
|
-
input
|
|
6124
|
-
});
|
|
6125
|
-
}
|
|
6126
|
-
async resolveQueue(input) {
|
|
6127
|
-
const preference = this.config.queue ?? true;
|
|
6128
|
-
if (preference === false) {
|
|
6129
|
-
this.currentQueueName = undefined;
|
|
6130
|
-
return;
|
|
6131
|
-
}
|
|
6132
|
-
if (typeof preference === "string") {
|
|
6133
|
-
const registeredQueue2 = getTaskQueueRegistry().getQueue(preference);
|
|
6134
|
-
if (registeredQueue2) {
|
|
6135
|
-
this.currentQueueName = registeredQueue2.server.queueName;
|
|
6136
|
-
return registeredQueue2;
|
|
6137
|
-
}
|
|
6138
|
-
this.currentQueueName = preference;
|
|
6139
|
-
return;
|
|
6140
|
-
}
|
|
6141
|
-
const queueName = await this.getDefaultQueueName(input);
|
|
6142
|
-
if (!queueName) {
|
|
6143
|
-
this.currentQueueName = undefined;
|
|
6144
|
-
return;
|
|
6145
|
-
}
|
|
6146
|
-
this.currentQueueName = queueName;
|
|
6147
|
-
let registeredQueue = getTaskQueueRegistry().getQueue(queueName);
|
|
6148
|
-
if (!registeredQueue) {
|
|
6149
|
-
registeredQueue = await this.createAndRegisterQueue(queueName, input);
|
|
6150
|
-
await registeredQueue.server.start();
|
|
6151
|
-
}
|
|
6152
|
-
return registeredQueue;
|
|
6153
|
-
}
|
|
6154
|
-
async getDefaultQueueName(_input) {
|
|
6155
|
-
return this.type;
|
|
6156
|
-
}
|
|
6157
|
-
async createAndRegisterQueue(queueName, input) {
|
|
6158
|
-
const factory = getJobQueueFactory();
|
|
6159
|
-
let registeredQueue = await factory({
|
|
6160
|
-
queueName,
|
|
6161
|
-
jobClass: this.jobClass,
|
|
6162
|
-
input,
|
|
6163
|
-
config: this.config,
|
|
6164
|
-
task: this
|
|
6165
|
-
});
|
|
6166
|
-
const registry = getTaskQueueRegistry();
|
|
6167
|
-
try {
|
|
6168
|
-
registry.registerQueue(registeredQueue);
|
|
6169
|
-
} catch (err) {
|
|
6170
|
-
if (err instanceof Error && err.message.includes("already exists")) {
|
|
6171
|
-
const existing = registry.getQueue(queueName);
|
|
6172
|
-
if (existing) {
|
|
6173
|
-
registeredQueue = existing;
|
|
6174
|
-
}
|
|
6175
|
-
} else {
|
|
6176
|
-
throw err;
|
|
6177
|
-
}
|
|
6178
|
-
}
|
|
6179
|
-
return registeredQueue;
|
|
6180
|
-
}
|
|
6181
|
-
abort() {
|
|
6182
|
-
if (this.currentQueueName && this.currentJobId) {
|
|
6183
|
-
const registeredQueue = getTaskQueueRegistry().getQueue(this.currentQueueName);
|
|
6184
|
-
if (registeredQueue) {
|
|
6185
|
-
registeredQueue.client.abort(this.currentJobId).catch((err) => {
|
|
6186
|
-
console.warn(`Failed to abort remote job ${this.currentJobId}`, err);
|
|
6187
|
-
});
|
|
6188
|
-
}
|
|
6189
|
-
}
|
|
6190
|
-
super.abort();
|
|
6191
|
-
}
|
|
6192
|
-
}
|
|
6193
6162
|
// src/task/MapTask.ts
|
|
6194
6163
|
var mapTaskConfigSchema = {
|
|
6195
6164
|
type: "object",
|
|
@@ -6410,13 +6379,19 @@ function resolveTaskFromRegistry(id, _format, registry) {
|
|
|
6410
6379
|
registerInputResolver("tasks", resolveTaskFromRegistry);
|
|
6411
6380
|
|
|
6412
6381
|
// src/task/TaskJSON.ts
|
|
6413
|
-
var createSingleTaskFromJSON = (item, registry) => {
|
|
6382
|
+
var createSingleTaskFromJSON = (item, registry, options) => {
|
|
6414
6383
|
if (!item.id)
|
|
6415
6384
|
throw new TaskJSONError("Task id required");
|
|
6416
6385
|
if (!item.type)
|
|
6417
6386
|
throw new TaskJSONError("Task type required");
|
|
6418
6387
|
if (item.defaults && Array.isArray(item.defaults))
|
|
6419
6388
|
throw new TaskJSONError("Task defaults must be an object");
|
|
6389
|
+
if (options?.allowedTypes) {
|
|
6390
|
+
const allowed = options.allowedTypes instanceof Set ? options.allowedTypes : new Set(options.allowedTypes);
|
|
6391
|
+
if (!allowed.has(item.type)) {
|
|
6392
|
+
throw new TaskJSONError(`Task type "${item.type}" is not in the allowed types list`);
|
|
6393
|
+
}
|
|
6394
|
+
}
|
|
6420
6395
|
const constructors = getTaskConstructors(registry);
|
|
6421
6396
|
const taskClass = constructors.get(item.type);
|
|
6422
6397
|
if (!taskClass)
|
|
@@ -6428,43 +6403,90 @@ var createSingleTaskFromJSON = (item, registry) => {
|
|
|
6428
6403
|
const task = new taskClass(item.defaults ?? {}, taskConfig, registry ? { registry } : {});
|
|
6429
6404
|
return task;
|
|
6430
6405
|
};
|
|
6431
|
-
var createTaskFromDependencyJSON = (item, registry) => {
|
|
6432
|
-
const task = createSingleTaskFromJSON(item, registry);
|
|
6406
|
+
var createTaskFromDependencyJSON = (item, registry, options) => {
|
|
6407
|
+
const task = createSingleTaskFromJSON(item, registry, options);
|
|
6433
6408
|
if (item.subtasks && item.subtasks.length > 0) {
|
|
6434
6409
|
if (!(task instanceof GraphAsTask)) {
|
|
6435
6410
|
throw new TaskConfigurationError("Subgraph is only supported for CompoundTasks");
|
|
6436
6411
|
}
|
|
6437
|
-
task.subGraph = createGraphFromDependencyJSON(item.subtasks, registry);
|
|
6412
|
+
task.subGraph = createGraphFromDependencyJSON(item.subtasks, registry, options);
|
|
6438
6413
|
}
|
|
6439
6414
|
return task;
|
|
6440
6415
|
};
|
|
6441
|
-
var createGraphFromDependencyJSON = (jsonItems, registry) => {
|
|
6416
|
+
var createGraphFromDependencyJSON = (jsonItems, registry, options) => {
|
|
6442
6417
|
const subGraph = new TaskGraph;
|
|
6443
6418
|
for (const subitem of jsonItems) {
|
|
6444
|
-
subGraph.addTask(createTaskFromDependencyJSON(subitem, registry));
|
|
6419
|
+
subGraph.addTask(createTaskFromDependencyJSON(subitem, registry, options));
|
|
6445
6420
|
}
|
|
6446
6421
|
return subGraph;
|
|
6447
6422
|
};
|
|
6448
|
-
var createTaskFromGraphJSON = (item, registry) => {
|
|
6449
|
-
const task = createSingleTaskFromJSON(item, registry);
|
|
6423
|
+
var createTaskFromGraphJSON = (item, registry, options) => {
|
|
6424
|
+
const task = createSingleTaskFromJSON(item, registry, options);
|
|
6450
6425
|
if (item.subgraph) {
|
|
6451
6426
|
if (!(task instanceof GraphAsTask)) {
|
|
6452
6427
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
6453
6428
|
}
|
|
6454
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph, registry);
|
|
6429
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, registry, options);
|
|
6455
6430
|
}
|
|
6456
6431
|
return task;
|
|
6457
6432
|
};
|
|
6458
|
-
var createGraphFromGraphJSON = (graphJsonObj, registry) => {
|
|
6433
|
+
var createGraphFromGraphJSON = (graphJsonObj, registry, options) => {
|
|
6459
6434
|
const subGraph = new TaskGraph;
|
|
6460
6435
|
for (const subitem of graphJsonObj.tasks) {
|
|
6461
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem, registry));
|
|
6436
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, registry, options));
|
|
6462
6437
|
}
|
|
6463
6438
|
for (const subitem of graphJsonObj.dataflows) {
|
|
6464
6439
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
6465
6440
|
}
|
|
6466
6441
|
return subGraph;
|
|
6467
6442
|
};
|
|
6443
|
+
// src/task/TaskQueueRegistry.ts
|
|
6444
|
+
import { EventEmitter as EventEmitter6 } from "@workglow/util";
|
|
6445
|
+
var taskQueueRegistry = null;
|
|
6446
|
+
|
|
6447
|
+
class TaskQueueRegistry {
|
|
6448
|
+
emitter = new EventEmitter6;
|
|
6449
|
+
queues = new Map;
|
|
6450
|
+
registerQueue(queue) {
|
|
6451
|
+
const queueName = queue.server.queueName;
|
|
6452
|
+
if (this.queues.has(queueName)) {
|
|
6453
|
+
throw new Error(`Queue with name ${queueName} already exists`);
|
|
6454
|
+
}
|
|
6455
|
+
this.queues.set(queueName, queue);
|
|
6456
|
+
this.emitter.emit("queue_registered", queueName);
|
|
6457
|
+
}
|
|
6458
|
+
getQueue(queueName) {
|
|
6459
|
+
return this.queues.get(queueName);
|
|
6460
|
+
}
|
|
6461
|
+
async startQueues() {
|
|
6462
|
+
for (const queue of this.queues.values()) {
|
|
6463
|
+
await queue.server.start();
|
|
6464
|
+
}
|
|
6465
|
+
}
|
|
6466
|
+
async stopQueues() {
|
|
6467
|
+
for (const queue of this.queues.values()) {
|
|
6468
|
+
await queue.server.stop();
|
|
6469
|
+
}
|
|
6470
|
+
}
|
|
6471
|
+
async clearQueues() {
|
|
6472
|
+
for (const queue of this.queues.values()) {
|
|
6473
|
+
await queue.storage.deleteAll();
|
|
6474
|
+
}
|
|
6475
|
+
}
|
|
6476
|
+
}
|
|
6477
|
+
function getTaskQueueRegistry() {
|
|
6478
|
+
if (!taskQueueRegistry) {
|
|
6479
|
+
taskQueueRegistry = new TaskQueueRegistry;
|
|
6480
|
+
}
|
|
6481
|
+
return taskQueueRegistry;
|
|
6482
|
+
}
|
|
6483
|
+
async function setTaskQueueRegistry(registry) {
|
|
6484
|
+
if (taskQueueRegistry) {
|
|
6485
|
+
await taskQueueRegistry.stopQueues();
|
|
6486
|
+
await taskQueueRegistry.clearQueues();
|
|
6487
|
+
}
|
|
6488
|
+
taskQueueRegistry = registry;
|
|
6489
|
+
}
|
|
6468
6490
|
// src/task/index.ts
|
|
6469
6491
|
var registerBaseTasks = () => {
|
|
6470
6492
|
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
@@ -6472,14 +6494,14 @@ var registerBaseTasks = () => {
|
|
|
6472
6494
|
return tasks;
|
|
6473
6495
|
};
|
|
6474
6496
|
// src/storage/TaskGraphRepository.ts
|
|
6475
|
-
import { createServiceToken as createServiceToken4, EventEmitter as
|
|
6497
|
+
import { createServiceToken as createServiceToken4, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
6476
6498
|
var TASK_GRAPH_REPOSITORY = createServiceToken4("taskgraph.taskGraphRepository");
|
|
6477
6499
|
|
|
6478
6500
|
class TaskGraphRepository {
|
|
6479
6501
|
type = "TaskGraphRepository";
|
|
6480
6502
|
get events() {
|
|
6481
6503
|
if (!this._events) {
|
|
6482
|
-
this._events = new
|
|
6504
|
+
this._events = new EventEmitter7;
|
|
6483
6505
|
}
|
|
6484
6506
|
return this._events;
|
|
6485
6507
|
}
|
|
@@ -6637,6 +6659,7 @@ export {
|
|
|
6637
6659
|
setTaskQueueRegistry,
|
|
6638
6660
|
setGlobalTaskConstructors,
|
|
6639
6661
|
serialGraph,
|
|
6662
|
+
schemaHasFormatAnnotations,
|
|
6640
6663
|
schemaAcceptsArray,
|
|
6641
6664
|
resolveSchemaInputs,
|
|
6642
6665
|
resetMethodNameCache,
|
|
@@ -6648,7 +6671,6 @@ export {
|
|
|
6648
6671
|
parallel,
|
|
6649
6672
|
mergeChainedOutputToInput,
|
|
6650
6673
|
mapTaskConfigSchema,
|
|
6651
|
-
jobQueueTaskConfigSchema,
|
|
6652
6674
|
iteratorTaskConfigSchema,
|
|
6653
6675
|
isTaskStreamable,
|
|
6654
6676
|
isStrictArraySchema,
|
|
@@ -6705,6 +6727,7 @@ export {
|
|
|
6705
6727
|
WHILE_CONTEXT_SCHEMA,
|
|
6706
6728
|
TaskTimeoutError,
|
|
6707
6729
|
TaskStatus,
|
|
6730
|
+
TaskSerializationError,
|
|
6708
6731
|
TaskRegistry,
|
|
6709
6732
|
TaskQueueRegistry,
|
|
6710
6733
|
TaskOutputTabularRepository,
|
|
@@ -6713,6 +6736,7 @@ export {
|
|
|
6713
6736
|
TaskOutputPrimaryKeyNames,
|
|
6714
6737
|
TaskJSONError,
|
|
6715
6738
|
TaskInvalidInputError,
|
|
6739
|
+
TaskGraphTimeoutError,
|
|
6716
6740
|
TaskGraphTabularRepository,
|
|
6717
6741
|
TaskGraphSchema,
|
|
6718
6742
|
TaskGraphRunner,
|
|
@@ -6732,7 +6756,6 @@ export {
|
|
|
6732
6756
|
PROPERTY_ARRAY,
|
|
6733
6757
|
MapTask,
|
|
6734
6758
|
JobTaskFailedError,
|
|
6735
|
-
JobQueueTask,
|
|
6736
6759
|
JOB_QUEUE_FACTORY,
|
|
6737
6760
|
IteratorTaskRunner,
|
|
6738
6761
|
IteratorTask,
|
|
@@ -6755,4 +6778,4 @@ export {
|
|
|
6755
6778
|
ConditionalTask
|
|
6756
6779
|
};
|
|
6757
6780
|
|
|
6758
|
-
//# debugId=
|
|
6781
|
+
//# debugId=8BC5BADB1209138664756E2164756E21
|