@workglow/task-graph 0.0.126 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -24
- package/dist/browser.js +289 -266
- package/dist/browser.js.map +23 -24
- package/dist/bun.js +289 -266
- package/dist/bun.js.map +22 -23
- package/dist/debug/console/ConsoleFormatters.d.ts.map +1 -1
- package/dist/node.js +289 -266
- package/dist/node.js.map +22 -23
- package/dist/task/ConditionalTask.d.ts +1 -0
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +0 -1
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +6 -0
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts +6 -0
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +11 -0
- package/dist/task/IteratorTask.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/JobQueueFactory.d.ts +3 -4
- package/dist/task/JobQueueFactory.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -0
- package/dist/task/MapTask.d.ts.map +1 -1
- package/dist/task/ReduceTask.d.ts +4 -0
- package/dist/task/ReduceTask.d.ts.map +1 -1
- package/dist/task/Task.d.ts +13 -1
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +23 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +1 -1
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +17 -7
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskQueueRegistry.d.ts +8 -0
- package/dist/task/TaskQueueRegistry.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +1 -0
- package/dist/task/WhileTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +0 -1
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/Dataflow.d.ts +11 -0
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/GraphSchemaUtils.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +10 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +15 -2
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/task/README.md +0 -17
- package/dist/task/JobQueueTask.d.ts +0 -130
- package/dist/task/JobQueueTask.d.ts.map +0 -1
package/dist/node.js
CHANGED
|
@@ -118,6 +118,7 @@ class Dataflow {
|
|
|
118
118
|
this.error = undefined;
|
|
119
119
|
this.value = undefined;
|
|
120
120
|
this.stream = undefined;
|
|
121
|
+
this._compatibilityCache = undefined;
|
|
121
122
|
this.emit("reset");
|
|
122
123
|
this.emit("status", this.status);
|
|
123
124
|
}
|
|
@@ -178,9 +179,19 @@ class Dataflow {
|
|
|
178
179
|
targetTaskPortId: this.targetTaskPortId
|
|
179
180
|
};
|
|
180
181
|
}
|
|
182
|
+
_compatibilityCache;
|
|
183
|
+
invalidateCompatibilityCache() {
|
|
184
|
+
this._compatibilityCache = undefined;
|
|
185
|
+
}
|
|
181
186
|
semanticallyCompatible(graph, dataflow) {
|
|
182
|
-
const
|
|
183
|
-
const
|
|
187
|
+
const sourceTask = graph.getTask(dataflow.sourceTaskId);
|
|
188
|
+
const targetTask = graph.getTask(dataflow.targetTaskId);
|
|
189
|
+
const shouldCache = !(sourceTask.constructor.hasDynamicSchemas ?? true) && !(targetTask.constructor.hasDynamicSchemas ?? true);
|
|
190
|
+
if (shouldCache && this._compatibilityCache !== undefined) {
|
|
191
|
+
return this._compatibilityCache;
|
|
192
|
+
}
|
|
193
|
+
const targetSchema = targetTask.inputSchema();
|
|
194
|
+
const sourceSchema = sourceTask.outputSchema();
|
|
184
195
|
if (typeof targetSchema === "boolean") {
|
|
185
196
|
if (targetSchema === false) {
|
|
186
197
|
return "incompatible";
|
|
@@ -201,8 +212,11 @@ class Dataflow {
|
|
|
201
212
|
if (sourceSchemaProperty === undefined && sourceSchema.additionalProperties === true) {
|
|
202
213
|
sourceSchemaProperty = true;
|
|
203
214
|
}
|
|
204
|
-
const
|
|
205
|
-
|
|
215
|
+
const result = areSemanticallyCompatible(sourceSchemaProperty, targetSchemaProperty);
|
|
216
|
+
if (shouldCache) {
|
|
217
|
+
this._compatibilityCache = result;
|
|
218
|
+
}
|
|
219
|
+
return result;
|
|
206
220
|
}
|
|
207
221
|
get events() {
|
|
208
222
|
if (!this._events) {
|
|
@@ -595,12 +609,13 @@ import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
|
595
609
|
import { EventEmitter as EventEmitter4, uuid4 as uuid44 } from "@workglow/util";
|
|
596
610
|
|
|
597
611
|
// src/task/GraphAsTask.ts
|
|
612
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
598
613
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
599
614
|
|
|
600
615
|
// src/task-graph/TaskGraphRunner.ts
|
|
601
616
|
import {
|
|
602
617
|
collectPropertyValues,
|
|
603
|
-
getLogger as
|
|
618
|
+
getLogger as getLogger3,
|
|
604
619
|
getTelemetryProvider as getTelemetryProvider2,
|
|
605
620
|
globalServiceRegistry as globalServiceRegistry2,
|
|
606
621
|
ServiceRegistry as ServiceRegistry2,
|
|
@@ -639,7 +654,7 @@ class TaskOutputRepository {
|
|
|
639
654
|
}
|
|
640
655
|
|
|
641
656
|
// src/task/ConditionalTask.ts
|
|
642
|
-
import { getLogger } from "@workglow/util";
|
|
657
|
+
import { getLogger as getLogger2 } from "@workglow/util";
|
|
643
658
|
|
|
644
659
|
// src/task/ConditionUtils.ts
|
|
645
660
|
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
@@ -717,6 +732,8 @@ import { BaseError } from "@workglow/util";
|
|
|
717
732
|
|
|
718
733
|
class TaskError extends BaseError {
|
|
719
734
|
static type = "TaskError";
|
|
735
|
+
taskType;
|
|
736
|
+
taskId;
|
|
720
737
|
constructor(message) {
|
|
721
738
|
super(message);
|
|
722
739
|
}
|
|
@@ -750,6 +767,14 @@ class TaskTimeoutError extends TaskAbortedError {
|
|
|
750
767
|
}
|
|
751
768
|
}
|
|
752
769
|
|
|
770
|
+
class TaskGraphTimeoutError extends TaskTimeoutError {
|
|
771
|
+
static type = "TaskGraphTimeoutError";
|
|
772
|
+
constructor(timeoutMs) {
|
|
773
|
+
super(timeoutMs);
|
|
774
|
+
this.message = timeoutMs ? `Graph execution timed out after ${timeoutMs}ms` : "Graph execution timed out";
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
753
778
|
class TaskFailedError extends TaskError {
|
|
754
779
|
static type = "TaskFailedError";
|
|
755
780
|
constructor(message = "Task failed") {
|
|
@@ -780,8 +805,16 @@ class TaskInvalidInputError extends TaskError {
|
|
|
780
805
|
}
|
|
781
806
|
}
|
|
782
807
|
|
|
808
|
+
class TaskSerializationError extends TaskError {
|
|
809
|
+
static type = "TaskSerializationError";
|
|
810
|
+
constructor(taskType) {
|
|
811
|
+
super(`Task "${taskType}" cannot be serialized: config contains non-serializable values. ` + `Use a declarative config alternative or remove function-valued config properties.`);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
783
815
|
// src/task/TaskRunner.ts
|
|
784
816
|
import {
|
|
817
|
+
getLogger,
|
|
785
818
|
getTelemetryProvider,
|
|
786
819
|
globalServiceRegistry,
|
|
787
820
|
SpanStatusCode
|
|
@@ -831,6 +864,18 @@ function getFormatPrefix(format) {
|
|
|
831
864
|
const colonIndex = format.indexOf(":");
|
|
832
865
|
return colonIndex >= 0 ? format.substring(0, colonIndex) : format;
|
|
833
866
|
}
|
|
867
|
+
function schemaHasFormatAnnotations(schema) {
|
|
868
|
+
if (typeof schema === "boolean")
|
|
869
|
+
return false;
|
|
870
|
+
const properties = schema.properties;
|
|
871
|
+
if (!properties || typeof properties !== "object")
|
|
872
|
+
return false;
|
|
873
|
+
for (const propSchema of Object.values(properties)) {
|
|
874
|
+
if (getSchemaFormat(propSchema) !== undefined)
|
|
875
|
+
return true;
|
|
876
|
+
}
|
|
877
|
+
return false;
|
|
878
|
+
}
|
|
834
879
|
async function resolveSchemaInputs(input, schema, config) {
|
|
835
880
|
if (typeof schema === "boolean")
|
|
836
881
|
return input;
|
|
@@ -996,6 +1041,12 @@ class TaskRunner {
|
|
|
996
1041
|
await this.handleStart(config);
|
|
997
1042
|
try {
|
|
998
1043
|
this.task.setInput(overrides);
|
|
1044
|
+
const configSchema = this.task.constructor.configSchema();
|
|
1045
|
+
if (schemaHasFormatAnnotations(configSchema)) {
|
|
1046
|
+
const source = this.task.originalConfig ?? this.task.config;
|
|
1047
|
+
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
1048
|
+
Object.assign(this.task.config, resolved);
|
|
1049
|
+
}
|
|
999
1050
|
const schema = this.task.constructor.inputSchema();
|
|
1000
1051
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1001
1052
|
const inputs = this.task.runInputData;
|
|
@@ -1009,6 +1060,12 @@ class TaskRunner {
|
|
|
1009
1060
|
}
|
|
1010
1061
|
let outputs;
|
|
1011
1062
|
const isStreamable = isTaskStreamable(this.task);
|
|
1063
|
+
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
1064
|
+
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
1065
|
+
if (streamMode !== "none") {
|
|
1066
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1012
1069
|
if (this.task.cacheable) {
|
|
1013
1070
|
outputs = await this.outputCache?.getOutput(this.task.type, inputs);
|
|
1014
1071
|
if (outputs) {
|
|
@@ -1047,6 +1104,12 @@ class TaskRunner {
|
|
|
1047
1104
|
return this.task.runOutputData;
|
|
1048
1105
|
}
|
|
1049
1106
|
this.task.setInput(overrides);
|
|
1107
|
+
const configSchema = this.task.constructor.configSchema();
|
|
1108
|
+
if (schemaHasFormatAnnotations(configSchema)) {
|
|
1109
|
+
const source = this.task.originalConfig ?? this.task.config;
|
|
1110
|
+
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
1111
|
+
Object.assign(this.task.config, resolved);
|
|
1112
|
+
}
|
|
1050
1113
|
const schema = this.task.constructor.inputSchema();
|
|
1051
1114
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1052
1115
|
await this.handleStartReactive();
|
|
@@ -1275,6 +1338,11 @@ class TaskRunner {
|
|
|
1275
1338
|
this.telemetrySpan.end();
|
|
1276
1339
|
this.telemetrySpan = undefined;
|
|
1277
1340
|
}
|
|
1341
|
+
if (typeof this.task.cleanup === "function") {
|
|
1342
|
+
try {
|
|
1343
|
+
await this.task.cleanup();
|
|
1344
|
+
} catch {}
|
|
1345
|
+
}
|
|
1278
1346
|
this.task.emit("abort", this.task.error);
|
|
1279
1347
|
this.task.emit("status", this.task.status);
|
|
1280
1348
|
}
|
|
@@ -1327,7 +1395,15 @@ class TaskRunner {
|
|
|
1327
1395
|
this.task.completedAt = new Date;
|
|
1328
1396
|
this.task.progress = 100;
|
|
1329
1397
|
this.task.status = TaskStatus.FAILED;
|
|
1330
|
-
|
|
1398
|
+
if (err instanceof TaskError) {
|
|
1399
|
+
this.task.error = err;
|
|
1400
|
+
} else {
|
|
1401
|
+
this.task.error = new TaskFailedError(`Task "${this.task.type}" (${this.task.id}): ${err?.message || "Task failed"}`);
|
|
1402
|
+
}
|
|
1403
|
+
if (this.task.error instanceof TaskError) {
|
|
1404
|
+
this.task.error.taskType ??= this.task.type;
|
|
1405
|
+
this.task.error.taskId ??= this.task.id;
|
|
1406
|
+
}
|
|
1331
1407
|
this.abortController = undefined;
|
|
1332
1408
|
if (this.telemetrySpan) {
|
|
1333
1409
|
this.telemetrySpan.setStatus(SpanStatusCode.ERROR, this.task.error.message);
|
|
@@ -1431,6 +1507,7 @@ class Task {
|
|
|
1431
1507
|
runInputData = {};
|
|
1432
1508
|
runOutputData = {};
|
|
1433
1509
|
config;
|
|
1510
|
+
originalConfig;
|
|
1434
1511
|
get id() {
|
|
1435
1512
|
return this.config.id;
|
|
1436
1513
|
}
|
|
@@ -1461,6 +1538,11 @@ class Task {
|
|
|
1461
1538
|
baseConfig.id = uuid42();
|
|
1462
1539
|
}
|
|
1463
1540
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
1541
|
+
try {
|
|
1542
|
+
this.originalConfig = Object.freeze(structuredClone(this.config));
|
|
1543
|
+
} catch {
|
|
1544
|
+
this.originalConfig = undefined;
|
|
1545
|
+
}
|
|
1464
1546
|
this.runConfig = runConfig;
|
|
1465
1547
|
}
|
|
1466
1548
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
@@ -1731,7 +1813,10 @@ class Task {
|
|
|
1731
1813
|
const path = e.data.pointer || "";
|
|
1732
1814
|
return `${e.message}${path ? ` (${path})` : ""}`;
|
|
1733
1815
|
});
|
|
1734
|
-
|
|
1816
|
+
const err = new TaskInvalidInputError(`Task "${this.type}" (${this.id}): Input ${JSON.stringify(Object.keys(input))} does not match schema: ${errorMessages.join(", ")}`);
|
|
1817
|
+
err.taskType = this.type;
|
|
1818
|
+
err.taskId = this.id;
|
|
1819
|
+
throw err;
|
|
1735
1820
|
}
|
|
1736
1821
|
return true;
|
|
1737
1822
|
}
|
|
@@ -1756,8 +1841,14 @@ class Task {
|
|
|
1756
1841
|
}
|
|
1757
1842
|
return obj;
|
|
1758
1843
|
}
|
|
1844
|
+
canSerializeConfig() {
|
|
1845
|
+
return true;
|
|
1846
|
+
}
|
|
1759
1847
|
toJSON(_options) {
|
|
1760
1848
|
const ctor = this.constructor;
|
|
1849
|
+
if (!this.canSerializeConfig() || !this.originalConfig) {
|
|
1850
|
+
throw new TaskSerializationError(this.type);
|
|
1851
|
+
}
|
|
1761
1852
|
const schema = ctor.configSchema();
|
|
1762
1853
|
const schemaProperties = typeof schema !== "boolean" && schema?.properties ? schema.properties : {};
|
|
1763
1854
|
const config = {};
|
|
@@ -1767,7 +1858,7 @@ class Task {
|
|
|
1767
1858
|
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
1768
1859
|
continue;
|
|
1769
1860
|
}
|
|
1770
|
-
const value = this.
|
|
1861
|
+
const value = this.originalConfig[key];
|
|
1771
1862
|
if (value === undefined)
|
|
1772
1863
|
continue;
|
|
1773
1864
|
if (typeof value === "function" || typeof value === "symbol")
|
|
@@ -1851,6 +1942,11 @@ class ConditionalTask extends Task {
|
|
|
1851
1942
|
static configSchema() {
|
|
1852
1943
|
return conditionalTaskConfigSchema;
|
|
1853
1944
|
}
|
|
1945
|
+
canSerializeConfig() {
|
|
1946
|
+
if (!this.config.branches)
|
|
1947
|
+
return true;
|
|
1948
|
+
return !this.config.branches.some((b) => typeof b.condition === "function");
|
|
1949
|
+
}
|
|
1854
1950
|
activeBranches = new Set;
|
|
1855
1951
|
buildBranchesFromConditionConfig(conditionConfig) {
|
|
1856
1952
|
if (!conditionConfig?.branches || conditionConfig.branches.length === 0) {
|
|
@@ -1913,7 +2009,7 @@ class ConditionalTask extends Task {
|
|
|
1913
2009
|
}
|
|
1914
2010
|
}
|
|
1915
2011
|
} catch (error) {
|
|
1916
|
-
|
|
2012
|
+
getLogger2().warn(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
1917
2013
|
}
|
|
1918
2014
|
}
|
|
1919
2015
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -2203,6 +2299,8 @@ class TaskGraphRunner {
|
|
|
2203
2299
|
inProgressFunctions = new Map;
|
|
2204
2300
|
failedTaskErrors = new Map;
|
|
2205
2301
|
telemetrySpan;
|
|
2302
|
+
graphTimeoutTimer;
|
|
2303
|
+
pendingGraphTimeoutError;
|
|
2206
2304
|
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph), reactiveScheduler = new TopologicalScheduler(graph)) {
|
|
2207
2305
|
this.processScheduler = processScheduler;
|
|
2208
2306
|
this.reactiveScheduler = reactiveScheduler;
|
|
@@ -2253,10 +2351,14 @@ class TaskGraphRunner {
|
|
|
2253
2351
|
}
|
|
2254
2352
|
} catch (err) {
|
|
2255
2353
|
error = err;
|
|
2256
|
-
|
|
2354
|
+
getLogger3().error("Error running graph", { error });
|
|
2257
2355
|
}
|
|
2258
2356
|
await Promise.allSettled(Array.from(this.inProgressTasks.values()));
|
|
2259
2357
|
await Promise.allSettled(Array.from(this.inProgressFunctions.values()));
|
|
2358
|
+
if (this.pendingGraphTimeoutError) {
|
|
2359
|
+
await this.handleAbort();
|
|
2360
|
+
throw this.pendingGraphTimeoutError;
|
|
2361
|
+
}
|
|
2260
2362
|
if (this.failedTaskErrors.size > 0) {
|
|
2261
2363
|
const latestError = this.failedTaskErrors.values().next().value;
|
|
2262
2364
|
this.handleError(latestError);
|
|
@@ -2344,6 +2446,7 @@ class TaskGraphRunner {
|
|
|
2344
2446
|
}
|
|
2345
2447
|
copyInputFromEdgesToNode(task) {
|
|
2346
2448
|
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2449
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2347
2450
|
for (const dataflow of dataflows) {
|
|
2348
2451
|
this.addInputData(task, dataflow.getPortData());
|
|
2349
2452
|
}
|
|
@@ -2352,7 +2455,7 @@ class TaskGraphRunner {
|
|
|
2352
2455
|
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2353
2456
|
for (const dataflow of dataflows) {
|
|
2354
2457
|
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow);
|
|
2355
|
-
|
|
2458
|
+
getLogger3().debug("pushOutputFromNodeToEdges", {
|
|
2356
2459
|
dataflowId: dataflow.id,
|
|
2357
2460
|
compatibility,
|
|
2358
2461
|
resultsKeys: Object.keys(results)
|
|
@@ -2366,7 +2469,7 @@ class TaskGraphRunner {
|
|
|
2366
2469
|
} else {
|
|
2367
2470
|
const resultsKeys = Object.keys(results);
|
|
2368
2471
|
if (resultsKeys.length > 0) {
|
|
2369
|
-
|
|
2472
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2370
2473
|
dataflowId: dataflow.id,
|
|
2371
2474
|
compatibility,
|
|
2372
2475
|
resultsKeys
|
|
@@ -2672,6 +2775,12 @@ class TaskGraphRunner {
|
|
|
2672
2775
|
}
|
|
2673
2776
|
this.graph.outputCache = this.outputCache;
|
|
2674
2777
|
}
|
|
2778
|
+
if (config?.maxTasks !== undefined && config.maxTasks > 0) {
|
|
2779
|
+
const taskCount = this.graph.getTasks().length;
|
|
2780
|
+
if (taskCount > config.maxTasks) {
|
|
2781
|
+
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2675
2784
|
if (this.running || this.reactiveRunning) {
|
|
2676
2785
|
throw new TaskConfigurationError("Graph is already running");
|
|
2677
2786
|
}
|
|
@@ -2680,6 +2789,13 @@ class TaskGraphRunner {
|
|
|
2680
2789
|
this.abortController.signal.addEventListener("abort", () => {
|
|
2681
2790
|
this.handleAbort();
|
|
2682
2791
|
});
|
|
2792
|
+
if (config?.timeout !== undefined && config.timeout > 0) {
|
|
2793
|
+
this.pendingGraphTimeoutError = undefined;
|
|
2794
|
+
this.graphTimeoutTimer = setTimeout(() => {
|
|
2795
|
+
this.pendingGraphTimeoutError = new TaskGraphTimeoutError(config.timeout);
|
|
2796
|
+
this.abortController?.abort();
|
|
2797
|
+
}, config.timeout);
|
|
2798
|
+
}
|
|
2683
2799
|
if (config?.parentSignal?.aborted) {
|
|
2684
2800
|
this.abortController.abort();
|
|
2685
2801
|
return;
|
|
@@ -2713,10 +2829,23 @@ class TaskGraphRunner {
|
|
|
2713
2829
|
if (config?.registry !== undefined) {
|
|
2714
2830
|
this.registry = config.registry;
|
|
2715
2831
|
}
|
|
2832
|
+
if (config?.maxTasks !== undefined && config.maxTasks > 0) {
|
|
2833
|
+
const taskCount = this.graph.getTasks().length;
|
|
2834
|
+
if (taskCount > config.maxTasks) {
|
|
2835
|
+
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
2836
|
+
}
|
|
2837
|
+
}
|
|
2716
2838
|
this.reactiveScheduler.reset();
|
|
2717
2839
|
this.reactiveRunning = true;
|
|
2718
2840
|
}
|
|
2841
|
+
clearGraphTimeout() {
|
|
2842
|
+
if (this.graphTimeoutTimer !== undefined) {
|
|
2843
|
+
clearTimeout(this.graphTimeoutTimer);
|
|
2844
|
+
this.graphTimeoutTimer = undefined;
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2719
2847
|
async handleComplete() {
|
|
2848
|
+
this.clearGraphTimeout();
|
|
2720
2849
|
this.running = false;
|
|
2721
2850
|
if (this.telemetrySpan) {
|
|
2722
2851
|
this.telemetrySpan.setStatus(SpanStatusCode2.OK);
|
|
@@ -2729,6 +2858,7 @@ class TaskGraphRunner {
|
|
|
2729
2858
|
this.reactiveRunning = false;
|
|
2730
2859
|
}
|
|
2731
2860
|
async handleError(error) {
|
|
2861
|
+
this.clearGraphTimeout();
|
|
2732
2862
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
2733
2863
|
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
2734
2864
|
return task.abort();
|
|
@@ -2747,6 +2877,7 @@ class TaskGraphRunner {
|
|
|
2747
2877
|
this.reactiveRunning = false;
|
|
2748
2878
|
}
|
|
2749
2879
|
async handleAbort() {
|
|
2880
|
+
this.clearGraphTimeout();
|
|
2750
2881
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
2751
2882
|
if (task.status === TaskStatus.PROCESSING || task.status === TaskStatus.STREAMING) {
|
|
2752
2883
|
return task.abort();
|
|
@@ -2891,7 +3022,7 @@ class GraphAsTask extends Task {
|
|
|
2891
3022
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
2892
3023
|
this._inputSchemaNode = schemaNode;
|
|
2893
3024
|
} catch (error) {
|
|
2894
|
-
|
|
3025
|
+
getLogger4().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
2895
3026
|
this._inputSchemaNode = compileSchema2({});
|
|
2896
3027
|
}
|
|
2897
3028
|
}
|
|
@@ -3150,7 +3281,9 @@ class TaskGraph {
|
|
|
3150
3281
|
outputCache: config?.outputCache || this.outputCache,
|
|
3151
3282
|
parentSignal: config?.parentSignal || undefined,
|
|
3152
3283
|
accumulateLeafOutputs: config?.accumulateLeafOutputs,
|
|
3153
|
-
registry: config?.registry
|
|
3284
|
+
registry: config?.registry,
|
|
3285
|
+
timeout: config?.timeout,
|
|
3286
|
+
maxTasks: config?.maxTasks
|
|
3154
3287
|
});
|
|
3155
3288
|
}
|
|
3156
3289
|
runReactive(input = {}, config = {}) {
|
|
@@ -3190,18 +3323,12 @@ class TaskGraph {
|
|
|
3190
3323
|
return this._dag.addEdges(addedEdges);
|
|
3191
3324
|
}
|
|
3192
3325
|
getDataflow(id) {
|
|
3193
|
-
for (const
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
if (maybeEdges !== null) {
|
|
3197
|
-
for (const edge of maybeEdges) {
|
|
3198
|
-
if (this._dag.edgeIdentity(edge, "", "") == id) {
|
|
3199
|
-
return edge;
|
|
3200
|
-
}
|
|
3201
|
-
}
|
|
3202
|
-
}
|
|
3326
|
+
for (const [, , edge] of this._dag.getEdges()) {
|
|
3327
|
+
if (edge.id === id) {
|
|
3328
|
+
return edge;
|
|
3203
3329
|
}
|
|
3204
3330
|
}
|
|
3331
|
+
return;
|
|
3205
3332
|
}
|
|
3206
3333
|
getDataflows() {
|
|
3207
3334
|
return this._dag.getEdges().map((edge) => edge[2]);
|
|
@@ -3389,9 +3516,9 @@ class TaskGraph {
|
|
|
3389
3516
|
emit(name, ...args) {
|
|
3390
3517
|
const dagEvent = EventTaskGraphToDagMapping[name];
|
|
3391
3518
|
if (dagEvent) {
|
|
3392
|
-
return this.emit_dag(name, ...args);
|
|
3519
|
+
return this.emit_dag.call(this, name, ...args);
|
|
3393
3520
|
} else {
|
|
3394
|
-
return this.emit_local(name, ...args);
|
|
3521
|
+
return this.emit_local.call(this, name, ...args);
|
|
3395
3522
|
}
|
|
3396
3523
|
}
|
|
3397
3524
|
emit_local(name, ...args) {
|
|
@@ -3418,7 +3545,7 @@ function serialGraph(tasks, inputHandle, outputHandle) {
|
|
|
3418
3545
|
// src/task-graph/Workflow.ts
|
|
3419
3546
|
import {
|
|
3420
3547
|
EventEmitter as EventEmitter5,
|
|
3421
|
-
getLogger as
|
|
3548
|
+
getLogger as getLogger5,
|
|
3422
3549
|
uuid4 as uuid45
|
|
3423
3550
|
} from "@workglow/util";
|
|
3424
3551
|
function getLastTask(workflow) {
|
|
@@ -3566,7 +3693,7 @@ class Workflow {
|
|
|
3566
3693
|
const taskSchema = task.inputSchema();
|
|
3567
3694
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
3568
3695
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3569
|
-
|
|
3696
|
+
getLogger5().error(this._error);
|
|
3570
3697
|
return;
|
|
3571
3698
|
}
|
|
3572
3699
|
dataflow.targetTaskId = task.id;
|
|
@@ -3591,10 +3718,10 @@ class Workflow {
|
|
|
3591
3718
|
if (result.error) {
|
|
3592
3719
|
if (this.isLoopBuilder) {
|
|
3593
3720
|
this._error = result.error;
|
|
3594
|
-
|
|
3721
|
+
getLogger5().warn(this._error);
|
|
3595
3722
|
} else {
|
|
3596
3723
|
this._error = result.error + " Task not added.";
|
|
3597
|
-
|
|
3724
|
+
getLogger5().error(this._error);
|
|
3598
3725
|
this.graph.removeTask(task.id);
|
|
3599
3726
|
}
|
|
3600
3727
|
}
|
|
@@ -3682,7 +3809,7 @@ class Workflow {
|
|
|
3682
3809
|
const nodes = this._graph.getTasks();
|
|
3683
3810
|
if (nodes.length === 0) {
|
|
3684
3811
|
this._error = "No tasks to remove";
|
|
3685
|
-
|
|
3812
|
+
getLogger5().error(this._error);
|
|
3686
3813
|
return this;
|
|
3687
3814
|
}
|
|
3688
3815
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -3713,7 +3840,7 @@ class Workflow {
|
|
|
3713
3840
|
if (-index > nodes.length) {
|
|
3714
3841
|
const errorMsg = `Back index greater than number of tasks`;
|
|
3715
3842
|
this._error = errorMsg;
|
|
3716
|
-
|
|
3843
|
+
getLogger5().error(this._error);
|
|
3717
3844
|
throw new WorkflowError(errorMsg);
|
|
3718
3845
|
}
|
|
3719
3846
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -3722,13 +3849,13 @@ class Workflow {
|
|
|
3722
3849
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
3723
3850
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
3724
3851
|
this._error = errorMsg;
|
|
3725
|
-
|
|
3852
|
+
getLogger5().error(this._error);
|
|
3726
3853
|
throw new WorkflowError(errorMsg);
|
|
3727
3854
|
}
|
|
3728
3855
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
3729
3856
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
3730
3857
|
this._error = errorMsg;
|
|
3731
|
-
|
|
3858
|
+
getLogger5().error(this._error);
|
|
3732
3859
|
throw new WorkflowError(errorMsg);
|
|
3733
3860
|
}
|
|
3734
3861
|
this._dataFlows.push(new Dataflow(lastNode.id, source, undefined, target));
|
|
@@ -3739,7 +3866,7 @@ class Workflow {
|
|
|
3739
3866
|
const parent = getLastTask(this);
|
|
3740
3867
|
if (!parent) {
|
|
3741
3868
|
this._error = "onError() requires a preceding task in the workflow";
|
|
3742
|
-
|
|
3869
|
+
getLogger5().error(this._error);
|
|
3743
3870
|
throw new WorkflowError(this._error);
|
|
3744
3871
|
}
|
|
3745
3872
|
const handlerTask = ensureTask(handler);
|
|
@@ -3837,7 +3964,7 @@ class Workflow {
|
|
|
3837
3964
|
const taskSchema = task.inputSchema();
|
|
3838
3965
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
3839
3966
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
3840
|
-
|
|
3967
|
+
getLogger5().error(this._error);
|
|
3841
3968
|
return;
|
|
3842
3969
|
}
|
|
3843
3970
|
dataflow.targetTaskId = task.id;
|
|
@@ -3867,7 +3994,7 @@ class Workflow {
|
|
|
3867
3994
|
});
|
|
3868
3995
|
if (result.error) {
|
|
3869
3996
|
this._error = result.error + " Task not added.";
|
|
3870
|
-
|
|
3997
|
+
getLogger5().error(this._error);
|
|
3871
3998
|
this.graph.removeTask(iteratorTask.id);
|
|
3872
3999
|
}
|
|
3873
4000
|
}
|
|
@@ -4713,7 +4840,11 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
4713
4840
|
mapPartialProgress = [];
|
|
4714
4841
|
mapPartialIterationCount = 0;
|
|
4715
4842
|
async executeTask(input) {
|
|
4716
|
-
|
|
4843
|
+
let analysis = this.task.analyzeIterationInput(input);
|
|
4844
|
+
const maxIterations = this.task.config.maxIterations;
|
|
4845
|
+
if (maxIterations !== undefined && maxIterations > 0 && analysis.iterationCount > maxIterations) {
|
|
4846
|
+
analysis = { ...analysis, iterationCount: maxIterations };
|
|
4847
|
+
}
|
|
4717
4848
|
if (analysis.iterationCount === 0) {
|
|
4718
4849
|
const emptyResult = this.task.getEmptyResult();
|
|
4719
4850
|
return this.executeTaskReactive(input, emptyResult);
|
|
@@ -4898,6 +5029,7 @@ var iteratorTaskConfigSchema = {
|
|
|
4898
5029
|
...graphAsTaskConfigSchema["properties"],
|
|
4899
5030
|
concurrencyLimit: { type: "integer", minimum: 1 },
|
|
4900
5031
|
batchSize: { type: "integer", minimum: 1 },
|
|
5032
|
+
maxIterations: { type: "integer", minimum: 1 },
|
|
4901
5033
|
iterationInputConfig: { type: "object", additionalProperties: true }
|
|
4902
5034
|
},
|
|
4903
5035
|
additionalProperties: false
|
|
@@ -4966,22 +5098,29 @@ function extractBaseSchema(schema) {
|
|
|
4966
5098
|
}
|
|
4967
5099
|
const variants = schema.oneOf ?? schema.anyOf;
|
|
4968
5100
|
if (Array.isArray(variants)) {
|
|
5101
|
+
let hasScalar = false;
|
|
5102
|
+
let hasArray = false;
|
|
5103
|
+
let scalarVariant;
|
|
5104
|
+
let arrayVariant;
|
|
4969
5105
|
for (const variant of variants) {
|
|
4970
5106
|
if (typeof variant === "object") {
|
|
4971
|
-
const
|
|
4972
|
-
if (
|
|
4973
|
-
|
|
5107
|
+
const v = variant;
|
|
5108
|
+
if (v.type === "array" || "items" in v) {
|
|
5109
|
+
hasArray = true;
|
|
5110
|
+
arrayVariant = variant;
|
|
5111
|
+
} else {
|
|
5112
|
+
hasScalar = true;
|
|
5113
|
+
scalarVariant = variant;
|
|
4974
5114
|
}
|
|
4975
5115
|
}
|
|
4976
5116
|
}
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
const variantType = variant.type;
|
|
4980
|
-
if (variantType === "array" && variant.items) {
|
|
4981
|
-
return variant.items;
|
|
4982
|
-
}
|
|
4983
|
-
}
|
|
5117
|
+
if (hasScalar && hasArray && variants.length === 2) {
|
|
5118
|
+
return scalarVariant;
|
|
4984
5119
|
}
|
|
5120
|
+
if (!hasScalar && hasArray && arrayVariant) {
|
|
5121
|
+
return arrayVariant.items;
|
|
5122
|
+
}
|
|
5123
|
+
return schema;
|
|
4985
5124
|
}
|
|
4986
5125
|
return schema;
|
|
4987
5126
|
}
|
|
@@ -5422,6 +5561,9 @@ class WhileTask extends GraphAsTask {
|
|
|
5422
5561
|
return WHILE_CONTEXT_SCHEMA;
|
|
5423
5562
|
}
|
|
5424
5563
|
_currentIteration = 0;
|
|
5564
|
+
canSerializeConfig() {
|
|
5565
|
+
return typeof this.config.condition !== "function";
|
|
5566
|
+
}
|
|
5425
5567
|
constructor(input = {}, config = {}) {
|
|
5426
5568
|
super(input, config);
|
|
5427
5569
|
}
|
|
@@ -5543,7 +5685,27 @@ class WhileTask extends GraphAsTask {
|
|
|
5543
5685
|
parentSignal: context.signal
|
|
5544
5686
|
});
|
|
5545
5687
|
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
5546
|
-
|
|
5688
|
+
let shouldContinue;
|
|
5689
|
+
try {
|
|
5690
|
+
shouldContinue = condition(currentOutput, this._currentIteration);
|
|
5691
|
+
} catch (err) {
|
|
5692
|
+
if (err instanceof TaskFailedError) {
|
|
5693
|
+
throw err;
|
|
5694
|
+
}
|
|
5695
|
+
const message = `${this.type}: Condition function threw at iteration ${this._currentIteration}: ${err instanceof Error ? err.message : String(err)}`;
|
|
5696
|
+
const wrappedError = new TaskFailedError(message);
|
|
5697
|
+
if (err instanceof Error && err.stack) {
|
|
5698
|
+
if (wrappedError.stack) {
|
|
5699
|
+
wrappedError.stack += `
|
|
5700
|
+
Caused by original error:
|
|
5701
|
+
${err.stack}`;
|
|
5702
|
+
} else {
|
|
5703
|
+
wrappedError.stack = err.stack;
|
|
5704
|
+
}
|
|
5705
|
+
}
|
|
5706
|
+
throw wrappedError;
|
|
5707
|
+
}
|
|
5708
|
+
if (!shouldContinue) {
|
|
5547
5709
|
break;
|
|
5548
5710
|
}
|
|
5549
5711
|
if (this.chainIterations) {
|
|
@@ -5587,7 +5749,13 @@ class WhileTask extends GraphAsTask {
|
|
|
5587
5749
|
parentSignal: context.signal
|
|
5588
5750
|
});
|
|
5589
5751
|
currentOutput = this.subGraph.mergeExecuteOutputsToRunOutput(results, this.compoundMerge);
|
|
5590
|
-
|
|
5752
|
+
let shouldContinue;
|
|
5753
|
+
try {
|
|
5754
|
+
shouldContinue = condition(currentOutput, this._currentIteration);
|
|
5755
|
+
} catch (err) {
|
|
5756
|
+
throw new TaskFailedError(`${this.type}: Condition function threw at iteration ${this._currentIteration}: ${err instanceof Error ? err.message : String(err)}`);
|
|
5757
|
+
}
|
|
5758
|
+
if (!shouldContinue) {
|
|
5591
5759
|
break;
|
|
5592
5760
|
}
|
|
5593
5761
|
if (this.chainIterations) {
|
|
@@ -5990,205 +6158,6 @@ function getJobQueueFactory() {
|
|
|
5990
6158
|
if (!globalServiceRegistry3.has(JOB_QUEUE_FACTORY)) {
|
|
5991
6159
|
registerJobQueueFactory(defaultJobQueueFactory);
|
|
5992
6160
|
}
|
|
5993
|
-
// src/task/JobQueueTask.ts
|
|
5994
|
-
import { Job as Job2 } from "@workglow/job-queue";
|
|
5995
|
-
|
|
5996
|
-
// src/task/TaskQueueRegistry.ts
|
|
5997
|
-
var taskQueueRegistry = null;
|
|
5998
|
-
|
|
5999
|
-
class TaskQueueRegistry {
|
|
6000
|
-
queues = new Map;
|
|
6001
|
-
registerQueue(queue) {
|
|
6002
|
-
const queueName = queue.server.queueName;
|
|
6003
|
-
if (this.queues.has(queueName)) {
|
|
6004
|
-
throw new Error(`Queue with name ${queueName} already exists`);
|
|
6005
|
-
}
|
|
6006
|
-
this.queues.set(queueName, queue);
|
|
6007
|
-
}
|
|
6008
|
-
getQueue(queueName) {
|
|
6009
|
-
return this.queues.get(queueName);
|
|
6010
|
-
}
|
|
6011
|
-
async startQueues() {
|
|
6012
|
-
for (const queue of this.queues.values()) {
|
|
6013
|
-
await queue.server.start();
|
|
6014
|
-
}
|
|
6015
|
-
}
|
|
6016
|
-
async stopQueues() {
|
|
6017
|
-
for (const queue of this.queues.values()) {
|
|
6018
|
-
await queue.server.stop();
|
|
6019
|
-
}
|
|
6020
|
-
}
|
|
6021
|
-
async clearQueues() {
|
|
6022
|
-
for (const queue of this.queues.values()) {
|
|
6023
|
-
await queue.storage.deleteAll();
|
|
6024
|
-
}
|
|
6025
|
-
}
|
|
6026
|
-
}
|
|
6027
|
-
function getTaskQueueRegistry() {
|
|
6028
|
-
if (!taskQueueRegistry) {
|
|
6029
|
-
taskQueueRegistry = new TaskQueueRegistry;
|
|
6030
|
-
}
|
|
6031
|
-
return taskQueueRegistry;
|
|
6032
|
-
}
|
|
6033
|
-
async function setTaskQueueRegistry(registry) {
|
|
6034
|
-
if (taskQueueRegistry) {
|
|
6035
|
-
await taskQueueRegistry.stopQueues();
|
|
6036
|
-
await taskQueueRegistry.clearQueues();
|
|
6037
|
-
}
|
|
6038
|
-
taskQueueRegistry = registry;
|
|
6039
|
-
}
|
|
6040
|
-
|
|
6041
|
-
// src/task/JobQueueTask.ts
|
|
6042
|
-
var jobQueueTaskConfigSchema = {
|
|
6043
|
-
type: "object",
|
|
6044
|
-
properties: {
|
|
6045
|
-
...graphAsTaskConfigSchema["properties"],
|
|
6046
|
-
queue: {
|
|
6047
|
-
oneOf: [{ type: "boolean" }, { type: "string" }],
|
|
6048
|
-
description: "Queue handling: false=run inline, true=use default, string=explicit queue name",
|
|
6049
|
-
"x-ui-hidden": true
|
|
6050
|
-
}
|
|
6051
|
-
},
|
|
6052
|
-
additionalProperties: false
|
|
6053
|
-
};
|
|
6054
|
-
|
|
6055
|
-
class JobQueueTask extends GraphAsTask {
|
|
6056
|
-
static type = "JobQueueTask";
|
|
6057
|
-
static canRunDirectly = true;
|
|
6058
|
-
static configSchema() {
|
|
6059
|
-
return jobQueueTaskConfigSchema;
|
|
6060
|
-
}
|
|
6061
|
-
currentQueueName;
|
|
6062
|
-
currentJobId;
|
|
6063
|
-
currentRunnerId;
|
|
6064
|
-
jobClass;
|
|
6065
|
-
constructor(input = {}, config = {}) {
|
|
6066
|
-
config.queue ??= true;
|
|
6067
|
-
super(input, config);
|
|
6068
|
-
this.jobClass = Job2;
|
|
6069
|
-
}
|
|
6070
|
-
async execute(input, executeContext) {
|
|
6071
|
-
let cleanup = () => {};
|
|
6072
|
-
try {
|
|
6073
|
-
if (this.config.queue === false && !this.constructor.canRunDirectly) {
|
|
6074
|
-
throw new TaskConfigurationError(`${this.type} cannot run directly without a queue`);
|
|
6075
|
-
}
|
|
6076
|
-
const registeredQueue = await this.resolveQueue(input);
|
|
6077
|
-
if (!registeredQueue) {
|
|
6078
|
-
if (!this.constructor.canRunDirectly) {
|
|
6079
|
-
const queueLabel = typeof this.config.queue === "string" ? this.config.queue : this.currentQueueName ?? this.type;
|
|
6080
|
-
throw new TaskConfigurationError(`Queue ${queueLabel} not found, and ${this.type} cannot run directly`);
|
|
6081
|
-
}
|
|
6082
|
-
this.currentJobId = undefined;
|
|
6083
|
-
const job = await this.createJob(input, this.currentQueueName);
|
|
6084
|
-
cleanup = job.onJobProgress((progress, message, details) => {
|
|
6085
|
-
executeContext.updateProgress(progress, message, details);
|
|
6086
|
-
});
|
|
6087
|
-
const output2 = await job.execute(job.input, {
|
|
6088
|
-
signal: executeContext.signal,
|
|
6089
|
-
updateProgress: executeContext.updateProgress.bind(this)
|
|
6090
|
-
});
|
|
6091
|
-
return output2;
|
|
6092
|
-
}
|
|
6093
|
-
const { client } = registeredQueue;
|
|
6094
|
-
const jobInput = await this.getJobInput(input);
|
|
6095
|
-
const handle = await client.submit(jobInput, {
|
|
6096
|
-
jobRunId: this.currentRunnerId,
|
|
6097
|
-
maxRetries: 10
|
|
6098
|
-
});
|
|
6099
|
-
this.currentJobId = handle.id;
|
|
6100
|
-
this.currentQueueName = client.queueName;
|
|
6101
|
-
cleanup = handle.onProgress((progress, message, details) => {
|
|
6102
|
-
executeContext.updateProgress(progress, message, details);
|
|
6103
|
-
});
|
|
6104
|
-
const output = await handle.waitFor();
|
|
6105
|
-
if (output === undefined) {
|
|
6106
|
-
throw new TaskConfigurationError("Job disabled, should not happen");
|
|
6107
|
-
}
|
|
6108
|
-
return output;
|
|
6109
|
-
} catch (err) {
|
|
6110
|
-
throw new JobTaskFailedError(err);
|
|
6111
|
-
} finally {
|
|
6112
|
-
cleanup();
|
|
6113
|
-
}
|
|
6114
|
-
}
|
|
6115
|
-
async getJobInput(input) {
|
|
6116
|
-
return input;
|
|
6117
|
-
}
|
|
6118
|
-
async createJob(input, queueName) {
|
|
6119
|
-
return new this.jobClass({
|
|
6120
|
-
queueName: queueName ?? this.currentQueueName,
|
|
6121
|
-
jobRunId: this.currentRunnerId,
|
|
6122
|
-
input
|
|
6123
|
-
});
|
|
6124
|
-
}
|
|
6125
|
-
async resolveQueue(input) {
|
|
6126
|
-
const preference = this.config.queue ?? true;
|
|
6127
|
-
if (preference === false) {
|
|
6128
|
-
this.currentQueueName = undefined;
|
|
6129
|
-
return;
|
|
6130
|
-
}
|
|
6131
|
-
if (typeof preference === "string") {
|
|
6132
|
-
const registeredQueue2 = getTaskQueueRegistry().getQueue(preference);
|
|
6133
|
-
if (registeredQueue2) {
|
|
6134
|
-
this.currentQueueName = registeredQueue2.server.queueName;
|
|
6135
|
-
return registeredQueue2;
|
|
6136
|
-
}
|
|
6137
|
-
this.currentQueueName = preference;
|
|
6138
|
-
return;
|
|
6139
|
-
}
|
|
6140
|
-
const queueName = await this.getDefaultQueueName(input);
|
|
6141
|
-
if (!queueName) {
|
|
6142
|
-
this.currentQueueName = undefined;
|
|
6143
|
-
return;
|
|
6144
|
-
}
|
|
6145
|
-
this.currentQueueName = queueName;
|
|
6146
|
-
let registeredQueue = getTaskQueueRegistry().getQueue(queueName);
|
|
6147
|
-
if (!registeredQueue) {
|
|
6148
|
-
registeredQueue = await this.createAndRegisterQueue(queueName, input);
|
|
6149
|
-
await registeredQueue.server.start();
|
|
6150
|
-
}
|
|
6151
|
-
return registeredQueue;
|
|
6152
|
-
}
|
|
6153
|
-
async getDefaultQueueName(_input) {
|
|
6154
|
-
return this.type;
|
|
6155
|
-
}
|
|
6156
|
-
async createAndRegisterQueue(queueName, input) {
|
|
6157
|
-
const factory = getJobQueueFactory();
|
|
6158
|
-
let registeredQueue = await factory({
|
|
6159
|
-
queueName,
|
|
6160
|
-
jobClass: this.jobClass,
|
|
6161
|
-
input,
|
|
6162
|
-
config: this.config,
|
|
6163
|
-
task: this
|
|
6164
|
-
});
|
|
6165
|
-
const registry = getTaskQueueRegistry();
|
|
6166
|
-
try {
|
|
6167
|
-
registry.registerQueue(registeredQueue);
|
|
6168
|
-
} catch (err) {
|
|
6169
|
-
if (err instanceof Error && err.message.includes("already exists")) {
|
|
6170
|
-
const existing = registry.getQueue(queueName);
|
|
6171
|
-
if (existing) {
|
|
6172
|
-
registeredQueue = existing;
|
|
6173
|
-
}
|
|
6174
|
-
} else {
|
|
6175
|
-
throw err;
|
|
6176
|
-
}
|
|
6177
|
-
}
|
|
6178
|
-
return registeredQueue;
|
|
6179
|
-
}
|
|
6180
|
-
abort() {
|
|
6181
|
-
if (this.currentQueueName && this.currentJobId) {
|
|
6182
|
-
const registeredQueue = getTaskQueueRegistry().getQueue(this.currentQueueName);
|
|
6183
|
-
if (registeredQueue) {
|
|
6184
|
-
registeredQueue.client.abort(this.currentJobId).catch((err) => {
|
|
6185
|
-
console.warn(`Failed to abort remote job ${this.currentJobId}`, err);
|
|
6186
|
-
});
|
|
6187
|
-
}
|
|
6188
|
-
}
|
|
6189
|
-
super.abort();
|
|
6190
|
-
}
|
|
6191
|
-
}
|
|
6192
6161
|
// src/task/MapTask.ts
|
|
6193
6162
|
var mapTaskConfigSchema = {
|
|
6194
6163
|
type: "object",
|
|
@@ -6409,13 +6378,19 @@ function resolveTaskFromRegistry(id, _format, registry) {
|
|
|
6409
6378
|
registerInputResolver("tasks", resolveTaskFromRegistry);
|
|
6410
6379
|
|
|
6411
6380
|
// src/task/TaskJSON.ts
|
|
6412
|
-
var createSingleTaskFromJSON = (item, registry) => {
|
|
6381
|
+
var createSingleTaskFromJSON = (item, registry, options) => {
|
|
6413
6382
|
if (!item.id)
|
|
6414
6383
|
throw new TaskJSONError("Task id required");
|
|
6415
6384
|
if (!item.type)
|
|
6416
6385
|
throw new TaskJSONError("Task type required");
|
|
6417
6386
|
if (item.defaults && Array.isArray(item.defaults))
|
|
6418
6387
|
throw new TaskJSONError("Task defaults must be an object");
|
|
6388
|
+
if (options?.allowedTypes) {
|
|
6389
|
+
const allowed = options.allowedTypes instanceof Set ? options.allowedTypes : new Set(options.allowedTypes);
|
|
6390
|
+
if (!allowed.has(item.type)) {
|
|
6391
|
+
throw new TaskJSONError(`Task type "${item.type}" is not in the allowed types list`);
|
|
6392
|
+
}
|
|
6393
|
+
}
|
|
6419
6394
|
const constructors = getTaskConstructors(registry);
|
|
6420
6395
|
const taskClass = constructors.get(item.type);
|
|
6421
6396
|
if (!taskClass)
|
|
@@ -6427,43 +6402,90 @@ var createSingleTaskFromJSON = (item, registry) => {
|
|
|
6427
6402
|
const task = new taskClass(item.defaults ?? {}, taskConfig, registry ? { registry } : {});
|
|
6428
6403
|
return task;
|
|
6429
6404
|
};
|
|
6430
|
-
var createTaskFromDependencyJSON = (item, registry) => {
|
|
6431
|
-
const task = createSingleTaskFromJSON(item, registry);
|
|
6405
|
+
var createTaskFromDependencyJSON = (item, registry, options) => {
|
|
6406
|
+
const task = createSingleTaskFromJSON(item, registry, options);
|
|
6432
6407
|
if (item.subtasks && item.subtasks.length > 0) {
|
|
6433
6408
|
if (!(task instanceof GraphAsTask)) {
|
|
6434
6409
|
throw new TaskConfigurationError("Subgraph is only supported for CompoundTasks");
|
|
6435
6410
|
}
|
|
6436
|
-
task.subGraph = createGraphFromDependencyJSON(item.subtasks, registry);
|
|
6411
|
+
task.subGraph = createGraphFromDependencyJSON(item.subtasks, registry, options);
|
|
6437
6412
|
}
|
|
6438
6413
|
return task;
|
|
6439
6414
|
};
|
|
6440
|
-
var createGraphFromDependencyJSON = (jsonItems, registry) => {
|
|
6415
|
+
var createGraphFromDependencyJSON = (jsonItems, registry, options) => {
|
|
6441
6416
|
const subGraph = new TaskGraph;
|
|
6442
6417
|
for (const subitem of jsonItems) {
|
|
6443
|
-
subGraph.addTask(createTaskFromDependencyJSON(subitem, registry));
|
|
6418
|
+
subGraph.addTask(createTaskFromDependencyJSON(subitem, registry, options));
|
|
6444
6419
|
}
|
|
6445
6420
|
return subGraph;
|
|
6446
6421
|
};
|
|
6447
|
-
var createTaskFromGraphJSON = (item, registry) => {
|
|
6448
|
-
const task = createSingleTaskFromJSON(item, registry);
|
|
6422
|
+
var createTaskFromGraphJSON = (item, registry, options) => {
|
|
6423
|
+
const task = createSingleTaskFromJSON(item, registry, options);
|
|
6449
6424
|
if (item.subgraph) {
|
|
6450
6425
|
if (!(task instanceof GraphAsTask)) {
|
|
6451
6426
|
throw new TaskConfigurationError("Subgraph is only supported for GraphAsTask");
|
|
6452
6427
|
}
|
|
6453
|
-
task.subGraph = createGraphFromGraphJSON(item.subgraph, registry);
|
|
6428
|
+
task.subGraph = createGraphFromGraphJSON(item.subgraph, registry, options);
|
|
6454
6429
|
}
|
|
6455
6430
|
return task;
|
|
6456
6431
|
};
|
|
6457
|
-
var createGraphFromGraphJSON = (graphJsonObj, registry) => {
|
|
6432
|
+
var createGraphFromGraphJSON = (graphJsonObj, registry, options) => {
|
|
6458
6433
|
const subGraph = new TaskGraph;
|
|
6459
6434
|
for (const subitem of graphJsonObj.tasks) {
|
|
6460
|
-
subGraph.addTask(createTaskFromGraphJSON(subitem, registry));
|
|
6435
|
+
subGraph.addTask(createTaskFromGraphJSON(subitem, registry, options));
|
|
6461
6436
|
}
|
|
6462
6437
|
for (const subitem of graphJsonObj.dataflows) {
|
|
6463
6438
|
subGraph.addDataflow(new Dataflow(subitem.sourceTaskId, subitem.sourceTaskPortId, subitem.targetTaskId, subitem.targetTaskPortId));
|
|
6464
6439
|
}
|
|
6465
6440
|
return subGraph;
|
|
6466
6441
|
};
|
|
6442
|
+
// src/task/TaskQueueRegistry.ts
|
|
6443
|
+
import { EventEmitter as EventEmitter6 } from "@workglow/util";
|
|
6444
|
+
var taskQueueRegistry = null;
|
|
6445
|
+
|
|
6446
|
+
class TaskQueueRegistry {
|
|
6447
|
+
emitter = new EventEmitter6;
|
|
6448
|
+
queues = new Map;
|
|
6449
|
+
registerQueue(queue) {
|
|
6450
|
+
const queueName = queue.server.queueName;
|
|
6451
|
+
if (this.queues.has(queueName)) {
|
|
6452
|
+
throw new Error(`Queue with name ${queueName} already exists`);
|
|
6453
|
+
}
|
|
6454
|
+
this.queues.set(queueName, queue);
|
|
6455
|
+
this.emitter.emit("queue_registered", queueName);
|
|
6456
|
+
}
|
|
6457
|
+
getQueue(queueName) {
|
|
6458
|
+
return this.queues.get(queueName);
|
|
6459
|
+
}
|
|
6460
|
+
async startQueues() {
|
|
6461
|
+
for (const queue of this.queues.values()) {
|
|
6462
|
+
await queue.server.start();
|
|
6463
|
+
}
|
|
6464
|
+
}
|
|
6465
|
+
async stopQueues() {
|
|
6466
|
+
for (const queue of this.queues.values()) {
|
|
6467
|
+
await queue.server.stop();
|
|
6468
|
+
}
|
|
6469
|
+
}
|
|
6470
|
+
async clearQueues() {
|
|
6471
|
+
for (const queue of this.queues.values()) {
|
|
6472
|
+
await queue.storage.deleteAll();
|
|
6473
|
+
}
|
|
6474
|
+
}
|
|
6475
|
+
}
|
|
6476
|
+
function getTaskQueueRegistry() {
|
|
6477
|
+
if (!taskQueueRegistry) {
|
|
6478
|
+
taskQueueRegistry = new TaskQueueRegistry;
|
|
6479
|
+
}
|
|
6480
|
+
return taskQueueRegistry;
|
|
6481
|
+
}
|
|
6482
|
+
async function setTaskQueueRegistry(registry) {
|
|
6483
|
+
if (taskQueueRegistry) {
|
|
6484
|
+
await taskQueueRegistry.stopQueues();
|
|
6485
|
+
await taskQueueRegistry.clearQueues();
|
|
6486
|
+
}
|
|
6487
|
+
taskQueueRegistry = registry;
|
|
6488
|
+
}
|
|
6467
6489
|
// src/task/index.ts
|
|
6468
6490
|
var registerBaseTasks = () => {
|
|
6469
6491
|
const tasks = [GraphAsTask, ConditionalTask, FallbackTask, MapTask, WhileTask, ReduceTask];
|
|
@@ -6471,14 +6493,14 @@ var registerBaseTasks = () => {
|
|
|
6471
6493
|
return tasks;
|
|
6472
6494
|
};
|
|
6473
6495
|
// src/storage/TaskGraphRepository.ts
|
|
6474
|
-
import { createServiceToken as createServiceToken4, EventEmitter as
|
|
6496
|
+
import { createServiceToken as createServiceToken4, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
6475
6497
|
var TASK_GRAPH_REPOSITORY = createServiceToken4("taskgraph.taskGraphRepository");
|
|
6476
6498
|
|
|
6477
6499
|
class TaskGraphRepository {
|
|
6478
6500
|
type = "TaskGraphRepository";
|
|
6479
6501
|
get events() {
|
|
6480
6502
|
if (!this._events) {
|
|
6481
|
-
this._events = new
|
|
6503
|
+
this._events = new EventEmitter7;
|
|
6482
6504
|
}
|
|
6483
6505
|
return this._events;
|
|
6484
6506
|
}
|
|
@@ -6636,6 +6658,7 @@ export {
|
|
|
6636
6658
|
setTaskQueueRegistry,
|
|
6637
6659
|
setGlobalTaskConstructors,
|
|
6638
6660
|
serialGraph,
|
|
6661
|
+
schemaHasFormatAnnotations,
|
|
6639
6662
|
schemaAcceptsArray,
|
|
6640
6663
|
resolveSchemaInputs,
|
|
6641
6664
|
resetMethodNameCache,
|
|
@@ -6647,7 +6670,6 @@ export {
|
|
|
6647
6670
|
parallel,
|
|
6648
6671
|
mergeChainedOutputToInput,
|
|
6649
6672
|
mapTaskConfigSchema,
|
|
6650
|
-
jobQueueTaskConfigSchema,
|
|
6651
6673
|
iteratorTaskConfigSchema,
|
|
6652
6674
|
isTaskStreamable,
|
|
6653
6675
|
isStrictArraySchema,
|
|
@@ -6704,6 +6726,7 @@ export {
|
|
|
6704
6726
|
WHILE_CONTEXT_SCHEMA,
|
|
6705
6727
|
TaskTimeoutError,
|
|
6706
6728
|
TaskStatus,
|
|
6729
|
+
TaskSerializationError,
|
|
6707
6730
|
TaskRegistry,
|
|
6708
6731
|
TaskQueueRegistry,
|
|
6709
6732
|
TaskOutputTabularRepository,
|
|
@@ -6712,6 +6735,7 @@ export {
|
|
|
6712
6735
|
TaskOutputPrimaryKeyNames,
|
|
6713
6736
|
TaskJSONError,
|
|
6714
6737
|
TaskInvalidInputError,
|
|
6738
|
+
TaskGraphTimeoutError,
|
|
6715
6739
|
TaskGraphTabularRepository,
|
|
6716
6740
|
TaskGraphSchema,
|
|
6717
6741
|
TaskGraphRunner,
|
|
@@ -6731,7 +6755,6 @@ export {
|
|
|
6731
6755
|
PROPERTY_ARRAY,
|
|
6732
6756
|
MapTask,
|
|
6733
6757
|
JobTaskFailedError,
|
|
6734
|
-
JobQueueTask,
|
|
6735
6758
|
JOB_QUEUE_FACTORY,
|
|
6736
6759
|
IteratorTaskRunner,
|
|
6737
6760
|
IteratorTask,
|
|
@@ -6754,4 +6777,4 @@ export {
|
|
|
6754
6777
|
ConditionalTask
|
|
6755
6778
|
};
|
|
6756
6779
|
|
|
6757
|
-
//# debugId=
|
|
6780
|
+
//# debugId=2758F4534C10DA3764756E2164756E21
|