@workglow/task-graph 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/browser.js +212 -102
- package/dist/browser.js.map +16 -13
- package/dist/bun.js +212 -102
- package/dist/bun.js.map +16 -13
- package/dist/common.d.ts +3 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +212 -102
- package/dist/node.js.map +16 -13
- package/dist/refcountable.d.ts +29 -0
- package/dist/refcountable.d.ts.map +1 -0
- package/dist/storage/PortCodecRegistry.d.ts +8 -0
- package/dist/storage/PortCodecRegistry.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +2 -2
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/GraphAsTaskRunner.d.ts +5 -5
- package/dist/task/GraphAsTaskRunner.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +3 -3
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/ITaskRunner.d.ts +2 -2
- package/dist/task/ITaskRunner.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +4 -3
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/Task.d.ts +8 -9
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +9 -9
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/WhileTaskRunner.d.ts +2 -2
- package/dist/task/WhileTaskRunner.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +1 -1
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +3 -3
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +13 -13
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +132 -74
- package/src/task/README.md +5 -4
- package/src/task-graph/README.md +1 -1
- package/dist/__tests__/public-exports.test.d.ts +0 -7
- package/dist/__tests__/public-exports.test.d.ts.map +0 -1
package/dist/node.js
CHANGED
|
@@ -584,6 +584,10 @@ function computeGraphEntitlements(graph, options) {
|
|
|
584
584
|
}
|
|
585
585
|
// src/task/InputResolver.ts
|
|
586
586
|
import { getInputResolvers } from "@workglow/util";
|
|
587
|
+
function isPlainObject(value) {
|
|
588
|
+
const proto = Object.getPrototypeOf(value);
|
|
589
|
+
return proto === Object.prototype || proto === null;
|
|
590
|
+
}
|
|
587
591
|
function getSchemaFormat(schema, visited = new WeakSet) {
|
|
588
592
|
if (typeof schema !== "object" || schema === null)
|
|
589
593
|
return;
|
|
@@ -671,6 +675,7 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
|
|
|
671
675
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
672
676
|
let value = resolved[key];
|
|
673
677
|
const format = getSchemaFormat(propSchema);
|
|
678
|
+
let phase1Transformed = false;
|
|
674
679
|
if (format) {
|
|
675
680
|
let resolver = resolvers.get(format);
|
|
676
681
|
if (!resolver) {
|
|
@@ -681,14 +686,18 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
|
|
|
681
686
|
if (typeof value === "string") {
|
|
682
687
|
value = await resolver(value, format, config.registry);
|
|
683
688
|
resolved[key] = value;
|
|
689
|
+
phase1Transformed = true;
|
|
684
690
|
} else if (Array.isArray(value) && value.some((item) => typeof item === "string")) {
|
|
685
691
|
const results = await Promise.all(value.map((item) => typeof item === "string" ? resolver(item, format, config.registry) : item));
|
|
686
692
|
value = results.filter((result) => result !== undefined);
|
|
687
693
|
resolved[key] = value;
|
|
694
|
+
phase1Transformed = true;
|
|
688
695
|
}
|
|
689
696
|
}
|
|
690
697
|
}
|
|
691
|
-
|
|
698
|
+
const hasFormatResolver = format ? !!(resolvers.get(format) ?? resolvers.get(getFormatPrefix(format))) : false;
|
|
699
|
+
const skipPhase2 = hasFormatResolver && !phase1Transformed;
|
|
700
|
+
if (!skipPhase2 && value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value) && isPlainObject(value)) {
|
|
692
701
|
const objectSchema = getObjectSchema(propSchema);
|
|
693
702
|
if (objectSchema && !visited.has(objectSchema)) {
|
|
694
703
|
visited.add(objectSchema);
|
|
@@ -1134,6 +1143,31 @@ import {
|
|
|
1134
1143
|
uuid4 as uuid43
|
|
1135
1144
|
} from "@workglow/util";
|
|
1136
1145
|
|
|
1146
|
+
// src/refcountable.ts
|
|
1147
|
+
var GLOBAL_KEY = Symbol.for("@workglow/task-graph/refcountable.predicates");
|
|
1148
|
+
var _g = globalThis;
|
|
1149
|
+
if (!Array.isArray(_g[GLOBAL_KEY])) {
|
|
1150
|
+
_g[GLOBAL_KEY] = [];
|
|
1151
|
+
}
|
|
1152
|
+
var predicates = _g[GLOBAL_KEY];
|
|
1153
|
+
function registerRefcountablePredicate(p) {
|
|
1154
|
+
predicates.push(p);
|
|
1155
|
+
}
|
|
1156
|
+
function asRefcountable(v) {
|
|
1157
|
+
if (v === null || v === undefined)
|
|
1158
|
+
return null;
|
|
1159
|
+
if (typeof v !== "object")
|
|
1160
|
+
return null;
|
|
1161
|
+
for (const p of predicates) {
|
|
1162
|
+
if (p(v))
|
|
1163
|
+
return v;
|
|
1164
|
+
}
|
|
1165
|
+
return null;
|
|
1166
|
+
}
|
|
1167
|
+
function _resetRefcountablePredicatesForTests() {
|
|
1168
|
+
predicates.length = 0;
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1137
1171
|
// src/storage/TaskOutputRepository.ts
|
|
1138
1172
|
import { createServiceToken as createServiceToken2, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1139
1173
|
var TASK_OUTPUT_REPOSITORY = createServiceToken2("taskgraph.taskOutputRepository");
|
|
@@ -1245,6 +1279,7 @@ import {
|
|
|
1245
1279
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1246
1280
|
SpanStatusCode
|
|
1247
1281
|
} from "@workglow/util";
|
|
1282
|
+
import { getPortCodec } from "@workglow/util";
|
|
1248
1283
|
|
|
1249
1284
|
// src/task/StreamTypes.ts
|
|
1250
1285
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1348,13 +1383,49 @@ function hasStructuredOutput(schema) {
|
|
|
1348
1383
|
}
|
|
1349
1384
|
|
|
1350
1385
|
// src/task/TaskRunner.ts
|
|
1386
|
+
async function serializeOutputPorts(output, schema) {
|
|
1387
|
+
if (!schema?.properties)
|
|
1388
|
+
return output;
|
|
1389
|
+
const out = { ...output };
|
|
1390
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1391
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1392
|
+
if (codec && out[key] !== undefined) {
|
|
1393
|
+
out[key] = await codec.serialize(out[key]);
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
return out;
|
|
1397
|
+
}
|
|
1398
|
+
async function deserializeOutputPorts(output, schema) {
|
|
1399
|
+
if (!schema?.properties)
|
|
1400
|
+
return output;
|
|
1401
|
+
const out = { ...output };
|
|
1402
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1403
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1404
|
+
if (codec && out[key] !== undefined) {
|
|
1405
|
+
out[key] = await codec.deserialize(out[key]);
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
return out;
|
|
1409
|
+
}
|
|
1410
|
+
async function normalizeInputsForCacheKey(inputs, schema) {
|
|
1411
|
+
if (!schema?.properties)
|
|
1412
|
+
return inputs;
|
|
1413
|
+
const out = { ...inputs };
|
|
1414
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1415
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1416
|
+
if (codec && out[key] !== undefined) {
|
|
1417
|
+
out[key] = await codec.serialize(out[key]);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
return out;
|
|
1421
|
+
}
|
|
1351
1422
|
function hasRunConfig(i) {
|
|
1352
1423
|
return i !== null && typeof i === "object" && "runConfig" in i;
|
|
1353
1424
|
}
|
|
1354
1425
|
|
|
1355
1426
|
class TaskRunner {
|
|
1356
1427
|
running = false;
|
|
1357
|
-
|
|
1428
|
+
previewRunning = false;
|
|
1358
1429
|
task;
|
|
1359
1430
|
abortController;
|
|
1360
1431
|
outputCache;
|
|
@@ -1372,6 +1443,10 @@ class TaskRunner {
|
|
|
1372
1443
|
}
|
|
1373
1444
|
async run(overrides = {}, config = {}) {
|
|
1374
1445
|
await this.handleStart(config);
|
|
1446
|
+
const proto = Object.getPrototypeOf(this.task);
|
|
1447
|
+
if (proto.execute === Task.prototype.execute && typeof proto.executeStream !== "function" && proto.executePreview !== Task.prototype.executePreview) {
|
|
1448
|
+
throw new TaskConfigurationError(`Task "${this.task.type}" implements only executePreview() and cannot be run via run(). ` + `After the run/runPreview split, run() requires execute() (or executeStream()). ` + `See docs/technical/02-dual-mode-execution.md.`);
|
|
1449
|
+
}
|
|
1375
1450
|
try {
|
|
1376
1451
|
this.task.setInput(overrides);
|
|
1377
1452
|
const configSchema = this.task.constructor.configSchema();
|
|
@@ -1399,18 +1474,21 @@ class TaskRunner {
|
|
|
1399
1474
|
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1400
1475
|
}
|
|
1401
1476
|
}
|
|
1477
|
+
const inputSchema = this.task.constructor.inputSchema();
|
|
1478
|
+
const outputSchema = this.task.constructor.outputSchema();
|
|
1479
|
+
const inputsForKey = this.outputCache ? await normalizeInputsForCacheKey(inputs, inputSchema) : inputs;
|
|
1402
1480
|
if (this.task.cacheable) {
|
|
1403
|
-
|
|
1404
|
-
if (
|
|
1481
|
+
const cached = await this.outputCache?.getOutput(this.task.type, inputsForKey);
|
|
1482
|
+
if (cached !== undefined) {
|
|
1483
|
+
outputs = await deserializeOutputPorts(cached, outputSchema);
|
|
1405
1484
|
this.telemetrySpan?.addEvent("workglow.task.cache_hit");
|
|
1406
1485
|
if (isStreamable) {
|
|
1407
1486
|
this.task.runOutputData = outputs;
|
|
1408
1487
|
this.task.emit("stream_start");
|
|
1409
1488
|
this.task.emit("stream_chunk", { type: "finish", data: outputs });
|
|
1410
1489
|
this.task.emit("stream_end", outputs);
|
|
1411
|
-
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
1412
1490
|
} else {
|
|
1413
|
-
this.task.runOutputData =
|
|
1491
|
+
this.task.runOutputData = outputs;
|
|
1414
1492
|
}
|
|
1415
1493
|
}
|
|
1416
1494
|
}
|
|
@@ -1421,7 +1499,8 @@ class TaskRunner {
|
|
|
1421
1499
|
outputs = await this.executeTask(inputs);
|
|
1422
1500
|
}
|
|
1423
1501
|
if (this.task.cacheable && outputs !== undefined) {
|
|
1424
|
-
await
|
|
1502
|
+
const wireOutputs = await serializeOutputPorts(outputs, outputSchema);
|
|
1503
|
+
await this.outputCache?.saveOutput(this.task.type, inputsForKey, wireOutputs);
|
|
1425
1504
|
}
|
|
1426
1505
|
this.task.runOutputData = outputs ?? {};
|
|
1427
1506
|
}
|
|
@@ -1432,7 +1511,7 @@ class TaskRunner {
|
|
|
1432
1511
|
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
1433
1512
|
}
|
|
1434
1513
|
}
|
|
1435
|
-
async
|
|
1514
|
+
async runPreview(overrides = {}) {
|
|
1436
1515
|
if (this.task.status === TaskStatus.PROCESSING) {
|
|
1437
1516
|
return this.task.runOutputData;
|
|
1438
1517
|
}
|
|
@@ -1445,18 +1524,21 @@ class TaskRunner {
|
|
|
1445
1524
|
}
|
|
1446
1525
|
const schema = this.task.constructor.inputSchema();
|
|
1447
1526
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1448
|
-
await this.
|
|
1527
|
+
await this.handleStartPreview();
|
|
1449
1528
|
try {
|
|
1450
1529
|
const inputs = this.task.runInputData;
|
|
1451
1530
|
const isValid = await this.task.validateInput(inputs);
|
|
1452
1531
|
if (!isValid) {
|
|
1453
1532
|
throw new TaskInvalidInputError("Invalid input data");
|
|
1454
1533
|
}
|
|
1455
|
-
const
|
|
1456
|
-
|
|
1457
|
-
|
|
1534
|
+
const resultPreview = await this.executeTaskPreview(inputs);
|
|
1535
|
+
if (resultPreview !== undefined) {
|
|
1536
|
+
this.task.runOutputData = resultPreview;
|
|
1537
|
+
}
|
|
1538
|
+
await this.handleCompletePreview();
|
|
1458
1539
|
} catch (err) {
|
|
1459
|
-
|
|
1540
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
1541
|
+
await this.handleErrorPreview();
|
|
1460
1542
|
} finally {
|
|
1461
1543
|
return this.task.runOutputData;
|
|
1462
1544
|
}
|
|
@@ -1490,11 +1572,10 @@ class TaskRunner {
|
|
|
1490
1572
|
registry: this.registry,
|
|
1491
1573
|
resourceScope: this.resourceScope
|
|
1492
1574
|
});
|
|
1493
|
-
return
|
|
1575
|
+
return result;
|
|
1494
1576
|
}
|
|
1495
|
-
async
|
|
1496
|
-
|
|
1497
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
1577
|
+
async executeTaskPreview(input) {
|
|
1578
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
1498
1579
|
}
|
|
1499
1580
|
async executeStreamingTask(input) {
|
|
1500
1581
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
@@ -1613,8 +1694,7 @@ class TaskRunner {
|
|
|
1613
1694
|
this.task.runOutputData = finalOutput;
|
|
1614
1695
|
}
|
|
1615
1696
|
this.task.emit("stream_end", this.task.runOutputData);
|
|
1616
|
-
|
|
1617
|
-
return reactiveResult;
|
|
1697
|
+
return this.task.runOutputData;
|
|
1618
1698
|
}
|
|
1619
1699
|
async handleStart(config = {}) {
|
|
1620
1700
|
if (this.task.status === TaskStatus.PROCESSING)
|
|
@@ -1677,8 +1757,8 @@ class TaskRunner {
|
|
|
1677
1757
|
this.task.emit("status", this.task.status);
|
|
1678
1758
|
}
|
|
1679
1759
|
updateProgress = async (_task, _progress, _message, ..._args) => {};
|
|
1680
|
-
async
|
|
1681
|
-
this.
|
|
1760
|
+
async handleStartPreview() {
|
|
1761
|
+
this.previewRunning = true;
|
|
1682
1762
|
}
|
|
1683
1763
|
clearTimeoutTimer() {
|
|
1684
1764
|
if (this.timeoutTimer !== undefined) {
|
|
@@ -1710,8 +1790,8 @@ class TaskRunner {
|
|
|
1710
1790
|
this.task.emit("abort", this.task.error);
|
|
1711
1791
|
this.task.emit("status", this.task.status);
|
|
1712
1792
|
}
|
|
1713
|
-
async
|
|
1714
|
-
this.
|
|
1793
|
+
async handleAbortPreview() {
|
|
1794
|
+
this.previewRunning = false;
|
|
1715
1795
|
}
|
|
1716
1796
|
async handleComplete() {
|
|
1717
1797
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
@@ -1730,8 +1810,8 @@ class TaskRunner {
|
|
|
1730
1810
|
this.task.emit("complete");
|
|
1731
1811
|
this.task.emit("status", this.task.status);
|
|
1732
1812
|
}
|
|
1733
|
-
async
|
|
1734
|
-
this.
|
|
1813
|
+
async handleCompletePreview() {
|
|
1814
|
+
this.previewRunning = false;
|
|
1735
1815
|
}
|
|
1736
1816
|
async handleDisable() {
|
|
1737
1817
|
if (this.task.status === TaskStatus.DISABLED)
|
|
@@ -1778,8 +1858,8 @@ class TaskRunner {
|
|
|
1778
1858
|
this.task.emit("error", this.task.error);
|
|
1779
1859
|
this.task.emit("status", this.task.status);
|
|
1780
1860
|
}
|
|
1781
|
-
async
|
|
1782
|
-
this.
|
|
1861
|
+
async handleErrorPreview() {
|
|
1862
|
+
this.previewRunning = false;
|
|
1783
1863
|
}
|
|
1784
1864
|
async handleProgress(progress, message, ...args) {
|
|
1785
1865
|
this.task.progress = progress;
|
|
@@ -1827,8 +1907,8 @@ class Task {
|
|
|
1827
1907
|
}
|
|
1828
1908
|
return;
|
|
1829
1909
|
}
|
|
1830
|
-
async
|
|
1831
|
-
return
|
|
1910
|
+
async executePreview(_input, _context) {
|
|
1911
|
+
return;
|
|
1832
1912
|
}
|
|
1833
1913
|
_runner;
|
|
1834
1914
|
get runner() {
|
|
@@ -1840,8 +1920,8 @@ class Task {
|
|
|
1840
1920
|
async run(overrides = {}, runConfig = {}) {
|
|
1841
1921
|
return this.runner.run(overrides, { ...this.runConfig, ...runConfig });
|
|
1842
1922
|
}
|
|
1843
|
-
async
|
|
1844
|
-
return this.runner.
|
|
1923
|
+
async runPreview(overrides = {}) {
|
|
1924
|
+
return this.runner.runPreview(overrides);
|
|
1845
1925
|
}
|
|
1846
1926
|
abort() {
|
|
1847
1927
|
this.runner.abort();
|
|
@@ -2802,9 +2882,9 @@ var GRAPH_RESULT_ARRAY = "GRAPH_RESULT_ARRAY";
|
|
|
2802
2882
|
|
|
2803
2883
|
class TaskGraphRunner {
|
|
2804
2884
|
processScheduler;
|
|
2805
|
-
|
|
2885
|
+
previewScheduler;
|
|
2806
2886
|
running = false;
|
|
2807
|
-
|
|
2887
|
+
previewRunning = false;
|
|
2808
2888
|
graph;
|
|
2809
2889
|
outputCache;
|
|
2810
2890
|
accumulateLeafOutputs = true;
|
|
@@ -2818,9 +2898,9 @@ class TaskGraphRunner {
|
|
|
2818
2898
|
graphTimeoutTimer;
|
|
2819
2899
|
pendingGraphTimeoutError;
|
|
2820
2900
|
activeEnforcer;
|
|
2821
|
-
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph),
|
|
2901
|
+
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph), previewScheduler = new TopologicalScheduler(graph)) {
|
|
2822
2902
|
this.processScheduler = processScheduler;
|
|
2823
|
-
this.
|
|
2903
|
+
this.previewScheduler = previewScheduler;
|
|
2824
2904
|
this.graph = graph;
|
|
2825
2905
|
graph.outputCache = outputCache;
|
|
2826
2906
|
this.handleProgress = this.handleProgress.bind(this);
|
|
@@ -2888,16 +2968,16 @@ class TaskGraphRunner {
|
|
|
2888
2968
|
await this.handleComplete();
|
|
2889
2969
|
return this.filterLeafResults(results);
|
|
2890
2970
|
}
|
|
2891
|
-
async
|
|
2892
|
-
await this.
|
|
2971
|
+
async runGraphPreview(input = {}, config) {
|
|
2972
|
+
await this.handleStartPreview(config);
|
|
2893
2973
|
const telemetry = getTelemetryProvider2();
|
|
2894
2974
|
const telemetryEnabled = telemetry.isEnabled;
|
|
2895
|
-
const
|
|
2896
|
-
let
|
|
2975
|
+
const previewRunId = telemetryEnabled ? uuid43() : "";
|
|
2976
|
+
let previewSpan;
|
|
2897
2977
|
if (telemetryEnabled) {
|
|
2898
|
-
|
|
2978
|
+
previewSpan = telemetry.startSpan("workglow.graph.runPreview", {
|
|
2899
2979
|
attributes: {
|
|
2900
|
-
"workglow.graph.
|
|
2980
|
+
"workglow.graph.preview.run_id": previewRunId,
|
|
2901
2981
|
"workglow.graph.task_count": this.graph.getTasks().length,
|
|
2902
2982
|
"workglow.graph.dataflow_count": this.graph.getDataflows().length
|
|
2903
2983
|
}
|
|
@@ -2907,7 +2987,7 @@ class TaskGraphRunner {
|
|
|
2907
2987
|
const taskTimings = [];
|
|
2908
2988
|
const results = [];
|
|
2909
2989
|
try {
|
|
2910
|
-
for await (const task of this.
|
|
2990
|
+
for await (const task of this.previewScheduler.tasks()) {
|
|
2911
2991
|
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
2912
2992
|
if (task.status === TaskStatus.PENDING) {
|
|
2913
2993
|
task.resetInputData();
|
|
@@ -2916,13 +2996,13 @@ class TaskGraphRunner {
|
|
|
2916
2996
|
const taskInput = isRootTask ? input : {};
|
|
2917
2997
|
if (telemetryEnabled) {
|
|
2918
2998
|
const taskType = String(task.constructor.runtype || task.constructor.type || "?");
|
|
2919
|
-
const
|
|
2920
|
-
const taskResult = await task.
|
|
2921
|
-
const
|
|
2999
|
+
const tPreview = performance.now();
|
|
3000
|
+
const taskResult = await task.runPreview(taskInput);
|
|
3001
|
+
const runPreviewMs = performance.now() - tPreview;
|
|
2922
3002
|
const tPush = performance.now();
|
|
2923
3003
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
2924
3004
|
const pushOutputMs = performance.now() - tPush;
|
|
2925
|
-
taskTimings.push({ id: task.id, type: taskType,
|
|
3005
|
+
taskTimings.push({ id: task.id, type: taskType, runPreviewMs, pushOutputMs });
|
|
2926
3006
|
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
2927
3007
|
results.push({
|
|
2928
3008
|
id: task.id,
|
|
@@ -2931,7 +3011,7 @@ class TaskGraphRunner {
|
|
|
2931
3011
|
});
|
|
2932
3012
|
}
|
|
2933
3013
|
} else {
|
|
2934
|
-
const taskResult = await task.
|
|
3014
|
+
const taskResult = await task.runPreview(taskInput);
|
|
2935
3015
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
2936
3016
|
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
2937
3017
|
results.push({
|
|
@@ -2942,35 +3022,35 @@ class TaskGraphRunner {
|
|
|
2942
3022
|
}
|
|
2943
3023
|
}
|
|
2944
3024
|
}
|
|
2945
|
-
await this.
|
|
2946
|
-
if (
|
|
3025
|
+
await this.handleCompletePreview();
|
|
3026
|
+
if (previewSpan) {
|
|
2947
3027
|
const totalMs = performance.now() - t0;
|
|
2948
|
-
|
|
2949
|
-
"workglow.graph.
|
|
2950
|
-
"workglow.graph.
|
|
3028
|
+
previewSpan.setAttributes({
|
|
3029
|
+
"workglow.graph.preview.duration_ms": Math.round(totalMs * 1000) / 1000,
|
|
3030
|
+
"workglow.graph.preview.tasks_executed": taskTimings.length
|
|
2951
3031
|
});
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
getLogger3().debug("task graph
|
|
2955
|
-
|
|
3032
|
+
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3033
|
+
previewSpan.end();
|
|
3034
|
+
getLogger3().debug("task graph runPreview timings", {
|
|
3035
|
+
previewRunId,
|
|
2956
3036
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
2957
3037
|
taskTimings
|
|
2958
3038
|
});
|
|
2959
3039
|
}
|
|
2960
3040
|
return this.filterLeafResults(results);
|
|
2961
3041
|
} catch (error) {
|
|
2962
|
-
await this.
|
|
2963
|
-
if (
|
|
3042
|
+
await this.handleErrorPreview();
|
|
3043
|
+
if (previewSpan) {
|
|
2964
3044
|
const totalMs = performance.now() - t0;
|
|
2965
3045
|
const message = error instanceof Error ? error.message : String(error);
|
|
2966
|
-
|
|
2967
|
-
"workglow.graph.
|
|
2968
|
-
"workglow.graph.
|
|
3046
|
+
previewSpan.setAttributes({
|
|
3047
|
+
"workglow.graph.preview.duration_ms": Math.round(totalMs * 1000) / 1000,
|
|
3048
|
+
"workglow.graph.preview.tasks_executed": taskTimings.length
|
|
2969
3049
|
});
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
getLogger3().debug("task graph
|
|
2973
|
-
|
|
3050
|
+
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3051
|
+
previewSpan.end();
|
|
3052
|
+
getLogger3().debug("task graph runPreview failed", {
|
|
3053
|
+
previewRunId,
|
|
2974
3054
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
2975
3055
|
taskTimings,
|
|
2976
3056
|
error
|
|
@@ -3042,6 +3122,23 @@ class TaskGraphRunner {
|
|
|
3042
3122
|
}
|
|
3043
3123
|
async pushOutputFromNodeToEdges(node, results) {
|
|
3044
3124
|
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
3125
|
+
if (Object.keys(results).length > 0) {
|
|
3126
|
+
const consumerCounts = new Map;
|
|
3127
|
+
for (const dataflow of dataflows) {
|
|
3128
|
+
if (dataflow.stream !== undefined)
|
|
3129
|
+
continue;
|
|
3130
|
+
const port = dataflow.sourceTaskPortId;
|
|
3131
|
+
consumerCounts.set(port, (consumerCounts.get(port) ?? 0) + 1);
|
|
3132
|
+
}
|
|
3133
|
+
for (const [port, count] of consumerCounts) {
|
|
3134
|
+
if (count <= 1)
|
|
3135
|
+
continue;
|
|
3136
|
+
const value = results[port];
|
|
3137
|
+
const ref = asRefcountable(value);
|
|
3138
|
+
if (ref)
|
|
3139
|
+
ref.retain(count - 1);
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3045
3142
|
for (const dataflow of dataflows) {
|
|
3046
3143
|
if (dataflow.stream !== undefined)
|
|
3047
3144
|
continue;
|
|
@@ -3381,7 +3478,7 @@ class TaskGraphRunner {
|
|
|
3381
3478
|
}
|
|
3382
3479
|
this.graph.outputCache = this.outputCache;
|
|
3383
3480
|
}
|
|
3384
|
-
if (this.running || this.
|
|
3481
|
+
if (this.running || this.previewRunning) {
|
|
3385
3482
|
throw new TaskConfigurationError("Graph is already running");
|
|
3386
3483
|
}
|
|
3387
3484
|
this.running = true;
|
|
@@ -3455,9 +3552,9 @@ class TaskGraphRunner {
|
|
|
3455
3552
|
}
|
|
3456
3553
|
this.graph.emit("start");
|
|
3457
3554
|
}
|
|
3458
|
-
async
|
|
3459
|
-
if (this.
|
|
3460
|
-
throw new TaskConfigurationError("Graph is already running
|
|
3555
|
+
async handleStartPreview(config) {
|
|
3556
|
+
if (this.previewRunning) {
|
|
3557
|
+
throw new TaskConfigurationError("Graph is already running in preview");
|
|
3461
3558
|
}
|
|
3462
3559
|
if (config?.registry !== undefined) {
|
|
3463
3560
|
this.registry = config.registry;
|
|
@@ -3468,8 +3565,8 @@ class TaskGraphRunner {
|
|
|
3468
3565
|
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
3469
3566
|
}
|
|
3470
3567
|
}
|
|
3471
|
-
this.
|
|
3472
|
-
this.
|
|
3568
|
+
this.previewScheduler.reset();
|
|
3569
|
+
this.previewRunning = true;
|
|
3473
3570
|
}
|
|
3474
3571
|
clearGraphTimeout() {
|
|
3475
3572
|
if (this.graphTimeoutTimer !== undefined) {
|
|
@@ -3488,8 +3585,8 @@ class TaskGraphRunner {
|
|
|
3488
3585
|
}
|
|
3489
3586
|
this.graph.emit("complete");
|
|
3490
3587
|
}
|
|
3491
|
-
async
|
|
3492
|
-
this.
|
|
3588
|
+
async handleCompletePreview() {
|
|
3589
|
+
this.previewRunning = false;
|
|
3493
3590
|
}
|
|
3494
3591
|
async handleError(error) {
|
|
3495
3592
|
this.clearGraphTimeout();
|
|
@@ -3508,8 +3605,8 @@ class TaskGraphRunner {
|
|
|
3508
3605
|
}
|
|
3509
3606
|
this.graph.emit("error", error);
|
|
3510
3607
|
}
|
|
3511
|
-
async
|
|
3512
|
-
this.
|
|
3608
|
+
async handleErrorPreview() {
|
|
3609
|
+
this.previewRunning = false;
|
|
3513
3610
|
}
|
|
3514
3611
|
async handleAbort() {
|
|
3515
3612
|
this.clearGraphTimeout();
|
|
@@ -3528,8 +3625,8 @@ class TaskGraphRunner {
|
|
|
3528
3625
|
}
|
|
3529
3626
|
this.graph.emit("abort");
|
|
3530
3627
|
}
|
|
3531
|
-
async
|
|
3532
|
-
this.
|
|
3628
|
+
async handleAbortPreview() {
|
|
3629
|
+
this.previewRunning = false;
|
|
3533
3630
|
}
|
|
3534
3631
|
async handleDisable() {
|
|
3535
3632
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
@@ -3572,8 +3669,8 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
3572
3669
|
unsubscribe();
|
|
3573
3670
|
return results;
|
|
3574
3671
|
}
|
|
3575
|
-
async
|
|
3576
|
-
return this.task.subGraph.
|
|
3672
|
+
async executeTaskChildrenPreview() {
|
|
3673
|
+
return this.task.subGraph.runPreview(this.task.runInputData, {
|
|
3577
3674
|
registry: this.registry,
|
|
3578
3675
|
resourceScope: this.resourceScope
|
|
3579
3676
|
});
|
|
@@ -3594,15 +3691,18 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
3594
3691
|
}
|
|
3595
3692
|
return this.task.runOutputData;
|
|
3596
3693
|
}
|
|
3597
|
-
async
|
|
3694
|
+
async executeTaskPreview(input) {
|
|
3598
3695
|
if (this.task.hasChildren()) {
|
|
3599
|
-
const
|
|
3600
|
-
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(
|
|
3696
|
+
const previewResults = await this.executeTaskChildrenPreview();
|
|
3697
|
+
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(previewResults, this.task.compoundMerge);
|
|
3698
|
+
return this.task.runOutputData;
|
|
3601
3699
|
} else {
|
|
3602
|
-
const
|
|
3603
|
-
|
|
3700
|
+
const previewResult = await super.executeTaskPreview(input);
|
|
3701
|
+
if (previewResult !== undefined) {
|
|
3702
|
+
this.task.runOutputData = previewResult;
|
|
3703
|
+
}
|
|
3704
|
+
return this.task.runOutputData;
|
|
3604
3705
|
}
|
|
3605
|
-
return this.task.runOutputData;
|
|
3606
3706
|
}
|
|
3607
3707
|
}
|
|
3608
3708
|
|
|
@@ -4030,8 +4130,8 @@ class TaskGraph {
|
|
|
4030
4130
|
resourceScope: config?.resourceScope
|
|
4031
4131
|
});
|
|
4032
4132
|
}
|
|
4033
|
-
|
|
4034
|
-
return this.runner.
|
|
4133
|
+
runPreview(input = {}, config = {}) {
|
|
4134
|
+
return this.runner.runGraphPreview(input, config);
|
|
4035
4135
|
}
|
|
4036
4136
|
mergeExecuteOutputsToRunOutput(results, compoundMerge) {
|
|
4037
4137
|
return this.runner.mergeExecuteOutputsToRunOutput(results, compoundMerge);
|
|
@@ -5921,9 +6021,8 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5921
6021
|
}
|
|
5922
6022
|
return this.executeTaskFallback(input);
|
|
5923
6023
|
}
|
|
5924
|
-
async
|
|
5925
|
-
|
|
5926
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6024
|
+
async executeTaskPreview(input) {
|
|
6025
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
5927
6026
|
}
|
|
5928
6027
|
async executeTaskFallback(input) {
|
|
5929
6028
|
const tasks = this.task.subGraph.getTasks();
|
|
@@ -5943,7 +6042,7 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5943
6042
|
this.resetTask(alternativeTask);
|
|
5944
6043
|
const result = await alternativeTask.run(input);
|
|
5945
6044
|
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
5946
|
-
return
|
|
6045
|
+
return result;
|
|
5947
6046
|
} catch (error) {
|
|
5948
6047
|
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
5949
6048
|
throw error;
|
|
@@ -5985,7 +6084,7 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5985
6084
|
});
|
|
5986
6085
|
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
5987
6086
|
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
5988
|
-
return
|
|
6087
|
+
return mergedOutput;
|
|
5989
6088
|
} catch (error) {
|
|
5990
6089
|
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
5991
6090
|
throw error;
|
|
@@ -6213,15 +6312,13 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
6213
6312
|
analysis = { ...analysis, iterationCount: maxIterations };
|
|
6214
6313
|
}
|
|
6215
6314
|
if (analysis.iterationCount === 0) {
|
|
6216
|
-
|
|
6217
|
-
return this.executeTaskReactive(input, emptyResult);
|
|
6315
|
+
return this.task.getEmptyResult();
|
|
6218
6316
|
}
|
|
6219
6317
|
const result = this.task.isReduceTask() ? await this.executeReduceIterations(analysis) : await this.executeCollectIterations(analysis);
|
|
6220
|
-
return
|
|
6318
|
+
return result;
|
|
6221
6319
|
}
|
|
6222
|
-
async
|
|
6223
|
-
|
|
6224
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6320
|
+
async executeTaskPreview(input) {
|
|
6321
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
6225
6322
|
}
|
|
6226
6323
|
async executeCollectIterations(analysis) {
|
|
6227
6324
|
const iterationCount = analysis.iterationCount;
|
|
@@ -6888,9 +6985,8 @@ class WhileTaskRunner extends GraphAsTaskRunner {
|
|
|
6888
6985
|
});
|
|
6889
6986
|
return result;
|
|
6890
6987
|
}
|
|
6891
|
-
async
|
|
6892
|
-
|
|
6893
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6988
|
+
async executeTaskPreview(input) {
|
|
6989
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
6894
6990
|
}
|
|
6895
6991
|
}
|
|
6896
6992
|
|
|
@@ -8118,6 +8214,14 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8118
8214
|
this.emit("output_pruned");
|
|
8119
8215
|
}
|
|
8120
8216
|
}
|
|
8217
|
+
// src/storage/PortCodecRegistry.ts
|
|
8218
|
+
import {
|
|
8219
|
+
registerPortCodec,
|
|
8220
|
+
getPortCodec as getPortCodec2,
|
|
8221
|
+
_resetPortCodecsForTests
|
|
8222
|
+
} from "@workglow/util";
|
|
8223
|
+
// src/node.ts
|
|
8224
|
+
registerRefcountablePredicate((v) => !!v && typeof v === "object" && ("backend" in v) && ("retain" in v) && ("release" in v) && ("materialize" in v));
|
|
8121
8225
|
export {
|
|
8122
8226
|
wrapSchemaInArray,
|
|
8123
8227
|
whileTaskConfigSchema,
|
|
@@ -8140,6 +8244,8 @@ export {
|
|
|
8140
8244
|
resolveIterationBound,
|
|
8141
8245
|
resetMethodNameCache,
|
|
8142
8246
|
removeIterationProperties,
|
|
8247
|
+
registerRefcountablePredicate,
|
|
8248
|
+
registerPortCodec,
|
|
8143
8249
|
registerJobQueueFactory,
|
|
8144
8250
|
registerBuiltInTransforms,
|
|
8145
8251
|
registerBaseTasks,
|
|
@@ -8175,6 +8281,7 @@ export {
|
|
|
8175
8281
|
getSchemaFormat,
|
|
8176
8282
|
getProfileGrants,
|
|
8177
8283
|
getPortStreamMode,
|
|
8284
|
+
getPortCodec2 as getPortCodec,
|
|
8178
8285
|
getOutputStreamMode,
|
|
8179
8286
|
getObjectSchema,
|
|
8180
8287
|
getObjectPortId,
|
|
@@ -8221,9 +8328,12 @@ export {
|
|
|
8221
8328
|
calculateNodeDepths,
|
|
8222
8329
|
buildIterationInputSchema,
|
|
8223
8330
|
autoConnect,
|
|
8331
|
+
asRefcountable,
|
|
8224
8332
|
addIterationContextToSchema,
|
|
8225
8333
|
addBoundaryNodesToGraphJson,
|
|
8226
8334
|
addBoundaryNodesToDependencyJson,
|
|
8335
|
+
_resetRefcountablePredicatesForTests,
|
|
8336
|
+
_resetPortCodecsForTests,
|
|
8227
8337
|
WorkflowError,
|
|
8228
8338
|
Workflow,
|
|
8229
8339
|
WhileTaskRunner,
|
|
@@ -8297,4 +8407,4 @@ export {
|
|
|
8297
8407
|
BROWSER_GRANTS
|
|
8298
8408
|
};
|
|
8299
8409
|
|
|
8300
|
-
//# debugId=
|
|
8410
|
+
//# debugId=3044ECF32B8CA32464756E2164756E21
|