@workglow/task-graph 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/browser.js +212 -102
- package/dist/browser.js.map +16 -13
- package/dist/bun.js +212 -102
- package/dist/bun.js.map +16 -13
- package/dist/common.d.ts +3 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +212 -102
- package/dist/node.js.map +16 -13
- package/dist/refcountable.d.ts +29 -0
- package/dist/refcountable.d.ts.map +1 -0
- package/dist/storage/PortCodecRegistry.d.ts +8 -0
- package/dist/storage/PortCodecRegistry.d.ts.map +1 -0
- package/dist/task/FallbackTaskRunner.d.ts +2 -2
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/GraphAsTaskRunner.d.ts +5 -5
- package/dist/task/GraphAsTaskRunner.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +3 -3
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/ITaskRunner.d.ts +2 -2
- package/dist/task/ITaskRunner.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTaskRunner.d.ts +4 -3
- package/dist/task/IteratorTaskRunner.d.ts.map +1 -1
- package/dist/task/Task.d.ts +8 -9
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +9 -9
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/WhileTaskRunner.d.ts +2 -2
- package/dist/task/WhileTaskRunner.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +1 -1
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +3 -3
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +13 -13
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +132 -74
- package/src/task/README.md +5 -4
- package/src/task-graph/README.md +1 -1
- package/dist/__tests__/public-exports.test.d.ts +0 -7
- package/dist/__tests__/public-exports.test.d.ts.map +0 -1
package/dist/bun.js
CHANGED
|
@@ -585,6 +585,10 @@ function computeGraphEntitlements(graph, options) {
|
|
|
585
585
|
}
|
|
586
586
|
// src/task/InputResolver.ts
|
|
587
587
|
import { getInputResolvers } from "@workglow/util";
|
|
588
|
+
function isPlainObject(value) {
|
|
589
|
+
const proto = Object.getPrototypeOf(value);
|
|
590
|
+
return proto === Object.prototype || proto === null;
|
|
591
|
+
}
|
|
588
592
|
function getSchemaFormat(schema, visited = new WeakSet) {
|
|
589
593
|
if (typeof schema !== "object" || schema === null)
|
|
590
594
|
return;
|
|
@@ -672,6 +676,7 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
|
|
|
672
676
|
for (const [key, propSchema] of Object.entries(properties)) {
|
|
673
677
|
let value = resolved[key];
|
|
674
678
|
const format = getSchemaFormat(propSchema);
|
|
679
|
+
let phase1Transformed = false;
|
|
675
680
|
if (format) {
|
|
676
681
|
let resolver = resolvers.get(format);
|
|
677
682
|
if (!resolver) {
|
|
@@ -682,14 +687,18 @@ async function resolveSchemaInputs(input, schema, config, visited = new Set) {
|
|
|
682
687
|
if (typeof value === "string") {
|
|
683
688
|
value = await resolver(value, format, config.registry);
|
|
684
689
|
resolved[key] = value;
|
|
690
|
+
phase1Transformed = true;
|
|
685
691
|
} else if (Array.isArray(value) && value.some((item) => typeof item === "string")) {
|
|
686
692
|
const results = await Promise.all(value.map((item) => typeof item === "string" ? resolver(item, format, config.registry) : item));
|
|
687
693
|
value = results.filter((result) => result !== undefined);
|
|
688
694
|
resolved[key] = value;
|
|
695
|
+
phase1Transformed = true;
|
|
689
696
|
}
|
|
690
697
|
}
|
|
691
698
|
}
|
|
692
|
-
|
|
699
|
+
const hasFormatResolver = format ? !!(resolvers.get(format) ?? resolvers.get(getFormatPrefix(format))) : false;
|
|
700
|
+
const skipPhase2 = hasFormatResolver && !phase1Transformed;
|
|
701
|
+
if (!skipPhase2 && value !== null && value !== undefined && typeof value === "object" && !Array.isArray(value) && isPlainObject(value)) {
|
|
693
702
|
const objectSchema = getObjectSchema(propSchema);
|
|
694
703
|
if (objectSchema && !visited.has(objectSchema)) {
|
|
695
704
|
visited.add(objectSchema);
|
|
@@ -1135,6 +1144,31 @@ import {
|
|
|
1135
1144
|
uuid4 as uuid43
|
|
1136
1145
|
} from "@workglow/util";
|
|
1137
1146
|
|
|
1147
|
+
// src/refcountable.ts
|
|
1148
|
+
var GLOBAL_KEY = Symbol.for("@workglow/task-graph/refcountable.predicates");
|
|
1149
|
+
var _g = globalThis;
|
|
1150
|
+
if (!Array.isArray(_g[GLOBAL_KEY])) {
|
|
1151
|
+
_g[GLOBAL_KEY] = [];
|
|
1152
|
+
}
|
|
1153
|
+
var predicates = _g[GLOBAL_KEY];
|
|
1154
|
+
function registerRefcountablePredicate(p) {
|
|
1155
|
+
predicates.push(p);
|
|
1156
|
+
}
|
|
1157
|
+
function asRefcountable(v) {
|
|
1158
|
+
if (v === null || v === undefined)
|
|
1159
|
+
return null;
|
|
1160
|
+
if (typeof v !== "object")
|
|
1161
|
+
return null;
|
|
1162
|
+
for (const p of predicates) {
|
|
1163
|
+
if (p(v))
|
|
1164
|
+
return v;
|
|
1165
|
+
}
|
|
1166
|
+
return null;
|
|
1167
|
+
}
|
|
1168
|
+
function _resetRefcountablePredicatesForTests() {
|
|
1169
|
+
predicates.length = 0;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1138
1172
|
// src/storage/TaskOutputRepository.ts
|
|
1139
1173
|
import { createServiceToken as createServiceToken2, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1140
1174
|
var TASK_OUTPUT_REPOSITORY = createServiceToken2("taskgraph.taskOutputRepository");
|
|
@@ -1246,6 +1280,7 @@ import {
|
|
|
1246
1280
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1247
1281
|
SpanStatusCode
|
|
1248
1282
|
} from "@workglow/util";
|
|
1283
|
+
import { getPortCodec } from "@workglow/util";
|
|
1249
1284
|
|
|
1250
1285
|
// src/task/StreamTypes.ts
|
|
1251
1286
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1349,13 +1384,49 @@ function hasStructuredOutput(schema) {
|
|
|
1349
1384
|
}
|
|
1350
1385
|
|
|
1351
1386
|
// src/task/TaskRunner.ts
|
|
1387
|
+
async function serializeOutputPorts(output, schema) {
|
|
1388
|
+
if (!schema?.properties)
|
|
1389
|
+
return output;
|
|
1390
|
+
const out = { ...output };
|
|
1391
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1392
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1393
|
+
if (codec && out[key] !== undefined) {
|
|
1394
|
+
out[key] = await codec.serialize(out[key]);
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
return out;
|
|
1398
|
+
}
|
|
1399
|
+
async function deserializeOutputPorts(output, schema) {
|
|
1400
|
+
if (!schema?.properties)
|
|
1401
|
+
return output;
|
|
1402
|
+
const out = { ...output };
|
|
1403
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1404
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1405
|
+
if (codec && out[key] !== undefined) {
|
|
1406
|
+
out[key] = await codec.deserialize(out[key]);
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
return out;
|
|
1410
|
+
}
|
|
1411
|
+
async function normalizeInputsForCacheKey(inputs, schema) {
|
|
1412
|
+
if (!schema?.properties)
|
|
1413
|
+
return inputs;
|
|
1414
|
+
const out = { ...inputs };
|
|
1415
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
1416
|
+
const codec = prop.format ? getPortCodec(prop.format) : undefined;
|
|
1417
|
+
if (codec && out[key] !== undefined) {
|
|
1418
|
+
out[key] = await codec.serialize(out[key]);
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
return out;
|
|
1422
|
+
}
|
|
1352
1423
|
function hasRunConfig(i) {
|
|
1353
1424
|
return i !== null && typeof i === "object" && "runConfig" in i;
|
|
1354
1425
|
}
|
|
1355
1426
|
|
|
1356
1427
|
class TaskRunner {
|
|
1357
1428
|
running = false;
|
|
1358
|
-
|
|
1429
|
+
previewRunning = false;
|
|
1359
1430
|
task;
|
|
1360
1431
|
abortController;
|
|
1361
1432
|
outputCache;
|
|
@@ -1373,6 +1444,10 @@ class TaskRunner {
|
|
|
1373
1444
|
}
|
|
1374
1445
|
async run(overrides = {}, config = {}) {
|
|
1375
1446
|
await this.handleStart(config);
|
|
1447
|
+
const proto = Object.getPrototypeOf(this.task);
|
|
1448
|
+
if (proto.execute === Task.prototype.execute && typeof proto.executeStream !== "function" && proto.executePreview !== Task.prototype.executePreview) {
|
|
1449
|
+
throw new TaskConfigurationError(`Task "${this.task.type}" implements only executePreview() and cannot be run via run(). ` + `After the run/runPreview split, run() requires execute() (or executeStream()). ` + `See docs/technical/02-dual-mode-execution.md.`);
|
|
1450
|
+
}
|
|
1376
1451
|
try {
|
|
1377
1452
|
this.task.setInput(overrides);
|
|
1378
1453
|
const configSchema = this.task.constructor.configSchema();
|
|
@@ -1400,18 +1475,21 @@ class TaskRunner {
|
|
|
1400
1475
|
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1401
1476
|
}
|
|
1402
1477
|
}
|
|
1478
|
+
const inputSchema = this.task.constructor.inputSchema();
|
|
1479
|
+
const outputSchema = this.task.constructor.outputSchema();
|
|
1480
|
+
const inputsForKey = this.outputCache ? await normalizeInputsForCacheKey(inputs, inputSchema) : inputs;
|
|
1403
1481
|
if (this.task.cacheable) {
|
|
1404
|
-
|
|
1405
|
-
if (
|
|
1482
|
+
const cached = await this.outputCache?.getOutput(this.task.type, inputsForKey);
|
|
1483
|
+
if (cached !== undefined) {
|
|
1484
|
+
outputs = await deserializeOutputPorts(cached, outputSchema);
|
|
1406
1485
|
this.telemetrySpan?.addEvent("workglow.task.cache_hit");
|
|
1407
1486
|
if (isStreamable) {
|
|
1408
1487
|
this.task.runOutputData = outputs;
|
|
1409
1488
|
this.task.emit("stream_start");
|
|
1410
1489
|
this.task.emit("stream_chunk", { type: "finish", data: outputs });
|
|
1411
1490
|
this.task.emit("stream_end", outputs);
|
|
1412
|
-
this.task.runOutputData = await this.executeTaskReactive(inputs, outputs);
|
|
1413
1491
|
} else {
|
|
1414
|
-
this.task.runOutputData =
|
|
1492
|
+
this.task.runOutputData = outputs;
|
|
1415
1493
|
}
|
|
1416
1494
|
}
|
|
1417
1495
|
}
|
|
@@ -1422,7 +1500,8 @@ class TaskRunner {
|
|
|
1422
1500
|
outputs = await this.executeTask(inputs);
|
|
1423
1501
|
}
|
|
1424
1502
|
if (this.task.cacheable && outputs !== undefined) {
|
|
1425
|
-
await
|
|
1503
|
+
const wireOutputs = await serializeOutputPorts(outputs, outputSchema);
|
|
1504
|
+
await this.outputCache?.saveOutput(this.task.type, inputsForKey, wireOutputs);
|
|
1426
1505
|
}
|
|
1427
1506
|
this.task.runOutputData = outputs ?? {};
|
|
1428
1507
|
}
|
|
@@ -1433,7 +1512,7 @@ class TaskRunner {
|
|
|
1433
1512
|
throw this.task.error instanceof TaskTimeoutError ? this.task.error : err;
|
|
1434
1513
|
}
|
|
1435
1514
|
}
|
|
1436
|
-
async
|
|
1515
|
+
async runPreview(overrides = {}) {
|
|
1437
1516
|
if (this.task.status === TaskStatus.PROCESSING) {
|
|
1438
1517
|
return this.task.runOutputData;
|
|
1439
1518
|
}
|
|
@@ -1446,18 +1525,21 @@ class TaskRunner {
|
|
|
1446
1525
|
}
|
|
1447
1526
|
const schema = this.task.constructor.inputSchema();
|
|
1448
1527
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
1449
|
-
await this.
|
|
1528
|
+
await this.handleStartPreview();
|
|
1450
1529
|
try {
|
|
1451
1530
|
const inputs = this.task.runInputData;
|
|
1452
1531
|
const isValid = await this.task.validateInput(inputs);
|
|
1453
1532
|
if (!isValid) {
|
|
1454
1533
|
throw new TaskInvalidInputError("Invalid input data");
|
|
1455
1534
|
}
|
|
1456
|
-
const
|
|
1457
|
-
|
|
1458
|
-
|
|
1535
|
+
const resultPreview = await this.executeTaskPreview(inputs);
|
|
1536
|
+
if (resultPreview !== undefined) {
|
|
1537
|
+
this.task.runOutputData = resultPreview;
|
|
1538
|
+
}
|
|
1539
|
+
await this.handleCompletePreview();
|
|
1459
1540
|
} catch (err) {
|
|
1460
|
-
|
|
1541
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
1542
|
+
await this.handleErrorPreview();
|
|
1461
1543
|
} finally {
|
|
1462
1544
|
return this.task.runOutputData;
|
|
1463
1545
|
}
|
|
@@ -1491,11 +1573,10 @@ class TaskRunner {
|
|
|
1491
1573
|
registry: this.registry,
|
|
1492
1574
|
resourceScope: this.resourceScope
|
|
1493
1575
|
});
|
|
1494
|
-
return
|
|
1576
|
+
return result;
|
|
1495
1577
|
}
|
|
1496
|
-
async
|
|
1497
|
-
|
|
1498
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
1578
|
+
async executeTaskPreview(input) {
|
|
1579
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
1499
1580
|
}
|
|
1500
1581
|
async executeStreamingTask(input) {
|
|
1501
1582
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
@@ -1614,8 +1695,7 @@ class TaskRunner {
|
|
|
1614
1695
|
this.task.runOutputData = finalOutput;
|
|
1615
1696
|
}
|
|
1616
1697
|
this.task.emit("stream_end", this.task.runOutputData);
|
|
1617
|
-
|
|
1618
|
-
return reactiveResult;
|
|
1698
|
+
return this.task.runOutputData;
|
|
1619
1699
|
}
|
|
1620
1700
|
async handleStart(config = {}) {
|
|
1621
1701
|
if (this.task.status === TaskStatus.PROCESSING)
|
|
@@ -1678,8 +1758,8 @@ class TaskRunner {
|
|
|
1678
1758
|
this.task.emit("status", this.task.status);
|
|
1679
1759
|
}
|
|
1680
1760
|
updateProgress = async (_task, _progress, _message, ..._args) => {};
|
|
1681
|
-
async
|
|
1682
|
-
this.
|
|
1761
|
+
async handleStartPreview() {
|
|
1762
|
+
this.previewRunning = true;
|
|
1683
1763
|
}
|
|
1684
1764
|
clearTimeoutTimer() {
|
|
1685
1765
|
if (this.timeoutTimer !== undefined) {
|
|
@@ -1711,8 +1791,8 @@ class TaskRunner {
|
|
|
1711
1791
|
this.task.emit("abort", this.task.error);
|
|
1712
1792
|
this.task.emit("status", this.task.status);
|
|
1713
1793
|
}
|
|
1714
|
-
async
|
|
1715
|
-
this.
|
|
1794
|
+
async handleAbortPreview() {
|
|
1795
|
+
this.previewRunning = false;
|
|
1716
1796
|
}
|
|
1717
1797
|
async handleComplete() {
|
|
1718
1798
|
if (this.task.status === TaskStatus.COMPLETED)
|
|
@@ -1731,8 +1811,8 @@ class TaskRunner {
|
|
|
1731
1811
|
this.task.emit("complete");
|
|
1732
1812
|
this.task.emit("status", this.task.status);
|
|
1733
1813
|
}
|
|
1734
|
-
async
|
|
1735
|
-
this.
|
|
1814
|
+
async handleCompletePreview() {
|
|
1815
|
+
this.previewRunning = false;
|
|
1736
1816
|
}
|
|
1737
1817
|
async handleDisable() {
|
|
1738
1818
|
if (this.task.status === TaskStatus.DISABLED)
|
|
@@ -1779,8 +1859,8 @@ class TaskRunner {
|
|
|
1779
1859
|
this.task.emit("error", this.task.error);
|
|
1780
1860
|
this.task.emit("status", this.task.status);
|
|
1781
1861
|
}
|
|
1782
|
-
async
|
|
1783
|
-
this.
|
|
1862
|
+
async handleErrorPreview() {
|
|
1863
|
+
this.previewRunning = false;
|
|
1784
1864
|
}
|
|
1785
1865
|
async handleProgress(progress, message, ...args) {
|
|
1786
1866
|
this.task.progress = progress;
|
|
@@ -1828,8 +1908,8 @@ class Task {
|
|
|
1828
1908
|
}
|
|
1829
1909
|
return;
|
|
1830
1910
|
}
|
|
1831
|
-
async
|
|
1832
|
-
return
|
|
1911
|
+
async executePreview(_input, _context) {
|
|
1912
|
+
return;
|
|
1833
1913
|
}
|
|
1834
1914
|
_runner;
|
|
1835
1915
|
get runner() {
|
|
@@ -1841,8 +1921,8 @@ class Task {
|
|
|
1841
1921
|
async run(overrides = {}, runConfig = {}) {
|
|
1842
1922
|
return this.runner.run(overrides, { ...this.runConfig, ...runConfig });
|
|
1843
1923
|
}
|
|
1844
|
-
async
|
|
1845
|
-
return this.runner.
|
|
1924
|
+
async runPreview(overrides = {}) {
|
|
1925
|
+
return this.runner.runPreview(overrides);
|
|
1846
1926
|
}
|
|
1847
1927
|
abort() {
|
|
1848
1928
|
this.runner.abort();
|
|
@@ -2803,9 +2883,9 @@ var GRAPH_RESULT_ARRAY = "GRAPH_RESULT_ARRAY";
|
|
|
2803
2883
|
|
|
2804
2884
|
class TaskGraphRunner {
|
|
2805
2885
|
processScheduler;
|
|
2806
|
-
|
|
2886
|
+
previewScheduler;
|
|
2807
2887
|
running = false;
|
|
2808
|
-
|
|
2888
|
+
previewRunning = false;
|
|
2809
2889
|
graph;
|
|
2810
2890
|
outputCache;
|
|
2811
2891
|
accumulateLeafOutputs = true;
|
|
@@ -2819,9 +2899,9 @@ class TaskGraphRunner {
|
|
|
2819
2899
|
graphTimeoutTimer;
|
|
2820
2900
|
pendingGraphTimeoutError;
|
|
2821
2901
|
activeEnforcer;
|
|
2822
|
-
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph),
|
|
2902
|
+
constructor(graph, outputCache, processScheduler = new DependencyBasedScheduler(graph), previewScheduler = new TopologicalScheduler(graph)) {
|
|
2823
2903
|
this.processScheduler = processScheduler;
|
|
2824
|
-
this.
|
|
2904
|
+
this.previewScheduler = previewScheduler;
|
|
2825
2905
|
this.graph = graph;
|
|
2826
2906
|
graph.outputCache = outputCache;
|
|
2827
2907
|
this.handleProgress = this.handleProgress.bind(this);
|
|
@@ -2889,16 +2969,16 @@ class TaskGraphRunner {
|
|
|
2889
2969
|
await this.handleComplete();
|
|
2890
2970
|
return this.filterLeafResults(results);
|
|
2891
2971
|
}
|
|
2892
|
-
async
|
|
2893
|
-
await this.
|
|
2972
|
+
async runGraphPreview(input = {}, config) {
|
|
2973
|
+
await this.handleStartPreview(config);
|
|
2894
2974
|
const telemetry = getTelemetryProvider2();
|
|
2895
2975
|
const telemetryEnabled = telemetry.isEnabled;
|
|
2896
|
-
const
|
|
2897
|
-
let
|
|
2976
|
+
const previewRunId = telemetryEnabled ? uuid43() : "";
|
|
2977
|
+
let previewSpan;
|
|
2898
2978
|
if (telemetryEnabled) {
|
|
2899
|
-
|
|
2979
|
+
previewSpan = telemetry.startSpan("workglow.graph.runPreview", {
|
|
2900
2980
|
attributes: {
|
|
2901
|
-
"workglow.graph.
|
|
2981
|
+
"workglow.graph.preview.run_id": previewRunId,
|
|
2902
2982
|
"workglow.graph.task_count": this.graph.getTasks().length,
|
|
2903
2983
|
"workglow.graph.dataflow_count": this.graph.getDataflows().length
|
|
2904
2984
|
}
|
|
@@ -2908,7 +2988,7 @@ class TaskGraphRunner {
|
|
|
2908
2988
|
const taskTimings = [];
|
|
2909
2989
|
const results = [];
|
|
2910
2990
|
try {
|
|
2911
|
-
for await (const task of this.
|
|
2991
|
+
for await (const task of this.previewScheduler.tasks()) {
|
|
2912
2992
|
const isRootTask = this.graph.getSourceDataflows(task.id).length === 0;
|
|
2913
2993
|
if (task.status === TaskStatus.PENDING) {
|
|
2914
2994
|
task.resetInputData();
|
|
@@ -2917,13 +2997,13 @@ class TaskGraphRunner {
|
|
|
2917
2997
|
const taskInput = isRootTask ? input : {};
|
|
2918
2998
|
if (telemetryEnabled) {
|
|
2919
2999
|
const taskType = String(task.constructor.runtype || task.constructor.type || "?");
|
|
2920
|
-
const
|
|
2921
|
-
const taskResult = await task.
|
|
2922
|
-
const
|
|
3000
|
+
const tPreview = performance.now();
|
|
3001
|
+
const taskResult = await task.runPreview(taskInput);
|
|
3002
|
+
const runPreviewMs = performance.now() - tPreview;
|
|
2923
3003
|
const tPush = performance.now();
|
|
2924
3004
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
2925
3005
|
const pushOutputMs = performance.now() - tPush;
|
|
2926
|
-
taskTimings.push({ id: task.id, type: taskType,
|
|
3006
|
+
taskTimings.push({ id: task.id, type: taskType, runPreviewMs, pushOutputMs });
|
|
2927
3007
|
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
2928
3008
|
results.push({
|
|
2929
3009
|
id: task.id,
|
|
@@ -2932,7 +3012,7 @@ class TaskGraphRunner {
|
|
|
2932
3012
|
});
|
|
2933
3013
|
}
|
|
2934
3014
|
} else {
|
|
2935
|
-
const taskResult = await task.
|
|
3015
|
+
const taskResult = await task.runPreview(taskInput);
|
|
2936
3016
|
await this.pushOutputFromNodeToEdges(task, taskResult);
|
|
2937
3017
|
if (this.graph.getTargetDataflows(task.id).length === 0) {
|
|
2938
3018
|
results.push({
|
|
@@ -2943,35 +3023,35 @@ class TaskGraphRunner {
|
|
|
2943
3023
|
}
|
|
2944
3024
|
}
|
|
2945
3025
|
}
|
|
2946
|
-
await this.
|
|
2947
|
-
if (
|
|
3026
|
+
await this.handleCompletePreview();
|
|
3027
|
+
if (previewSpan) {
|
|
2948
3028
|
const totalMs = performance.now() - t0;
|
|
2949
|
-
|
|
2950
|
-
"workglow.graph.
|
|
2951
|
-
"workglow.graph.
|
|
3029
|
+
previewSpan.setAttributes({
|
|
3030
|
+
"workglow.graph.preview.duration_ms": Math.round(totalMs * 1000) / 1000,
|
|
3031
|
+
"workglow.graph.preview.tasks_executed": taskTimings.length
|
|
2952
3032
|
});
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
getLogger3().debug("task graph
|
|
2956
|
-
|
|
3033
|
+
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3034
|
+
previewSpan.end();
|
|
3035
|
+
getLogger3().debug("task graph runPreview timings", {
|
|
3036
|
+
previewRunId,
|
|
2957
3037
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
2958
3038
|
taskTimings
|
|
2959
3039
|
});
|
|
2960
3040
|
}
|
|
2961
3041
|
return this.filterLeafResults(results);
|
|
2962
3042
|
} catch (error) {
|
|
2963
|
-
await this.
|
|
2964
|
-
if (
|
|
3043
|
+
await this.handleErrorPreview();
|
|
3044
|
+
if (previewSpan) {
|
|
2965
3045
|
const totalMs = performance.now() - t0;
|
|
2966
3046
|
const message = error instanceof Error ? error.message : String(error);
|
|
2967
|
-
|
|
2968
|
-
"workglow.graph.
|
|
2969
|
-
"workglow.graph.
|
|
3047
|
+
previewSpan.setAttributes({
|
|
3048
|
+
"workglow.graph.preview.duration_ms": Math.round(totalMs * 1000) / 1000,
|
|
3049
|
+
"workglow.graph.preview.tasks_executed": taskTimings.length
|
|
2970
3050
|
});
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
getLogger3().debug("task graph
|
|
2974
|
-
|
|
3051
|
+
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3052
|
+
previewSpan.end();
|
|
3053
|
+
getLogger3().debug("task graph runPreview failed", {
|
|
3054
|
+
previewRunId,
|
|
2975
3055
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
2976
3056
|
taskTimings,
|
|
2977
3057
|
error
|
|
@@ -3043,6 +3123,23 @@ class TaskGraphRunner {
|
|
|
3043
3123
|
}
|
|
3044
3124
|
async pushOutputFromNodeToEdges(node, results) {
|
|
3045
3125
|
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
3126
|
+
if (Object.keys(results).length > 0) {
|
|
3127
|
+
const consumerCounts = new Map;
|
|
3128
|
+
for (const dataflow of dataflows) {
|
|
3129
|
+
if (dataflow.stream !== undefined)
|
|
3130
|
+
continue;
|
|
3131
|
+
const port = dataflow.sourceTaskPortId;
|
|
3132
|
+
consumerCounts.set(port, (consumerCounts.get(port) ?? 0) + 1);
|
|
3133
|
+
}
|
|
3134
|
+
for (const [port, count] of consumerCounts) {
|
|
3135
|
+
if (count <= 1)
|
|
3136
|
+
continue;
|
|
3137
|
+
const value = results[port];
|
|
3138
|
+
const ref = asRefcountable(value);
|
|
3139
|
+
if (ref)
|
|
3140
|
+
ref.retain(count - 1);
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3046
3143
|
for (const dataflow of dataflows) {
|
|
3047
3144
|
if (dataflow.stream !== undefined)
|
|
3048
3145
|
continue;
|
|
@@ -3382,7 +3479,7 @@ class TaskGraphRunner {
|
|
|
3382
3479
|
}
|
|
3383
3480
|
this.graph.outputCache = this.outputCache;
|
|
3384
3481
|
}
|
|
3385
|
-
if (this.running || this.
|
|
3482
|
+
if (this.running || this.previewRunning) {
|
|
3386
3483
|
throw new TaskConfigurationError("Graph is already running");
|
|
3387
3484
|
}
|
|
3388
3485
|
this.running = true;
|
|
@@ -3456,9 +3553,9 @@ class TaskGraphRunner {
|
|
|
3456
3553
|
}
|
|
3457
3554
|
this.graph.emit("start");
|
|
3458
3555
|
}
|
|
3459
|
-
async
|
|
3460
|
-
if (this.
|
|
3461
|
-
throw new TaskConfigurationError("Graph is already running
|
|
3556
|
+
async handleStartPreview(config) {
|
|
3557
|
+
if (this.previewRunning) {
|
|
3558
|
+
throw new TaskConfigurationError("Graph is already running in preview");
|
|
3462
3559
|
}
|
|
3463
3560
|
if (config?.registry !== undefined) {
|
|
3464
3561
|
this.registry = config.registry;
|
|
@@ -3469,8 +3566,8 @@ class TaskGraphRunner {
|
|
|
3469
3566
|
throw new TaskConfigurationError(`Graph has ${taskCount} tasks, exceeding the limit of ${config.maxTasks}`);
|
|
3470
3567
|
}
|
|
3471
3568
|
}
|
|
3472
|
-
this.
|
|
3473
|
-
this.
|
|
3569
|
+
this.previewScheduler.reset();
|
|
3570
|
+
this.previewRunning = true;
|
|
3474
3571
|
}
|
|
3475
3572
|
clearGraphTimeout() {
|
|
3476
3573
|
if (this.graphTimeoutTimer !== undefined) {
|
|
@@ -3489,8 +3586,8 @@ class TaskGraphRunner {
|
|
|
3489
3586
|
}
|
|
3490
3587
|
this.graph.emit("complete");
|
|
3491
3588
|
}
|
|
3492
|
-
async
|
|
3493
|
-
this.
|
|
3589
|
+
async handleCompletePreview() {
|
|
3590
|
+
this.previewRunning = false;
|
|
3494
3591
|
}
|
|
3495
3592
|
async handleError(error) {
|
|
3496
3593
|
this.clearGraphTimeout();
|
|
@@ -3509,8 +3606,8 @@ class TaskGraphRunner {
|
|
|
3509
3606
|
}
|
|
3510
3607
|
this.graph.emit("error", error);
|
|
3511
3608
|
}
|
|
3512
|
-
async
|
|
3513
|
-
this.
|
|
3609
|
+
async handleErrorPreview() {
|
|
3610
|
+
this.previewRunning = false;
|
|
3514
3611
|
}
|
|
3515
3612
|
async handleAbort() {
|
|
3516
3613
|
this.clearGraphTimeout();
|
|
@@ -3529,8 +3626,8 @@ class TaskGraphRunner {
|
|
|
3529
3626
|
}
|
|
3530
3627
|
this.graph.emit("abort");
|
|
3531
3628
|
}
|
|
3532
|
-
async
|
|
3533
|
-
this.
|
|
3629
|
+
async handleAbortPreview() {
|
|
3630
|
+
this.previewRunning = false;
|
|
3534
3631
|
}
|
|
3535
3632
|
async handleDisable() {
|
|
3536
3633
|
await Promise.allSettled(this.graph.getTasks().map(async (task) => {
|
|
@@ -3573,8 +3670,8 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
3573
3670
|
unsubscribe();
|
|
3574
3671
|
return results;
|
|
3575
3672
|
}
|
|
3576
|
-
async
|
|
3577
|
-
return this.task.subGraph.
|
|
3673
|
+
async executeTaskChildrenPreview() {
|
|
3674
|
+
return this.task.subGraph.runPreview(this.task.runInputData, {
|
|
3578
3675
|
registry: this.registry,
|
|
3579
3676
|
resourceScope: this.resourceScope
|
|
3580
3677
|
});
|
|
@@ -3595,15 +3692,18 @@ class GraphAsTaskRunner extends TaskRunner {
|
|
|
3595
3692
|
}
|
|
3596
3693
|
return this.task.runOutputData;
|
|
3597
3694
|
}
|
|
3598
|
-
async
|
|
3695
|
+
async executeTaskPreview(input) {
|
|
3599
3696
|
if (this.task.hasChildren()) {
|
|
3600
|
-
const
|
|
3601
|
-
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(
|
|
3697
|
+
const previewResults = await this.executeTaskChildrenPreview();
|
|
3698
|
+
this.task.runOutputData = this.task.subGraph.mergeExecuteOutputsToRunOutput(previewResults, this.task.compoundMerge);
|
|
3699
|
+
return this.task.runOutputData;
|
|
3602
3700
|
} else {
|
|
3603
|
-
const
|
|
3604
|
-
|
|
3701
|
+
const previewResult = await super.executeTaskPreview(input);
|
|
3702
|
+
if (previewResult !== undefined) {
|
|
3703
|
+
this.task.runOutputData = previewResult;
|
|
3704
|
+
}
|
|
3705
|
+
return this.task.runOutputData;
|
|
3605
3706
|
}
|
|
3606
|
-
return this.task.runOutputData;
|
|
3607
3707
|
}
|
|
3608
3708
|
}
|
|
3609
3709
|
|
|
@@ -4031,8 +4131,8 @@ class TaskGraph {
|
|
|
4031
4131
|
resourceScope: config?.resourceScope
|
|
4032
4132
|
});
|
|
4033
4133
|
}
|
|
4034
|
-
|
|
4035
|
-
return this.runner.
|
|
4134
|
+
runPreview(input = {}, config = {}) {
|
|
4135
|
+
return this.runner.runGraphPreview(input, config);
|
|
4036
4136
|
}
|
|
4037
4137
|
mergeExecuteOutputsToRunOutput(results, compoundMerge) {
|
|
4038
4138
|
return this.runner.mergeExecuteOutputsToRunOutput(results, compoundMerge);
|
|
@@ -5922,9 +6022,8 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5922
6022
|
}
|
|
5923
6023
|
return this.executeTaskFallback(input);
|
|
5924
6024
|
}
|
|
5925
|
-
async
|
|
5926
|
-
|
|
5927
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6025
|
+
async executeTaskPreview(input) {
|
|
6026
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
5928
6027
|
}
|
|
5929
6028
|
async executeTaskFallback(input) {
|
|
5930
6029
|
const tasks = this.task.subGraph.getTasks();
|
|
@@ -5944,7 +6043,7 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5944
6043
|
this.resetTask(alternativeTask);
|
|
5945
6044
|
const result = await alternativeTask.run(input);
|
|
5946
6045
|
await this.handleProgress(100, `Alternative ${attemptNumber}/${totalAttempts} succeeded: ${alternativeTask.type}`);
|
|
5947
|
-
return
|
|
6046
|
+
return result;
|
|
5948
6047
|
} catch (error) {
|
|
5949
6048
|
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
5950
6049
|
throw error;
|
|
@@ -5986,7 +6085,7 @@ class FallbackTaskRunner extends GraphAsTaskRunner {
|
|
|
5986
6085
|
});
|
|
5987
6086
|
const mergedOutput = this.task.subGraph.mergeExecuteOutputsToRunOutput(results, this.task.compoundMerge);
|
|
5988
6087
|
await this.handleProgress(100, `Data alternative ${attemptNumber}/${totalAttempts} succeeded`);
|
|
5989
|
-
return
|
|
6088
|
+
return mergedOutput;
|
|
5990
6089
|
} catch (error) {
|
|
5991
6090
|
if (error instanceof TaskAbortedError && !(error instanceof TaskTimeoutError)) {
|
|
5992
6091
|
throw error;
|
|
@@ -6214,15 +6313,13 @@ class IteratorTaskRunner extends GraphAsTaskRunner {
|
|
|
6214
6313
|
analysis = { ...analysis, iterationCount: maxIterations };
|
|
6215
6314
|
}
|
|
6216
6315
|
if (analysis.iterationCount === 0) {
|
|
6217
|
-
|
|
6218
|
-
return this.executeTaskReactive(input, emptyResult);
|
|
6316
|
+
return this.task.getEmptyResult();
|
|
6219
6317
|
}
|
|
6220
6318
|
const result = this.task.isReduceTask() ? await this.executeReduceIterations(analysis) : await this.executeCollectIterations(analysis);
|
|
6221
|
-
return
|
|
6319
|
+
return result;
|
|
6222
6320
|
}
|
|
6223
|
-
async
|
|
6224
|
-
|
|
6225
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6321
|
+
async executeTaskPreview(input) {
|
|
6322
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
6226
6323
|
}
|
|
6227
6324
|
async executeCollectIterations(analysis) {
|
|
6228
6325
|
const iterationCount = analysis.iterationCount;
|
|
@@ -6889,9 +6986,8 @@ class WhileTaskRunner extends GraphAsTaskRunner {
|
|
|
6889
6986
|
});
|
|
6890
6987
|
return result;
|
|
6891
6988
|
}
|
|
6892
|
-
async
|
|
6893
|
-
|
|
6894
|
-
return Object.assign({}, output, reactiveResult ?? {});
|
|
6989
|
+
async executeTaskPreview(input) {
|
|
6990
|
+
return this.task.executePreview?.(input, { own: this.own });
|
|
6895
6991
|
}
|
|
6896
6992
|
}
|
|
6897
6993
|
|
|
@@ -8119,6 +8215,14 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8119
8215
|
this.emit("output_pruned");
|
|
8120
8216
|
}
|
|
8121
8217
|
}
|
|
8218
|
+
// src/storage/PortCodecRegistry.ts
|
|
8219
|
+
import {
|
|
8220
|
+
registerPortCodec,
|
|
8221
|
+
getPortCodec as getPortCodec2,
|
|
8222
|
+
_resetPortCodecsForTests
|
|
8223
|
+
} from "@workglow/util";
|
|
8224
|
+
// src/bun.ts
|
|
8225
|
+
registerRefcountablePredicate((v) => !!v && typeof v === "object" && ("backend" in v) && ("retain" in v) && ("release" in v) && ("materialize" in v));
|
|
8122
8226
|
export {
|
|
8123
8227
|
wrapSchemaInArray,
|
|
8124
8228
|
whileTaskConfigSchema,
|
|
@@ -8141,6 +8245,8 @@ export {
|
|
|
8141
8245
|
resolveIterationBound,
|
|
8142
8246
|
resetMethodNameCache,
|
|
8143
8247
|
removeIterationProperties,
|
|
8248
|
+
registerRefcountablePredicate,
|
|
8249
|
+
registerPortCodec,
|
|
8144
8250
|
registerJobQueueFactory,
|
|
8145
8251
|
registerBuiltInTransforms,
|
|
8146
8252
|
registerBaseTasks,
|
|
@@ -8176,6 +8282,7 @@ export {
|
|
|
8176
8282
|
getSchemaFormat,
|
|
8177
8283
|
getProfileGrants,
|
|
8178
8284
|
getPortStreamMode,
|
|
8285
|
+
getPortCodec2 as getPortCodec,
|
|
8179
8286
|
getOutputStreamMode,
|
|
8180
8287
|
getObjectSchema,
|
|
8181
8288
|
getObjectPortId,
|
|
@@ -8222,9 +8329,12 @@ export {
|
|
|
8222
8329
|
calculateNodeDepths,
|
|
8223
8330
|
buildIterationInputSchema,
|
|
8224
8331
|
autoConnect,
|
|
8332
|
+
asRefcountable,
|
|
8225
8333
|
addIterationContextToSchema,
|
|
8226
8334
|
addBoundaryNodesToGraphJson,
|
|
8227
8335
|
addBoundaryNodesToDependencyJson,
|
|
8336
|
+
_resetRefcountablePredicatesForTests,
|
|
8337
|
+
_resetPortCodecsForTests,
|
|
8228
8338
|
WorkflowError,
|
|
8229
8339
|
Workflow,
|
|
8230
8340
|
WhileTaskRunner,
|
|
@@ -8298,4 +8408,4 @@ export {
|
|
|
8298
8408
|
BROWSER_GRANTS
|
|
8299
8409
|
};
|
|
8300
8410
|
|
|
8301
|
-
//# debugId=
|
|
8411
|
+
//# debugId=791136A32061C4D664756E2164756E21
|