@workglow/task-graph 0.2.37 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -46
- package/dist/browser.js +639 -368
- package/dist/browser.js.map +19 -15
- package/dist/bun.js +639 -368
- package/dist/bun.js.map +19 -15
- package/dist/cache/CacheJanitor.d.ts +27 -0
- package/dist/cache/CacheJanitor.d.ts.map +1 -0
- package/dist/cache/CachePolicy.d.ts +16 -0
- package/dist/cache/CachePolicy.d.ts.map +1 -0
- package/dist/cache/CacheRegistry.d.ts +30 -0
- package/dist/cache/CacheRegistry.d.ts.map +1 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts +56 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts.map +1 -0
- package/dist/cache/index.d.ts +10 -0
- package/dist/cache/index.d.ts.map +1 -0
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +639 -368
- package/dist/node.js.map +19 -15
- package/dist/storage/TaskOutputRepository.d.ts +40 -4
- package/dist/storage/TaskOutputRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputTabularRepository.d.ts +27 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts.map +1 -1
- package/dist/task/CacheCoordinator.d.ts +17 -0
- package/dist/task/CacheCoordinator.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +0 -1
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/FallbackTaskRunner.d.ts +8 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +21 -1
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/Task.d.ts +50 -0
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +0 -13
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +28 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task-graph/Conversions.d.ts.map +1 -1
- package/dist/task-graph/StreamPump.d.ts +8 -0
- package/dist/task-graph/StreamPump.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +7 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +45 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +91 -2
package/dist/browser.js
CHANGED
|
@@ -1277,14 +1277,14 @@ import { EventEmitter as EventEmitter4, uuid4 as uuid45 } from "@workglow/util";
|
|
|
1277
1277
|
import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
1278
1278
|
|
|
1279
1279
|
// src/task/GraphAsTask.ts
|
|
1280
|
-
import { getLogger as
|
|
1280
|
+
import { getLogger as getLogger7 } from "@workglow/util";
|
|
1281
1281
|
import { CycleError } from "@workglow/util/graph";
|
|
1282
1282
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
1283
1283
|
|
|
1284
1284
|
// src/task-graph/TaskGraphRunner.ts
|
|
1285
1285
|
import {
|
|
1286
1286
|
collectPropertyValues,
|
|
1287
|
-
getLogger as
|
|
1287
|
+
getLogger as getLogger6,
|
|
1288
1288
|
getTelemetryProvider as getTelemetryProvider2,
|
|
1289
1289
|
globalServiceRegistry as globalServiceRegistry3,
|
|
1290
1290
|
ResourceScope as ResourceScope2,
|
|
@@ -1293,9 +1293,39 @@ import {
|
|
|
1293
1293
|
uuid4 as uuid44
|
|
1294
1294
|
} from "@workglow/util";
|
|
1295
1295
|
|
|
1296
|
+
// src/cache/CacheJanitor.ts
|
|
1297
|
+
class CacheJanitor {
|
|
1298
|
+
privateBacking;
|
|
1299
|
+
constructor({ privateBacking }) {
|
|
1300
|
+
this.privateBacking = privateBacking;
|
|
1301
|
+
}
|
|
1302
|
+
async sweepStaleRunPrivate(olderThanMs) {
|
|
1303
|
+
await this.privateBacking.clearOlderThanWithTaskTypePrefix("__run:", olderThanMs);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
// src/cache/CachePolicy.ts
|
|
1307
|
+
var DEFAULT_CACHE_POLICY = { kind: "deterministic" };
|
|
1308
|
+
function isPolicyCached(policy) {
|
|
1309
|
+
return policy.kind !== "none";
|
|
1310
|
+
}
|
|
1311
|
+
function isPolicyPrivate(policy) {
|
|
1312
|
+
return policy.kind === "private";
|
|
1313
|
+
}
|
|
1314
|
+
// src/cache/CacheRegistry.ts
|
|
1315
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
1316
|
+
var CACHE_REGISTRY = createServiceToken2("taskgraph.cacheRegistry");
|
|
1317
|
+
|
|
1318
|
+
class DefaultCacheRegistry {
|
|
1319
|
+
deterministic;
|
|
1320
|
+
private;
|
|
1321
|
+
constructor(init = {}) {
|
|
1322
|
+
this.deterministic = init.deterministic;
|
|
1323
|
+
this.private = init.private;
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1296
1326
|
// src/storage/TaskOutputRepository.ts
|
|
1297
|
-
import { createServiceToken as
|
|
1298
|
-
var TASK_OUTPUT_REPOSITORY =
|
|
1327
|
+
import { createServiceToken as createServiceToken3, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1328
|
+
var TASK_OUTPUT_REPOSITORY = createServiceToken3("taskgraph.taskOutputRepository");
|
|
1299
1329
|
|
|
1300
1330
|
class TaskOutputRepository {
|
|
1301
1331
|
outputCompression;
|
|
@@ -1321,10 +1351,53 @@ class TaskOutputRepository {
|
|
|
1321
1351
|
emit(name, ...args) {
|
|
1322
1352
|
this._events?.emit(name, ...args);
|
|
1323
1353
|
}
|
|
1354
|
+
async deleteByTaskTypePrefix(_prefix) {
|
|
1355
|
+
throw new Error(`${this.constructor.name}: deleteByTaskTypePrefix is not supported by this repository.`);
|
|
1356
|
+
}
|
|
1357
|
+
async clearOlderThanWithTaskTypePrefix(_prefix, _olderThanMs) {
|
|
1358
|
+
throw new Error(`${this.constructor.name}: clearOlderThanWithTaskTypePrefix is not supported by this repository.`);
|
|
1359
|
+
}
|
|
1360
|
+
async sizeByTaskTypePrefix(_prefix) {
|
|
1361
|
+
throw new Error(`${this.constructor.name}: sizeByTaskTypePrefix is not supported by this repository.`);
|
|
1362
|
+
}
|
|
1324
1363
|
}
|
|
1325
1364
|
|
|
1365
|
+
// src/cache/RunPrivateCacheRepo.ts
|
|
1366
|
+
class RunPrivateCacheRepo extends TaskOutputRepository {
|
|
1367
|
+
backing;
|
|
1368
|
+
runId;
|
|
1369
|
+
constructor({ backing, runId }) {
|
|
1370
|
+
super({ outputCompression: backing.outputCompression });
|
|
1371
|
+
this.backing = backing;
|
|
1372
|
+
this.runId = runId;
|
|
1373
|
+
}
|
|
1374
|
+
ns(taskType) {
|
|
1375
|
+
return `__run:${this.runId}::${taskType}`;
|
|
1376
|
+
}
|
|
1377
|
+
async saveOutput(taskType, inputs, output, createdAt) {
|
|
1378
|
+
await this.backing.saveOutput(this.ns(taskType), inputs, output, createdAt);
|
|
1379
|
+
}
|
|
1380
|
+
async getOutput(taskType, inputs) {
|
|
1381
|
+
return this.backing.getOutput(this.ns(taskType), inputs);
|
|
1382
|
+
}
|
|
1383
|
+
async clear() {
|
|
1384
|
+
await this.clearRun();
|
|
1385
|
+
}
|
|
1386
|
+
async clearRun() {
|
|
1387
|
+
await this.backing.deleteByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1388
|
+
}
|
|
1389
|
+
async size() {
|
|
1390
|
+
return this.backing.sizeByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1391
|
+
}
|
|
1392
|
+
async clearOlderThan(olderThanInMs) {
|
|
1393
|
+
await this.backing.clearOlderThanWithTaskTypePrefix(`__run:${this.runId}::`, olderThanInMs);
|
|
1394
|
+
}
|
|
1395
|
+
isDurable() {
|
|
1396
|
+
return this.backing.isDurable();
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1326
1399
|
// src/task/EntitlementEnforcer.ts
|
|
1327
|
-
import { createServiceToken as
|
|
1400
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1328
1401
|
|
|
1329
1402
|
// src/task/EntitlementPolicy.ts
|
|
1330
1403
|
var EMPTY_POLICY = Object.freeze({
|
|
@@ -1368,7 +1441,7 @@ function can(policy, id, resources) {
|
|
|
1368
1441
|
}
|
|
1369
1442
|
|
|
1370
1443
|
// src/task/EntitlementResolver.ts
|
|
1371
|
-
import { createServiceToken as
|
|
1444
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1372
1445
|
var PERMISSIVE_RESOLVER = {
|
|
1373
1446
|
lookup: () => "grant",
|
|
1374
1447
|
prompt: async () => "grant",
|
|
@@ -1379,7 +1452,7 @@ var DENY_ALL_RESOLVER = {
|
|
|
1379
1452
|
prompt: async () => "deny",
|
|
1380
1453
|
save: () => {}
|
|
1381
1454
|
};
|
|
1382
|
-
var ENTITLEMENT_RESOLVER =
|
|
1455
|
+
var ENTITLEMENT_RESOLVER = createServiceToken4("workglow.entitlementResolver");
|
|
1383
1456
|
|
|
1384
1457
|
// src/task/EntitlementEnforcer.ts
|
|
1385
1458
|
function formatEntitlementDenial(denial) {
|
|
@@ -1452,7 +1525,7 @@ function createScopedEnforcer(grants) {
|
|
|
1452
1525
|
function createGrantListEnforcer(grants) {
|
|
1453
1526
|
return createScopedEnforcer(grants.map((id) => ({ id })));
|
|
1454
1527
|
}
|
|
1455
|
-
var ENTITLEMENT_ENFORCER =
|
|
1528
|
+
var ENTITLEMENT_ENFORCER = createServiceToken5("workglow.entitlementEnforcer");
|
|
1456
1529
|
|
|
1457
1530
|
// src/task/StreamTypes.ts
|
|
1458
1531
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1555,246 +1628,13 @@ function hasStructuredOutput(schema) {
|
|
|
1555
1628
|
return getStructuredOutputSchemas(schema).size > 0;
|
|
1556
1629
|
}
|
|
1557
1630
|
|
|
1558
|
-
// src/task-graph/EdgeMaterializer.ts
|
|
1559
|
-
import { getLogger } from "@workglow/util";
|
|
1560
|
-
import { previewSource } from "@workglow/util/media";
|
|
1561
|
-
class EdgeMaterializer {
|
|
1562
|
-
graph;
|
|
1563
|
-
runner;
|
|
1564
|
-
constructor(graph, runner) {
|
|
1565
|
-
this.graph = graph;
|
|
1566
|
-
this.runner = runner;
|
|
1567
|
-
}
|
|
1568
|
-
filterInputForTask(task, input) {
|
|
1569
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1570
|
-
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1571
|
-
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1572
|
-
const filteredInput = {};
|
|
1573
|
-
for (const [key, value] of Object.entries(input)) {
|
|
1574
|
-
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
1575
|
-
filteredInput[key] = value;
|
|
1576
|
-
}
|
|
1577
|
-
}
|
|
1578
|
-
return filteredInput;
|
|
1579
|
-
}
|
|
1580
|
-
copyInputFromEdgesToNode(task) {
|
|
1581
|
-
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1582
|
-
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
1583
|
-
for (const dataflow of dataflows) {
|
|
1584
|
-
const live = dataflow.getCurrentValue();
|
|
1585
|
-
const port = dataflow.targetTaskPortId;
|
|
1586
|
-
let portData;
|
|
1587
|
-
if (port === DATAFLOW_ALL_PORTS) {
|
|
1588
|
-
portData = live;
|
|
1589
|
-
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
1590
|
-
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
1591
|
-
} else {
|
|
1592
|
-
portData = { [port]: live };
|
|
1593
|
-
}
|
|
1594
|
-
this.runner.addInputData(task, portData);
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
async pushOutputFromNodeToEdges(node, results) {
|
|
1598
|
-
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1599
|
-
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
1600
|
-
for (const port of Object.keys(results)) {
|
|
1601
|
-
const value = results[port];
|
|
1602
|
-
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
1603
|
-
results[port] = await previewSource(value);
|
|
1604
|
-
}
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
1607
|
-
for (const dataflow of dataflows) {
|
|
1608
|
-
if (dataflow.stream !== undefined)
|
|
1609
|
-
continue;
|
|
1610
|
-
const registry = this.runner["registry"];
|
|
1611
|
-
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
1612
|
-
if (compatibility === "static") {
|
|
1613
|
-
dataflow.setPortData(results);
|
|
1614
|
-
await dataflow.applyTransforms(registry);
|
|
1615
|
-
} else if (compatibility === "runtime") {
|
|
1616
|
-
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
1617
|
-
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
1618
|
-
dataflow.setPortData(narrowed);
|
|
1619
|
-
await dataflow.applyTransforms(registry);
|
|
1620
|
-
} else {
|
|
1621
|
-
const resultsKeys = Object.keys(results);
|
|
1622
|
-
if (resultsKeys.length > 0) {
|
|
1623
|
-
getLogger().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
1624
|
-
dataflowId: dataflow.id,
|
|
1625
|
-
compatibility,
|
|
1626
|
-
resultsKeys
|
|
1627
|
-
});
|
|
1628
|
-
}
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
pushErrorFromNodeToEdges(node) {
|
|
1633
|
-
if (!node?.config?.id)
|
|
1634
|
-
return;
|
|
1635
|
-
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1636
|
-
dataflow.error = node.error;
|
|
1637
|
-
});
|
|
1638
|
-
}
|
|
1639
|
-
hasErrorOutputEdges(task) {
|
|
1640
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1641
|
-
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
1642
|
-
}
|
|
1643
|
-
pushErrorOutputToEdges(task) {
|
|
1644
|
-
const taskError = task.error;
|
|
1645
|
-
const errorData = {
|
|
1646
|
-
error: taskError?.message ?? "Unknown error",
|
|
1647
|
-
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
1648
|
-
};
|
|
1649
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1650
|
-
for (const df of dataflows) {
|
|
1651
|
-
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
1652
|
-
df.value = errorData;
|
|
1653
|
-
df.setStatus(TaskStatus.COMPLETED);
|
|
1654
|
-
} else {
|
|
1655
|
-
df.setStatus(TaskStatus.DISABLED);
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
1659
|
-
}
|
|
1660
|
-
resetTask(graph, task, runId) {
|
|
1661
|
-
task.status = TaskStatus.PENDING;
|
|
1662
|
-
task.resetInputData();
|
|
1663
|
-
task.runOutputData = {};
|
|
1664
|
-
task.error = undefined;
|
|
1665
|
-
task.progress = 0;
|
|
1666
|
-
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
1667
|
-
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
1668
|
-
if (task?.config?.id) {
|
|
1669
|
-
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1670
|
-
dataflow.error = task.error;
|
|
1671
|
-
});
|
|
1672
|
-
}
|
|
1673
|
-
task.emit("reset");
|
|
1674
|
-
task.emit("status", task.status);
|
|
1675
|
-
}
|
|
1676
|
-
static isImageValueShape(v) {
|
|
1677
|
-
if (v === null || typeof v !== "object")
|
|
1678
|
-
return false;
|
|
1679
|
-
const o = v;
|
|
1680
|
-
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
1681
|
-
}
|
|
1682
|
-
}
|
|
1683
|
-
|
|
1684
|
-
// src/task-graph/RunContext.ts
|
|
1685
|
-
import { uuid4 as uuid42 } from "@workglow/util";
|
|
1686
|
-
|
|
1687
|
-
class RunContext {
|
|
1688
|
-
runId;
|
|
1689
|
-
abortController;
|
|
1690
|
-
inProgressTasks = new Map;
|
|
1691
|
-
inProgressFunctions = new Map;
|
|
1692
|
-
failedTaskErrors = new Map;
|
|
1693
|
-
telemetrySpan;
|
|
1694
|
-
graphTimeoutTimer;
|
|
1695
|
-
pendingGraphTimeoutError;
|
|
1696
|
-
activeEnforcer;
|
|
1697
|
-
parentSignalCleanup;
|
|
1698
|
-
constructor(parentSignal) {
|
|
1699
|
-
this.runId = uuid42();
|
|
1700
|
-
this.abortController = new AbortController;
|
|
1701
|
-
if (parentSignal) {
|
|
1702
|
-
const onParentAbort = () => this.abortController.abort();
|
|
1703
|
-
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
1704
|
-
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
1705
|
-
if (parentSignal.aborted) {
|
|
1706
|
-
this.parentSignalCleanup();
|
|
1707
|
-
this.parentSignalCleanup = undefined;
|
|
1708
|
-
this.abortController.abort();
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
}
|
|
1712
|
-
dispose() {
|
|
1713
|
-
this.parentSignalCleanup?.();
|
|
1714
|
-
this.parentSignalCleanup = undefined;
|
|
1715
|
-
}
|
|
1716
|
-
}
|
|
1717
|
-
|
|
1718
|
-
// src/task-graph/RunScheduler.ts
|
|
1719
|
-
import { getLogger as getLogger4 } from "@workglow/util";
|
|
1720
|
-
|
|
1721
|
-
// src/task/ConditionalTask.ts
|
|
1722
|
-
import { getLogger as getLogger3 } from "@workglow/util";
|
|
1723
|
-
|
|
1724
|
-
// src/task/ConditionUtils.ts
|
|
1725
|
-
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
1726
|
-
if (fieldValue === null || fieldValue === undefined) {
|
|
1727
|
-
switch (operator) {
|
|
1728
|
-
case "is_empty":
|
|
1729
|
-
return true;
|
|
1730
|
-
case "is_not_empty":
|
|
1731
|
-
return false;
|
|
1732
|
-
case "is_true":
|
|
1733
|
-
return false;
|
|
1734
|
-
case "is_false":
|
|
1735
|
-
return true;
|
|
1736
|
-
default:
|
|
1737
|
-
return false;
|
|
1738
|
-
}
|
|
1739
|
-
}
|
|
1740
|
-
const strValue = String(fieldValue);
|
|
1741
|
-
const numValue = Number(fieldValue);
|
|
1742
|
-
switch (operator) {
|
|
1743
|
-
case "equals":
|
|
1744
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1745
|
-
return numValue === Number(compareValue);
|
|
1746
|
-
}
|
|
1747
|
-
return strValue === compareValue;
|
|
1748
|
-
case "not_equals":
|
|
1749
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1750
|
-
return numValue !== Number(compareValue);
|
|
1751
|
-
}
|
|
1752
|
-
return strValue !== compareValue;
|
|
1753
|
-
case "greater_than":
|
|
1754
|
-
return numValue > Number(compareValue);
|
|
1755
|
-
case "greater_or_equal":
|
|
1756
|
-
return numValue >= Number(compareValue);
|
|
1757
|
-
case "less_than":
|
|
1758
|
-
return numValue < Number(compareValue);
|
|
1759
|
-
case "less_or_equal":
|
|
1760
|
-
return numValue <= Number(compareValue);
|
|
1761
|
-
case "contains":
|
|
1762
|
-
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
1763
|
-
case "starts_with":
|
|
1764
|
-
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
1765
|
-
case "ends_with":
|
|
1766
|
-
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
1767
|
-
case "is_empty":
|
|
1768
|
-
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
1769
|
-
case "is_not_empty":
|
|
1770
|
-
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
1771
|
-
case "is_true":
|
|
1772
|
-
return Boolean(fieldValue) === true;
|
|
1773
|
-
case "is_false":
|
|
1774
|
-
return Boolean(fieldValue) === false;
|
|
1775
|
-
default:
|
|
1776
|
-
return false;
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
function getNestedValue(obj, path) {
|
|
1780
|
-
const parts = path.split(".");
|
|
1781
|
-
let current = obj;
|
|
1782
|
-
for (const part of parts) {
|
|
1783
|
-
if (current === null || current === undefined || typeof current !== "object") {
|
|
1784
|
-
return;
|
|
1785
|
-
}
|
|
1786
|
-
current = current[part];
|
|
1787
|
-
}
|
|
1788
|
-
return current;
|
|
1789
|
-
}
|
|
1790
|
-
|
|
1791
1631
|
// src/task/Task.ts
|
|
1792
|
-
import { deepEqual, EventEmitter as EventEmitter3, uuid4 as
|
|
1632
|
+
import { deepEqual, EventEmitter as EventEmitter3, getLogger as getLogger2, uuid4 as uuid42 } from "@workglow/util";
|
|
1793
1633
|
import { compileSchema } from "@workglow/util/schema";
|
|
1794
1634
|
|
|
1795
1635
|
// src/task/TaskRunner.ts
|
|
1796
1636
|
import {
|
|
1797
|
-
getLogger
|
|
1637
|
+
getLogger,
|
|
1798
1638
|
getTelemetryProvider,
|
|
1799
1639
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1800
1640
|
ResourceScope,
|
|
@@ -1803,7 +1643,6 @@ import {
|
|
|
1803
1643
|
|
|
1804
1644
|
// src/task/CacheCoordinator.ts
|
|
1805
1645
|
import { getPortCodec } from "@workglow/util";
|
|
1806
|
-
|
|
1807
1646
|
class CacheCoordinator {
|
|
1808
1647
|
task;
|
|
1809
1648
|
constructor(task) {
|
|
@@ -1813,7 +1652,9 @@ class CacheCoordinator {
|
|
|
1813
1652
|
if (!outputCache)
|
|
1814
1653
|
return inputs;
|
|
1815
1654
|
const inputSchema = this.task.constructor.inputSchema();
|
|
1816
|
-
|
|
1655
|
+
const normalized = await CacheCoordinator.normalizeInputsForCacheKey(inputs, inputSchema);
|
|
1656
|
+
normalized.__cv = this.task.getCacheVersion();
|
|
1657
|
+
return normalized;
|
|
1817
1658
|
}
|
|
1818
1659
|
async lookup(keyInputs, outputCache, isStreamable, ctx) {
|
|
1819
1660
|
if (!outputCache || !this.task.cacheable)
|
|
@@ -1841,6 +1682,20 @@ class CacheCoordinator {
|
|
|
1841
1682
|
const wireOutputs = await CacheCoordinator.serializeOutputPorts(output, outputSchema);
|
|
1842
1683
|
await outputCache.saveOutput(this.task.type, keyInputs, wireOutputs);
|
|
1843
1684
|
}
|
|
1685
|
+
repoFor(registry, policy) {
|
|
1686
|
+
if (!registry || !isPolicyCached(policy))
|
|
1687
|
+
return;
|
|
1688
|
+
return isPolicyPrivate(policy) ? registry.private : registry.deterministic;
|
|
1689
|
+
}
|
|
1690
|
+
async buildKeyForPolicy(inputs, registry, policy) {
|
|
1691
|
+
return this.buildKey(inputs, this.repoFor(registry, policy));
|
|
1692
|
+
}
|
|
1693
|
+
async lookupByPolicy(keyInputs, registry, policy, isStreamable, ctx) {
|
|
1694
|
+
return this.lookup(keyInputs, this.repoFor(registry, policy), isStreamable, ctx);
|
|
1695
|
+
}
|
|
1696
|
+
async saveByPolicy(keyInputs, output, registry, policy) {
|
|
1697
|
+
return this.save(keyInputs, output, this.repoFor(registry, policy));
|
|
1698
|
+
}
|
|
1844
1699
|
static async serializeOutputPorts(output, schema) {
|
|
1845
1700
|
if (!schema?.properties)
|
|
1846
1701
|
return output;
|
|
@@ -2076,12 +1931,15 @@ class TaskRunner {
|
|
|
2076
1931
|
task;
|
|
2077
1932
|
currentCtx;
|
|
2078
1933
|
outputCache;
|
|
1934
|
+
cacheRegistry;
|
|
2079
1935
|
cacheCoordinator;
|
|
2080
1936
|
streamProcessor;
|
|
2081
1937
|
registry = globalServiceRegistry2;
|
|
2082
1938
|
resourceScope;
|
|
2083
1939
|
ownsResourceScope = false;
|
|
2084
1940
|
inputStreams;
|
|
1941
|
+
runId;
|
|
1942
|
+
static __privateWithoutRunIdWarned = new Set;
|
|
2085
1943
|
constructor(task) {
|
|
2086
1944
|
this.task = task;
|
|
2087
1945
|
this.own = this.own.bind(this);
|
|
@@ -2119,11 +1977,23 @@ class TaskRunner {
|
|
|
2119
1977
|
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
2120
1978
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
2121
1979
|
if (streamMode !== "none") {
|
|
2122
|
-
|
|
1980
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
let policy = this.task.getCachePolicy(inputs);
|
|
1984
|
+
if (policy.kind === "private" && !this.runId && this.cacheRegistry?.private !== undefined && !(this.cacheRegistry.private instanceof RunPrivateCacheRepo)) {
|
|
1985
|
+
const taskType = this.task.type;
|
|
1986
|
+
if (!TaskRunner.__privateWithoutRunIdWarned.has(taskType)) {
|
|
1987
|
+
TaskRunner.__privateWithoutRunIdWarned.add(taskType);
|
|
1988
|
+
getLogger().warn(`TaskRunner: task "${taskType}" has a private cache policy but no runId was ` + `provided. Private cache writes are skipped for this run — use TaskGraphRunner ` + `with runId for run-namespaced private caching, or provide an already namespaced ` + `private cache repo in CACHE_REGISTRY.`);
|
|
2123
1989
|
}
|
|
1990
|
+
policy = { kind: "none" };
|
|
2124
1991
|
}
|
|
2125
|
-
|
|
2126
|
-
|
|
1992
|
+
this.currentCtx?.telemetrySpan?.setAttributes({
|
|
1993
|
+
"workglow.task.cache_policy": policy.kind
|
|
1994
|
+
});
|
|
1995
|
+
const keyInputs = await this.cacheCoordinator.buildKeyForPolicy(inputs, this.cacheRegistry, policy);
|
|
1996
|
+
let outputs = await this.cacheCoordinator.lookupByPolicy(keyInputs, this.cacheRegistry, policy, isStreamable, ctx);
|
|
2127
1997
|
if (outputs === undefined) {
|
|
2128
1998
|
outputs = isStreamable ? await this.streamProcessor.run(inputs, ctx, {
|
|
2129
1999
|
registry: this.registry,
|
|
@@ -2132,7 +2002,7 @@ class TaskRunner {
|
|
|
2132
2002
|
onProgress: this.handleProgress.bind(this),
|
|
2133
2003
|
own: this.own
|
|
2134
2004
|
}) : await this.executeTask(inputs, ctx);
|
|
2135
|
-
await this.cacheCoordinator.
|
|
2005
|
+
await this.cacheCoordinator.saveByPolicy(keyInputs, outputs, this.cacheRegistry, policy);
|
|
2136
2006
|
this.task.runOutputData = outputs ?? {};
|
|
2137
2007
|
}
|
|
2138
2008
|
await this.handleComplete();
|
|
@@ -2169,7 +2039,7 @@ class TaskRunner {
|
|
|
2169
2039
|
}
|
|
2170
2040
|
await this.handleCompletePreview();
|
|
2171
2041
|
} catch (err) {
|
|
2172
|
-
|
|
2042
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
2173
2043
|
await this.handleErrorPreview();
|
|
2174
2044
|
} finally {
|
|
2175
2045
|
ctx.dispose();
|
|
@@ -2254,7 +2124,7 @@ class TaskRunner {
|
|
|
2254
2124
|
const out = await this.runPreview(overrides);
|
|
2255
2125
|
yield out;
|
|
2256
2126
|
} catch (err) {
|
|
2257
|
-
|
|
2127
|
+
getLogger().debug("runPreviewStream iteration failed", {
|
|
2258
2128
|
taskId: this.task.config?.id,
|
|
2259
2129
|
error: err
|
|
2260
2130
|
});
|
|
@@ -2297,7 +2167,8 @@ class TaskRunner {
|
|
|
2297
2167
|
updateProgress: this.handleProgress.bind(this),
|
|
2298
2168
|
own: this.own,
|
|
2299
2169
|
registry: this.registry,
|
|
2300
|
-
resourceScope: this.resourceScope
|
|
2170
|
+
resourceScope: this.resourceScope,
|
|
2171
|
+
runId: this.runId
|
|
2301
2172
|
});
|
|
2302
2173
|
return result;
|
|
2303
2174
|
}
|
|
@@ -2325,22 +2196,29 @@ class TaskRunner {
|
|
|
2325
2196
|
ctx.abortController.signal.addEventListener("abort", () => {
|
|
2326
2197
|
this.handleAbort();
|
|
2327
2198
|
});
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2199
|
+
if (config.registry) {
|
|
2200
|
+
this.registry = config.registry;
|
|
2201
|
+
}
|
|
2202
|
+
this.runId = config.runId;
|
|
2203
|
+
const legacy = config.outputCache ?? this.task.runConfig?.outputCache;
|
|
2204
|
+
if (legacy === false) {
|
|
2205
|
+
this.cacheRegistry = undefined;
|
|
2206
|
+
this.outputCache = undefined;
|
|
2207
|
+
} else if (legacy instanceof TaskOutputRepository) {
|
|
2208
|
+
this.cacheRegistry = new DefaultCacheRegistry({ deterministic: legacy });
|
|
2209
|
+
this.outputCache = legacy;
|
|
2210
|
+
} else if (legacy === true) {
|
|
2211
|
+
const instance = globalServiceRegistry2.has(TASK_OUTPUT_REPOSITORY) ? globalServiceRegistry2.get(TASK_OUTPUT_REPOSITORY) : undefined;
|
|
2331
2212
|
this.outputCache = instance;
|
|
2332
|
-
|
|
2213
|
+
this.cacheRegistry = instance ? new DefaultCacheRegistry({ deterministic: instance }) : undefined;
|
|
2214
|
+
} else {
|
|
2333
2215
|
this.outputCache = undefined;
|
|
2334
|
-
|
|
2335
|
-
this.outputCache = cache;
|
|
2216
|
+
this.cacheRegistry = this.registry.has(CACHE_REGISTRY) ? this.registry.get(CACHE_REGISTRY) : undefined;
|
|
2336
2217
|
}
|
|
2337
2218
|
ctx.shouldAccumulate = config.shouldAccumulate !== false;
|
|
2338
2219
|
if (config.updateProgress) {
|
|
2339
2220
|
this.updateProgress = config.updateProgress;
|
|
2340
2221
|
}
|
|
2341
|
-
if (config.registry) {
|
|
2342
|
-
this.registry = config.registry;
|
|
2343
|
-
}
|
|
2344
2222
|
if (config.resourceScope) {
|
|
2345
2223
|
this.resourceScope = config.resourceScope;
|
|
2346
2224
|
}
|
|
@@ -2362,7 +2240,6 @@ class TaskRunner {
|
|
|
2362
2240
|
attributes: {
|
|
2363
2241
|
"workglow.task.type": this.task.type,
|
|
2364
2242
|
"workglow.task.id": String(this.task.config.id),
|
|
2365
|
-
"workglow.task.cacheable": this.task.cacheable,
|
|
2366
2243
|
"workglow.task.title": this.task.title || undefined
|
|
2367
2244
|
}
|
|
2368
2245
|
});
|
|
@@ -2493,12 +2370,15 @@ class Task {
|
|
|
2493
2370
|
static title = "";
|
|
2494
2371
|
static description = "";
|
|
2495
2372
|
static cacheable = true;
|
|
2373
|
+
static version = 1;
|
|
2374
|
+
static cachePolicy = { kind: "deterministic" };
|
|
2496
2375
|
static hasDynamicSchemas = false;
|
|
2497
2376
|
static passthroughInputsToOutputs = false;
|
|
2498
2377
|
static customizable = false;
|
|
2499
2378
|
static isGraphOutput = false;
|
|
2500
2379
|
static isPassthrough = false;
|
|
2501
2380
|
static hasDynamicEntitlements = false;
|
|
2381
|
+
static __cacheableDeprecationWarned = new Set;
|
|
2502
2382
|
static entitlements() {
|
|
2503
2383
|
return EMPTY_ENTITLEMENTS;
|
|
2504
2384
|
}
|
|
@@ -2577,7 +2457,39 @@ class Task {
|
|
|
2577
2457
|
return this.config?.description ?? this.constructor.description;
|
|
2578
2458
|
}
|
|
2579
2459
|
get cacheable() {
|
|
2580
|
-
|
|
2460
|
+
if (this.runConfig?.cacheable !== undefined)
|
|
2461
|
+
return this.runConfig.cacheable;
|
|
2462
|
+
if (this.config?.cacheable !== undefined)
|
|
2463
|
+
return this.config.cacheable;
|
|
2464
|
+
return this.getCachePolicy(this.runInputData ?? {}).kind !== "none";
|
|
2465
|
+
}
|
|
2466
|
+
getCacheVersion() {
|
|
2467
|
+
const versions = [];
|
|
2468
|
+
let ctor = this.constructor;
|
|
2469
|
+
while (ctor && ctor !== Function.prototype) {
|
|
2470
|
+
if (Object.prototype.hasOwnProperty.call(ctor, "version") && typeof ctor.version === "number") {
|
|
2471
|
+
versions.push(ctor.version);
|
|
2472
|
+
}
|
|
2473
|
+
ctor = Object.getPrototypeOf(ctor);
|
|
2474
|
+
}
|
|
2475
|
+
if (versions.length === 0)
|
|
2476
|
+
versions.push(1);
|
|
2477
|
+
return versions.join(".");
|
|
2478
|
+
}
|
|
2479
|
+
getCachePolicy(_inputs) {
|
|
2480
|
+
const ctor = this.constructor;
|
|
2481
|
+
const hasLegacyOverride = Object.prototype.hasOwnProperty.call(ctor, "cacheable") && ctor.cacheable === false;
|
|
2482
|
+
const hasPolicyOverride = Object.prototype.hasOwnProperty.call(ctor, "cachePolicy");
|
|
2483
|
+
if (hasLegacyOverride && !hasPolicyOverride) {
|
|
2484
|
+
if (!Task.__cacheableDeprecationWarned.has(ctor.type)) {
|
|
2485
|
+
Task.__cacheableDeprecationWarned.add(ctor.type);
|
|
2486
|
+
getLogger2().warn(`Task "${ctor.type}": static \`cacheable = false\` is deprecated. ` + `Use \`static cachePolicy: CachePolicy = { kind: "none" }\` instead.`);
|
|
2487
|
+
}
|
|
2488
|
+
return { kind: "none" };
|
|
2489
|
+
}
|
|
2490
|
+
if (this.runConfig?.cacheable === false || this.config?.cacheable === false)
|
|
2491
|
+
return { kind: "none" };
|
|
2492
|
+
return ctor.cachePolicy ?? DEFAULT_CACHE_POLICY;
|
|
2581
2493
|
}
|
|
2582
2494
|
defaults;
|
|
2583
2495
|
runInputData = {};
|
|
@@ -2612,7 +2524,7 @@ class Task {
|
|
|
2612
2524
|
...title ? { title } : {}
|
|
2613
2525
|
}, restConfig);
|
|
2614
2526
|
if (baseConfig.id === undefined) {
|
|
2615
|
-
baseConfig.id =
|
|
2527
|
+
baseConfig.id = uuid42();
|
|
2616
2528
|
}
|
|
2617
2529
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
2618
2530
|
try {
|
|
@@ -2621,6 +2533,10 @@ class Task {
|
|
|
2621
2533
|
this.originalConfig = undefined;
|
|
2622
2534
|
}
|
|
2623
2535
|
this.runConfig = runConfig;
|
|
2536
|
+
const inputSchema = this.constructor.inputSchema();
|
|
2537
|
+
if (inputSchema && typeof inputSchema !== "boolean" && inputSchema.properties && Object.prototype.hasOwnProperty.call(inputSchema.properties, "__cv")) {
|
|
2538
|
+
throw new TaskConfigurationError(`Task "${this.constructor.type}": input port name '__cv' is reserved ` + `for cache versioning. Rename the port to avoid collision with the cache key sentinel.`);
|
|
2539
|
+
}
|
|
2624
2540
|
}
|
|
2625
2541
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
2626
2542
|
const schema = this.inputSchema();
|
|
@@ -2918,91 +2834,320 @@ class Task {
|
|
|
2918
2834
|
result[key] = this.stripSymbols(obj[key]);
|
|
2919
2835
|
}
|
|
2920
2836
|
}
|
|
2921
|
-
return result;
|
|
2837
|
+
return result;
|
|
2838
|
+
}
|
|
2839
|
+
return obj;
|
|
2840
|
+
}
|
|
2841
|
+
canSerializeConfig() {
|
|
2842
|
+
return true;
|
|
2843
|
+
}
|
|
2844
|
+
toJSON(_options) {
|
|
2845
|
+
const ctor = this.constructor;
|
|
2846
|
+
if (!this.canSerializeConfig() || !this.originalConfig) {
|
|
2847
|
+
throw new TaskSerializationError(this.type);
|
|
2848
|
+
}
|
|
2849
|
+
const schema = ctor.configSchema();
|
|
2850
|
+
const schemaProperties = typeof schema !== "boolean" && schema?.properties ? schema.properties : {};
|
|
2851
|
+
const config = {};
|
|
2852
|
+
for (const [key, propSchema] of Object.entries(schemaProperties)) {
|
|
2853
|
+
if (key === "id")
|
|
2854
|
+
continue;
|
|
2855
|
+
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
2856
|
+
continue;
|
|
2857
|
+
}
|
|
2858
|
+
const value = this.originalConfig[key];
|
|
2859
|
+
if (value === undefined)
|
|
2860
|
+
continue;
|
|
2861
|
+
if (typeof value === "function" || typeof value === "symbol")
|
|
2862
|
+
continue;
|
|
2863
|
+
config[key] = value;
|
|
2864
|
+
}
|
|
2865
|
+
if (config.title === ctor.title)
|
|
2866
|
+
delete config.title;
|
|
2867
|
+
if (config.description === ctor.description)
|
|
2868
|
+
delete config.description;
|
|
2869
|
+
const extras = config.extras;
|
|
2870
|
+
if (!extras || Object.keys(extras).length === 0)
|
|
2871
|
+
delete config.extras;
|
|
2872
|
+
const base = {
|
|
2873
|
+
id: this.id,
|
|
2874
|
+
type: this.type,
|
|
2875
|
+
defaults: this.defaults
|
|
2876
|
+
};
|
|
2877
|
+
if (Object.keys(config).length > 0) {
|
|
2878
|
+
base.config = config;
|
|
2879
|
+
}
|
|
2880
|
+
return this.stripSymbols(base);
|
|
2881
|
+
}
|
|
2882
|
+
toDependencyJSON(options) {
|
|
2883
|
+
const json = this.toJSON(options);
|
|
2884
|
+
return json;
|
|
2885
|
+
}
|
|
2886
|
+
hasChildren() {
|
|
2887
|
+
return this._subGraph !== undefined && this._subGraph !== null && this._subGraph.getTasks().length > 0;
|
|
2888
|
+
}
|
|
2889
|
+
_taskAddedListener = () => {
|
|
2890
|
+
this.emit("regenerate");
|
|
2891
|
+
};
|
|
2892
|
+
_subGraph = undefined;
|
|
2893
|
+
set subGraph(subGraph) {
|
|
2894
|
+
if (this._subGraph) {
|
|
2895
|
+
this._subGraph.off("task_added", this._taskAddedListener);
|
|
2896
|
+
}
|
|
2897
|
+
this._subGraph = subGraph;
|
|
2898
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2899
|
+
}
|
|
2900
|
+
get subGraph() {
|
|
2901
|
+
if (!this._subGraph) {
|
|
2902
|
+
this._subGraph = new TaskGraph;
|
|
2903
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2904
|
+
}
|
|
2905
|
+
return this._subGraph;
|
|
2906
|
+
}
|
|
2907
|
+
regenerateGraph() {
|
|
2908
|
+
if (this.hasChildren()) {
|
|
2909
|
+
for (const dataflow of this.subGraph.getDataflows()) {
|
|
2910
|
+
this.subGraph.removeDataflow(dataflow);
|
|
2911
|
+
}
|
|
2912
|
+
for (const child of this.subGraph.getTasks()) {
|
|
2913
|
+
this.subGraph.removeTask(child.id);
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
this.events.emit("regenerate");
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
|
|
2920
|
+
// src/task-graph/EdgeMaterializer.ts
|
|
2921
|
+
import { getLogger as getLogger3 } from "@workglow/util";
|
|
2922
|
+
import { previewSource } from "@workglow/util/media";
|
|
2923
|
+
class EdgeMaterializer {
|
|
2924
|
+
graph;
|
|
2925
|
+
runner;
|
|
2926
|
+
constructor(graph, runner) {
|
|
2927
|
+
this.graph = graph;
|
|
2928
|
+
this.runner = runner;
|
|
2929
|
+
}
|
|
2930
|
+
filterInputForTask(task, input) {
|
|
2931
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
2932
|
+
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
2933
|
+
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
2934
|
+
const filteredInput = {};
|
|
2935
|
+
for (const [key, value] of Object.entries(input)) {
|
|
2936
|
+
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
2937
|
+
filteredInput[key] = value;
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
return filteredInput;
|
|
2941
|
+
}
|
|
2942
|
+
copyInputFromEdgesToNode(task) {
|
|
2943
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2944
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2945
|
+
for (const dataflow of dataflows) {
|
|
2946
|
+
const live = dataflow.getCurrentValue();
|
|
2947
|
+
const port = dataflow.targetTaskPortId;
|
|
2948
|
+
let portData;
|
|
2949
|
+
if (port === DATAFLOW_ALL_PORTS) {
|
|
2950
|
+
portData = live;
|
|
2951
|
+
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
2952
|
+
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
2953
|
+
} else {
|
|
2954
|
+
portData = { [port]: live };
|
|
2955
|
+
}
|
|
2956
|
+
this.runner.addInputData(task, portData);
|
|
2957
|
+
}
|
|
2958
|
+
}
|
|
2959
|
+
async pushOutputFromNodeToEdges(node, results) {
|
|
2960
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2961
|
+
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
2962
|
+
for (const port of Object.keys(results)) {
|
|
2963
|
+
const value = results[port];
|
|
2964
|
+
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
2965
|
+
results[port] = await previewSource(value);
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
for (const dataflow of dataflows) {
|
|
2970
|
+
if (dataflow.stream !== undefined)
|
|
2971
|
+
continue;
|
|
2972
|
+
const registry = this.runner["registry"];
|
|
2973
|
+
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
2974
|
+
if (compatibility === "static") {
|
|
2975
|
+
dataflow.setPortData(results);
|
|
2976
|
+
await dataflow.applyTransforms(registry);
|
|
2977
|
+
} else if (compatibility === "runtime") {
|
|
2978
|
+
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
2979
|
+
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
2980
|
+
dataflow.setPortData(narrowed);
|
|
2981
|
+
await dataflow.applyTransforms(registry);
|
|
2982
|
+
} else {
|
|
2983
|
+
const resultsKeys = Object.keys(results);
|
|
2984
|
+
if (resultsKeys.length > 0) {
|
|
2985
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2986
|
+
dataflowId: dataflow.id,
|
|
2987
|
+
compatibility,
|
|
2988
|
+
resultsKeys
|
|
2989
|
+
});
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2922
2992
|
}
|
|
2923
|
-
return obj;
|
|
2924
2993
|
}
|
|
2925
|
-
|
|
2926
|
-
|
|
2994
|
+
pushErrorFromNodeToEdges(node) {
|
|
2995
|
+
if (!node?.config?.id)
|
|
2996
|
+
return;
|
|
2997
|
+
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
2998
|
+
dataflow.error = node.error;
|
|
2999
|
+
});
|
|
2927
3000
|
}
|
|
2928
|
-
|
|
2929
|
-
const
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
const
|
|
2934
|
-
const
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
if (key === "id")
|
|
2938
|
-
continue;
|
|
2939
|
-
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
2940
|
-
continue;
|
|
2941
|
-
}
|
|
2942
|
-
const value = this.originalConfig[key];
|
|
2943
|
-
if (value === undefined)
|
|
2944
|
-
continue;
|
|
2945
|
-
if (typeof value === "function" || typeof value === "symbol")
|
|
2946
|
-
continue;
|
|
2947
|
-
config[key] = value;
|
|
2948
|
-
}
|
|
2949
|
-
if (config.title === ctor.title)
|
|
2950
|
-
delete config.title;
|
|
2951
|
-
if (config.description === ctor.description)
|
|
2952
|
-
delete config.description;
|
|
2953
|
-
const extras = config.extras;
|
|
2954
|
-
if (!extras || Object.keys(extras).length === 0)
|
|
2955
|
-
delete config.extras;
|
|
2956
|
-
const base = {
|
|
2957
|
-
id: this.id,
|
|
2958
|
-
type: this.type,
|
|
2959
|
-
defaults: this.defaults
|
|
3001
|
+
hasErrorOutputEdges(task) {
|
|
3002
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3003
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
3004
|
+
}
|
|
3005
|
+
pushErrorOutputToEdges(task) {
|
|
3006
|
+
const taskError = task.error;
|
|
3007
|
+
const errorData = {
|
|
3008
|
+
error: taskError?.message ?? "Unknown error",
|
|
3009
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2960
3010
|
};
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
3011
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3012
|
+
for (const df of dataflows) {
|
|
3013
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
3014
|
+
df.value = errorData;
|
|
3015
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
3016
|
+
} else {
|
|
3017
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
3018
|
+
}
|
|
2967
3019
|
}
|
|
2968
|
-
|
|
3020
|
+
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
2969
3021
|
}
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
3022
|
+
resetTask(graph, task, runId) {
|
|
3023
|
+
task.status = TaskStatus.PENDING;
|
|
3024
|
+
task.resetInputData();
|
|
3025
|
+
task.runOutputData = {};
|
|
3026
|
+
task.error = undefined;
|
|
3027
|
+
task.progress = 0;
|
|
3028
|
+
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
3029
|
+
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
3030
|
+
if (task?.config?.id) {
|
|
3031
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
3032
|
+
dataflow.error = task.error;
|
|
3033
|
+
});
|
|
3034
|
+
}
|
|
3035
|
+
task.emit("reset");
|
|
3036
|
+
task.emit("status", task.status);
|
|
2973
3037
|
}
|
|
2974
|
-
|
|
2975
|
-
|
|
3038
|
+
static isImageValueShape(v) {
|
|
3039
|
+
if (v === null || typeof v !== "object")
|
|
3040
|
+
return false;
|
|
3041
|
+
const o = v;
|
|
3042
|
+
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
2976
3043
|
}
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
3044
|
+
}
|
|
3045
|
+
|
|
3046
|
+
// src/task-graph/RunContext.ts
|
|
3047
|
+
import { uuid4 as uuid43 } from "@workglow/util";
|
|
3048
|
+
|
|
3049
|
+
class RunContext {
|
|
3050
|
+
runId;
|
|
3051
|
+
abortController;
|
|
3052
|
+
inProgressTasks = new Map;
|
|
3053
|
+
inProgressFunctions = new Map;
|
|
3054
|
+
failedTaskErrors = new Map;
|
|
3055
|
+
telemetrySpan;
|
|
3056
|
+
graphTimeoutTimer;
|
|
3057
|
+
pendingGraphTimeoutError;
|
|
3058
|
+
activeEnforcer;
|
|
3059
|
+
parentSignalCleanup;
|
|
3060
|
+
constructor(parentSignal) {
|
|
3061
|
+
this.runId = uuid43();
|
|
3062
|
+
this.abortController = new AbortController;
|
|
3063
|
+
if (parentSignal) {
|
|
3064
|
+
const onParentAbort = () => this.abortController.abort();
|
|
3065
|
+
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
3066
|
+
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
3067
|
+
if (parentSignal.aborted) {
|
|
3068
|
+
this.parentSignalCleanup();
|
|
3069
|
+
this.parentSignalCleanup = undefined;
|
|
3070
|
+
this.abortController.abort();
|
|
3071
|
+
}
|
|
2984
3072
|
}
|
|
2985
|
-
this._subGraph = subGraph;
|
|
2986
|
-
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2987
3073
|
}
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
3074
|
+
dispose() {
|
|
3075
|
+
this.parentSignalCleanup?.();
|
|
3076
|
+
this.parentSignalCleanup = undefined;
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
|
|
3080
|
+
// src/task-graph/RunScheduler.ts
|
|
3081
|
+
import { getLogger as getLogger5 } from "@workglow/util";
|
|
3082
|
+
|
|
3083
|
+
// src/task/ConditionalTask.ts
|
|
3084
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
3085
|
+
|
|
3086
|
+
// src/task/ConditionUtils.ts
|
|
3087
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
3088
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
3089
|
+
switch (operator) {
|
|
3090
|
+
case "is_empty":
|
|
3091
|
+
return true;
|
|
3092
|
+
case "is_not_empty":
|
|
3093
|
+
return false;
|
|
3094
|
+
case "is_true":
|
|
3095
|
+
return false;
|
|
3096
|
+
case "is_false":
|
|
3097
|
+
return true;
|
|
3098
|
+
default:
|
|
3099
|
+
return false;
|
|
2992
3100
|
}
|
|
2993
|
-
return this._subGraph;
|
|
2994
3101
|
}
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
3102
|
+
const strValue = String(fieldValue);
|
|
3103
|
+
const numValue = Number(fieldValue);
|
|
3104
|
+
switch (operator) {
|
|
3105
|
+
case "equals":
|
|
3106
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3107
|
+
return numValue === Number(compareValue);
|
|
2999
3108
|
}
|
|
3000
|
-
|
|
3001
|
-
|
|
3109
|
+
return strValue === compareValue;
|
|
3110
|
+
case "not_equals":
|
|
3111
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3112
|
+
return numValue !== Number(compareValue);
|
|
3002
3113
|
}
|
|
3114
|
+
return strValue !== compareValue;
|
|
3115
|
+
case "greater_than":
|
|
3116
|
+
return numValue > Number(compareValue);
|
|
3117
|
+
case "greater_or_equal":
|
|
3118
|
+
return numValue >= Number(compareValue);
|
|
3119
|
+
case "less_than":
|
|
3120
|
+
return numValue < Number(compareValue);
|
|
3121
|
+
case "less_or_equal":
|
|
3122
|
+
return numValue <= Number(compareValue);
|
|
3123
|
+
case "contains":
|
|
3124
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
3125
|
+
case "starts_with":
|
|
3126
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
3127
|
+
case "ends_with":
|
|
3128
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
3129
|
+
case "is_empty":
|
|
3130
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
3131
|
+
case "is_not_empty":
|
|
3132
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
3133
|
+
case "is_true":
|
|
3134
|
+
return Boolean(fieldValue) === true;
|
|
3135
|
+
case "is_false":
|
|
3136
|
+
return Boolean(fieldValue) === false;
|
|
3137
|
+
default:
|
|
3138
|
+
return false;
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
function getNestedValue(obj, path) {
|
|
3142
|
+
const parts = path.split(".");
|
|
3143
|
+
let current = obj;
|
|
3144
|
+
for (const part of parts) {
|
|
3145
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
3146
|
+
return;
|
|
3003
3147
|
}
|
|
3004
|
-
|
|
3148
|
+
current = current[part];
|
|
3005
3149
|
}
|
|
3150
|
+
return current;
|
|
3006
3151
|
}
|
|
3007
3152
|
|
|
3008
3153
|
// src/task/ConditionalTask.ts
|
|
@@ -3094,7 +3239,7 @@ class ConditionalTask extends Task {
|
|
|
3094
3239
|
}
|
|
3095
3240
|
}
|
|
3096
3241
|
} catch (error) {
|
|
3097
|
-
|
|
3242
|
+
getLogger4().error(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
3098
3243
|
}
|
|
3099
3244
|
}
|
|
3100
3245
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -3366,7 +3511,7 @@ class RunScheduler {
|
|
|
3366
3511
|
ctx.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
3367
3512
|
}
|
|
3368
3513
|
} catch (err) {
|
|
3369
|
-
|
|
3514
|
+
getLogger5().error("Error running graph", { error: err });
|
|
3370
3515
|
}
|
|
3371
3516
|
await Promise.allSettled(Array.from(ctx.inProgressTasks.values()));
|
|
3372
3517
|
await Promise.allSettled(Array.from(ctx.inProgressFunctions.values()));
|
|
@@ -3442,11 +3587,12 @@ class StreamPump {
|
|
|
3442
3587
|
task.on("stream_end", onStreamEnd);
|
|
3443
3588
|
try {
|
|
3444
3589
|
const results = await task.runner.run(input, {
|
|
3445
|
-
outputCache: options.
|
|
3590
|
+
outputCache: options.legacyCacheExplicitlyDisabled ? false : options.outputCache,
|
|
3446
3591
|
shouldAccumulate,
|
|
3447
3592
|
updateProgress: options.updateProgress,
|
|
3448
3593
|
registry: options.registry,
|
|
3449
|
-
resourceScope: options.resourceScope
|
|
3594
|
+
resourceScope: options.resourceScope,
|
|
3595
|
+
runId: options.runId
|
|
3450
3596
|
});
|
|
3451
3597
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3452
3598
|
return {
|
|
@@ -3715,10 +3861,14 @@ class TaskGraphRunner {
|
|
|
3715
3861
|
previewRunning = false;
|
|
3716
3862
|
graph;
|
|
3717
3863
|
outputCache;
|
|
3864
|
+
legacyCacheExplicitlyDisabled = false;
|
|
3718
3865
|
accumulateLeafOutputs = true;
|
|
3719
3866
|
registry = globalServiceRegistry3;
|
|
3720
3867
|
resourceScope;
|
|
3721
3868
|
currentCtx;
|
|
3869
|
+
runId;
|
|
3870
|
+
currentRunPrivate;
|
|
3871
|
+
baseRegistryForRun;
|
|
3722
3872
|
edgeMaterializer;
|
|
3723
3873
|
streamPump;
|
|
3724
3874
|
runScheduler;
|
|
@@ -3758,6 +3908,15 @@ class TaskGraphRunner {
|
|
|
3758
3908
|
throw new TaskAbortedError;
|
|
3759
3909
|
}
|
|
3760
3910
|
await this.handleComplete();
|
|
3911
|
+
const runPrivateToClean = this.currentRunPrivate;
|
|
3912
|
+
this.currentRunPrivate = undefined;
|
|
3913
|
+
if (runPrivateToClean) {
|
|
3914
|
+
try {
|
|
3915
|
+
await runPrivateToClean.clearRun();
|
|
3916
|
+
} catch (e) {
|
|
3917
|
+
getLogger6().warn("RunPrivateCacheRepo.clearRun failed", { error: e });
|
|
3918
|
+
}
|
|
3919
|
+
}
|
|
3761
3920
|
return this.filterLeafResults(results);
|
|
3762
3921
|
} finally {
|
|
3763
3922
|
if (ownsScope) {
|
|
@@ -3829,7 +3988,7 @@ class TaskGraphRunner {
|
|
|
3829
3988
|
});
|
|
3830
3989
|
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3831
3990
|
previewSpan.end();
|
|
3832
|
-
|
|
3991
|
+
getLogger6().debug("task graph runPreview timings", {
|
|
3833
3992
|
previewRunId,
|
|
3834
3993
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3835
3994
|
taskTimings
|
|
@@ -3847,7 +4006,7 @@ class TaskGraphRunner {
|
|
|
3847
4006
|
});
|
|
3848
4007
|
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3849
4008
|
previewSpan.end();
|
|
3850
|
-
|
|
4009
|
+
getLogger6().debug("task graph runPreview failed", {
|
|
3851
4010
|
previewRunId,
|
|
3852
4011
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3853
4012
|
taskTimings,
|
|
@@ -3918,14 +4077,17 @@ class TaskGraphRunner {
|
|
|
3918
4077
|
outputCache: this.outputCache,
|
|
3919
4078
|
resourceScope: this.resourceScope,
|
|
3920
4079
|
accumulateLeafOutputs: this.accumulateLeafOutputs,
|
|
3921
|
-
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a)
|
|
4080
|
+
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a),
|
|
4081
|
+
runId: this.runId,
|
|
4082
|
+
legacyCacheExplicitlyDisabled: this.legacyCacheExplicitlyDisabled
|
|
3922
4083
|
});
|
|
3923
4084
|
}
|
|
3924
4085
|
const results = await task.runner.run(input, {
|
|
3925
|
-
outputCache: this.
|
|
4086
|
+
outputCache: this.legacyCacheExplicitlyDisabled ? false : this.outputCache,
|
|
3926
4087
|
updateProgress: async (task2, progress, message, ...args) => await this.runScheduler.handleProgress(this.currentCtx, task2, progress, message, ...args),
|
|
3927
4088
|
registry: this.registry,
|
|
3928
|
-
resourceScope: this.resourceScope
|
|
4089
|
+
resourceScope: this.resourceScope,
|
|
4090
|
+
runId: this.runId
|
|
3929
4091
|
});
|
|
3930
4092
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3931
4093
|
return {
|
|
@@ -3946,6 +4108,19 @@ class TaskGraphRunner {
|
|
|
3946
4108
|
dataflow.reset();
|
|
3947
4109
|
});
|
|
3948
4110
|
}
|
|
4111
|
+
static __durabilityWarnedRepos = new WeakSet;
|
|
4112
|
+
static graphUsesPrivatePolicy(graph) {
|
|
4113
|
+
return graph.getTasks().some((t) => {
|
|
4114
|
+
const ctor = t.constructor;
|
|
4115
|
+
if (ctor.cachePolicy?.kind === "private")
|
|
4116
|
+
return true;
|
|
4117
|
+
const proto = ctor.prototype;
|
|
4118
|
+
if (typeof proto.getCachePolicy === "function" && proto.getCachePolicy !== Task.prototype.getCachePolicy) {
|
|
4119
|
+
return true;
|
|
4120
|
+
}
|
|
4121
|
+
return false;
|
|
4122
|
+
});
|
|
4123
|
+
}
|
|
3949
4124
|
async handleStart(config) {
|
|
3950
4125
|
if (config?.registry !== undefined) {
|
|
3951
4126
|
this.registry = config.registry;
|
|
@@ -3955,18 +4130,61 @@ class TaskGraphRunner {
|
|
|
3955
4130
|
if (config?.resourceScope !== undefined) {
|
|
3956
4131
|
this.resourceScope = config.resourceScope;
|
|
3957
4132
|
}
|
|
4133
|
+
this.baseRegistryForRun = this.registry;
|
|
4134
|
+
this.runId = config?.runId;
|
|
4135
|
+
if (this.registry.has(CACHE_REGISTRY)) {
|
|
4136
|
+
const checkRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4137
|
+
if (checkRegistry.private && !checkRegistry.private.isDurable()) {
|
|
4138
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4139
|
+
const repo = checkRegistry.private;
|
|
4140
|
+
if (!TaskGraphRunner.__durabilityWarnedRepos.has(repo)) {
|
|
4141
|
+
TaskGraphRunner.__durabilityWarnedRepos.add(repo);
|
|
4142
|
+
getLogger6().warn("TaskGraphRunner: private cache repo may be used but is non-durable — " + "restart-survival will not work. Ensure the CacheRegistry 'private' " + "slot is backed by a durable storage backend.");
|
|
4143
|
+
}
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
if (!this.runId && checkRegistry.private) {
|
|
4147
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4148
|
+
throw new TaskConfigurationError("TaskGraphRunner: graph contains a private-policy task but no runId was provided. " + "Provide `runId` in TaskGraphRunConfig so private cache entries can be namespaced.");
|
|
4149
|
+
}
|
|
4150
|
+
}
|
|
4151
|
+
}
|
|
4152
|
+
if (this.runId && this.registry.has(CACHE_REGISTRY)) {
|
|
4153
|
+
const baseRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4154
|
+
if (baseRegistry.private) {
|
|
4155
|
+
const wrappedPrivate = new RunPrivateCacheRepo({
|
|
4156
|
+
backing: baseRegistry.private,
|
|
4157
|
+
runId: this.runId
|
|
4158
|
+
});
|
|
4159
|
+
this.currentRunPrivate = wrappedPrivate;
|
|
4160
|
+
const wrappedRegistry = new DefaultCacheRegistry({
|
|
4161
|
+
deterministic: baseRegistry.deterministic,
|
|
4162
|
+
private: wrappedPrivate
|
|
4163
|
+
});
|
|
4164
|
+
const childContainer = this.registry.container.createChildContainer();
|
|
4165
|
+
const runtimeServices = new ServiceRegistry3(childContainer);
|
|
4166
|
+
runtimeServices.registerInstance(CACHE_REGISTRY, wrappedRegistry);
|
|
4167
|
+
this.registry = runtimeServices;
|
|
4168
|
+
this.resourceScope?.register(`cache:private:${this.runId}`, async () => {});
|
|
4169
|
+
}
|
|
4170
|
+
}
|
|
3958
4171
|
this.accumulateLeafOutputs = config?.accumulateLeafOutputs !== false;
|
|
3959
4172
|
if (config?.outputCache !== undefined) {
|
|
3960
4173
|
if (typeof config.outputCache === "boolean") {
|
|
3961
4174
|
if (config.outputCache === true) {
|
|
3962
4175
|
this.outputCache = this.registry.get(TASK_OUTPUT_REPOSITORY);
|
|
4176
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3963
4177
|
} else {
|
|
3964
4178
|
this.outputCache = undefined;
|
|
4179
|
+
this.legacyCacheExplicitlyDisabled = true;
|
|
3965
4180
|
}
|
|
3966
4181
|
} else {
|
|
3967
4182
|
this.outputCache = config.outputCache;
|
|
4183
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3968
4184
|
}
|
|
3969
4185
|
this.graph.outputCache = this.outputCache;
|
|
4186
|
+
} else {
|
|
4187
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3970
4188
|
}
|
|
3971
4189
|
if (this.running || this.previewRunning) {
|
|
3972
4190
|
throw new TaskConfigurationError("Graph is already running");
|
|
@@ -4055,6 +4273,10 @@ class TaskGraphRunner {
|
|
|
4055
4273
|
}
|
|
4056
4274
|
ctx?.dispose();
|
|
4057
4275
|
this.currentCtx = undefined;
|
|
4276
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4277
|
+
this.registry = this.baseRegistryForRun;
|
|
4278
|
+
this.baseRegistryForRun = undefined;
|
|
4279
|
+
}
|
|
4058
4280
|
this.graph.emit("complete");
|
|
4059
4281
|
}
|
|
4060
4282
|
async handleCompletePreview() {
|
|
@@ -4067,6 +4289,7 @@ class TaskGraphRunner {
|
|
|
4067
4289
|
return task.abort();
|
|
4068
4290
|
}
|
|
4069
4291
|
}));
|
|
4292
|
+
this.currentRunPrivate = undefined;
|
|
4070
4293
|
const ctx = this.currentCtx;
|
|
4071
4294
|
this.running = false;
|
|
4072
4295
|
if (ctx?.telemetrySpan) {
|
|
@@ -4076,6 +4299,10 @@ class TaskGraphRunner {
|
|
|
4076
4299
|
}
|
|
4077
4300
|
ctx?.dispose();
|
|
4078
4301
|
this.currentCtx = undefined;
|
|
4302
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4303
|
+
this.registry = this.baseRegistryForRun;
|
|
4304
|
+
this.baseRegistryForRun = undefined;
|
|
4305
|
+
}
|
|
4079
4306
|
this.graph.emit("error", error);
|
|
4080
4307
|
}
|
|
4081
4308
|
async handleErrorPreview() {
|
|
@@ -4091,6 +4318,7 @@ class TaskGraphRunner {
|
|
|
4091
4318
|
return task.abort();
|
|
4092
4319
|
}
|
|
4093
4320
|
}));
|
|
4321
|
+
this.currentRunPrivate = undefined;
|
|
4094
4322
|
const ctx = this.currentCtx;
|
|
4095
4323
|
if (ctx?.telemetrySpan) {
|
|
4096
4324
|
ctx.telemetrySpan.setStatus(SpanStatusCode2.ERROR, "aborted");
|
|
@@ -4099,6 +4327,10 @@ class TaskGraphRunner {
|
|
|
4099
4327
|
}
|
|
4100
4328
|
ctx?.dispose();
|
|
4101
4329
|
this.currentCtx = undefined;
|
|
4330
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4331
|
+
this.registry = this.baseRegistryForRun;
|
|
4332
|
+
this.baseRegistryForRun = undefined;
|
|
4333
|
+
}
|
|
4102
4334
|
this.graph.emit("abort");
|
|
4103
4335
|
}
|
|
4104
4336
|
async handleAbortPreview() {
|
|
@@ -4222,7 +4454,7 @@ class GraphAsTask extends Task {
|
|
|
4222
4454
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
4223
4455
|
this._inputSchemaNode = schemaNode;
|
|
4224
4456
|
} catch (error) {
|
|
4225
|
-
|
|
4457
|
+
getLogger7().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
4226
4458
|
this._inputSchemaNode = compileSchema2({});
|
|
4227
4459
|
}
|
|
4228
4460
|
}
|
|
@@ -4506,7 +4738,7 @@ function convertPipeFunctionToTask(fn, config) {
|
|
|
4506
4738
|
additionalProperties: true
|
|
4507
4739
|
};
|
|
4508
4740
|
};
|
|
4509
|
-
static
|
|
4741
|
+
static cachePolicy = { kind: "none" };
|
|
4510
4742
|
async execute(input, context) {
|
|
4511
4743
|
return fn(input, context);
|
|
4512
4744
|
}
|
|
@@ -4589,7 +4821,8 @@ class TaskGraph {
|
|
|
4589
4821
|
timeout: config?.timeout,
|
|
4590
4822
|
maxTasks: config?.maxTasks,
|
|
4591
4823
|
resourceScope: config?.resourceScope,
|
|
4592
|
-
disposeStrategy: config?.disposeStrategy
|
|
4824
|
+
disposeStrategy: config?.disposeStrategy,
|
|
4825
|
+
runId: config?.runId
|
|
4593
4826
|
});
|
|
4594
4827
|
}
|
|
4595
4828
|
runPreview(input = {}, config = {}) {
|
|
@@ -5138,7 +5371,7 @@ function autoConnect(graph, sourceTask, targetTask, options) {
|
|
|
5138
5371
|
}
|
|
5139
5372
|
|
|
5140
5373
|
// src/task-graph/LoopBuilderContext.ts
|
|
5141
|
-
import { getLogger as
|
|
5374
|
+
import { getLogger as getLogger8 } from "@workglow/util";
|
|
5142
5375
|
function runLoopAutoConnect(parentGraph, pending) {
|
|
5143
5376
|
const { parent, iteratorTask } = pending;
|
|
5144
5377
|
if (parentGraph.getTargetDataflows(parent.id).length !== 0)
|
|
@@ -5152,7 +5385,7 @@ function runLoopAutoConnect(parentGraph, pending) {
|
|
|
5152
5385
|
const result = autoConnect(parentGraph, parent, iteratorTask, { earlierTasks });
|
|
5153
5386
|
if (result.error) {
|
|
5154
5387
|
const message = result.error + " Task not added.";
|
|
5155
|
-
|
|
5388
|
+
getLogger8().error(message);
|
|
5156
5389
|
parentGraph.removeTask(iteratorTask.id);
|
|
5157
5390
|
return message;
|
|
5158
5391
|
}
|
|
@@ -5191,7 +5424,7 @@ class LoopBuilderContext {
|
|
|
5191
5424
|
}
|
|
5192
5425
|
|
|
5193
5426
|
// src/task-graph/WorkflowBuilder.ts
|
|
5194
|
-
import { getLogger as
|
|
5427
|
+
import { getLogger as getLogger9, uuid4 as uuid47 } from "@workglow/util";
|
|
5195
5428
|
|
|
5196
5429
|
// src/task-graph/ConditionalBuilder.ts
|
|
5197
5430
|
import { uuid4 as uuid46 } from "@workglow/util";
|
|
@@ -5350,7 +5583,7 @@ class WorkflowBuilder {
|
|
|
5350
5583
|
const taskSchema = task.inputSchema();
|
|
5351
5584
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5352
5585
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5353
|
-
|
|
5586
|
+
getLogger9().error(this._error);
|
|
5354
5587
|
return;
|
|
5355
5588
|
}
|
|
5356
5589
|
dataflow.targetTaskId = task.id;
|
|
@@ -5375,10 +5608,10 @@ class WorkflowBuilder {
|
|
|
5375
5608
|
if (result.error) {
|
|
5376
5609
|
if (this._loopContext !== undefined) {
|
|
5377
5610
|
this._error = result.error;
|
|
5378
|
-
|
|
5611
|
+
getLogger9().warn(this._error);
|
|
5379
5612
|
} else {
|
|
5380
5613
|
this._error = result.error + " Task not added.";
|
|
5381
|
-
|
|
5614
|
+
getLogger9().error(this._error);
|
|
5382
5615
|
this._facade.graph.removeTask(task.id);
|
|
5383
5616
|
}
|
|
5384
5617
|
}
|
|
@@ -5401,7 +5634,7 @@ class WorkflowBuilder {
|
|
|
5401
5634
|
const taskSchema = task.inputSchema();
|
|
5402
5635
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5403
5636
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5404
|
-
|
|
5637
|
+
getLogger9().error(this._error);
|
|
5405
5638
|
return;
|
|
5406
5639
|
}
|
|
5407
5640
|
dataflow.targetTaskId = task.id;
|
|
@@ -5453,7 +5686,7 @@ class WorkflowBuilder {
|
|
|
5453
5686
|
if (-index > nodes.length) {
|
|
5454
5687
|
const errorMsg = `Back index greater than number of tasks`;
|
|
5455
5688
|
this._error = errorMsg;
|
|
5456
|
-
|
|
5689
|
+
getLogger9().error(this._error);
|
|
5457
5690
|
throw new WorkflowError(errorMsg);
|
|
5458
5691
|
}
|
|
5459
5692
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -5462,13 +5695,13 @@ class WorkflowBuilder {
|
|
|
5462
5695
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
5463
5696
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
5464
5697
|
this._error = errorMsg;
|
|
5465
|
-
|
|
5698
|
+
getLogger9().error(this._error);
|
|
5466
5699
|
throw new WorkflowError(errorMsg);
|
|
5467
5700
|
}
|
|
5468
5701
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
5469
5702
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
5470
5703
|
this._error = errorMsg;
|
|
5471
|
-
|
|
5704
|
+
getLogger9().error(this._error);
|
|
5472
5705
|
throw new WorkflowError(errorMsg);
|
|
5473
5706
|
}
|
|
5474
5707
|
const df = new Dataflow(lastNode.id, source, undefined, target);
|
|
@@ -5481,7 +5714,7 @@ class WorkflowBuilder {
|
|
|
5481
5714
|
const parent = getLastTask(this._facade);
|
|
5482
5715
|
if (!parent) {
|
|
5483
5716
|
this._error = "onError() requires a preceding task in the workflow";
|
|
5484
|
-
|
|
5717
|
+
getLogger9().error(this._error);
|
|
5485
5718
|
throw new WorkflowError(this._error);
|
|
5486
5719
|
}
|
|
5487
5720
|
const handlerTask = ensureTask(handler);
|
|
@@ -5495,7 +5728,7 @@ class WorkflowBuilder {
|
|
|
5495
5728
|
const nodes = this._facade.graph.getTasks();
|
|
5496
5729
|
if (nodes.length === 0) {
|
|
5497
5730
|
this._error = "No tasks to remove";
|
|
5498
|
-
|
|
5731
|
+
getLogger9().error(this._error);
|
|
5499
5732
|
return;
|
|
5500
5733
|
}
|
|
5501
5734
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -6459,7 +6692,7 @@ function registerBuiltInTransforms() {
|
|
|
6459
6692
|
TransformRegistry.registerTransform(t);
|
|
6460
6693
|
}
|
|
6461
6694
|
// src/task/EntitlementProfile.ts
|
|
6462
|
-
import { createServiceToken as
|
|
6695
|
+
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
6463
6696
|
var STATIC_SIGNAL_SOURCE = Object.freeze({
|
|
6464
6697
|
subscribe(_listener) {
|
|
6465
6698
|
return () => {};
|
|
@@ -6562,7 +6795,7 @@ function createPolicyProfile(name, policy, options = {}) {
|
|
|
6562
6795
|
};
|
|
6563
6796
|
return profile;
|
|
6564
6797
|
}
|
|
6565
|
-
var ENTITLEMENT_PROFILE =
|
|
6798
|
+
var ENTITLEMENT_PROFILE = createServiceToken6("workglow.entitlementProfile");
|
|
6566
6799
|
// src/task/EntitlementProfiles.ts
|
|
6567
6800
|
var BROWSER_GRANTS = [
|
|
6568
6801
|
{ id: Entitlements.NETWORK_HTTP },
|
|
@@ -8184,8 +8417,8 @@ import {
|
|
|
8184
8417
|
JobQueueClient,
|
|
8185
8418
|
JobQueueServer
|
|
8186
8419
|
} from "@workglow/job-queue";
|
|
8187
|
-
import { createServiceToken as
|
|
8188
|
-
var JOB_QUEUE_FACTORY =
|
|
8420
|
+
import { createServiceToken as createServiceToken7, globalServiceRegistry as globalServiceRegistry4 } from "@workglow/util";
|
|
8421
|
+
var JOB_QUEUE_FACTORY = createServiceToken7("taskgraph.jobQueueFactory");
|
|
8189
8422
|
var defaultJobQueueFactory = async ({
|
|
8190
8423
|
queueName,
|
|
8191
8424
|
jobClass,
|
|
@@ -8450,8 +8683,8 @@ queueMicrotask(() => {
|
|
|
8450
8683
|
});
|
|
8451
8684
|
// src/task/TaskRegistry.ts
|
|
8452
8685
|
import {
|
|
8453
|
-
createServiceToken as
|
|
8454
|
-
getLogger as
|
|
8686
|
+
createServiceToken as createServiceToken8,
|
|
8687
|
+
getLogger as getLogger10,
|
|
8455
8688
|
globalServiceRegistry as globalServiceRegistry5,
|
|
8456
8689
|
registerInputCompactor,
|
|
8457
8690
|
registerInputResolver
|
|
@@ -8474,7 +8707,7 @@ function registerTask(baseClass) {
|
|
|
8474
8707
|
const result = validateSchema(schema);
|
|
8475
8708
|
if (!result.valid) {
|
|
8476
8709
|
const messages = result.errors.map((e) => `${e.path}: ${e.message}`).join("; ");
|
|
8477
|
-
|
|
8710
|
+
getLogger10().warn(`Task "${baseClass.type}" has invalid ${name}: ${messages}`, {
|
|
8478
8711
|
taskType: baseClass.type,
|
|
8479
8712
|
schemaName: name,
|
|
8480
8713
|
errors: result.errors
|
|
@@ -8490,7 +8723,7 @@ var TaskRegistry = {
|
|
|
8490
8723
|
registerTask,
|
|
8491
8724
|
unregisterTask
|
|
8492
8725
|
};
|
|
8493
|
-
var TASK_CONSTRUCTORS =
|
|
8726
|
+
var TASK_CONSTRUCTORS = createServiceToken8("task.constructors");
|
|
8494
8727
|
function getGlobalTaskConstructors() {
|
|
8495
8728
|
return globalServiceRegistry5.get(TASK_CONSTRUCTORS);
|
|
8496
8729
|
}
|
|
@@ -8659,8 +8892,8 @@ var registerBaseTasks = () => {
|
|
|
8659
8892
|
return tasks;
|
|
8660
8893
|
};
|
|
8661
8894
|
// src/storage/TaskGraphRepository.ts
|
|
8662
|
-
import { createServiceToken as
|
|
8663
|
-
var TASK_GRAPH_REPOSITORY =
|
|
8895
|
+
import { createServiceToken as createServiceToken9, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
8896
|
+
var TASK_GRAPH_REPOSITORY = createServiceToken9("taskgraph.taskGraphRepository");
|
|
8664
8897
|
|
|
8665
8898
|
class TaskGraphRepository {
|
|
8666
8899
|
type = "TaskGraphRepository";
|
|
@@ -8756,6 +8989,10 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8756
8989
|
this.tabularRepository = tabularRepository;
|
|
8757
8990
|
this.outputCompression = outputCompression;
|
|
8758
8991
|
}
|
|
8992
|
+
isDurable() {
|
|
8993
|
+
const backing = this.tabularRepository;
|
|
8994
|
+
return backing.isDurable?.() ?? true;
|
|
8995
|
+
}
|
|
8759
8996
|
async setupDatabase() {
|
|
8760
8997
|
await this.tabularRepository.setupDatabase?.();
|
|
8761
8998
|
}
|
|
@@ -8816,6 +9053,33 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8816
9053
|
await this.tabularRepository.deleteSearch({ createdAt: { value: date, operator: "<" } });
|
|
8817
9054
|
this.emit("output_pruned");
|
|
8818
9055
|
}
|
|
9056
|
+
async deleteByTaskTypePrefix(prefix) {
|
|
9057
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9058
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9059
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9060
|
+
}
|
|
9061
|
+
}
|
|
9062
|
+
}
|
|
9063
|
+
async clearOlderThanWithTaskTypePrefix(prefix, olderThanInMs) {
|
|
9064
|
+
const cutoff = Date.now() - olderThanInMs;
|
|
9065
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9066
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9067
|
+
const ts = typeof row.createdAt === "string" ? new Date(row.createdAt).getTime() : NaN;
|
|
9068
|
+
if (!isNaN(ts) && ts < cutoff) {
|
|
9069
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9070
|
+
}
|
|
9071
|
+
}
|
|
9072
|
+
}
|
|
9073
|
+
}
|
|
9074
|
+
async sizeByTaskTypePrefix(prefix) {
|
|
9075
|
+
let count = 0;
|
|
9076
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9077
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9078
|
+
count++;
|
|
9079
|
+
}
|
|
9080
|
+
}
|
|
9081
|
+
return count;
|
|
9082
|
+
}
|
|
8819
9083
|
}
|
|
8820
9084
|
// src/storage/PortCodecRegistry.ts
|
|
8821
9085
|
import { _resetPortCodecsForTests, getPortCodec as getPortCodec2, registerPortCodec } from "@workglow/util";
|
|
@@ -9500,6 +9764,8 @@ export {
|
|
|
9500
9764
|
isoDateToUnixTransform,
|
|
9501
9765
|
isTaskStreamable,
|
|
9502
9766
|
isStrictArraySchema,
|
|
9767
|
+
isPolicyPrivate,
|
|
9768
|
+
isPolicyCached,
|
|
9503
9769
|
isIterationProperty,
|
|
9504
9770
|
isFlexibleSchema,
|
|
9505
9771
|
isDarkMode,
|
|
@@ -9615,6 +9881,7 @@ export {
|
|
|
9615
9881
|
STATIC_SIGNAL_SOURCE,
|
|
9616
9882
|
SERVER_GRANTS,
|
|
9617
9883
|
RunScheduler,
|
|
9884
|
+
RunPrivateCacheRepo,
|
|
9618
9885
|
RunContext,
|
|
9619
9886
|
ReduceTask,
|
|
9620
9887
|
PROPERTY_ARRAY,
|
|
@@ -9642,10 +9909,12 @@ export {
|
|
|
9642
9909
|
ENTITLEMENT_ENFORCER,
|
|
9643
9910
|
EMPTY_POLICY,
|
|
9644
9911
|
EMPTY_ENTITLEMENTS,
|
|
9912
|
+
DefaultCacheRegistry,
|
|
9645
9913
|
DataflowArrow,
|
|
9646
9914
|
Dataflow,
|
|
9647
9915
|
DESKTOP_GRANTS,
|
|
9648
9916
|
DENY_ALL_RESOLVER,
|
|
9917
|
+
DEFAULT_CACHE_POLICY,
|
|
9649
9918
|
DATAFLOW_ERROR_PORT,
|
|
9650
9919
|
DATAFLOW_ALL_PORTS,
|
|
9651
9920
|
CreateWorkflow,
|
|
@@ -9653,8 +9922,10 @@ export {
|
|
|
9653
9922
|
CreateEndLoopWorkflow,
|
|
9654
9923
|
CreateAdaptiveWorkflow,
|
|
9655
9924
|
ConditionalTask,
|
|
9925
|
+
CacheJanitor,
|
|
9656
9926
|
CacheCoordinator,
|
|
9927
|
+
CACHE_REGISTRY,
|
|
9657
9928
|
BROWSER_GRANTS
|
|
9658
9929
|
};
|
|
9659
9930
|
|
|
9660
|
-
//# debugId=
|
|
9931
|
+
//# debugId=4C6784BFF95D87B164756E2164756E21
|