@workglow/task-graph 0.2.37 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -46
- package/dist/browser.js +639 -368
- package/dist/browser.js.map +19 -15
- package/dist/bun.js +639 -368
- package/dist/bun.js.map +19 -15
- package/dist/cache/CacheJanitor.d.ts +27 -0
- package/dist/cache/CacheJanitor.d.ts.map +1 -0
- package/dist/cache/CachePolicy.d.ts +16 -0
- package/dist/cache/CachePolicy.d.ts.map +1 -0
- package/dist/cache/CacheRegistry.d.ts +30 -0
- package/dist/cache/CacheRegistry.d.ts.map +1 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts +56 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts.map +1 -0
- package/dist/cache/index.d.ts +10 -0
- package/dist/cache/index.d.ts.map +1 -0
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +639 -368
- package/dist/node.js.map +19 -15
- package/dist/storage/TaskOutputRepository.d.ts +40 -4
- package/dist/storage/TaskOutputRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputTabularRepository.d.ts +27 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts.map +1 -1
- package/dist/task/CacheCoordinator.d.ts +17 -0
- package/dist/task/CacheCoordinator.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +0 -1
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/FallbackTaskRunner.d.ts +8 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +21 -1
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/Task.d.ts +50 -0
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +0 -13
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +28 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task-graph/Conversions.d.ts.map +1 -1
- package/dist/task-graph/StreamPump.d.ts +8 -0
- package/dist/task-graph/StreamPump.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +7 -0
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +45 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +91 -2
package/dist/bun.js
CHANGED
|
@@ -1278,14 +1278,14 @@ import { EventEmitter as EventEmitter4, uuid4 as uuid45 } from "@workglow/util";
|
|
|
1278
1278
|
import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
1279
1279
|
|
|
1280
1280
|
// src/task/GraphAsTask.ts
|
|
1281
|
-
import { getLogger as
|
|
1281
|
+
import { getLogger as getLogger7 } from "@workglow/util";
|
|
1282
1282
|
import { CycleError } from "@workglow/util/graph";
|
|
1283
1283
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
1284
1284
|
|
|
1285
1285
|
// src/task-graph/TaskGraphRunner.ts
|
|
1286
1286
|
import {
|
|
1287
1287
|
collectPropertyValues,
|
|
1288
|
-
getLogger as
|
|
1288
|
+
getLogger as getLogger6,
|
|
1289
1289
|
getTelemetryProvider as getTelemetryProvider2,
|
|
1290
1290
|
globalServiceRegistry as globalServiceRegistry3,
|
|
1291
1291
|
ResourceScope as ResourceScope2,
|
|
@@ -1294,9 +1294,39 @@ import {
|
|
|
1294
1294
|
uuid4 as uuid44
|
|
1295
1295
|
} from "@workglow/util";
|
|
1296
1296
|
|
|
1297
|
+
// src/cache/CacheJanitor.ts
|
|
1298
|
+
class CacheJanitor {
|
|
1299
|
+
privateBacking;
|
|
1300
|
+
constructor({ privateBacking }) {
|
|
1301
|
+
this.privateBacking = privateBacking;
|
|
1302
|
+
}
|
|
1303
|
+
async sweepStaleRunPrivate(olderThanMs) {
|
|
1304
|
+
await this.privateBacking.clearOlderThanWithTaskTypePrefix("__run:", olderThanMs);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
// src/cache/CachePolicy.ts
|
|
1308
|
+
var DEFAULT_CACHE_POLICY = { kind: "deterministic" };
|
|
1309
|
+
function isPolicyCached(policy) {
|
|
1310
|
+
return policy.kind !== "none";
|
|
1311
|
+
}
|
|
1312
|
+
function isPolicyPrivate(policy) {
|
|
1313
|
+
return policy.kind === "private";
|
|
1314
|
+
}
|
|
1315
|
+
// src/cache/CacheRegistry.ts
|
|
1316
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
1317
|
+
var CACHE_REGISTRY = createServiceToken2("taskgraph.cacheRegistry");
|
|
1318
|
+
|
|
1319
|
+
class DefaultCacheRegistry {
|
|
1320
|
+
deterministic;
|
|
1321
|
+
private;
|
|
1322
|
+
constructor(init = {}) {
|
|
1323
|
+
this.deterministic = init.deterministic;
|
|
1324
|
+
this.private = init.private;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1297
1327
|
// src/storage/TaskOutputRepository.ts
|
|
1298
|
-
import { createServiceToken as
|
|
1299
|
-
var TASK_OUTPUT_REPOSITORY =
|
|
1328
|
+
import { createServiceToken as createServiceToken3, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1329
|
+
var TASK_OUTPUT_REPOSITORY = createServiceToken3("taskgraph.taskOutputRepository");
|
|
1300
1330
|
|
|
1301
1331
|
class TaskOutputRepository {
|
|
1302
1332
|
outputCompression;
|
|
@@ -1322,10 +1352,53 @@ class TaskOutputRepository {
|
|
|
1322
1352
|
emit(name, ...args) {
|
|
1323
1353
|
this._events?.emit(name, ...args);
|
|
1324
1354
|
}
|
|
1355
|
+
async deleteByTaskTypePrefix(_prefix) {
|
|
1356
|
+
throw new Error(`${this.constructor.name}: deleteByTaskTypePrefix is not supported by this repository.`);
|
|
1357
|
+
}
|
|
1358
|
+
async clearOlderThanWithTaskTypePrefix(_prefix, _olderThanMs) {
|
|
1359
|
+
throw new Error(`${this.constructor.name}: clearOlderThanWithTaskTypePrefix is not supported by this repository.`);
|
|
1360
|
+
}
|
|
1361
|
+
async sizeByTaskTypePrefix(_prefix) {
|
|
1362
|
+
throw new Error(`${this.constructor.name}: sizeByTaskTypePrefix is not supported by this repository.`);
|
|
1363
|
+
}
|
|
1325
1364
|
}
|
|
1326
1365
|
|
|
1366
|
+
// src/cache/RunPrivateCacheRepo.ts
|
|
1367
|
+
class RunPrivateCacheRepo extends TaskOutputRepository {
|
|
1368
|
+
backing;
|
|
1369
|
+
runId;
|
|
1370
|
+
constructor({ backing, runId }) {
|
|
1371
|
+
super({ outputCompression: backing.outputCompression });
|
|
1372
|
+
this.backing = backing;
|
|
1373
|
+
this.runId = runId;
|
|
1374
|
+
}
|
|
1375
|
+
ns(taskType) {
|
|
1376
|
+
return `__run:${this.runId}::${taskType}`;
|
|
1377
|
+
}
|
|
1378
|
+
async saveOutput(taskType, inputs, output, createdAt) {
|
|
1379
|
+
await this.backing.saveOutput(this.ns(taskType), inputs, output, createdAt);
|
|
1380
|
+
}
|
|
1381
|
+
async getOutput(taskType, inputs) {
|
|
1382
|
+
return this.backing.getOutput(this.ns(taskType), inputs);
|
|
1383
|
+
}
|
|
1384
|
+
async clear() {
|
|
1385
|
+
await this.clearRun();
|
|
1386
|
+
}
|
|
1387
|
+
async clearRun() {
|
|
1388
|
+
await this.backing.deleteByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1389
|
+
}
|
|
1390
|
+
async size() {
|
|
1391
|
+
return this.backing.sizeByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1392
|
+
}
|
|
1393
|
+
async clearOlderThan(olderThanInMs) {
|
|
1394
|
+
await this.backing.clearOlderThanWithTaskTypePrefix(`__run:${this.runId}::`, olderThanInMs);
|
|
1395
|
+
}
|
|
1396
|
+
isDurable() {
|
|
1397
|
+
return this.backing.isDurable();
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1327
1400
|
// src/task/EntitlementEnforcer.ts
|
|
1328
|
-
import { createServiceToken as
|
|
1401
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1329
1402
|
|
|
1330
1403
|
// src/task/EntitlementPolicy.ts
|
|
1331
1404
|
var EMPTY_POLICY = Object.freeze({
|
|
@@ -1369,7 +1442,7 @@ function can(policy, id, resources) {
|
|
|
1369
1442
|
}
|
|
1370
1443
|
|
|
1371
1444
|
// src/task/EntitlementResolver.ts
|
|
1372
|
-
import { createServiceToken as
|
|
1445
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1373
1446
|
var PERMISSIVE_RESOLVER = {
|
|
1374
1447
|
lookup: () => "grant",
|
|
1375
1448
|
prompt: async () => "grant",
|
|
@@ -1380,7 +1453,7 @@ var DENY_ALL_RESOLVER = {
|
|
|
1380
1453
|
prompt: async () => "deny",
|
|
1381
1454
|
save: () => {}
|
|
1382
1455
|
};
|
|
1383
|
-
var ENTITLEMENT_RESOLVER =
|
|
1456
|
+
var ENTITLEMENT_RESOLVER = createServiceToken4("workglow.entitlementResolver");
|
|
1384
1457
|
|
|
1385
1458
|
// src/task/EntitlementEnforcer.ts
|
|
1386
1459
|
function formatEntitlementDenial(denial) {
|
|
@@ -1453,7 +1526,7 @@ function createScopedEnforcer(grants) {
|
|
|
1453
1526
|
function createGrantListEnforcer(grants) {
|
|
1454
1527
|
return createScopedEnforcer(grants.map((id) => ({ id })));
|
|
1455
1528
|
}
|
|
1456
|
-
var ENTITLEMENT_ENFORCER =
|
|
1529
|
+
var ENTITLEMENT_ENFORCER = createServiceToken5("workglow.entitlementEnforcer");
|
|
1457
1530
|
|
|
1458
1531
|
// src/task/StreamTypes.ts
|
|
1459
1532
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1556,246 +1629,13 @@ function hasStructuredOutput(schema) {
|
|
|
1556
1629
|
return getStructuredOutputSchemas(schema).size > 0;
|
|
1557
1630
|
}
|
|
1558
1631
|
|
|
1559
|
-
// src/task-graph/EdgeMaterializer.ts
|
|
1560
|
-
import { getLogger } from "@workglow/util";
|
|
1561
|
-
import { previewSource } from "@workglow/util/media";
|
|
1562
|
-
class EdgeMaterializer {
|
|
1563
|
-
graph;
|
|
1564
|
-
runner;
|
|
1565
|
-
constructor(graph, runner) {
|
|
1566
|
-
this.graph = graph;
|
|
1567
|
-
this.runner = runner;
|
|
1568
|
-
}
|
|
1569
|
-
filterInputForTask(task, input) {
|
|
1570
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1571
|
-
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1572
|
-
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1573
|
-
const filteredInput = {};
|
|
1574
|
-
for (const [key, value] of Object.entries(input)) {
|
|
1575
|
-
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
1576
|
-
filteredInput[key] = value;
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
return filteredInput;
|
|
1580
|
-
}
|
|
1581
|
-
copyInputFromEdgesToNode(task) {
|
|
1582
|
-
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1583
|
-
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
1584
|
-
for (const dataflow of dataflows) {
|
|
1585
|
-
const live = dataflow.getCurrentValue();
|
|
1586
|
-
const port = dataflow.targetTaskPortId;
|
|
1587
|
-
let portData;
|
|
1588
|
-
if (port === DATAFLOW_ALL_PORTS) {
|
|
1589
|
-
portData = live;
|
|
1590
|
-
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
1591
|
-
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
1592
|
-
} else {
|
|
1593
|
-
portData = { [port]: live };
|
|
1594
|
-
}
|
|
1595
|
-
this.runner.addInputData(task, portData);
|
|
1596
|
-
}
|
|
1597
|
-
}
|
|
1598
|
-
async pushOutputFromNodeToEdges(node, results) {
|
|
1599
|
-
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1600
|
-
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
1601
|
-
for (const port of Object.keys(results)) {
|
|
1602
|
-
const value = results[port];
|
|
1603
|
-
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
1604
|
-
results[port] = await previewSource(value);
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
for (const dataflow of dataflows) {
|
|
1609
|
-
if (dataflow.stream !== undefined)
|
|
1610
|
-
continue;
|
|
1611
|
-
const registry = this.runner["registry"];
|
|
1612
|
-
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
1613
|
-
if (compatibility === "static") {
|
|
1614
|
-
dataflow.setPortData(results);
|
|
1615
|
-
await dataflow.applyTransforms(registry);
|
|
1616
|
-
} else if (compatibility === "runtime") {
|
|
1617
|
-
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
1618
|
-
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
1619
|
-
dataflow.setPortData(narrowed);
|
|
1620
|
-
await dataflow.applyTransforms(registry);
|
|
1621
|
-
} else {
|
|
1622
|
-
const resultsKeys = Object.keys(results);
|
|
1623
|
-
if (resultsKeys.length > 0) {
|
|
1624
|
-
getLogger().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
1625
|
-
dataflowId: dataflow.id,
|
|
1626
|
-
compatibility,
|
|
1627
|
-
resultsKeys
|
|
1628
|
-
});
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
}
|
|
1633
|
-
pushErrorFromNodeToEdges(node) {
|
|
1634
|
-
if (!node?.config?.id)
|
|
1635
|
-
return;
|
|
1636
|
-
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1637
|
-
dataflow.error = node.error;
|
|
1638
|
-
});
|
|
1639
|
-
}
|
|
1640
|
-
hasErrorOutputEdges(task) {
|
|
1641
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1642
|
-
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
1643
|
-
}
|
|
1644
|
-
pushErrorOutputToEdges(task) {
|
|
1645
|
-
const taskError = task.error;
|
|
1646
|
-
const errorData = {
|
|
1647
|
-
error: taskError?.message ?? "Unknown error",
|
|
1648
|
-
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
1649
|
-
};
|
|
1650
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1651
|
-
for (const df of dataflows) {
|
|
1652
|
-
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
1653
|
-
df.value = errorData;
|
|
1654
|
-
df.setStatus(TaskStatus.COMPLETED);
|
|
1655
|
-
} else {
|
|
1656
|
-
df.setStatus(TaskStatus.DISABLED);
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
|
-
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
1660
|
-
}
|
|
1661
|
-
resetTask(graph, task, runId) {
|
|
1662
|
-
task.status = TaskStatus.PENDING;
|
|
1663
|
-
task.resetInputData();
|
|
1664
|
-
task.runOutputData = {};
|
|
1665
|
-
task.error = undefined;
|
|
1666
|
-
task.progress = 0;
|
|
1667
|
-
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
1668
|
-
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
1669
|
-
if (task?.config?.id) {
|
|
1670
|
-
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1671
|
-
dataflow.error = task.error;
|
|
1672
|
-
});
|
|
1673
|
-
}
|
|
1674
|
-
task.emit("reset");
|
|
1675
|
-
task.emit("status", task.status);
|
|
1676
|
-
}
|
|
1677
|
-
static isImageValueShape(v) {
|
|
1678
|
-
if (v === null || typeof v !== "object")
|
|
1679
|
-
return false;
|
|
1680
|
-
const o = v;
|
|
1681
|
-
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
// src/task-graph/RunContext.ts
|
|
1686
|
-
import { uuid4 as uuid42 } from "@workglow/util";
|
|
1687
|
-
|
|
1688
|
-
class RunContext {
|
|
1689
|
-
runId;
|
|
1690
|
-
abortController;
|
|
1691
|
-
inProgressTasks = new Map;
|
|
1692
|
-
inProgressFunctions = new Map;
|
|
1693
|
-
failedTaskErrors = new Map;
|
|
1694
|
-
telemetrySpan;
|
|
1695
|
-
graphTimeoutTimer;
|
|
1696
|
-
pendingGraphTimeoutError;
|
|
1697
|
-
activeEnforcer;
|
|
1698
|
-
parentSignalCleanup;
|
|
1699
|
-
constructor(parentSignal) {
|
|
1700
|
-
this.runId = uuid42();
|
|
1701
|
-
this.abortController = new AbortController;
|
|
1702
|
-
if (parentSignal) {
|
|
1703
|
-
const onParentAbort = () => this.abortController.abort();
|
|
1704
|
-
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
1705
|
-
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
1706
|
-
if (parentSignal.aborted) {
|
|
1707
|
-
this.parentSignalCleanup();
|
|
1708
|
-
this.parentSignalCleanup = undefined;
|
|
1709
|
-
this.abortController.abort();
|
|
1710
|
-
}
|
|
1711
|
-
}
|
|
1712
|
-
}
|
|
1713
|
-
dispose() {
|
|
1714
|
-
this.parentSignalCleanup?.();
|
|
1715
|
-
this.parentSignalCleanup = undefined;
|
|
1716
|
-
}
|
|
1717
|
-
}
|
|
1718
|
-
|
|
1719
|
-
// src/task-graph/RunScheduler.ts
|
|
1720
|
-
import { getLogger as getLogger4 } from "@workglow/util";
|
|
1721
|
-
|
|
1722
|
-
// src/task/ConditionalTask.ts
|
|
1723
|
-
import { getLogger as getLogger3 } from "@workglow/util";
|
|
1724
|
-
|
|
1725
|
-
// src/task/ConditionUtils.ts
|
|
1726
|
-
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
1727
|
-
if (fieldValue === null || fieldValue === undefined) {
|
|
1728
|
-
switch (operator) {
|
|
1729
|
-
case "is_empty":
|
|
1730
|
-
return true;
|
|
1731
|
-
case "is_not_empty":
|
|
1732
|
-
return false;
|
|
1733
|
-
case "is_true":
|
|
1734
|
-
return false;
|
|
1735
|
-
case "is_false":
|
|
1736
|
-
return true;
|
|
1737
|
-
default:
|
|
1738
|
-
return false;
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
const strValue = String(fieldValue);
|
|
1742
|
-
const numValue = Number(fieldValue);
|
|
1743
|
-
switch (operator) {
|
|
1744
|
-
case "equals":
|
|
1745
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1746
|
-
return numValue === Number(compareValue);
|
|
1747
|
-
}
|
|
1748
|
-
return strValue === compareValue;
|
|
1749
|
-
case "not_equals":
|
|
1750
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1751
|
-
return numValue !== Number(compareValue);
|
|
1752
|
-
}
|
|
1753
|
-
return strValue !== compareValue;
|
|
1754
|
-
case "greater_than":
|
|
1755
|
-
return numValue > Number(compareValue);
|
|
1756
|
-
case "greater_or_equal":
|
|
1757
|
-
return numValue >= Number(compareValue);
|
|
1758
|
-
case "less_than":
|
|
1759
|
-
return numValue < Number(compareValue);
|
|
1760
|
-
case "less_or_equal":
|
|
1761
|
-
return numValue <= Number(compareValue);
|
|
1762
|
-
case "contains":
|
|
1763
|
-
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
1764
|
-
case "starts_with":
|
|
1765
|
-
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
1766
|
-
case "ends_with":
|
|
1767
|
-
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
1768
|
-
case "is_empty":
|
|
1769
|
-
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
1770
|
-
case "is_not_empty":
|
|
1771
|
-
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
1772
|
-
case "is_true":
|
|
1773
|
-
return Boolean(fieldValue) === true;
|
|
1774
|
-
case "is_false":
|
|
1775
|
-
return Boolean(fieldValue) === false;
|
|
1776
|
-
default:
|
|
1777
|
-
return false;
|
|
1778
|
-
}
|
|
1779
|
-
}
|
|
1780
|
-
function getNestedValue(obj, path) {
|
|
1781
|
-
const parts = path.split(".");
|
|
1782
|
-
let current = obj;
|
|
1783
|
-
for (const part of parts) {
|
|
1784
|
-
if (current === null || current === undefined || typeof current !== "object") {
|
|
1785
|
-
return;
|
|
1786
|
-
}
|
|
1787
|
-
current = current[part];
|
|
1788
|
-
}
|
|
1789
|
-
return current;
|
|
1790
|
-
}
|
|
1791
|
-
|
|
1792
1632
|
// src/task/Task.ts
|
|
1793
|
-
import { deepEqual, EventEmitter as EventEmitter3, uuid4 as
|
|
1633
|
+
import { deepEqual, EventEmitter as EventEmitter3, getLogger as getLogger2, uuid4 as uuid42 } from "@workglow/util";
|
|
1794
1634
|
import { compileSchema } from "@workglow/util/schema";
|
|
1795
1635
|
|
|
1796
1636
|
// src/task/TaskRunner.ts
|
|
1797
1637
|
import {
|
|
1798
|
-
getLogger
|
|
1638
|
+
getLogger,
|
|
1799
1639
|
getTelemetryProvider,
|
|
1800
1640
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1801
1641
|
ResourceScope,
|
|
@@ -1804,7 +1644,6 @@ import {
|
|
|
1804
1644
|
|
|
1805
1645
|
// src/task/CacheCoordinator.ts
|
|
1806
1646
|
import { getPortCodec } from "@workglow/util";
|
|
1807
|
-
|
|
1808
1647
|
class CacheCoordinator {
|
|
1809
1648
|
task;
|
|
1810
1649
|
constructor(task) {
|
|
@@ -1814,7 +1653,9 @@ class CacheCoordinator {
|
|
|
1814
1653
|
if (!outputCache)
|
|
1815
1654
|
return inputs;
|
|
1816
1655
|
const inputSchema = this.task.constructor.inputSchema();
|
|
1817
|
-
|
|
1656
|
+
const normalized = await CacheCoordinator.normalizeInputsForCacheKey(inputs, inputSchema);
|
|
1657
|
+
normalized.__cv = this.task.getCacheVersion();
|
|
1658
|
+
return normalized;
|
|
1818
1659
|
}
|
|
1819
1660
|
async lookup(keyInputs, outputCache, isStreamable, ctx) {
|
|
1820
1661
|
if (!outputCache || !this.task.cacheable)
|
|
@@ -1842,6 +1683,20 @@ class CacheCoordinator {
|
|
|
1842
1683
|
const wireOutputs = await CacheCoordinator.serializeOutputPorts(output, outputSchema);
|
|
1843
1684
|
await outputCache.saveOutput(this.task.type, keyInputs, wireOutputs);
|
|
1844
1685
|
}
|
|
1686
|
+
repoFor(registry, policy) {
|
|
1687
|
+
if (!registry || !isPolicyCached(policy))
|
|
1688
|
+
return;
|
|
1689
|
+
return isPolicyPrivate(policy) ? registry.private : registry.deterministic;
|
|
1690
|
+
}
|
|
1691
|
+
async buildKeyForPolicy(inputs, registry, policy) {
|
|
1692
|
+
return this.buildKey(inputs, this.repoFor(registry, policy));
|
|
1693
|
+
}
|
|
1694
|
+
async lookupByPolicy(keyInputs, registry, policy, isStreamable, ctx) {
|
|
1695
|
+
return this.lookup(keyInputs, this.repoFor(registry, policy), isStreamable, ctx);
|
|
1696
|
+
}
|
|
1697
|
+
async saveByPolicy(keyInputs, output, registry, policy) {
|
|
1698
|
+
return this.save(keyInputs, output, this.repoFor(registry, policy));
|
|
1699
|
+
}
|
|
1845
1700
|
static async serializeOutputPorts(output, schema) {
|
|
1846
1701
|
if (!schema?.properties)
|
|
1847
1702
|
return output;
|
|
@@ -2077,12 +1932,15 @@ class TaskRunner {
|
|
|
2077
1932
|
task;
|
|
2078
1933
|
currentCtx;
|
|
2079
1934
|
outputCache;
|
|
1935
|
+
cacheRegistry;
|
|
2080
1936
|
cacheCoordinator;
|
|
2081
1937
|
streamProcessor;
|
|
2082
1938
|
registry = globalServiceRegistry2;
|
|
2083
1939
|
resourceScope;
|
|
2084
1940
|
ownsResourceScope = false;
|
|
2085
1941
|
inputStreams;
|
|
1942
|
+
runId;
|
|
1943
|
+
static __privateWithoutRunIdWarned = new Set;
|
|
2086
1944
|
constructor(task) {
|
|
2087
1945
|
this.task = task;
|
|
2088
1946
|
this.own = this.own.bind(this);
|
|
@@ -2120,11 +1978,23 @@ class TaskRunner {
|
|
|
2120
1978
|
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
2121
1979
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
2122
1980
|
if (streamMode !== "none") {
|
|
2123
|
-
|
|
1981
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
let policy = this.task.getCachePolicy(inputs);
|
|
1985
|
+
if (policy.kind === "private" && !this.runId && this.cacheRegistry?.private !== undefined && !(this.cacheRegistry.private instanceof RunPrivateCacheRepo)) {
|
|
1986
|
+
const taskType = this.task.type;
|
|
1987
|
+
if (!TaskRunner.__privateWithoutRunIdWarned.has(taskType)) {
|
|
1988
|
+
TaskRunner.__privateWithoutRunIdWarned.add(taskType);
|
|
1989
|
+
getLogger().warn(`TaskRunner: task "${taskType}" has a private cache policy but no runId was ` + `provided. Private cache writes are skipped for this run \u2014 use TaskGraphRunner ` + `with runId for run-namespaced private caching, or provide an already namespaced ` + `private cache repo in CACHE_REGISTRY.`);
|
|
2124
1990
|
}
|
|
1991
|
+
policy = { kind: "none" };
|
|
2125
1992
|
}
|
|
2126
|
-
|
|
2127
|
-
|
|
1993
|
+
this.currentCtx?.telemetrySpan?.setAttributes({
|
|
1994
|
+
"workglow.task.cache_policy": policy.kind
|
|
1995
|
+
});
|
|
1996
|
+
const keyInputs = await this.cacheCoordinator.buildKeyForPolicy(inputs, this.cacheRegistry, policy);
|
|
1997
|
+
let outputs = await this.cacheCoordinator.lookupByPolicy(keyInputs, this.cacheRegistry, policy, isStreamable, ctx);
|
|
2128
1998
|
if (outputs === undefined) {
|
|
2129
1999
|
outputs = isStreamable ? await this.streamProcessor.run(inputs, ctx, {
|
|
2130
2000
|
registry: this.registry,
|
|
@@ -2133,7 +2003,7 @@ class TaskRunner {
|
|
|
2133
2003
|
onProgress: this.handleProgress.bind(this),
|
|
2134
2004
|
own: this.own
|
|
2135
2005
|
}) : await this.executeTask(inputs, ctx);
|
|
2136
|
-
await this.cacheCoordinator.
|
|
2006
|
+
await this.cacheCoordinator.saveByPolicy(keyInputs, outputs, this.cacheRegistry, policy);
|
|
2137
2007
|
this.task.runOutputData = outputs ?? {};
|
|
2138
2008
|
}
|
|
2139
2009
|
await this.handleComplete();
|
|
@@ -2170,7 +2040,7 @@ class TaskRunner {
|
|
|
2170
2040
|
}
|
|
2171
2041
|
await this.handleCompletePreview();
|
|
2172
2042
|
} catch (err) {
|
|
2173
|
-
|
|
2043
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
2174
2044
|
await this.handleErrorPreview();
|
|
2175
2045
|
} finally {
|
|
2176
2046
|
ctx.dispose();
|
|
@@ -2255,7 +2125,7 @@ class TaskRunner {
|
|
|
2255
2125
|
const out = await this.runPreview(overrides);
|
|
2256
2126
|
yield out;
|
|
2257
2127
|
} catch (err) {
|
|
2258
|
-
|
|
2128
|
+
getLogger().debug("runPreviewStream iteration failed", {
|
|
2259
2129
|
taskId: this.task.config?.id,
|
|
2260
2130
|
error: err
|
|
2261
2131
|
});
|
|
@@ -2298,7 +2168,8 @@ class TaskRunner {
|
|
|
2298
2168
|
updateProgress: this.handleProgress.bind(this),
|
|
2299
2169
|
own: this.own,
|
|
2300
2170
|
registry: this.registry,
|
|
2301
|
-
resourceScope: this.resourceScope
|
|
2171
|
+
resourceScope: this.resourceScope,
|
|
2172
|
+
runId: this.runId
|
|
2302
2173
|
});
|
|
2303
2174
|
return result;
|
|
2304
2175
|
}
|
|
@@ -2326,22 +2197,29 @@ class TaskRunner {
|
|
|
2326
2197
|
ctx.abortController.signal.addEventListener("abort", () => {
|
|
2327
2198
|
this.handleAbort();
|
|
2328
2199
|
});
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2200
|
+
if (config.registry) {
|
|
2201
|
+
this.registry = config.registry;
|
|
2202
|
+
}
|
|
2203
|
+
this.runId = config.runId;
|
|
2204
|
+
const legacy = config.outputCache ?? this.task.runConfig?.outputCache;
|
|
2205
|
+
if (legacy === false) {
|
|
2206
|
+
this.cacheRegistry = undefined;
|
|
2207
|
+
this.outputCache = undefined;
|
|
2208
|
+
} else if (legacy instanceof TaskOutputRepository) {
|
|
2209
|
+
this.cacheRegistry = new DefaultCacheRegistry({ deterministic: legacy });
|
|
2210
|
+
this.outputCache = legacy;
|
|
2211
|
+
} else if (legacy === true) {
|
|
2212
|
+
const instance = globalServiceRegistry2.has(TASK_OUTPUT_REPOSITORY) ? globalServiceRegistry2.get(TASK_OUTPUT_REPOSITORY) : undefined;
|
|
2332
2213
|
this.outputCache = instance;
|
|
2333
|
-
|
|
2214
|
+
this.cacheRegistry = instance ? new DefaultCacheRegistry({ deterministic: instance }) : undefined;
|
|
2215
|
+
} else {
|
|
2334
2216
|
this.outputCache = undefined;
|
|
2335
|
-
|
|
2336
|
-
this.outputCache = cache;
|
|
2217
|
+
this.cacheRegistry = this.registry.has(CACHE_REGISTRY) ? this.registry.get(CACHE_REGISTRY) : undefined;
|
|
2337
2218
|
}
|
|
2338
2219
|
ctx.shouldAccumulate = config.shouldAccumulate !== false;
|
|
2339
2220
|
if (config.updateProgress) {
|
|
2340
2221
|
this.updateProgress = config.updateProgress;
|
|
2341
2222
|
}
|
|
2342
|
-
if (config.registry) {
|
|
2343
|
-
this.registry = config.registry;
|
|
2344
|
-
}
|
|
2345
2223
|
if (config.resourceScope) {
|
|
2346
2224
|
this.resourceScope = config.resourceScope;
|
|
2347
2225
|
}
|
|
@@ -2363,7 +2241,6 @@ class TaskRunner {
|
|
|
2363
2241
|
attributes: {
|
|
2364
2242
|
"workglow.task.type": this.task.type,
|
|
2365
2243
|
"workglow.task.id": String(this.task.config.id),
|
|
2366
|
-
"workglow.task.cacheable": this.task.cacheable,
|
|
2367
2244
|
"workglow.task.title": this.task.title || undefined
|
|
2368
2245
|
}
|
|
2369
2246
|
});
|
|
@@ -2494,12 +2371,15 @@ class Task {
|
|
|
2494
2371
|
static title = "";
|
|
2495
2372
|
static description = "";
|
|
2496
2373
|
static cacheable = true;
|
|
2374
|
+
static version = 1;
|
|
2375
|
+
static cachePolicy = { kind: "deterministic" };
|
|
2497
2376
|
static hasDynamicSchemas = false;
|
|
2498
2377
|
static passthroughInputsToOutputs = false;
|
|
2499
2378
|
static customizable = false;
|
|
2500
2379
|
static isGraphOutput = false;
|
|
2501
2380
|
static isPassthrough = false;
|
|
2502
2381
|
static hasDynamicEntitlements = false;
|
|
2382
|
+
static __cacheableDeprecationWarned = new Set;
|
|
2503
2383
|
static entitlements() {
|
|
2504
2384
|
return EMPTY_ENTITLEMENTS;
|
|
2505
2385
|
}
|
|
@@ -2578,7 +2458,39 @@ class Task {
|
|
|
2578
2458
|
return this.config?.description ?? this.constructor.description;
|
|
2579
2459
|
}
|
|
2580
2460
|
get cacheable() {
|
|
2581
|
-
|
|
2461
|
+
if (this.runConfig?.cacheable !== undefined)
|
|
2462
|
+
return this.runConfig.cacheable;
|
|
2463
|
+
if (this.config?.cacheable !== undefined)
|
|
2464
|
+
return this.config.cacheable;
|
|
2465
|
+
return this.getCachePolicy(this.runInputData ?? {}).kind !== "none";
|
|
2466
|
+
}
|
|
2467
|
+
getCacheVersion() {
|
|
2468
|
+
const versions = [];
|
|
2469
|
+
let ctor = this.constructor;
|
|
2470
|
+
while (ctor && ctor !== Function.prototype) {
|
|
2471
|
+
if (Object.prototype.hasOwnProperty.call(ctor, "version") && typeof ctor.version === "number") {
|
|
2472
|
+
versions.push(ctor.version);
|
|
2473
|
+
}
|
|
2474
|
+
ctor = Object.getPrototypeOf(ctor);
|
|
2475
|
+
}
|
|
2476
|
+
if (versions.length === 0)
|
|
2477
|
+
versions.push(1);
|
|
2478
|
+
return versions.join(".");
|
|
2479
|
+
}
|
|
2480
|
+
getCachePolicy(_inputs) {
|
|
2481
|
+
const ctor = this.constructor;
|
|
2482
|
+
const hasLegacyOverride = Object.prototype.hasOwnProperty.call(ctor, "cacheable") && ctor.cacheable === false;
|
|
2483
|
+
const hasPolicyOverride = Object.prototype.hasOwnProperty.call(ctor, "cachePolicy");
|
|
2484
|
+
if (hasLegacyOverride && !hasPolicyOverride) {
|
|
2485
|
+
if (!Task.__cacheableDeprecationWarned.has(ctor.type)) {
|
|
2486
|
+
Task.__cacheableDeprecationWarned.add(ctor.type);
|
|
2487
|
+
getLogger2().warn(`Task "${ctor.type}": static \`cacheable = false\` is deprecated. ` + `Use \`static cachePolicy: CachePolicy = { kind: "none" }\` instead.`);
|
|
2488
|
+
}
|
|
2489
|
+
return { kind: "none" };
|
|
2490
|
+
}
|
|
2491
|
+
if (this.runConfig?.cacheable === false || this.config?.cacheable === false)
|
|
2492
|
+
return { kind: "none" };
|
|
2493
|
+
return ctor.cachePolicy ?? DEFAULT_CACHE_POLICY;
|
|
2582
2494
|
}
|
|
2583
2495
|
defaults;
|
|
2584
2496
|
runInputData = {};
|
|
@@ -2613,7 +2525,7 @@ class Task {
|
|
|
2613
2525
|
...title ? { title } : {}
|
|
2614
2526
|
}, restConfig);
|
|
2615
2527
|
if (baseConfig.id === undefined) {
|
|
2616
|
-
baseConfig.id =
|
|
2528
|
+
baseConfig.id = uuid42();
|
|
2617
2529
|
}
|
|
2618
2530
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
2619
2531
|
try {
|
|
@@ -2622,6 +2534,10 @@ class Task {
|
|
|
2622
2534
|
this.originalConfig = undefined;
|
|
2623
2535
|
}
|
|
2624
2536
|
this.runConfig = runConfig;
|
|
2537
|
+
const inputSchema = this.constructor.inputSchema();
|
|
2538
|
+
if (inputSchema && typeof inputSchema !== "boolean" && inputSchema.properties && Object.prototype.hasOwnProperty.call(inputSchema.properties, "__cv")) {
|
|
2539
|
+
throw new TaskConfigurationError(`Task "${this.constructor.type}": input port name '__cv' is reserved ` + `for cache versioning. Rename the port to avoid collision with the cache key sentinel.`);
|
|
2540
|
+
}
|
|
2625
2541
|
}
|
|
2626
2542
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
2627
2543
|
const schema = this.inputSchema();
|
|
@@ -2919,91 +2835,320 @@ class Task {
|
|
|
2919
2835
|
result[key] = this.stripSymbols(obj[key]);
|
|
2920
2836
|
}
|
|
2921
2837
|
}
|
|
2922
|
-
return result;
|
|
2838
|
+
return result;
|
|
2839
|
+
}
|
|
2840
|
+
return obj;
|
|
2841
|
+
}
|
|
2842
|
+
canSerializeConfig() {
|
|
2843
|
+
return true;
|
|
2844
|
+
}
|
|
2845
|
+
toJSON(_options) {
|
|
2846
|
+
const ctor = this.constructor;
|
|
2847
|
+
if (!this.canSerializeConfig() || !this.originalConfig) {
|
|
2848
|
+
throw new TaskSerializationError(this.type);
|
|
2849
|
+
}
|
|
2850
|
+
const schema = ctor.configSchema();
|
|
2851
|
+
const schemaProperties = typeof schema !== "boolean" && schema?.properties ? schema.properties : {};
|
|
2852
|
+
const config = {};
|
|
2853
|
+
for (const [key, propSchema] of Object.entries(schemaProperties)) {
|
|
2854
|
+
if (key === "id")
|
|
2855
|
+
continue;
|
|
2856
|
+
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
2857
|
+
continue;
|
|
2858
|
+
}
|
|
2859
|
+
const value = this.originalConfig[key];
|
|
2860
|
+
if (value === undefined)
|
|
2861
|
+
continue;
|
|
2862
|
+
if (typeof value === "function" || typeof value === "symbol")
|
|
2863
|
+
continue;
|
|
2864
|
+
config[key] = value;
|
|
2865
|
+
}
|
|
2866
|
+
if (config.title === ctor.title)
|
|
2867
|
+
delete config.title;
|
|
2868
|
+
if (config.description === ctor.description)
|
|
2869
|
+
delete config.description;
|
|
2870
|
+
const extras = config.extras;
|
|
2871
|
+
if (!extras || Object.keys(extras).length === 0)
|
|
2872
|
+
delete config.extras;
|
|
2873
|
+
const base = {
|
|
2874
|
+
id: this.id,
|
|
2875
|
+
type: this.type,
|
|
2876
|
+
defaults: this.defaults
|
|
2877
|
+
};
|
|
2878
|
+
if (Object.keys(config).length > 0) {
|
|
2879
|
+
base.config = config;
|
|
2880
|
+
}
|
|
2881
|
+
return this.stripSymbols(base);
|
|
2882
|
+
}
|
|
2883
|
+
toDependencyJSON(options) {
|
|
2884
|
+
const json = this.toJSON(options);
|
|
2885
|
+
return json;
|
|
2886
|
+
}
|
|
2887
|
+
hasChildren() {
|
|
2888
|
+
return this._subGraph !== undefined && this._subGraph !== null && this._subGraph.getTasks().length > 0;
|
|
2889
|
+
}
|
|
2890
|
+
_taskAddedListener = () => {
|
|
2891
|
+
this.emit("regenerate");
|
|
2892
|
+
};
|
|
2893
|
+
_subGraph = undefined;
|
|
2894
|
+
set subGraph(subGraph) {
|
|
2895
|
+
if (this._subGraph) {
|
|
2896
|
+
this._subGraph.off("task_added", this._taskAddedListener);
|
|
2897
|
+
}
|
|
2898
|
+
this._subGraph = subGraph;
|
|
2899
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2900
|
+
}
|
|
2901
|
+
get subGraph() {
|
|
2902
|
+
if (!this._subGraph) {
|
|
2903
|
+
this._subGraph = new TaskGraph;
|
|
2904
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2905
|
+
}
|
|
2906
|
+
return this._subGraph;
|
|
2907
|
+
}
|
|
2908
|
+
regenerateGraph() {
|
|
2909
|
+
if (this.hasChildren()) {
|
|
2910
|
+
for (const dataflow of this.subGraph.getDataflows()) {
|
|
2911
|
+
this.subGraph.removeDataflow(dataflow);
|
|
2912
|
+
}
|
|
2913
|
+
for (const child of this.subGraph.getTasks()) {
|
|
2914
|
+
this.subGraph.removeTask(child.id);
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
this.events.emit("regenerate");
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
// src/task-graph/EdgeMaterializer.ts
|
|
2922
|
+
import { getLogger as getLogger3 } from "@workglow/util";
|
|
2923
|
+
import { previewSource } from "@workglow/util/media";
|
|
2924
|
+
class EdgeMaterializer {
|
|
2925
|
+
graph;
|
|
2926
|
+
runner;
|
|
2927
|
+
constructor(graph, runner) {
|
|
2928
|
+
this.graph = graph;
|
|
2929
|
+
this.runner = runner;
|
|
2930
|
+
}
|
|
2931
|
+
filterInputForTask(task, input) {
|
|
2932
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
2933
|
+
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
2934
|
+
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
2935
|
+
const filteredInput = {};
|
|
2936
|
+
for (const [key, value] of Object.entries(input)) {
|
|
2937
|
+
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
2938
|
+
filteredInput[key] = value;
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
return filteredInput;
|
|
2942
|
+
}
|
|
2943
|
+
copyInputFromEdgesToNode(task) {
|
|
2944
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2945
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2946
|
+
for (const dataflow of dataflows) {
|
|
2947
|
+
const live = dataflow.getCurrentValue();
|
|
2948
|
+
const port = dataflow.targetTaskPortId;
|
|
2949
|
+
let portData;
|
|
2950
|
+
if (port === DATAFLOW_ALL_PORTS) {
|
|
2951
|
+
portData = live;
|
|
2952
|
+
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
2953
|
+
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
2954
|
+
} else {
|
|
2955
|
+
portData = { [port]: live };
|
|
2956
|
+
}
|
|
2957
|
+
this.runner.addInputData(task, portData);
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
async pushOutputFromNodeToEdges(node, results) {
|
|
2961
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2962
|
+
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
2963
|
+
for (const port of Object.keys(results)) {
|
|
2964
|
+
const value = results[port];
|
|
2965
|
+
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
2966
|
+
results[port] = await previewSource(value);
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
for (const dataflow of dataflows) {
|
|
2971
|
+
if (dataflow.stream !== undefined)
|
|
2972
|
+
continue;
|
|
2973
|
+
const registry = this.runner["registry"];
|
|
2974
|
+
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
2975
|
+
if (compatibility === "static") {
|
|
2976
|
+
dataflow.setPortData(results);
|
|
2977
|
+
await dataflow.applyTransforms(registry);
|
|
2978
|
+
} else if (compatibility === "runtime") {
|
|
2979
|
+
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
2980
|
+
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
2981
|
+
dataflow.setPortData(narrowed);
|
|
2982
|
+
await dataflow.applyTransforms(registry);
|
|
2983
|
+
} else {
|
|
2984
|
+
const resultsKeys = Object.keys(results);
|
|
2985
|
+
if (resultsKeys.length > 0) {
|
|
2986
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2987
|
+
dataflowId: dataflow.id,
|
|
2988
|
+
compatibility,
|
|
2989
|
+
resultsKeys
|
|
2990
|
+
});
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2923
2993
|
}
|
|
2924
|
-
return obj;
|
|
2925
2994
|
}
|
|
2926
|
-
|
|
2927
|
-
|
|
2995
|
+
pushErrorFromNodeToEdges(node) {
|
|
2996
|
+
if (!node?.config?.id)
|
|
2997
|
+
return;
|
|
2998
|
+
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
2999
|
+
dataflow.error = node.error;
|
|
3000
|
+
});
|
|
2928
3001
|
}
|
|
2929
|
-
|
|
2930
|
-
const
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
const
|
|
2935
|
-
const
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
if (key === "id")
|
|
2939
|
-
continue;
|
|
2940
|
-
if (propSchema?.["x-ui-hidden"] === true && key !== "inputSchema" && key !== "outputSchema" && key !== "extras") {
|
|
2941
|
-
continue;
|
|
2942
|
-
}
|
|
2943
|
-
const value = this.originalConfig[key];
|
|
2944
|
-
if (value === undefined)
|
|
2945
|
-
continue;
|
|
2946
|
-
if (typeof value === "function" || typeof value === "symbol")
|
|
2947
|
-
continue;
|
|
2948
|
-
config[key] = value;
|
|
2949
|
-
}
|
|
2950
|
-
if (config.title === ctor.title)
|
|
2951
|
-
delete config.title;
|
|
2952
|
-
if (config.description === ctor.description)
|
|
2953
|
-
delete config.description;
|
|
2954
|
-
const extras = config.extras;
|
|
2955
|
-
if (!extras || Object.keys(extras).length === 0)
|
|
2956
|
-
delete config.extras;
|
|
2957
|
-
const base = {
|
|
2958
|
-
id: this.id,
|
|
2959
|
-
type: this.type,
|
|
2960
|
-
defaults: this.defaults
|
|
3002
|
+
hasErrorOutputEdges(task) {
|
|
3003
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3004
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
3005
|
+
}
|
|
3006
|
+
pushErrorOutputToEdges(task) {
|
|
3007
|
+
const taskError = task.error;
|
|
3008
|
+
const errorData = {
|
|
3009
|
+
error: taskError?.message ?? "Unknown error",
|
|
3010
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
2961
3011
|
};
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
3012
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3013
|
+
for (const df of dataflows) {
|
|
3014
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
3015
|
+
df.value = errorData;
|
|
3016
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
3017
|
+
} else {
|
|
3018
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
3019
|
+
}
|
|
2968
3020
|
}
|
|
2969
|
-
|
|
3021
|
+
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
2970
3022
|
}
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
3023
|
+
resetTask(graph, task, runId) {
|
|
3024
|
+
task.status = TaskStatus.PENDING;
|
|
3025
|
+
task.resetInputData();
|
|
3026
|
+
task.runOutputData = {};
|
|
3027
|
+
task.error = undefined;
|
|
3028
|
+
task.progress = 0;
|
|
3029
|
+
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
3030
|
+
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
3031
|
+
if (task?.config?.id) {
|
|
3032
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
3033
|
+
dataflow.error = task.error;
|
|
3034
|
+
});
|
|
3035
|
+
}
|
|
3036
|
+
task.emit("reset");
|
|
3037
|
+
task.emit("status", task.status);
|
|
2974
3038
|
}
|
|
2975
|
-
|
|
2976
|
-
|
|
3039
|
+
static isImageValueShape(v) {
|
|
3040
|
+
if (v === null || typeof v !== "object")
|
|
3041
|
+
return false;
|
|
3042
|
+
const o = v;
|
|
3043
|
+
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
2977
3044
|
}
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
3045
|
+
}
|
|
3046
|
+
|
|
3047
|
+
// src/task-graph/RunContext.ts
|
|
3048
|
+
import { uuid4 as uuid43 } from "@workglow/util";
|
|
3049
|
+
|
|
3050
|
+
class RunContext {
|
|
3051
|
+
runId;
|
|
3052
|
+
abortController;
|
|
3053
|
+
inProgressTasks = new Map;
|
|
3054
|
+
inProgressFunctions = new Map;
|
|
3055
|
+
failedTaskErrors = new Map;
|
|
3056
|
+
telemetrySpan;
|
|
3057
|
+
graphTimeoutTimer;
|
|
3058
|
+
pendingGraphTimeoutError;
|
|
3059
|
+
activeEnforcer;
|
|
3060
|
+
parentSignalCleanup;
|
|
3061
|
+
constructor(parentSignal) {
|
|
3062
|
+
this.runId = uuid43();
|
|
3063
|
+
this.abortController = new AbortController;
|
|
3064
|
+
if (parentSignal) {
|
|
3065
|
+
const onParentAbort = () => this.abortController.abort();
|
|
3066
|
+
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
3067
|
+
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
3068
|
+
if (parentSignal.aborted) {
|
|
3069
|
+
this.parentSignalCleanup();
|
|
3070
|
+
this.parentSignalCleanup = undefined;
|
|
3071
|
+
this.abortController.abort();
|
|
3072
|
+
}
|
|
2985
3073
|
}
|
|
2986
|
-
this._subGraph = subGraph;
|
|
2987
|
-
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2988
3074
|
}
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
3075
|
+
dispose() {
|
|
3076
|
+
this.parentSignalCleanup?.();
|
|
3077
|
+
this.parentSignalCleanup = undefined;
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
// src/task-graph/RunScheduler.ts
|
|
3082
|
+
import { getLogger as getLogger5 } from "@workglow/util";
|
|
3083
|
+
|
|
3084
|
+
// src/task/ConditionalTask.ts
|
|
3085
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
3086
|
+
|
|
3087
|
+
// src/task/ConditionUtils.ts
|
|
3088
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
3089
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
3090
|
+
switch (operator) {
|
|
3091
|
+
case "is_empty":
|
|
3092
|
+
return true;
|
|
3093
|
+
case "is_not_empty":
|
|
3094
|
+
return false;
|
|
3095
|
+
case "is_true":
|
|
3096
|
+
return false;
|
|
3097
|
+
case "is_false":
|
|
3098
|
+
return true;
|
|
3099
|
+
default:
|
|
3100
|
+
return false;
|
|
2993
3101
|
}
|
|
2994
|
-
return this._subGraph;
|
|
2995
3102
|
}
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3103
|
+
const strValue = String(fieldValue);
|
|
3104
|
+
const numValue = Number(fieldValue);
|
|
3105
|
+
switch (operator) {
|
|
3106
|
+
case "equals":
|
|
3107
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3108
|
+
return numValue === Number(compareValue);
|
|
3000
3109
|
}
|
|
3001
|
-
|
|
3002
|
-
|
|
3110
|
+
return strValue === compareValue;
|
|
3111
|
+
case "not_equals":
|
|
3112
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3113
|
+
return numValue !== Number(compareValue);
|
|
3003
3114
|
}
|
|
3115
|
+
return strValue !== compareValue;
|
|
3116
|
+
case "greater_than":
|
|
3117
|
+
return numValue > Number(compareValue);
|
|
3118
|
+
case "greater_or_equal":
|
|
3119
|
+
return numValue >= Number(compareValue);
|
|
3120
|
+
case "less_than":
|
|
3121
|
+
return numValue < Number(compareValue);
|
|
3122
|
+
case "less_or_equal":
|
|
3123
|
+
return numValue <= Number(compareValue);
|
|
3124
|
+
case "contains":
|
|
3125
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
3126
|
+
case "starts_with":
|
|
3127
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
3128
|
+
case "ends_with":
|
|
3129
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
3130
|
+
case "is_empty":
|
|
3131
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
3132
|
+
case "is_not_empty":
|
|
3133
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
3134
|
+
case "is_true":
|
|
3135
|
+
return Boolean(fieldValue) === true;
|
|
3136
|
+
case "is_false":
|
|
3137
|
+
return Boolean(fieldValue) === false;
|
|
3138
|
+
default:
|
|
3139
|
+
return false;
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
function getNestedValue(obj, path) {
|
|
3143
|
+
const parts = path.split(".");
|
|
3144
|
+
let current = obj;
|
|
3145
|
+
for (const part of parts) {
|
|
3146
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
3147
|
+
return;
|
|
3004
3148
|
}
|
|
3005
|
-
|
|
3149
|
+
current = current[part];
|
|
3006
3150
|
}
|
|
3151
|
+
return current;
|
|
3007
3152
|
}
|
|
3008
3153
|
|
|
3009
3154
|
// src/task/ConditionalTask.ts
|
|
@@ -3095,7 +3240,7 @@ class ConditionalTask extends Task {
|
|
|
3095
3240
|
}
|
|
3096
3241
|
}
|
|
3097
3242
|
} catch (error) {
|
|
3098
|
-
|
|
3243
|
+
getLogger4().error(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
3099
3244
|
}
|
|
3100
3245
|
}
|
|
3101
3246
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -3367,7 +3512,7 @@ class RunScheduler {
|
|
|
3367
3512
|
ctx.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
3368
3513
|
}
|
|
3369
3514
|
} catch (err) {
|
|
3370
|
-
|
|
3515
|
+
getLogger5().error("Error running graph", { error: err });
|
|
3371
3516
|
}
|
|
3372
3517
|
await Promise.allSettled(Array.from(ctx.inProgressTasks.values()));
|
|
3373
3518
|
await Promise.allSettled(Array.from(ctx.inProgressFunctions.values()));
|
|
@@ -3443,11 +3588,12 @@ class StreamPump {
|
|
|
3443
3588
|
task.on("stream_end", onStreamEnd);
|
|
3444
3589
|
try {
|
|
3445
3590
|
const results = await task.runner.run(input, {
|
|
3446
|
-
outputCache: options.
|
|
3591
|
+
outputCache: options.legacyCacheExplicitlyDisabled ? false : options.outputCache,
|
|
3447
3592
|
shouldAccumulate,
|
|
3448
3593
|
updateProgress: options.updateProgress,
|
|
3449
3594
|
registry: options.registry,
|
|
3450
|
-
resourceScope: options.resourceScope
|
|
3595
|
+
resourceScope: options.resourceScope,
|
|
3596
|
+
runId: options.runId
|
|
3451
3597
|
});
|
|
3452
3598
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3453
3599
|
return {
|
|
@@ -3716,10 +3862,14 @@ class TaskGraphRunner {
|
|
|
3716
3862
|
previewRunning = false;
|
|
3717
3863
|
graph;
|
|
3718
3864
|
outputCache;
|
|
3865
|
+
legacyCacheExplicitlyDisabled = false;
|
|
3719
3866
|
accumulateLeafOutputs = true;
|
|
3720
3867
|
registry = globalServiceRegistry3;
|
|
3721
3868
|
resourceScope;
|
|
3722
3869
|
currentCtx;
|
|
3870
|
+
runId;
|
|
3871
|
+
currentRunPrivate;
|
|
3872
|
+
baseRegistryForRun;
|
|
3723
3873
|
edgeMaterializer;
|
|
3724
3874
|
streamPump;
|
|
3725
3875
|
runScheduler;
|
|
@@ -3759,6 +3909,15 @@ class TaskGraphRunner {
|
|
|
3759
3909
|
throw new TaskAbortedError;
|
|
3760
3910
|
}
|
|
3761
3911
|
await this.handleComplete();
|
|
3912
|
+
const runPrivateToClean = this.currentRunPrivate;
|
|
3913
|
+
this.currentRunPrivate = undefined;
|
|
3914
|
+
if (runPrivateToClean) {
|
|
3915
|
+
try {
|
|
3916
|
+
await runPrivateToClean.clearRun();
|
|
3917
|
+
} catch (e) {
|
|
3918
|
+
getLogger6().warn("RunPrivateCacheRepo.clearRun failed", { error: e });
|
|
3919
|
+
}
|
|
3920
|
+
}
|
|
3762
3921
|
return this.filterLeafResults(results);
|
|
3763
3922
|
} finally {
|
|
3764
3923
|
if (ownsScope) {
|
|
@@ -3830,7 +3989,7 @@ class TaskGraphRunner {
|
|
|
3830
3989
|
});
|
|
3831
3990
|
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3832
3991
|
previewSpan.end();
|
|
3833
|
-
|
|
3992
|
+
getLogger6().debug("task graph runPreview timings", {
|
|
3834
3993
|
previewRunId,
|
|
3835
3994
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3836
3995
|
taskTimings
|
|
@@ -3848,7 +4007,7 @@ class TaskGraphRunner {
|
|
|
3848
4007
|
});
|
|
3849
4008
|
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3850
4009
|
previewSpan.end();
|
|
3851
|
-
|
|
4010
|
+
getLogger6().debug("task graph runPreview failed", {
|
|
3852
4011
|
previewRunId,
|
|
3853
4012
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3854
4013
|
taskTimings,
|
|
@@ -3919,14 +4078,17 @@ class TaskGraphRunner {
|
|
|
3919
4078
|
outputCache: this.outputCache,
|
|
3920
4079
|
resourceScope: this.resourceScope,
|
|
3921
4080
|
accumulateLeafOutputs: this.accumulateLeafOutputs,
|
|
3922
|
-
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a)
|
|
4081
|
+
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a),
|
|
4082
|
+
runId: this.runId,
|
|
4083
|
+
legacyCacheExplicitlyDisabled: this.legacyCacheExplicitlyDisabled
|
|
3923
4084
|
});
|
|
3924
4085
|
}
|
|
3925
4086
|
const results = await task.runner.run(input, {
|
|
3926
|
-
outputCache: this.
|
|
4087
|
+
outputCache: this.legacyCacheExplicitlyDisabled ? false : this.outputCache,
|
|
3927
4088
|
updateProgress: async (task2, progress, message, ...args) => await this.runScheduler.handleProgress(this.currentCtx, task2, progress, message, ...args),
|
|
3928
4089
|
registry: this.registry,
|
|
3929
|
-
resourceScope: this.resourceScope
|
|
4090
|
+
resourceScope: this.resourceScope,
|
|
4091
|
+
runId: this.runId
|
|
3930
4092
|
});
|
|
3931
4093
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3932
4094
|
return {
|
|
@@ -3947,6 +4109,19 @@ class TaskGraphRunner {
|
|
|
3947
4109
|
dataflow.reset();
|
|
3948
4110
|
});
|
|
3949
4111
|
}
|
|
4112
|
+
static __durabilityWarnedRepos = new WeakSet;
|
|
4113
|
+
static graphUsesPrivatePolicy(graph) {
|
|
4114
|
+
return graph.getTasks().some((t) => {
|
|
4115
|
+
const ctor = t.constructor;
|
|
4116
|
+
if (ctor.cachePolicy?.kind === "private")
|
|
4117
|
+
return true;
|
|
4118
|
+
const proto = ctor.prototype;
|
|
4119
|
+
if (typeof proto.getCachePolicy === "function" && proto.getCachePolicy !== Task.prototype.getCachePolicy) {
|
|
4120
|
+
return true;
|
|
4121
|
+
}
|
|
4122
|
+
return false;
|
|
4123
|
+
});
|
|
4124
|
+
}
|
|
3950
4125
|
async handleStart(config) {
|
|
3951
4126
|
if (config?.registry !== undefined) {
|
|
3952
4127
|
this.registry = config.registry;
|
|
@@ -3956,18 +4131,61 @@ class TaskGraphRunner {
|
|
|
3956
4131
|
if (config?.resourceScope !== undefined) {
|
|
3957
4132
|
this.resourceScope = config.resourceScope;
|
|
3958
4133
|
}
|
|
4134
|
+
this.baseRegistryForRun = this.registry;
|
|
4135
|
+
this.runId = config?.runId;
|
|
4136
|
+
if (this.registry.has(CACHE_REGISTRY)) {
|
|
4137
|
+
const checkRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4138
|
+
if (checkRegistry.private && !checkRegistry.private.isDurable()) {
|
|
4139
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4140
|
+
const repo = checkRegistry.private;
|
|
4141
|
+
if (!TaskGraphRunner.__durabilityWarnedRepos.has(repo)) {
|
|
4142
|
+
TaskGraphRunner.__durabilityWarnedRepos.add(repo);
|
|
4143
|
+
getLogger6().warn("TaskGraphRunner: private cache repo may be used but is non-durable \u2014 " + "restart-survival will not work. Ensure the CacheRegistry 'private' " + "slot is backed by a durable storage backend.");
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
}
|
|
4147
|
+
if (!this.runId && checkRegistry.private) {
|
|
4148
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4149
|
+
throw new TaskConfigurationError("TaskGraphRunner: graph contains a private-policy task but no runId was provided. " + "Provide `runId` in TaskGraphRunConfig so private cache entries can be namespaced.");
|
|
4150
|
+
}
|
|
4151
|
+
}
|
|
4152
|
+
}
|
|
4153
|
+
if (this.runId && this.registry.has(CACHE_REGISTRY)) {
|
|
4154
|
+
const baseRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4155
|
+
if (baseRegistry.private) {
|
|
4156
|
+
const wrappedPrivate = new RunPrivateCacheRepo({
|
|
4157
|
+
backing: baseRegistry.private,
|
|
4158
|
+
runId: this.runId
|
|
4159
|
+
});
|
|
4160
|
+
this.currentRunPrivate = wrappedPrivate;
|
|
4161
|
+
const wrappedRegistry = new DefaultCacheRegistry({
|
|
4162
|
+
deterministic: baseRegistry.deterministic,
|
|
4163
|
+
private: wrappedPrivate
|
|
4164
|
+
});
|
|
4165
|
+
const childContainer = this.registry.container.createChildContainer();
|
|
4166
|
+
const runtimeServices = new ServiceRegistry3(childContainer);
|
|
4167
|
+
runtimeServices.registerInstance(CACHE_REGISTRY, wrappedRegistry);
|
|
4168
|
+
this.registry = runtimeServices;
|
|
4169
|
+
this.resourceScope?.register(`cache:private:${this.runId}`, async () => {});
|
|
4170
|
+
}
|
|
4171
|
+
}
|
|
3959
4172
|
this.accumulateLeafOutputs = config?.accumulateLeafOutputs !== false;
|
|
3960
4173
|
if (config?.outputCache !== undefined) {
|
|
3961
4174
|
if (typeof config.outputCache === "boolean") {
|
|
3962
4175
|
if (config.outputCache === true) {
|
|
3963
4176
|
this.outputCache = this.registry.get(TASK_OUTPUT_REPOSITORY);
|
|
4177
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3964
4178
|
} else {
|
|
3965
4179
|
this.outputCache = undefined;
|
|
4180
|
+
this.legacyCacheExplicitlyDisabled = true;
|
|
3966
4181
|
}
|
|
3967
4182
|
} else {
|
|
3968
4183
|
this.outputCache = config.outputCache;
|
|
4184
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3969
4185
|
}
|
|
3970
4186
|
this.graph.outputCache = this.outputCache;
|
|
4187
|
+
} else {
|
|
4188
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3971
4189
|
}
|
|
3972
4190
|
if (this.running || this.previewRunning) {
|
|
3973
4191
|
throw new TaskConfigurationError("Graph is already running");
|
|
@@ -4056,6 +4274,10 @@ class TaskGraphRunner {
|
|
|
4056
4274
|
}
|
|
4057
4275
|
ctx?.dispose();
|
|
4058
4276
|
this.currentCtx = undefined;
|
|
4277
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4278
|
+
this.registry = this.baseRegistryForRun;
|
|
4279
|
+
this.baseRegistryForRun = undefined;
|
|
4280
|
+
}
|
|
4059
4281
|
this.graph.emit("complete");
|
|
4060
4282
|
}
|
|
4061
4283
|
async handleCompletePreview() {
|
|
@@ -4068,6 +4290,7 @@ class TaskGraphRunner {
|
|
|
4068
4290
|
return task.abort();
|
|
4069
4291
|
}
|
|
4070
4292
|
}));
|
|
4293
|
+
this.currentRunPrivate = undefined;
|
|
4071
4294
|
const ctx = this.currentCtx;
|
|
4072
4295
|
this.running = false;
|
|
4073
4296
|
if (ctx?.telemetrySpan) {
|
|
@@ -4077,6 +4300,10 @@ class TaskGraphRunner {
|
|
|
4077
4300
|
}
|
|
4078
4301
|
ctx?.dispose();
|
|
4079
4302
|
this.currentCtx = undefined;
|
|
4303
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4304
|
+
this.registry = this.baseRegistryForRun;
|
|
4305
|
+
this.baseRegistryForRun = undefined;
|
|
4306
|
+
}
|
|
4080
4307
|
this.graph.emit("error", error);
|
|
4081
4308
|
}
|
|
4082
4309
|
async handleErrorPreview() {
|
|
@@ -4092,6 +4319,7 @@ class TaskGraphRunner {
|
|
|
4092
4319
|
return task.abort();
|
|
4093
4320
|
}
|
|
4094
4321
|
}));
|
|
4322
|
+
this.currentRunPrivate = undefined;
|
|
4095
4323
|
const ctx = this.currentCtx;
|
|
4096
4324
|
if (ctx?.telemetrySpan) {
|
|
4097
4325
|
ctx.telemetrySpan.setStatus(SpanStatusCode2.ERROR, "aborted");
|
|
@@ -4100,6 +4328,10 @@ class TaskGraphRunner {
|
|
|
4100
4328
|
}
|
|
4101
4329
|
ctx?.dispose();
|
|
4102
4330
|
this.currentCtx = undefined;
|
|
4331
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4332
|
+
this.registry = this.baseRegistryForRun;
|
|
4333
|
+
this.baseRegistryForRun = undefined;
|
|
4334
|
+
}
|
|
4103
4335
|
this.graph.emit("abort");
|
|
4104
4336
|
}
|
|
4105
4337
|
async handleAbortPreview() {
|
|
@@ -4223,7 +4455,7 @@ class GraphAsTask extends Task {
|
|
|
4223
4455
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
4224
4456
|
this._inputSchemaNode = schemaNode;
|
|
4225
4457
|
} catch (error) {
|
|
4226
|
-
|
|
4458
|
+
getLogger7().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
4227
4459
|
this._inputSchemaNode = compileSchema2({});
|
|
4228
4460
|
}
|
|
4229
4461
|
}
|
|
@@ -4507,7 +4739,7 @@ function convertPipeFunctionToTask(fn, config) {
|
|
|
4507
4739
|
additionalProperties: true
|
|
4508
4740
|
};
|
|
4509
4741
|
};
|
|
4510
|
-
static
|
|
4742
|
+
static cachePolicy = { kind: "none" };
|
|
4511
4743
|
async execute(input, context) {
|
|
4512
4744
|
return fn(input, context);
|
|
4513
4745
|
}
|
|
@@ -4590,7 +4822,8 @@ class TaskGraph {
|
|
|
4590
4822
|
timeout: config?.timeout,
|
|
4591
4823
|
maxTasks: config?.maxTasks,
|
|
4592
4824
|
resourceScope: config?.resourceScope,
|
|
4593
|
-
disposeStrategy: config?.disposeStrategy
|
|
4825
|
+
disposeStrategy: config?.disposeStrategy,
|
|
4826
|
+
runId: config?.runId
|
|
4594
4827
|
});
|
|
4595
4828
|
}
|
|
4596
4829
|
runPreview(input = {}, config = {}) {
|
|
@@ -5139,7 +5372,7 @@ function autoConnect(graph, sourceTask, targetTask, options) {
|
|
|
5139
5372
|
}
|
|
5140
5373
|
|
|
5141
5374
|
// src/task-graph/LoopBuilderContext.ts
|
|
5142
|
-
import { getLogger as
|
|
5375
|
+
import { getLogger as getLogger8 } from "@workglow/util";
|
|
5143
5376
|
function runLoopAutoConnect(parentGraph, pending) {
|
|
5144
5377
|
const { parent, iteratorTask } = pending;
|
|
5145
5378
|
if (parentGraph.getTargetDataflows(parent.id).length !== 0)
|
|
@@ -5153,7 +5386,7 @@ function runLoopAutoConnect(parentGraph, pending) {
|
|
|
5153
5386
|
const result = autoConnect(parentGraph, parent, iteratorTask, { earlierTasks });
|
|
5154
5387
|
if (result.error) {
|
|
5155
5388
|
const message = result.error + " Task not added.";
|
|
5156
|
-
|
|
5389
|
+
getLogger8().error(message);
|
|
5157
5390
|
parentGraph.removeTask(iteratorTask.id);
|
|
5158
5391
|
return message;
|
|
5159
5392
|
}
|
|
@@ -5192,7 +5425,7 @@ class LoopBuilderContext {
|
|
|
5192
5425
|
}
|
|
5193
5426
|
|
|
5194
5427
|
// src/task-graph/WorkflowBuilder.ts
|
|
5195
|
-
import { getLogger as
|
|
5428
|
+
import { getLogger as getLogger9, uuid4 as uuid47 } from "@workglow/util";
|
|
5196
5429
|
|
|
5197
5430
|
// src/task-graph/ConditionalBuilder.ts
|
|
5198
5431
|
import { uuid4 as uuid46 } from "@workglow/util";
|
|
@@ -5351,7 +5584,7 @@ class WorkflowBuilder {
|
|
|
5351
5584
|
const taskSchema = task.inputSchema();
|
|
5352
5585
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5353
5586
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5354
|
-
|
|
5587
|
+
getLogger9().error(this._error);
|
|
5355
5588
|
return;
|
|
5356
5589
|
}
|
|
5357
5590
|
dataflow.targetTaskId = task.id;
|
|
@@ -5376,10 +5609,10 @@ class WorkflowBuilder {
|
|
|
5376
5609
|
if (result.error) {
|
|
5377
5610
|
if (this._loopContext !== undefined) {
|
|
5378
5611
|
this._error = result.error;
|
|
5379
|
-
|
|
5612
|
+
getLogger9().warn(this._error);
|
|
5380
5613
|
} else {
|
|
5381
5614
|
this._error = result.error + " Task not added.";
|
|
5382
|
-
|
|
5615
|
+
getLogger9().error(this._error);
|
|
5383
5616
|
this._facade.graph.removeTask(task.id);
|
|
5384
5617
|
}
|
|
5385
5618
|
}
|
|
@@ -5402,7 +5635,7 @@ class WorkflowBuilder {
|
|
|
5402
5635
|
const taskSchema = task.inputSchema();
|
|
5403
5636
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5404
5637
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5405
|
-
|
|
5638
|
+
getLogger9().error(this._error);
|
|
5406
5639
|
return;
|
|
5407
5640
|
}
|
|
5408
5641
|
dataflow.targetTaskId = task.id;
|
|
@@ -5454,7 +5687,7 @@ class WorkflowBuilder {
|
|
|
5454
5687
|
if (-index > nodes.length) {
|
|
5455
5688
|
const errorMsg = `Back index greater than number of tasks`;
|
|
5456
5689
|
this._error = errorMsg;
|
|
5457
|
-
|
|
5690
|
+
getLogger9().error(this._error);
|
|
5458
5691
|
throw new WorkflowError(errorMsg);
|
|
5459
5692
|
}
|
|
5460
5693
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -5463,13 +5696,13 @@ class WorkflowBuilder {
|
|
|
5463
5696
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
5464
5697
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
5465
5698
|
this._error = errorMsg;
|
|
5466
|
-
|
|
5699
|
+
getLogger9().error(this._error);
|
|
5467
5700
|
throw new WorkflowError(errorMsg);
|
|
5468
5701
|
}
|
|
5469
5702
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
5470
5703
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
5471
5704
|
this._error = errorMsg;
|
|
5472
|
-
|
|
5705
|
+
getLogger9().error(this._error);
|
|
5473
5706
|
throw new WorkflowError(errorMsg);
|
|
5474
5707
|
}
|
|
5475
5708
|
const df = new Dataflow(lastNode.id, source, undefined, target);
|
|
@@ -5482,7 +5715,7 @@ class WorkflowBuilder {
|
|
|
5482
5715
|
const parent = getLastTask(this._facade);
|
|
5483
5716
|
if (!parent) {
|
|
5484
5717
|
this._error = "onError() requires a preceding task in the workflow";
|
|
5485
|
-
|
|
5718
|
+
getLogger9().error(this._error);
|
|
5486
5719
|
throw new WorkflowError(this._error);
|
|
5487
5720
|
}
|
|
5488
5721
|
const handlerTask = ensureTask(handler);
|
|
@@ -5496,7 +5729,7 @@ class WorkflowBuilder {
|
|
|
5496
5729
|
const nodes = this._facade.graph.getTasks();
|
|
5497
5730
|
if (nodes.length === 0) {
|
|
5498
5731
|
this._error = "No tasks to remove";
|
|
5499
|
-
|
|
5732
|
+
getLogger9().error(this._error);
|
|
5500
5733
|
return;
|
|
5501
5734
|
}
|
|
5502
5735
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -6460,7 +6693,7 @@ function registerBuiltInTransforms() {
|
|
|
6460
6693
|
TransformRegistry.registerTransform(t);
|
|
6461
6694
|
}
|
|
6462
6695
|
// src/task/EntitlementProfile.ts
|
|
6463
|
-
import { createServiceToken as
|
|
6696
|
+
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
6464
6697
|
var STATIC_SIGNAL_SOURCE = Object.freeze({
|
|
6465
6698
|
subscribe(_listener) {
|
|
6466
6699
|
return () => {};
|
|
@@ -6563,7 +6796,7 @@ function createPolicyProfile(name, policy, options = {}) {
|
|
|
6563
6796
|
};
|
|
6564
6797
|
return profile;
|
|
6565
6798
|
}
|
|
6566
|
-
var ENTITLEMENT_PROFILE =
|
|
6799
|
+
var ENTITLEMENT_PROFILE = createServiceToken6("workglow.entitlementProfile");
|
|
6567
6800
|
// src/task/EntitlementProfiles.ts
|
|
6568
6801
|
var BROWSER_GRANTS = [
|
|
6569
6802
|
{ id: Entitlements.NETWORK_HTTP },
|
|
@@ -8185,8 +8418,8 @@ import {
|
|
|
8185
8418
|
JobQueueClient,
|
|
8186
8419
|
JobQueueServer
|
|
8187
8420
|
} from "@workglow/job-queue";
|
|
8188
|
-
import { createServiceToken as
|
|
8189
|
-
var JOB_QUEUE_FACTORY =
|
|
8421
|
+
import { createServiceToken as createServiceToken7, globalServiceRegistry as globalServiceRegistry4 } from "@workglow/util";
|
|
8422
|
+
var JOB_QUEUE_FACTORY = createServiceToken7("taskgraph.jobQueueFactory");
|
|
8190
8423
|
var defaultJobQueueFactory = async ({
|
|
8191
8424
|
queueName,
|
|
8192
8425
|
jobClass,
|
|
@@ -8451,8 +8684,8 @@ queueMicrotask(() => {
|
|
|
8451
8684
|
});
|
|
8452
8685
|
// src/task/TaskRegistry.ts
|
|
8453
8686
|
import {
|
|
8454
|
-
createServiceToken as
|
|
8455
|
-
getLogger as
|
|
8687
|
+
createServiceToken as createServiceToken8,
|
|
8688
|
+
getLogger as getLogger10,
|
|
8456
8689
|
globalServiceRegistry as globalServiceRegistry5,
|
|
8457
8690
|
registerInputCompactor,
|
|
8458
8691
|
registerInputResolver
|
|
@@ -8475,7 +8708,7 @@ function registerTask(baseClass) {
|
|
|
8475
8708
|
const result = validateSchema(schema);
|
|
8476
8709
|
if (!result.valid) {
|
|
8477
8710
|
const messages = result.errors.map((e) => `${e.path}: ${e.message}`).join("; ");
|
|
8478
|
-
|
|
8711
|
+
getLogger10().warn(`Task "${baseClass.type}" has invalid ${name}: ${messages}`, {
|
|
8479
8712
|
taskType: baseClass.type,
|
|
8480
8713
|
schemaName: name,
|
|
8481
8714
|
errors: result.errors
|
|
@@ -8491,7 +8724,7 @@ var TaskRegistry = {
|
|
|
8491
8724
|
registerTask,
|
|
8492
8725
|
unregisterTask
|
|
8493
8726
|
};
|
|
8494
|
-
var TASK_CONSTRUCTORS =
|
|
8727
|
+
var TASK_CONSTRUCTORS = createServiceToken8("task.constructors");
|
|
8495
8728
|
function getGlobalTaskConstructors() {
|
|
8496
8729
|
return globalServiceRegistry5.get(TASK_CONSTRUCTORS);
|
|
8497
8730
|
}
|
|
@@ -8660,8 +8893,8 @@ var registerBaseTasks = () => {
|
|
|
8660
8893
|
return tasks;
|
|
8661
8894
|
};
|
|
8662
8895
|
// src/storage/TaskGraphRepository.ts
|
|
8663
|
-
import { createServiceToken as
|
|
8664
|
-
var TASK_GRAPH_REPOSITORY =
|
|
8896
|
+
import { createServiceToken as createServiceToken9, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
8897
|
+
var TASK_GRAPH_REPOSITORY = createServiceToken9("taskgraph.taskGraphRepository");
|
|
8665
8898
|
|
|
8666
8899
|
class TaskGraphRepository {
|
|
8667
8900
|
type = "TaskGraphRepository";
|
|
@@ -8757,6 +8990,10 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8757
8990
|
this.tabularRepository = tabularRepository;
|
|
8758
8991
|
this.outputCompression = outputCompression;
|
|
8759
8992
|
}
|
|
8993
|
+
isDurable() {
|
|
8994
|
+
const backing = this.tabularRepository;
|
|
8995
|
+
return backing.isDurable?.() ?? true;
|
|
8996
|
+
}
|
|
8760
8997
|
async setupDatabase() {
|
|
8761
8998
|
await this.tabularRepository.setupDatabase?.();
|
|
8762
8999
|
}
|
|
@@ -8817,6 +9054,33 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8817
9054
|
await this.tabularRepository.deleteSearch({ createdAt: { value: date, operator: "<" } });
|
|
8818
9055
|
this.emit("output_pruned");
|
|
8819
9056
|
}
|
|
9057
|
+
async deleteByTaskTypePrefix(prefix) {
|
|
9058
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9059
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9060
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9061
|
+
}
|
|
9062
|
+
}
|
|
9063
|
+
}
|
|
9064
|
+
async clearOlderThanWithTaskTypePrefix(prefix, olderThanInMs) {
|
|
9065
|
+
const cutoff = Date.now() - olderThanInMs;
|
|
9066
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9067
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9068
|
+
const ts = typeof row.createdAt === "string" ? new Date(row.createdAt).getTime() : NaN;
|
|
9069
|
+
if (!isNaN(ts) && ts < cutoff) {
|
|
9070
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9071
|
+
}
|
|
9072
|
+
}
|
|
9073
|
+
}
|
|
9074
|
+
}
|
|
9075
|
+
async sizeByTaskTypePrefix(prefix) {
|
|
9076
|
+
let count = 0;
|
|
9077
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9078
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9079
|
+
count++;
|
|
9080
|
+
}
|
|
9081
|
+
}
|
|
9082
|
+
return count;
|
|
9083
|
+
}
|
|
8820
9084
|
}
|
|
8821
9085
|
// src/storage/PortCodecRegistry.ts
|
|
8822
9086
|
import { _resetPortCodecsForTests, getPortCodec as getPortCodec2, registerPortCodec } from "@workglow/util";
|
|
@@ -8866,6 +9130,8 @@ export {
|
|
|
8866
9130
|
isoDateToUnixTransform,
|
|
8867
9131
|
isTaskStreamable,
|
|
8868
9132
|
isStrictArraySchema,
|
|
9133
|
+
isPolicyPrivate,
|
|
9134
|
+
isPolicyCached,
|
|
8869
9135
|
isIterationProperty,
|
|
8870
9136
|
isFlexibleSchema,
|
|
8871
9137
|
indexTransform,
|
|
@@ -8979,6 +9245,7 @@ export {
|
|
|
8979
9245
|
STATIC_SIGNAL_SOURCE,
|
|
8980
9246
|
SERVER_GRANTS,
|
|
8981
9247
|
RunScheduler,
|
|
9248
|
+
RunPrivateCacheRepo,
|
|
8982
9249
|
RunContext,
|
|
8983
9250
|
ReduceTask,
|
|
8984
9251
|
PROPERTY_ARRAY,
|
|
@@ -9006,10 +9273,12 @@ export {
|
|
|
9006
9273
|
ENTITLEMENT_ENFORCER,
|
|
9007
9274
|
EMPTY_POLICY,
|
|
9008
9275
|
EMPTY_ENTITLEMENTS,
|
|
9276
|
+
DefaultCacheRegistry,
|
|
9009
9277
|
DataflowArrow,
|
|
9010
9278
|
Dataflow,
|
|
9011
9279
|
DESKTOP_GRANTS,
|
|
9012
9280
|
DENY_ALL_RESOLVER,
|
|
9281
|
+
DEFAULT_CACHE_POLICY,
|
|
9013
9282
|
DATAFLOW_ERROR_PORT,
|
|
9014
9283
|
DATAFLOW_ALL_PORTS,
|
|
9015
9284
|
CreateWorkflow,
|
|
@@ -9017,8 +9286,10 @@ export {
|
|
|
9017
9286
|
CreateEndLoopWorkflow,
|
|
9018
9287
|
CreateAdaptiveWorkflow,
|
|
9019
9288
|
ConditionalTask,
|
|
9289
|
+
CacheJanitor,
|
|
9020
9290
|
CacheCoordinator,
|
|
9291
|
+
CACHE_REGISTRY,
|
|
9021
9292
|
BROWSER_GRANTS
|
|
9022
9293
|
};
|
|
9023
9294
|
|
|
9024
|
-
//# debugId=
|
|
9295
|
+
//# debugId=E2522767BD1162D964756E2164756E21
|