@workglow/task-graph 0.2.36 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -46
- package/dist/browser.js +792 -506
- package/dist/browser.js.map +40 -36
- package/dist/bun.js +792 -506
- package/dist/bun.js.map +40 -36
- package/dist/cache/CacheJanitor.d.ts +27 -0
- package/dist/cache/CacheJanitor.d.ts.map +1 -0
- package/dist/cache/CachePolicy.d.ts +16 -0
- package/dist/cache/CachePolicy.d.ts.map +1 -0
- package/dist/cache/CacheRegistry.d.ts +30 -0
- package/dist/cache/CacheRegistry.d.ts.map +1 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts +56 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts.map +1 -0
- package/dist/cache/index.d.ts +10 -0
- package/dist/cache/index.d.ts.map +1 -0
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +792 -506
- package/dist/node.js.map +40 -36
- package/dist/storage/PortCodecRegistry.d.ts +1 -1
- package/dist/storage/PortCodecRegistry.d.ts.map +1 -1
- package/dist/storage/TaskGraphTabularRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputRepository.d.ts +40 -4
- package/dist/storage/TaskOutputRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputTabularRepository.d.ts +27 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts.map +1 -1
- package/dist/task/CacheCoordinator.d.ts +18 -1
- package/dist/task/CacheCoordinator.d.ts.map +1 -1
- package/dist/task/ConditionalTask.d.ts +14 -14
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/EntitlementProfile.d.ts.map +1 -1
- package/dist/task/EntitlementProfiles.d.ts +1 -1
- package/dist/task/EntitlementProfiles.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +8 -9
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/FallbackTaskRunner.d.ts +8 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts +4 -4
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +28 -2
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputCompactor.d.ts +1 -1
- package/dist/task/InputCompactor.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +4 -4
- package/dist/task/JobQueueFactory.d.ts +1 -1
- package/dist/task/JobQueueFactory.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -4
- package/dist/task/ReduceTask.d.ts +4 -4
- package/dist/task/StreamProcessor.d.ts +1 -1
- package/dist/task/StreamProcessor.d.ts.map +1 -1
- package/dist/task/Task.d.ts +50 -0
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +2 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +2 -2
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +0 -13
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskQueueRegistry.d.ts +1 -1
- package/dist/task/TaskQueueRegistry.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +28 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +1 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -4
- package/dist/task/iterationSchema.d.ts +1 -1
- package/dist/task/iterationSchema.d.ts.map +1 -1
- package/dist/task-graph/Conversions.d.ts.map +1 -1
- package/dist/task-graph/Dataflow.d.ts +1 -1
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/StreamPump.d.ts +8 -0
- package/dist/task-graph/StreamPump.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +10 -1
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +45 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/dist/task-graph/transforms/index.d.ts +6 -6
- package/dist/task-graph/transforms/index.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +91 -2
package/dist/bun.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/task-graph/Dataflow.ts
|
|
3
|
-
import { areSemanticallyCompatible } from "@workglow/util/schema";
|
|
4
3
|
import { EventEmitter } from "@workglow/util";
|
|
4
|
+
import { areSemanticallyCompatible } from "@workglow/util/schema";
|
|
5
5
|
|
|
6
6
|
// src/task/TaskError.ts
|
|
7
7
|
import { BaseError } from "@workglow/util";
|
|
@@ -61,9 +61,13 @@ class TaskFailedError extends TaskError {
|
|
|
61
61
|
class JobTaskFailedError extends TaskFailedError {
|
|
62
62
|
static type = "JobTaskFailedError";
|
|
63
63
|
jobError;
|
|
64
|
+
code;
|
|
64
65
|
constructor(err) {
|
|
65
66
|
super(String(err));
|
|
66
67
|
this.jobError = err;
|
|
68
|
+
if (err.code) {
|
|
69
|
+
this.code = err.code;
|
|
70
|
+
}
|
|
67
71
|
}
|
|
68
72
|
}
|
|
69
73
|
|
|
@@ -1274,14 +1278,14 @@ import { EventEmitter as EventEmitter4, uuid4 as uuid45 } from "@workglow/util";
|
|
|
1274
1278
|
import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
1275
1279
|
|
|
1276
1280
|
// src/task/GraphAsTask.ts
|
|
1277
|
-
import { getLogger as
|
|
1281
|
+
import { getLogger as getLogger7 } from "@workglow/util";
|
|
1278
1282
|
import { CycleError } from "@workglow/util/graph";
|
|
1279
1283
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
1280
1284
|
|
|
1281
1285
|
// src/task-graph/TaskGraphRunner.ts
|
|
1282
1286
|
import {
|
|
1283
1287
|
collectPropertyValues,
|
|
1284
|
-
getLogger as
|
|
1288
|
+
getLogger as getLogger6,
|
|
1285
1289
|
getTelemetryProvider as getTelemetryProvider2,
|
|
1286
1290
|
globalServiceRegistry as globalServiceRegistry3,
|
|
1287
1291
|
ResourceScope as ResourceScope2,
|
|
@@ -1290,9 +1294,39 @@ import {
|
|
|
1290
1294
|
uuid4 as uuid44
|
|
1291
1295
|
} from "@workglow/util";
|
|
1292
1296
|
|
|
1297
|
+
// src/cache/CacheJanitor.ts
|
|
1298
|
+
class CacheJanitor {
|
|
1299
|
+
privateBacking;
|
|
1300
|
+
constructor({ privateBacking }) {
|
|
1301
|
+
this.privateBacking = privateBacking;
|
|
1302
|
+
}
|
|
1303
|
+
async sweepStaleRunPrivate(olderThanMs) {
|
|
1304
|
+
await this.privateBacking.clearOlderThanWithTaskTypePrefix("__run:", olderThanMs);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
// src/cache/CachePolicy.ts
|
|
1308
|
+
var DEFAULT_CACHE_POLICY = { kind: "deterministic" };
|
|
1309
|
+
function isPolicyCached(policy) {
|
|
1310
|
+
return policy.kind !== "none";
|
|
1311
|
+
}
|
|
1312
|
+
function isPolicyPrivate(policy) {
|
|
1313
|
+
return policy.kind === "private";
|
|
1314
|
+
}
|
|
1315
|
+
// src/cache/CacheRegistry.ts
|
|
1316
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
1317
|
+
var CACHE_REGISTRY = createServiceToken2("taskgraph.cacheRegistry");
|
|
1318
|
+
|
|
1319
|
+
class DefaultCacheRegistry {
|
|
1320
|
+
deterministic;
|
|
1321
|
+
private;
|
|
1322
|
+
constructor(init = {}) {
|
|
1323
|
+
this.deterministic = init.deterministic;
|
|
1324
|
+
this.private = init.private;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1293
1327
|
// src/storage/TaskOutputRepository.ts
|
|
1294
|
-
import { createServiceToken as
|
|
1295
|
-
var TASK_OUTPUT_REPOSITORY =
|
|
1328
|
+
import { createServiceToken as createServiceToken3, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1329
|
+
var TASK_OUTPUT_REPOSITORY = createServiceToken3("taskgraph.taskOutputRepository");
|
|
1296
1330
|
|
|
1297
1331
|
class TaskOutputRepository {
|
|
1298
1332
|
outputCompression;
|
|
@@ -1318,10 +1352,53 @@ class TaskOutputRepository {
|
|
|
1318
1352
|
emit(name, ...args) {
|
|
1319
1353
|
this._events?.emit(name, ...args);
|
|
1320
1354
|
}
|
|
1355
|
+
async deleteByTaskTypePrefix(_prefix) {
|
|
1356
|
+
throw new Error(`${this.constructor.name}: deleteByTaskTypePrefix is not supported by this repository.`);
|
|
1357
|
+
}
|
|
1358
|
+
async clearOlderThanWithTaskTypePrefix(_prefix, _olderThanMs) {
|
|
1359
|
+
throw new Error(`${this.constructor.name}: clearOlderThanWithTaskTypePrefix is not supported by this repository.`);
|
|
1360
|
+
}
|
|
1361
|
+
async sizeByTaskTypePrefix(_prefix) {
|
|
1362
|
+
throw new Error(`${this.constructor.name}: sizeByTaskTypePrefix is not supported by this repository.`);
|
|
1363
|
+
}
|
|
1321
1364
|
}
|
|
1322
1365
|
|
|
1366
|
+
// src/cache/RunPrivateCacheRepo.ts
|
|
1367
|
+
class RunPrivateCacheRepo extends TaskOutputRepository {
|
|
1368
|
+
backing;
|
|
1369
|
+
runId;
|
|
1370
|
+
constructor({ backing, runId }) {
|
|
1371
|
+
super({ outputCompression: backing.outputCompression });
|
|
1372
|
+
this.backing = backing;
|
|
1373
|
+
this.runId = runId;
|
|
1374
|
+
}
|
|
1375
|
+
ns(taskType) {
|
|
1376
|
+
return `__run:${this.runId}::${taskType}`;
|
|
1377
|
+
}
|
|
1378
|
+
async saveOutput(taskType, inputs, output, createdAt) {
|
|
1379
|
+
await this.backing.saveOutput(this.ns(taskType), inputs, output, createdAt);
|
|
1380
|
+
}
|
|
1381
|
+
async getOutput(taskType, inputs) {
|
|
1382
|
+
return this.backing.getOutput(this.ns(taskType), inputs);
|
|
1383
|
+
}
|
|
1384
|
+
async clear() {
|
|
1385
|
+
await this.clearRun();
|
|
1386
|
+
}
|
|
1387
|
+
async clearRun() {
|
|
1388
|
+
await this.backing.deleteByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1389
|
+
}
|
|
1390
|
+
async size() {
|
|
1391
|
+
return this.backing.sizeByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1392
|
+
}
|
|
1393
|
+
async clearOlderThan(olderThanInMs) {
|
|
1394
|
+
await this.backing.clearOlderThanWithTaskTypePrefix(`__run:${this.runId}::`, olderThanInMs);
|
|
1395
|
+
}
|
|
1396
|
+
isDurable() {
|
|
1397
|
+
return this.backing.isDurable();
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1323
1400
|
// src/task/EntitlementEnforcer.ts
|
|
1324
|
-
import { createServiceToken as
|
|
1401
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1325
1402
|
|
|
1326
1403
|
// src/task/EntitlementPolicy.ts
|
|
1327
1404
|
var EMPTY_POLICY = Object.freeze({
|
|
@@ -1365,7 +1442,7 @@ function can(policy, id, resources) {
|
|
|
1365
1442
|
}
|
|
1366
1443
|
|
|
1367
1444
|
// src/task/EntitlementResolver.ts
|
|
1368
|
-
import { createServiceToken as
|
|
1445
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1369
1446
|
var PERMISSIVE_RESOLVER = {
|
|
1370
1447
|
lookup: () => "grant",
|
|
1371
1448
|
prompt: async () => "grant",
|
|
@@ -1376,7 +1453,7 @@ var DENY_ALL_RESOLVER = {
|
|
|
1376
1453
|
prompt: async () => "deny",
|
|
1377
1454
|
save: () => {}
|
|
1378
1455
|
};
|
|
1379
|
-
var ENTITLEMENT_RESOLVER =
|
|
1456
|
+
var ENTITLEMENT_RESOLVER = createServiceToken4("workglow.entitlementResolver");
|
|
1380
1457
|
|
|
1381
1458
|
// src/task/EntitlementEnforcer.ts
|
|
1382
1459
|
function formatEntitlementDenial(denial) {
|
|
@@ -1449,7 +1526,7 @@ function createScopedEnforcer(grants) {
|
|
|
1449
1526
|
function createGrantListEnforcer(grants) {
|
|
1450
1527
|
return createScopedEnforcer(grants.map((id) => ({ id })));
|
|
1451
1528
|
}
|
|
1452
|
-
var ENTITLEMENT_ENFORCER =
|
|
1529
|
+
var ENTITLEMENT_ENFORCER = createServiceToken5("workglow.entitlementEnforcer");
|
|
1453
1530
|
|
|
1454
1531
|
// src/task/StreamTypes.ts
|
|
1455
1532
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1552,246 +1629,13 @@ function hasStructuredOutput(schema) {
|
|
|
1552
1629
|
return getStructuredOutputSchemas(schema).size > 0;
|
|
1553
1630
|
}
|
|
1554
1631
|
|
|
1555
|
-
// src/task-graph/EdgeMaterializer.ts
|
|
1556
|
-
import { getLogger } from "@workglow/util";
|
|
1557
|
-
import { previewSource } from "@workglow/util/media";
|
|
1558
|
-
class EdgeMaterializer {
|
|
1559
|
-
graph;
|
|
1560
|
-
runner;
|
|
1561
|
-
constructor(graph, runner) {
|
|
1562
|
-
this.graph = graph;
|
|
1563
|
-
this.runner = runner;
|
|
1564
|
-
}
|
|
1565
|
-
filterInputForTask(task, input) {
|
|
1566
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1567
|
-
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1568
|
-
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1569
|
-
const filteredInput = {};
|
|
1570
|
-
for (const [key, value] of Object.entries(input)) {
|
|
1571
|
-
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
1572
|
-
filteredInput[key] = value;
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
return filteredInput;
|
|
1576
|
-
}
|
|
1577
|
-
copyInputFromEdgesToNode(task) {
|
|
1578
|
-
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1579
|
-
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
1580
|
-
for (const dataflow of dataflows) {
|
|
1581
|
-
const live = dataflow.getCurrentValue();
|
|
1582
|
-
const port = dataflow.targetTaskPortId;
|
|
1583
|
-
let portData;
|
|
1584
|
-
if (port === DATAFLOW_ALL_PORTS) {
|
|
1585
|
-
portData = live;
|
|
1586
|
-
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
1587
|
-
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
1588
|
-
} else {
|
|
1589
|
-
portData = { [port]: live };
|
|
1590
|
-
}
|
|
1591
|
-
this.runner.addInputData(task, portData);
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
async pushOutputFromNodeToEdges(node, results) {
|
|
1595
|
-
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1596
|
-
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
1597
|
-
for (const port of Object.keys(results)) {
|
|
1598
|
-
const value = results[port];
|
|
1599
|
-
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
1600
|
-
results[port] = await previewSource(value);
|
|
1601
|
-
}
|
|
1602
|
-
}
|
|
1603
|
-
}
|
|
1604
|
-
for (const dataflow of dataflows) {
|
|
1605
|
-
if (dataflow.stream !== undefined)
|
|
1606
|
-
continue;
|
|
1607
|
-
const registry = this.runner["registry"];
|
|
1608
|
-
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
1609
|
-
if (compatibility === "static") {
|
|
1610
|
-
dataflow.setPortData(results);
|
|
1611
|
-
await dataflow.applyTransforms(registry);
|
|
1612
|
-
} else if (compatibility === "runtime") {
|
|
1613
|
-
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
1614
|
-
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
1615
|
-
dataflow.setPortData(narrowed);
|
|
1616
|
-
await dataflow.applyTransforms(registry);
|
|
1617
|
-
} else {
|
|
1618
|
-
const resultsKeys = Object.keys(results);
|
|
1619
|
-
if (resultsKeys.length > 0) {
|
|
1620
|
-
getLogger().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
1621
|
-
dataflowId: dataflow.id,
|
|
1622
|
-
compatibility,
|
|
1623
|
-
resultsKeys
|
|
1624
|
-
});
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
pushErrorFromNodeToEdges(node) {
|
|
1630
|
-
if (!node?.config?.id)
|
|
1631
|
-
return;
|
|
1632
|
-
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1633
|
-
dataflow.error = node.error;
|
|
1634
|
-
});
|
|
1635
|
-
}
|
|
1636
|
-
hasErrorOutputEdges(task) {
|
|
1637
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1638
|
-
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
1639
|
-
}
|
|
1640
|
-
pushErrorOutputToEdges(task) {
|
|
1641
|
-
const taskError = task.error;
|
|
1642
|
-
const errorData = {
|
|
1643
|
-
error: taskError?.message ?? "Unknown error",
|
|
1644
|
-
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
1645
|
-
};
|
|
1646
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1647
|
-
for (const df of dataflows) {
|
|
1648
|
-
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
1649
|
-
df.value = errorData;
|
|
1650
|
-
df.setStatus(TaskStatus.COMPLETED);
|
|
1651
|
-
} else {
|
|
1652
|
-
df.setStatus(TaskStatus.DISABLED);
|
|
1653
|
-
}
|
|
1654
|
-
}
|
|
1655
|
-
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
1656
|
-
}
|
|
1657
|
-
resetTask(graph, task, runId) {
|
|
1658
|
-
task.status = TaskStatus.PENDING;
|
|
1659
|
-
task.resetInputData();
|
|
1660
|
-
task.runOutputData = {};
|
|
1661
|
-
task.error = undefined;
|
|
1662
|
-
task.progress = 0;
|
|
1663
|
-
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
1664
|
-
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
1665
|
-
if (task?.config?.id) {
|
|
1666
|
-
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1667
|
-
dataflow.error = task.error;
|
|
1668
|
-
});
|
|
1669
|
-
}
|
|
1670
|
-
task.emit("reset");
|
|
1671
|
-
task.emit("status", task.status);
|
|
1672
|
-
}
|
|
1673
|
-
static isImageValueShape(v) {
|
|
1674
|
-
if (v === null || typeof v !== "object")
|
|
1675
|
-
return false;
|
|
1676
|
-
const o = v;
|
|
1677
|
-
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
|
|
1681
|
-
// src/task-graph/RunContext.ts
|
|
1682
|
-
import { uuid4 as uuid42 } from "@workglow/util";
|
|
1683
|
-
|
|
1684
|
-
class RunContext {
|
|
1685
|
-
runId;
|
|
1686
|
-
abortController;
|
|
1687
|
-
inProgressTasks = new Map;
|
|
1688
|
-
inProgressFunctions = new Map;
|
|
1689
|
-
failedTaskErrors = new Map;
|
|
1690
|
-
telemetrySpan;
|
|
1691
|
-
graphTimeoutTimer;
|
|
1692
|
-
pendingGraphTimeoutError;
|
|
1693
|
-
activeEnforcer;
|
|
1694
|
-
parentSignalCleanup;
|
|
1695
|
-
constructor(parentSignal) {
|
|
1696
|
-
this.runId = uuid42();
|
|
1697
|
-
this.abortController = new AbortController;
|
|
1698
|
-
if (parentSignal) {
|
|
1699
|
-
const onParentAbort = () => this.abortController.abort();
|
|
1700
|
-
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
1701
|
-
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
1702
|
-
if (parentSignal.aborted) {
|
|
1703
|
-
this.parentSignalCleanup();
|
|
1704
|
-
this.parentSignalCleanup = undefined;
|
|
1705
|
-
this.abortController.abort();
|
|
1706
|
-
}
|
|
1707
|
-
}
|
|
1708
|
-
}
|
|
1709
|
-
dispose() {
|
|
1710
|
-
this.parentSignalCleanup?.();
|
|
1711
|
-
this.parentSignalCleanup = undefined;
|
|
1712
|
-
}
|
|
1713
|
-
}
|
|
1714
|
-
|
|
1715
|
-
// src/task-graph/RunScheduler.ts
|
|
1716
|
-
import { getLogger as getLogger4 } from "@workglow/util";
|
|
1717
|
-
|
|
1718
|
-
// src/task/ConditionalTask.ts
|
|
1719
|
-
import { getLogger as getLogger3 } from "@workglow/util";
|
|
1720
|
-
|
|
1721
|
-
// src/task/ConditionUtils.ts
|
|
1722
|
-
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
1723
|
-
if (fieldValue === null || fieldValue === undefined) {
|
|
1724
|
-
switch (operator) {
|
|
1725
|
-
case "is_empty":
|
|
1726
|
-
return true;
|
|
1727
|
-
case "is_not_empty":
|
|
1728
|
-
return false;
|
|
1729
|
-
case "is_true":
|
|
1730
|
-
return false;
|
|
1731
|
-
case "is_false":
|
|
1732
|
-
return true;
|
|
1733
|
-
default:
|
|
1734
|
-
return false;
|
|
1735
|
-
}
|
|
1736
|
-
}
|
|
1737
|
-
const strValue = String(fieldValue);
|
|
1738
|
-
const numValue = Number(fieldValue);
|
|
1739
|
-
switch (operator) {
|
|
1740
|
-
case "equals":
|
|
1741
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1742
|
-
return numValue === Number(compareValue);
|
|
1743
|
-
}
|
|
1744
|
-
return strValue === compareValue;
|
|
1745
|
-
case "not_equals":
|
|
1746
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1747
|
-
return numValue !== Number(compareValue);
|
|
1748
|
-
}
|
|
1749
|
-
return strValue !== compareValue;
|
|
1750
|
-
case "greater_than":
|
|
1751
|
-
return numValue > Number(compareValue);
|
|
1752
|
-
case "greater_or_equal":
|
|
1753
|
-
return numValue >= Number(compareValue);
|
|
1754
|
-
case "less_than":
|
|
1755
|
-
return numValue < Number(compareValue);
|
|
1756
|
-
case "less_or_equal":
|
|
1757
|
-
return numValue <= Number(compareValue);
|
|
1758
|
-
case "contains":
|
|
1759
|
-
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
1760
|
-
case "starts_with":
|
|
1761
|
-
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
1762
|
-
case "ends_with":
|
|
1763
|
-
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
1764
|
-
case "is_empty":
|
|
1765
|
-
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
1766
|
-
case "is_not_empty":
|
|
1767
|
-
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
1768
|
-
case "is_true":
|
|
1769
|
-
return Boolean(fieldValue) === true;
|
|
1770
|
-
case "is_false":
|
|
1771
|
-
return Boolean(fieldValue) === false;
|
|
1772
|
-
default:
|
|
1773
|
-
return false;
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
function getNestedValue(obj, path) {
|
|
1777
|
-
const parts = path.split(".");
|
|
1778
|
-
let current = obj;
|
|
1779
|
-
for (const part of parts) {
|
|
1780
|
-
if (current === null || current === undefined || typeof current !== "object") {
|
|
1781
|
-
return;
|
|
1782
|
-
}
|
|
1783
|
-
current = current[part];
|
|
1784
|
-
}
|
|
1785
|
-
return current;
|
|
1786
|
-
}
|
|
1787
|
-
|
|
1788
1632
|
// src/task/Task.ts
|
|
1789
|
-
import { deepEqual, EventEmitter as EventEmitter3, uuid4 as
|
|
1633
|
+
import { deepEqual, EventEmitter as EventEmitter3, getLogger as getLogger2, uuid4 as uuid42 } from "@workglow/util";
|
|
1790
1634
|
import { compileSchema } from "@workglow/util/schema";
|
|
1791
1635
|
|
|
1792
1636
|
// src/task/TaskRunner.ts
|
|
1793
1637
|
import {
|
|
1794
|
-
getLogger
|
|
1638
|
+
getLogger,
|
|
1795
1639
|
getTelemetryProvider,
|
|
1796
1640
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1797
1641
|
ResourceScope,
|
|
@@ -1800,7 +1644,6 @@ import {
|
|
|
1800
1644
|
|
|
1801
1645
|
// src/task/CacheCoordinator.ts
|
|
1802
1646
|
import { getPortCodec } from "@workglow/util";
|
|
1803
|
-
|
|
1804
1647
|
class CacheCoordinator {
|
|
1805
1648
|
task;
|
|
1806
1649
|
constructor(task) {
|
|
@@ -1810,7 +1653,9 @@ class CacheCoordinator {
|
|
|
1810
1653
|
if (!outputCache)
|
|
1811
1654
|
return inputs;
|
|
1812
1655
|
const inputSchema = this.task.constructor.inputSchema();
|
|
1813
|
-
|
|
1656
|
+
const normalized = await CacheCoordinator.normalizeInputsForCacheKey(inputs, inputSchema);
|
|
1657
|
+
normalized.__cv = this.task.getCacheVersion();
|
|
1658
|
+
return normalized;
|
|
1814
1659
|
}
|
|
1815
1660
|
async lookup(keyInputs, outputCache, isStreamable, ctx) {
|
|
1816
1661
|
if (!outputCache || !this.task.cacheable)
|
|
@@ -1838,6 +1683,20 @@ class CacheCoordinator {
|
|
|
1838
1683
|
const wireOutputs = await CacheCoordinator.serializeOutputPorts(output, outputSchema);
|
|
1839
1684
|
await outputCache.saveOutput(this.task.type, keyInputs, wireOutputs);
|
|
1840
1685
|
}
|
|
1686
|
+
repoFor(registry, policy) {
|
|
1687
|
+
if (!registry || !isPolicyCached(policy))
|
|
1688
|
+
return;
|
|
1689
|
+
return isPolicyPrivate(policy) ? registry.private : registry.deterministic;
|
|
1690
|
+
}
|
|
1691
|
+
async buildKeyForPolicy(inputs, registry, policy) {
|
|
1692
|
+
return this.buildKey(inputs, this.repoFor(registry, policy));
|
|
1693
|
+
}
|
|
1694
|
+
async lookupByPolicy(keyInputs, registry, policy, isStreamable, ctx) {
|
|
1695
|
+
return this.lookup(keyInputs, this.repoFor(registry, policy), isStreamable, ctx);
|
|
1696
|
+
}
|
|
1697
|
+
async saveByPolicy(keyInputs, output, registry, policy) {
|
|
1698
|
+
return this.save(keyInputs, output, this.repoFor(registry, policy));
|
|
1699
|
+
}
|
|
1841
1700
|
static async serializeOutputPorts(output, schema) {
|
|
1842
1701
|
if (!schema?.properties)
|
|
1843
1702
|
return output;
|
|
@@ -2073,12 +1932,15 @@ class TaskRunner {
|
|
|
2073
1932
|
task;
|
|
2074
1933
|
currentCtx;
|
|
2075
1934
|
outputCache;
|
|
1935
|
+
cacheRegistry;
|
|
2076
1936
|
cacheCoordinator;
|
|
2077
1937
|
streamProcessor;
|
|
2078
1938
|
registry = globalServiceRegistry2;
|
|
2079
1939
|
resourceScope;
|
|
2080
1940
|
ownsResourceScope = false;
|
|
2081
1941
|
inputStreams;
|
|
1942
|
+
runId;
|
|
1943
|
+
static __privateWithoutRunIdWarned = new Set;
|
|
2082
1944
|
constructor(task) {
|
|
2083
1945
|
this.task = task;
|
|
2084
1946
|
this.own = this.own.bind(this);
|
|
@@ -2091,7 +1953,7 @@ class TaskRunner {
|
|
|
2091
1953
|
throw new TaskConfigurationError(`Task "${this.task.type}" is already running. Concurrent run() calls on the same TaskRunner are not supported.`);
|
|
2092
1954
|
}
|
|
2093
1955
|
const ownsScope = config.resourceScope === undefined;
|
|
2094
|
-
const effectiveConfig = ownsScope ? { ...config, resourceScope: new ResourceScope } : config;
|
|
1956
|
+
const effectiveConfig = ownsScope ? { ...config, resourceScope: new ResourceScope({ strategy: config.disposeStrategy }) } : config;
|
|
2095
1957
|
this.ownsResourceScope = ownsScope;
|
|
2096
1958
|
try {
|
|
2097
1959
|
await this.handleStart(effectiveConfig);
|
|
@@ -2116,11 +1978,23 @@ class TaskRunner {
|
|
|
2116
1978
|
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
2117
1979
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
2118
1980
|
if (streamMode !== "none") {
|
|
2119
|
-
|
|
1981
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
2120
1982
|
}
|
|
2121
1983
|
}
|
|
2122
|
-
|
|
2123
|
-
|
|
1984
|
+
let policy = this.task.getCachePolicy(inputs);
|
|
1985
|
+
if (policy.kind === "private" && !this.runId && this.cacheRegistry?.private !== undefined && !(this.cacheRegistry.private instanceof RunPrivateCacheRepo)) {
|
|
1986
|
+
const taskType = this.task.type;
|
|
1987
|
+
if (!TaskRunner.__privateWithoutRunIdWarned.has(taskType)) {
|
|
1988
|
+
TaskRunner.__privateWithoutRunIdWarned.add(taskType);
|
|
1989
|
+
getLogger().warn(`TaskRunner: task "${taskType}" has a private cache policy but no runId was ` + `provided. Private cache writes are skipped for this run \u2014 use TaskGraphRunner ` + `with runId for run-namespaced private caching, or provide an already namespaced ` + `private cache repo in CACHE_REGISTRY.`);
|
|
1990
|
+
}
|
|
1991
|
+
policy = { kind: "none" };
|
|
1992
|
+
}
|
|
1993
|
+
this.currentCtx?.telemetrySpan?.setAttributes({
|
|
1994
|
+
"workglow.task.cache_policy": policy.kind
|
|
1995
|
+
});
|
|
1996
|
+
const keyInputs = await this.cacheCoordinator.buildKeyForPolicy(inputs, this.cacheRegistry, policy);
|
|
1997
|
+
let outputs = await this.cacheCoordinator.lookupByPolicy(keyInputs, this.cacheRegistry, policy, isStreamable, ctx);
|
|
2124
1998
|
if (outputs === undefined) {
|
|
2125
1999
|
outputs = isStreamable ? await this.streamProcessor.run(inputs, ctx, {
|
|
2126
2000
|
registry: this.registry,
|
|
@@ -2129,7 +2003,7 @@ class TaskRunner {
|
|
|
2129
2003
|
onProgress: this.handleProgress.bind(this),
|
|
2130
2004
|
own: this.own
|
|
2131
2005
|
}) : await this.executeTask(inputs, ctx);
|
|
2132
|
-
await this.cacheCoordinator.
|
|
2006
|
+
await this.cacheCoordinator.saveByPolicy(keyInputs, outputs, this.cacheRegistry, policy);
|
|
2133
2007
|
this.task.runOutputData = outputs ?? {};
|
|
2134
2008
|
}
|
|
2135
2009
|
await this.handleComplete();
|
|
@@ -2140,7 +2014,7 @@ class TaskRunner {
|
|
|
2140
2014
|
}
|
|
2141
2015
|
} finally {
|
|
2142
2016
|
if (ownsScope) {
|
|
2143
|
-
await effectiveConfig.resourceScope.
|
|
2017
|
+
await effectiveConfig.resourceScope.runComplete();
|
|
2144
2018
|
this.resourceScope = undefined;
|
|
2145
2019
|
}
|
|
2146
2020
|
this.ownsResourceScope = false;
|
|
@@ -2166,7 +2040,7 @@ class TaskRunner {
|
|
|
2166
2040
|
}
|
|
2167
2041
|
await this.handleCompletePreview();
|
|
2168
2042
|
} catch (err) {
|
|
2169
|
-
|
|
2043
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
2170
2044
|
await this.handleErrorPreview();
|
|
2171
2045
|
} finally {
|
|
2172
2046
|
ctx.dispose();
|
|
@@ -2251,7 +2125,7 @@ class TaskRunner {
|
|
|
2251
2125
|
const out = await this.runPreview(overrides);
|
|
2252
2126
|
yield out;
|
|
2253
2127
|
} catch (err) {
|
|
2254
|
-
|
|
2128
|
+
getLogger().debug("runPreviewStream iteration failed", {
|
|
2255
2129
|
taskId: this.task.config?.id,
|
|
2256
2130
|
error: err
|
|
2257
2131
|
});
|
|
@@ -2294,7 +2168,8 @@ class TaskRunner {
|
|
|
2294
2168
|
updateProgress: this.handleProgress.bind(this),
|
|
2295
2169
|
own: this.own,
|
|
2296
2170
|
registry: this.registry,
|
|
2297
|
-
resourceScope: this.resourceScope
|
|
2171
|
+
resourceScope: this.resourceScope,
|
|
2172
|
+
runId: this.runId
|
|
2298
2173
|
});
|
|
2299
2174
|
return result;
|
|
2300
2175
|
}
|
|
@@ -2308,7 +2183,8 @@ class TaskRunner {
|
|
|
2308
2183
|
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
2309
2184
|
Object.assign(this.task.config, resolved);
|
|
2310
2185
|
}
|
|
2311
|
-
const
|
|
2186
|
+
const ctor = this.task.constructor;
|
|
2187
|
+
const schema = ctor.hasDynamicSchemas ? this.task.inputSchema() : ctor.inputSchema();
|
|
2312
2188
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
2313
2189
|
}
|
|
2314
2190
|
async handleStart(config = {}) {
|
|
@@ -2321,25 +2197,35 @@ class TaskRunner {
|
|
|
2321
2197
|
ctx.abortController.signal.addEventListener("abort", () => {
|
|
2322
2198
|
this.handleAbort();
|
|
2323
2199
|
});
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2200
|
+
if (config.registry) {
|
|
2201
|
+
this.registry = config.registry;
|
|
2202
|
+
}
|
|
2203
|
+
this.runId = config.runId;
|
|
2204
|
+
const legacy = config.outputCache ?? this.task.runConfig?.outputCache;
|
|
2205
|
+
if (legacy === false) {
|
|
2206
|
+
this.cacheRegistry = undefined;
|
|
2207
|
+
this.outputCache = undefined;
|
|
2208
|
+
} else if (legacy instanceof TaskOutputRepository) {
|
|
2209
|
+
this.cacheRegistry = new DefaultCacheRegistry({ deterministic: legacy });
|
|
2210
|
+
this.outputCache = legacy;
|
|
2211
|
+
} else if (legacy === true) {
|
|
2212
|
+
const instance = globalServiceRegistry2.has(TASK_OUTPUT_REPOSITORY) ? globalServiceRegistry2.get(TASK_OUTPUT_REPOSITORY) : undefined;
|
|
2327
2213
|
this.outputCache = instance;
|
|
2328
|
-
|
|
2214
|
+
this.cacheRegistry = instance ? new DefaultCacheRegistry({ deterministic: instance }) : undefined;
|
|
2215
|
+
} else {
|
|
2329
2216
|
this.outputCache = undefined;
|
|
2330
|
-
|
|
2331
|
-
this.outputCache = cache;
|
|
2217
|
+
this.cacheRegistry = this.registry.has(CACHE_REGISTRY) ? this.registry.get(CACHE_REGISTRY) : undefined;
|
|
2332
2218
|
}
|
|
2333
2219
|
ctx.shouldAccumulate = config.shouldAccumulate !== false;
|
|
2334
2220
|
if (config.updateProgress) {
|
|
2335
2221
|
this.updateProgress = config.updateProgress;
|
|
2336
2222
|
}
|
|
2337
|
-
if (config.registry) {
|
|
2338
|
-
this.registry = config.registry;
|
|
2339
|
-
}
|
|
2340
2223
|
if (config.resourceScope) {
|
|
2341
2224
|
this.resourceScope = config.resourceScope;
|
|
2342
2225
|
}
|
|
2226
|
+
if (this.resourceScope) {
|
|
2227
|
+
await this.resourceScope.runStart();
|
|
2228
|
+
}
|
|
2343
2229
|
if (ctx.abortController.signal.aborted)
|
|
2344
2230
|
return;
|
|
2345
2231
|
const timeout = this.task.config.timeout;
|
|
@@ -2355,7 +2241,6 @@ class TaskRunner {
|
|
|
2355
2241
|
attributes: {
|
|
2356
2242
|
"workglow.task.type": this.task.type,
|
|
2357
2243
|
"workglow.task.id": String(this.task.config.id),
|
|
2358
|
-
"workglow.task.cacheable": this.task.cacheable,
|
|
2359
2244
|
"workglow.task.title": this.task.title || undefined
|
|
2360
2245
|
}
|
|
2361
2246
|
});
|
|
@@ -2486,12 +2371,15 @@ class Task {
|
|
|
2486
2371
|
static title = "";
|
|
2487
2372
|
static description = "";
|
|
2488
2373
|
static cacheable = true;
|
|
2374
|
+
static version = 1;
|
|
2375
|
+
static cachePolicy = { kind: "deterministic" };
|
|
2489
2376
|
static hasDynamicSchemas = false;
|
|
2490
2377
|
static passthroughInputsToOutputs = false;
|
|
2491
2378
|
static customizable = false;
|
|
2492
2379
|
static isGraphOutput = false;
|
|
2493
2380
|
static isPassthrough = false;
|
|
2494
2381
|
static hasDynamicEntitlements = false;
|
|
2382
|
+
static __cacheableDeprecationWarned = new Set;
|
|
2495
2383
|
static entitlements() {
|
|
2496
2384
|
return EMPTY_ENTITLEMENTS;
|
|
2497
2385
|
}
|
|
@@ -2570,7 +2458,39 @@ class Task {
|
|
|
2570
2458
|
return this.config?.description ?? this.constructor.description;
|
|
2571
2459
|
}
|
|
2572
2460
|
get cacheable() {
|
|
2573
|
-
|
|
2461
|
+
if (this.runConfig?.cacheable !== undefined)
|
|
2462
|
+
return this.runConfig.cacheable;
|
|
2463
|
+
if (this.config?.cacheable !== undefined)
|
|
2464
|
+
return this.config.cacheable;
|
|
2465
|
+
return this.getCachePolicy(this.runInputData ?? {}).kind !== "none";
|
|
2466
|
+
}
|
|
2467
|
+
getCacheVersion() {
|
|
2468
|
+
const versions = [];
|
|
2469
|
+
let ctor = this.constructor;
|
|
2470
|
+
while (ctor && ctor !== Function.prototype) {
|
|
2471
|
+
if (Object.prototype.hasOwnProperty.call(ctor, "version") && typeof ctor.version === "number") {
|
|
2472
|
+
versions.push(ctor.version);
|
|
2473
|
+
}
|
|
2474
|
+
ctor = Object.getPrototypeOf(ctor);
|
|
2475
|
+
}
|
|
2476
|
+
if (versions.length === 0)
|
|
2477
|
+
versions.push(1);
|
|
2478
|
+
return versions.join(".");
|
|
2479
|
+
}
|
|
2480
|
+
getCachePolicy(_inputs) {
|
|
2481
|
+
const ctor = this.constructor;
|
|
2482
|
+
const hasLegacyOverride = Object.prototype.hasOwnProperty.call(ctor, "cacheable") && ctor.cacheable === false;
|
|
2483
|
+
const hasPolicyOverride = Object.prototype.hasOwnProperty.call(ctor, "cachePolicy");
|
|
2484
|
+
if (hasLegacyOverride && !hasPolicyOverride) {
|
|
2485
|
+
if (!Task.__cacheableDeprecationWarned.has(ctor.type)) {
|
|
2486
|
+
Task.__cacheableDeprecationWarned.add(ctor.type);
|
|
2487
|
+
getLogger2().warn(`Task "${ctor.type}": static \`cacheable = false\` is deprecated. ` + `Use \`static cachePolicy: CachePolicy = { kind: "none" }\` instead.`);
|
|
2488
|
+
}
|
|
2489
|
+
return { kind: "none" };
|
|
2490
|
+
}
|
|
2491
|
+
if (this.runConfig?.cacheable === false || this.config?.cacheable === false)
|
|
2492
|
+
return { kind: "none" };
|
|
2493
|
+
return ctor.cachePolicy ?? DEFAULT_CACHE_POLICY;
|
|
2574
2494
|
}
|
|
2575
2495
|
defaults;
|
|
2576
2496
|
runInputData = {};
|
|
@@ -2605,7 +2525,7 @@ class Task {
|
|
|
2605
2525
|
...title ? { title } : {}
|
|
2606
2526
|
}, restConfig);
|
|
2607
2527
|
if (baseConfig.id === undefined) {
|
|
2608
|
-
baseConfig.id =
|
|
2528
|
+
baseConfig.id = uuid42();
|
|
2609
2529
|
}
|
|
2610
2530
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
2611
2531
|
try {
|
|
@@ -2614,6 +2534,10 @@ class Task {
|
|
|
2614
2534
|
this.originalConfig = undefined;
|
|
2615
2535
|
}
|
|
2616
2536
|
this.runConfig = runConfig;
|
|
2537
|
+
const inputSchema = this.constructor.inputSchema();
|
|
2538
|
+
if (inputSchema && typeof inputSchema !== "boolean" && inputSchema.properties && Object.prototype.hasOwnProperty.call(inputSchema.properties, "__cv")) {
|
|
2539
|
+
throw new TaskConfigurationError(`Task "${this.constructor.type}": input port name '__cv' is reserved ` + `for cache versioning. Rename the port to avoid collision with the cache key sentinel.`);
|
|
2540
|
+
}
|
|
2617
2541
|
}
|
|
2618
2542
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
2619
2543
|
const schema = this.inputSchema();
|
|
@@ -2954,48 +2878,277 @@ class Task {
|
|
|
2954
2878
|
if (Object.keys(config).length > 0) {
|
|
2955
2879
|
base.config = config;
|
|
2956
2880
|
}
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2881
|
+
return this.stripSymbols(base);
|
|
2882
|
+
}
|
|
2883
|
+
toDependencyJSON(options) {
|
|
2884
|
+
const json = this.toJSON(options);
|
|
2885
|
+
return json;
|
|
2886
|
+
}
|
|
2887
|
+
hasChildren() {
|
|
2888
|
+
return this._subGraph !== undefined && this._subGraph !== null && this._subGraph.getTasks().length > 0;
|
|
2889
|
+
}
|
|
2890
|
+
_taskAddedListener = () => {
|
|
2891
|
+
this.emit("regenerate");
|
|
2892
|
+
};
|
|
2893
|
+
_subGraph = undefined;
|
|
2894
|
+
set subGraph(subGraph) {
|
|
2895
|
+
if (this._subGraph) {
|
|
2896
|
+
this._subGraph.off("task_added", this._taskAddedListener);
|
|
2897
|
+
}
|
|
2898
|
+
this._subGraph = subGraph;
|
|
2899
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2900
|
+
}
|
|
2901
|
+
get subGraph() {
|
|
2902
|
+
if (!this._subGraph) {
|
|
2903
|
+
this._subGraph = new TaskGraph;
|
|
2904
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2905
|
+
}
|
|
2906
|
+
return this._subGraph;
|
|
2907
|
+
}
|
|
2908
|
+
regenerateGraph() {
|
|
2909
|
+
if (this.hasChildren()) {
|
|
2910
|
+
for (const dataflow of this.subGraph.getDataflows()) {
|
|
2911
|
+
this.subGraph.removeDataflow(dataflow);
|
|
2912
|
+
}
|
|
2913
|
+
for (const child of this.subGraph.getTasks()) {
|
|
2914
|
+
this.subGraph.removeTask(child.id);
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
this.events.emit("regenerate");
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
// src/task-graph/EdgeMaterializer.ts
|
|
2922
|
+
import { getLogger as getLogger3 } from "@workglow/util";
|
|
2923
|
+
import { previewSource } from "@workglow/util/media";
|
|
2924
|
+
class EdgeMaterializer {
|
|
2925
|
+
graph;
|
|
2926
|
+
runner;
|
|
2927
|
+
constructor(graph, runner) {
|
|
2928
|
+
this.graph = graph;
|
|
2929
|
+
this.runner = runner;
|
|
2930
|
+
}
|
|
2931
|
+
filterInputForTask(task, input) {
|
|
2932
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
2933
|
+
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
2934
|
+
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
2935
|
+
const filteredInput = {};
|
|
2936
|
+
for (const [key, value] of Object.entries(input)) {
|
|
2937
|
+
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
2938
|
+
filteredInput[key] = value;
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
return filteredInput;
|
|
2942
|
+
}
|
|
2943
|
+
copyInputFromEdgesToNode(task) {
|
|
2944
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2945
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2946
|
+
for (const dataflow of dataflows) {
|
|
2947
|
+
const live = dataflow.getCurrentValue();
|
|
2948
|
+
const port = dataflow.targetTaskPortId;
|
|
2949
|
+
let portData;
|
|
2950
|
+
if (port === DATAFLOW_ALL_PORTS) {
|
|
2951
|
+
portData = live;
|
|
2952
|
+
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
2953
|
+
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
2954
|
+
} else {
|
|
2955
|
+
portData = { [port]: live };
|
|
2956
|
+
}
|
|
2957
|
+
this.runner.addInputData(task, portData);
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
async pushOutputFromNodeToEdges(node, results) {
|
|
2961
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2962
|
+
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
2963
|
+
for (const port of Object.keys(results)) {
|
|
2964
|
+
const value = results[port];
|
|
2965
|
+
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
2966
|
+
results[port] = await previewSource(value);
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
for (const dataflow of dataflows) {
|
|
2971
|
+
if (dataflow.stream !== undefined)
|
|
2972
|
+
continue;
|
|
2973
|
+
const registry = this.runner["registry"];
|
|
2974
|
+
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
2975
|
+
if (compatibility === "static") {
|
|
2976
|
+
dataflow.setPortData(results);
|
|
2977
|
+
await dataflow.applyTransforms(registry);
|
|
2978
|
+
} else if (compatibility === "runtime") {
|
|
2979
|
+
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
2980
|
+
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
2981
|
+
dataflow.setPortData(narrowed);
|
|
2982
|
+
await dataflow.applyTransforms(registry);
|
|
2983
|
+
} else {
|
|
2984
|
+
const resultsKeys = Object.keys(results);
|
|
2985
|
+
if (resultsKeys.length > 0) {
|
|
2986
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2987
|
+
dataflowId: dataflow.id,
|
|
2988
|
+
compatibility,
|
|
2989
|
+
resultsKeys
|
|
2990
|
+
});
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
}
|
|
2994
|
+
}
|
|
2995
|
+
pushErrorFromNodeToEdges(node) {
|
|
2996
|
+
if (!node?.config?.id)
|
|
2997
|
+
return;
|
|
2998
|
+
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
2999
|
+
dataflow.error = node.error;
|
|
3000
|
+
});
|
|
3001
|
+
}
|
|
3002
|
+
hasErrorOutputEdges(task) {
|
|
3003
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3004
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
3005
|
+
}
|
|
3006
|
+
pushErrorOutputToEdges(task) {
|
|
3007
|
+
const taskError = task.error;
|
|
3008
|
+
const errorData = {
|
|
3009
|
+
error: taskError?.message ?? "Unknown error",
|
|
3010
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
3011
|
+
};
|
|
3012
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3013
|
+
for (const df of dataflows) {
|
|
3014
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
3015
|
+
df.value = errorData;
|
|
3016
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
3017
|
+
} else {
|
|
3018
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
3019
|
+
}
|
|
3020
|
+
}
|
|
3021
|
+
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
3022
|
+
}
|
|
3023
|
+
resetTask(graph, task, runId) {
|
|
3024
|
+
task.status = TaskStatus.PENDING;
|
|
3025
|
+
task.resetInputData();
|
|
3026
|
+
task.runOutputData = {};
|
|
3027
|
+
task.error = undefined;
|
|
3028
|
+
task.progress = 0;
|
|
3029
|
+
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
3030
|
+
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
3031
|
+
if (task?.config?.id) {
|
|
3032
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
3033
|
+
dataflow.error = task.error;
|
|
3034
|
+
});
|
|
2960
3035
|
}
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
toDependencyJSON(options) {
|
|
2964
|
-
const json = this.toJSON(options);
|
|
2965
|
-
return json;
|
|
3036
|
+
task.emit("reset");
|
|
3037
|
+
task.emit("status", task.status);
|
|
2966
3038
|
}
|
|
2967
|
-
|
|
2968
|
-
|
|
3039
|
+
static isImageValueShape(v) {
|
|
3040
|
+
if (v === null || typeof v !== "object")
|
|
3041
|
+
return false;
|
|
3042
|
+
const o = v;
|
|
3043
|
+
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
2969
3044
|
}
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
3045
|
+
}
|
|
3046
|
+
|
|
3047
|
+
// src/task-graph/RunContext.ts
|
|
3048
|
+
import { uuid4 as uuid43 } from "@workglow/util";
|
|
3049
|
+
|
|
3050
|
+
class RunContext {
|
|
3051
|
+
runId;
|
|
3052
|
+
abortController;
|
|
3053
|
+
inProgressTasks = new Map;
|
|
3054
|
+
inProgressFunctions = new Map;
|
|
3055
|
+
failedTaskErrors = new Map;
|
|
3056
|
+
telemetrySpan;
|
|
3057
|
+
graphTimeoutTimer;
|
|
3058
|
+
pendingGraphTimeoutError;
|
|
3059
|
+
activeEnforcer;
|
|
3060
|
+
parentSignalCleanup;
|
|
3061
|
+
constructor(parentSignal) {
|
|
3062
|
+
this.runId = uuid43();
|
|
3063
|
+
this.abortController = new AbortController;
|
|
3064
|
+
if (parentSignal) {
|
|
3065
|
+
const onParentAbort = () => this.abortController.abort();
|
|
3066
|
+
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
3067
|
+
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
3068
|
+
if (parentSignal.aborted) {
|
|
3069
|
+
this.parentSignalCleanup();
|
|
3070
|
+
this.parentSignalCleanup = undefined;
|
|
3071
|
+
this.abortController.abort();
|
|
3072
|
+
}
|
|
2977
3073
|
}
|
|
2978
|
-
this._subGraph = subGraph;
|
|
2979
|
-
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2980
3074
|
}
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
3075
|
+
dispose() {
|
|
3076
|
+
this.parentSignalCleanup?.();
|
|
3077
|
+
this.parentSignalCleanup = undefined;
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
// src/task-graph/RunScheduler.ts
|
|
3082
|
+
import { getLogger as getLogger5 } from "@workglow/util";
|
|
3083
|
+
|
|
3084
|
+
// src/task/ConditionalTask.ts
|
|
3085
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
3086
|
+
|
|
3087
|
+
// src/task/ConditionUtils.ts
|
|
3088
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
3089
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
3090
|
+
switch (operator) {
|
|
3091
|
+
case "is_empty":
|
|
3092
|
+
return true;
|
|
3093
|
+
case "is_not_empty":
|
|
3094
|
+
return false;
|
|
3095
|
+
case "is_true":
|
|
3096
|
+
return false;
|
|
3097
|
+
case "is_false":
|
|
3098
|
+
return true;
|
|
3099
|
+
default:
|
|
3100
|
+
return false;
|
|
2985
3101
|
}
|
|
2986
|
-
return this._subGraph;
|
|
2987
3102
|
}
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
3103
|
+
const strValue = String(fieldValue);
|
|
3104
|
+
const numValue = Number(fieldValue);
|
|
3105
|
+
switch (operator) {
|
|
3106
|
+
case "equals":
|
|
3107
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3108
|
+
return numValue === Number(compareValue);
|
|
2992
3109
|
}
|
|
2993
|
-
|
|
2994
|
-
|
|
3110
|
+
return strValue === compareValue;
|
|
3111
|
+
case "not_equals":
|
|
3112
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3113
|
+
return numValue !== Number(compareValue);
|
|
2995
3114
|
}
|
|
3115
|
+
return strValue !== compareValue;
|
|
3116
|
+
case "greater_than":
|
|
3117
|
+
return numValue > Number(compareValue);
|
|
3118
|
+
case "greater_or_equal":
|
|
3119
|
+
return numValue >= Number(compareValue);
|
|
3120
|
+
case "less_than":
|
|
3121
|
+
return numValue < Number(compareValue);
|
|
3122
|
+
case "less_or_equal":
|
|
3123
|
+
return numValue <= Number(compareValue);
|
|
3124
|
+
case "contains":
|
|
3125
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
3126
|
+
case "starts_with":
|
|
3127
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
3128
|
+
case "ends_with":
|
|
3129
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
3130
|
+
case "is_empty":
|
|
3131
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
3132
|
+
case "is_not_empty":
|
|
3133
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
3134
|
+
case "is_true":
|
|
3135
|
+
return Boolean(fieldValue) === true;
|
|
3136
|
+
case "is_false":
|
|
3137
|
+
return Boolean(fieldValue) === false;
|
|
3138
|
+
default:
|
|
3139
|
+
return false;
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
function getNestedValue(obj, path) {
|
|
3143
|
+
const parts = path.split(".");
|
|
3144
|
+
let current = obj;
|
|
3145
|
+
for (const part of parts) {
|
|
3146
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
3147
|
+
return;
|
|
2996
3148
|
}
|
|
2997
|
-
|
|
3149
|
+
current = current[part];
|
|
2998
3150
|
}
|
|
3151
|
+
return current;
|
|
2999
3152
|
}
|
|
3000
3153
|
|
|
3001
3154
|
// src/task/ConditionalTask.ts
|
|
@@ -3087,7 +3240,7 @@ class ConditionalTask extends Task {
|
|
|
3087
3240
|
}
|
|
3088
3241
|
}
|
|
3089
3242
|
} catch (error) {
|
|
3090
|
-
|
|
3243
|
+
getLogger4().error(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
3091
3244
|
}
|
|
3092
3245
|
}
|
|
3093
3246
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -3359,7 +3512,7 @@ class RunScheduler {
|
|
|
3359
3512
|
ctx.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
3360
3513
|
}
|
|
3361
3514
|
} catch (err) {
|
|
3362
|
-
|
|
3515
|
+
getLogger5().error("Error running graph", { error: err });
|
|
3363
3516
|
}
|
|
3364
3517
|
await Promise.allSettled(Array.from(ctx.inProgressTasks.values()));
|
|
3365
3518
|
await Promise.allSettled(Array.from(ctx.inProgressFunctions.values()));
|
|
@@ -3435,11 +3588,12 @@ class StreamPump {
|
|
|
3435
3588
|
task.on("stream_end", onStreamEnd);
|
|
3436
3589
|
try {
|
|
3437
3590
|
const results = await task.runner.run(input, {
|
|
3438
|
-
outputCache: options.
|
|
3591
|
+
outputCache: options.legacyCacheExplicitlyDisabled ? false : options.outputCache,
|
|
3439
3592
|
shouldAccumulate,
|
|
3440
3593
|
updateProgress: options.updateProgress,
|
|
3441
3594
|
registry: options.registry,
|
|
3442
|
-
resourceScope: options.resourceScope
|
|
3595
|
+
resourceScope: options.resourceScope,
|
|
3596
|
+
runId: options.runId
|
|
3443
3597
|
});
|
|
3444
3598
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3445
3599
|
return {
|
|
@@ -3708,10 +3862,14 @@ class TaskGraphRunner {
|
|
|
3708
3862
|
previewRunning = false;
|
|
3709
3863
|
graph;
|
|
3710
3864
|
outputCache;
|
|
3865
|
+
legacyCacheExplicitlyDisabled = false;
|
|
3711
3866
|
accumulateLeafOutputs = true;
|
|
3712
3867
|
registry = globalServiceRegistry3;
|
|
3713
3868
|
resourceScope;
|
|
3714
3869
|
currentCtx;
|
|
3870
|
+
runId;
|
|
3871
|
+
currentRunPrivate;
|
|
3872
|
+
baseRegistryForRun;
|
|
3715
3873
|
edgeMaterializer;
|
|
3716
3874
|
streamPump;
|
|
3717
3875
|
runScheduler;
|
|
@@ -3728,7 +3886,10 @@ class TaskGraphRunner {
|
|
|
3728
3886
|
}
|
|
3729
3887
|
async runGraph(input = {}, config) {
|
|
3730
3888
|
const ownsScope = config?.resourceScope === undefined;
|
|
3731
|
-
const effectiveConfig = ownsScope ? {
|
|
3889
|
+
const effectiveConfig = ownsScope ? {
|
|
3890
|
+
...config,
|
|
3891
|
+
resourceScope: new ResourceScope2({ strategy: config?.disposeStrategy })
|
|
3892
|
+
} : config;
|
|
3732
3893
|
try {
|
|
3733
3894
|
await this.handleStart(effectiveConfig);
|
|
3734
3895
|
const ctx = this.currentCtx;
|
|
@@ -3748,10 +3909,19 @@ class TaskGraphRunner {
|
|
|
3748
3909
|
throw new TaskAbortedError;
|
|
3749
3910
|
}
|
|
3750
3911
|
await this.handleComplete();
|
|
3912
|
+
const runPrivateToClean = this.currentRunPrivate;
|
|
3913
|
+
this.currentRunPrivate = undefined;
|
|
3914
|
+
if (runPrivateToClean) {
|
|
3915
|
+
try {
|
|
3916
|
+
await runPrivateToClean.clearRun();
|
|
3917
|
+
} catch (e) {
|
|
3918
|
+
getLogger6().warn("RunPrivateCacheRepo.clearRun failed", { error: e });
|
|
3919
|
+
}
|
|
3920
|
+
}
|
|
3751
3921
|
return this.filterLeafResults(results);
|
|
3752
3922
|
} finally {
|
|
3753
3923
|
if (ownsScope) {
|
|
3754
|
-
await effectiveConfig.resourceScope.
|
|
3924
|
+
await effectiveConfig.resourceScope.runComplete();
|
|
3755
3925
|
this.resourceScope = undefined;
|
|
3756
3926
|
}
|
|
3757
3927
|
}
|
|
@@ -3819,7 +3989,7 @@ class TaskGraphRunner {
|
|
|
3819
3989
|
});
|
|
3820
3990
|
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3821
3991
|
previewSpan.end();
|
|
3822
|
-
|
|
3992
|
+
getLogger6().debug("task graph runPreview timings", {
|
|
3823
3993
|
previewRunId,
|
|
3824
3994
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3825
3995
|
taskTimings
|
|
@@ -3837,7 +4007,7 @@ class TaskGraphRunner {
|
|
|
3837
4007
|
});
|
|
3838
4008
|
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3839
4009
|
previewSpan.end();
|
|
3840
|
-
|
|
4010
|
+
getLogger6().debug("task graph runPreview failed", {
|
|
3841
4011
|
previewRunId,
|
|
3842
4012
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3843
4013
|
taskTimings,
|
|
@@ -3908,14 +4078,17 @@ class TaskGraphRunner {
|
|
|
3908
4078
|
outputCache: this.outputCache,
|
|
3909
4079
|
resourceScope: this.resourceScope,
|
|
3910
4080
|
accumulateLeafOutputs: this.accumulateLeafOutputs,
|
|
3911
|
-
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a)
|
|
4081
|
+
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a),
|
|
4082
|
+
runId: this.runId,
|
|
4083
|
+
legacyCacheExplicitlyDisabled: this.legacyCacheExplicitlyDisabled
|
|
3912
4084
|
});
|
|
3913
4085
|
}
|
|
3914
4086
|
const results = await task.runner.run(input, {
|
|
3915
|
-
outputCache: this.
|
|
4087
|
+
outputCache: this.legacyCacheExplicitlyDisabled ? false : this.outputCache,
|
|
3916
4088
|
updateProgress: async (task2, progress, message, ...args) => await this.runScheduler.handleProgress(this.currentCtx, task2, progress, message, ...args),
|
|
3917
4089
|
registry: this.registry,
|
|
3918
|
-
resourceScope: this.resourceScope
|
|
4090
|
+
resourceScope: this.resourceScope,
|
|
4091
|
+
runId: this.runId
|
|
3919
4092
|
});
|
|
3920
4093
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3921
4094
|
return {
|
|
@@ -3936,6 +4109,19 @@ class TaskGraphRunner {
|
|
|
3936
4109
|
dataflow.reset();
|
|
3937
4110
|
});
|
|
3938
4111
|
}
|
|
4112
|
+
static __durabilityWarnedRepos = new WeakSet;
|
|
4113
|
+
static graphUsesPrivatePolicy(graph) {
|
|
4114
|
+
return graph.getTasks().some((t) => {
|
|
4115
|
+
const ctor = t.constructor;
|
|
4116
|
+
if (ctor.cachePolicy?.kind === "private")
|
|
4117
|
+
return true;
|
|
4118
|
+
const proto = ctor.prototype;
|
|
4119
|
+
if (typeof proto.getCachePolicy === "function" && proto.getCachePolicy !== Task.prototype.getCachePolicy) {
|
|
4120
|
+
return true;
|
|
4121
|
+
}
|
|
4122
|
+
return false;
|
|
4123
|
+
});
|
|
4124
|
+
}
|
|
3939
4125
|
async handleStart(config) {
|
|
3940
4126
|
if (config?.registry !== undefined) {
|
|
3941
4127
|
this.registry = config.registry;
|
|
@@ -3945,18 +4131,61 @@ class TaskGraphRunner {
|
|
|
3945
4131
|
if (config?.resourceScope !== undefined) {
|
|
3946
4132
|
this.resourceScope = config.resourceScope;
|
|
3947
4133
|
}
|
|
4134
|
+
this.baseRegistryForRun = this.registry;
|
|
4135
|
+
this.runId = config?.runId;
|
|
4136
|
+
if (this.registry.has(CACHE_REGISTRY)) {
|
|
4137
|
+
const checkRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4138
|
+
if (checkRegistry.private && !checkRegistry.private.isDurable()) {
|
|
4139
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4140
|
+
const repo = checkRegistry.private;
|
|
4141
|
+
if (!TaskGraphRunner.__durabilityWarnedRepos.has(repo)) {
|
|
4142
|
+
TaskGraphRunner.__durabilityWarnedRepos.add(repo);
|
|
4143
|
+
getLogger6().warn("TaskGraphRunner: private cache repo may be used but is non-durable \u2014 " + "restart-survival will not work. Ensure the CacheRegistry 'private' " + "slot is backed by a durable storage backend.");
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
}
|
|
4147
|
+
if (!this.runId && checkRegistry.private) {
|
|
4148
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4149
|
+
throw new TaskConfigurationError("TaskGraphRunner: graph contains a private-policy task but no runId was provided. " + "Provide `runId` in TaskGraphRunConfig so private cache entries can be namespaced.");
|
|
4150
|
+
}
|
|
4151
|
+
}
|
|
4152
|
+
}
|
|
4153
|
+
if (this.runId && this.registry.has(CACHE_REGISTRY)) {
|
|
4154
|
+
const baseRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4155
|
+
if (baseRegistry.private) {
|
|
4156
|
+
const wrappedPrivate = new RunPrivateCacheRepo({
|
|
4157
|
+
backing: baseRegistry.private,
|
|
4158
|
+
runId: this.runId
|
|
4159
|
+
});
|
|
4160
|
+
this.currentRunPrivate = wrappedPrivate;
|
|
4161
|
+
const wrappedRegistry = new DefaultCacheRegistry({
|
|
4162
|
+
deterministic: baseRegistry.deterministic,
|
|
4163
|
+
private: wrappedPrivate
|
|
4164
|
+
});
|
|
4165
|
+
const childContainer = this.registry.container.createChildContainer();
|
|
4166
|
+
const runtimeServices = new ServiceRegistry3(childContainer);
|
|
4167
|
+
runtimeServices.registerInstance(CACHE_REGISTRY, wrappedRegistry);
|
|
4168
|
+
this.registry = runtimeServices;
|
|
4169
|
+
this.resourceScope?.register(`cache:private:${this.runId}`, async () => {});
|
|
4170
|
+
}
|
|
4171
|
+
}
|
|
3948
4172
|
this.accumulateLeafOutputs = config?.accumulateLeafOutputs !== false;
|
|
3949
4173
|
if (config?.outputCache !== undefined) {
|
|
3950
4174
|
if (typeof config.outputCache === "boolean") {
|
|
3951
4175
|
if (config.outputCache === true) {
|
|
3952
4176
|
this.outputCache = this.registry.get(TASK_OUTPUT_REPOSITORY);
|
|
4177
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3953
4178
|
} else {
|
|
3954
4179
|
this.outputCache = undefined;
|
|
4180
|
+
this.legacyCacheExplicitlyDisabled = true;
|
|
3955
4181
|
}
|
|
3956
4182
|
} else {
|
|
3957
4183
|
this.outputCache = config.outputCache;
|
|
4184
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3958
4185
|
}
|
|
3959
4186
|
this.graph.outputCache = this.outputCache;
|
|
4187
|
+
} else {
|
|
4188
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3960
4189
|
}
|
|
3961
4190
|
if (this.running || this.previewRunning) {
|
|
3962
4191
|
throw new TaskConfigurationError("Graph is already running");
|
|
@@ -3970,6 +4199,9 @@ class TaskGraphRunner {
|
|
|
3970
4199
|
if (config?.timeout !== undefined) {
|
|
3971
4200
|
this.runScheduler.armGraphTimeout(config.timeout, ctx);
|
|
3972
4201
|
}
|
|
4202
|
+
if (this.resourceScope) {
|
|
4203
|
+
await this.resourceScope.runStart();
|
|
4204
|
+
}
|
|
3973
4205
|
if (ctx.abortController.signal.aborted)
|
|
3974
4206
|
return;
|
|
3975
4207
|
this.resetGraph(this.graph, ctx.runId);
|
|
@@ -4042,6 +4274,10 @@ class TaskGraphRunner {
|
|
|
4042
4274
|
}
|
|
4043
4275
|
ctx?.dispose();
|
|
4044
4276
|
this.currentCtx = undefined;
|
|
4277
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4278
|
+
this.registry = this.baseRegistryForRun;
|
|
4279
|
+
this.baseRegistryForRun = undefined;
|
|
4280
|
+
}
|
|
4045
4281
|
this.graph.emit("complete");
|
|
4046
4282
|
}
|
|
4047
4283
|
async handleCompletePreview() {
|
|
@@ -4054,6 +4290,7 @@ class TaskGraphRunner {
|
|
|
4054
4290
|
return task.abort();
|
|
4055
4291
|
}
|
|
4056
4292
|
}));
|
|
4293
|
+
this.currentRunPrivate = undefined;
|
|
4057
4294
|
const ctx = this.currentCtx;
|
|
4058
4295
|
this.running = false;
|
|
4059
4296
|
if (ctx?.telemetrySpan) {
|
|
@@ -4063,6 +4300,10 @@ class TaskGraphRunner {
|
|
|
4063
4300
|
}
|
|
4064
4301
|
ctx?.dispose();
|
|
4065
4302
|
this.currentCtx = undefined;
|
|
4303
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4304
|
+
this.registry = this.baseRegistryForRun;
|
|
4305
|
+
this.baseRegistryForRun = undefined;
|
|
4306
|
+
}
|
|
4066
4307
|
this.graph.emit("error", error);
|
|
4067
4308
|
}
|
|
4068
4309
|
async handleErrorPreview() {
|
|
@@ -4078,6 +4319,7 @@ class TaskGraphRunner {
|
|
|
4078
4319
|
return task.abort();
|
|
4079
4320
|
}
|
|
4080
4321
|
}));
|
|
4322
|
+
this.currentRunPrivate = undefined;
|
|
4081
4323
|
const ctx = this.currentCtx;
|
|
4082
4324
|
if (ctx?.telemetrySpan) {
|
|
4083
4325
|
ctx.telemetrySpan.setStatus(SpanStatusCode2.ERROR, "aborted");
|
|
@@ -4086,6 +4328,10 @@ class TaskGraphRunner {
|
|
|
4086
4328
|
}
|
|
4087
4329
|
ctx?.dispose();
|
|
4088
4330
|
this.currentCtx = undefined;
|
|
4331
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4332
|
+
this.registry = this.baseRegistryForRun;
|
|
4333
|
+
this.baseRegistryForRun = undefined;
|
|
4334
|
+
}
|
|
4089
4335
|
this.graph.emit("abort");
|
|
4090
4336
|
}
|
|
4091
4337
|
async handleAbortPreview() {
|
|
@@ -4209,7 +4455,7 @@ class GraphAsTask extends Task {
|
|
|
4209
4455
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
4210
4456
|
this._inputSchemaNode = schemaNode;
|
|
4211
4457
|
} catch (error) {
|
|
4212
|
-
|
|
4458
|
+
getLogger7().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
4213
4459
|
this._inputSchemaNode = compileSchema2({});
|
|
4214
4460
|
}
|
|
4215
4461
|
}
|
|
@@ -4493,7 +4739,7 @@ function convertPipeFunctionToTask(fn, config) {
|
|
|
4493
4739
|
additionalProperties: true
|
|
4494
4740
|
};
|
|
4495
4741
|
};
|
|
4496
|
-
static
|
|
4742
|
+
static cachePolicy = { kind: "none" };
|
|
4497
4743
|
async execute(input, context) {
|
|
4498
4744
|
return fn(input, context);
|
|
4499
4745
|
}
|
|
@@ -4575,7 +4821,9 @@ class TaskGraph {
|
|
|
4575
4821
|
registry: config?.registry,
|
|
4576
4822
|
timeout: config?.timeout,
|
|
4577
4823
|
maxTasks: config?.maxTasks,
|
|
4578
|
-
resourceScope: config?.resourceScope
|
|
4824
|
+
resourceScope: config?.resourceScope,
|
|
4825
|
+
disposeStrategy: config?.disposeStrategy,
|
|
4826
|
+
runId: config?.runId
|
|
4579
4827
|
});
|
|
4580
4828
|
}
|
|
4581
4829
|
runPreview(input = {}, config = {}) {
|
|
@@ -5124,7 +5372,7 @@ function autoConnect(graph, sourceTask, targetTask, options) {
|
|
|
5124
5372
|
}
|
|
5125
5373
|
|
|
5126
5374
|
// src/task-graph/LoopBuilderContext.ts
|
|
5127
|
-
import { getLogger as
|
|
5375
|
+
import { getLogger as getLogger8 } from "@workglow/util";
|
|
5128
5376
|
function runLoopAutoConnect(parentGraph, pending) {
|
|
5129
5377
|
const { parent, iteratorTask } = pending;
|
|
5130
5378
|
if (parentGraph.getTargetDataflows(parent.id).length !== 0)
|
|
@@ -5138,7 +5386,7 @@ function runLoopAutoConnect(parentGraph, pending) {
|
|
|
5138
5386
|
const result = autoConnect(parentGraph, parent, iteratorTask, { earlierTasks });
|
|
5139
5387
|
if (result.error) {
|
|
5140
5388
|
const message = result.error + " Task not added.";
|
|
5141
|
-
|
|
5389
|
+
getLogger8().error(message);
|
|
5142
5390
|
parentGraph.removeTask(iteratorTask.id);
|
|
5143
5391
|
return message;
|
|
5144
5392
|
}
|
|
@@ -5177,7 +5425,7 @@ class LoopBuilderContext {
|
|
|
5177
5425
|
}
|
|
5178
5426
|
|
|
5179
5427
|
// src/task-graph/WorkflowBuilder.ts
|
|
5180
|
-
import { getLogger as
|
|
5428
|
+
import { getLogger as getLogger9, uuid4 as uuid47 } from "@workglow/util";
|
|
5181
5429
|
|
|
5182
5430
|
// src/task-graph/ConditionalBuilder.ts
|
|
5183
5431
|
import { uuid4 as uuid46 } from "@workglow/util";
|
|
@@ -5336,7 +5584,7 @@ class WorkflowBuilder {
|
|
|
5336
5584
|
const taskSchema = task.inputSchema();
|
|
5337
5585
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5338
5586
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5339
|
-
|
|
5587
|
+
getLogger9().error(this._error);
|
|
5340
5588
|
return;
|
|
5341
5589
|
}
|
|
5342
5590
|
dataflow.targetTaskId = task.id;
|
|
@@ -5361,10 +5609,10 @@ class WorkflowBuilder {
|
|
|
5361
5609
|
if (result.error) {
|
|
5362
5610
|
if (this._loopContext !== undefined) {
|
|
5363
5611
|
this._error = result.error;
|
|
5364
|
-
|
|
5612
|
+
getLogger9().warn(this._error);
|
|
5365
5613
|
} else {
|
|
5366
5614
|
this._error = result.error + " Task not added.";
|
|
5367
|
-
|
|
5615
|
+
getLogger9().error(this._error);
|
|
5368
5616
|
this._facade.graph.removeTask(task.id);
|
|
5369
5617
|
}
|
|
5370
5618
|
}
|
|
@@ -5387,7 +5635,7 @@ class WorkflowBuilder {
|
|
|
5387
5635
|
const taskSchema = task.inputSchema();
|
|
5388
5636
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5389
5637
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5390
|
-
|
|
5638
|
+
getLogger9().error(this._error);
|
|
5391
5639
|
return;
|
|
5392
5640
|
}
|
|
5393
5641
|
dataflow.targetTaskId = task.id;
|
|
@@ -5439,7 +5687,7 @@ class WorkflowBuilder {
|
|
|
5439
5687
|
if (-index > nodes.length) {
|
|
5440
5688
|
const errorMsg = `Back index greater than number of tasks`;
|
|
5441
5689
|
this._error = errorMsg;
|
|
5442
|
-
|
|
5690
|
+
getLogger9().error(this._error);
|
|
5443
5691
|
throw new WorkflowError(errorMsg);
|
|
5444
5692
|
}
|
|
5445
5693
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -5448,13 +5696,13 @@ class WorkflowBuilder {
|
|
|
5448
5696
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
5449
5697
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
5450
5698
|
this._error = errorMsg;
|
|
5451
|
-
|
|
5699
|
+
getLogger9().error(this._error);
|
|
5452
5700
|
throw new WorkflowError(errorMsg);
|
|
5453
5701
|
}
|
|
5454
5702
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
5455
5703
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
5456
5704
|
this._error = errorMsg;
|
|
5457
|
-
|
|
5705
|
+
getLogger9().error(this._error);
|
|
5458
5706
|
throw new WorkflowError(errorMsg);
|
|
5459
5707
|
}
|
|
5460
5708
|
const df = new Dataflow(lastNode.id, source, undefined, target);
|
|
@@ -5467,7 +5715,7 @@ class WorkflowBuilder {
|
|
|
5467
5715
|
const parent = getLastTask(this._facade);
|
|
5468
5716
|
if (!parent) {
|
|
5469
5717
|
this._error = "onError() requires a preceding task in the workflow";
|
|
5470
|
-
|
|
5718
|
+
getLogger9().error(this._error);
|
|
5471
5719
|
throw new WorkflowError(this._error);
|
|
5472
5720
|
}
|
|
5473
5721
|
const handlerTask = ensureTask(handler);
|
|
@@ -5481,7 +5729,7 @@ class WorkflowBuilder {
|
|
|
5481
5729
|
const nodes = this._facade.graph.getTasks();
|
|
5482
5730
|
if (nodes.length === 0) {
|
|
5483
5731
|
this._error = "No tasks to remove";
|
|
5484
|
-
|
|
5732
|
+
getLogger9().error(this._error);
|
|
5485
5733
|
return;
|
|
5486
5734
|
}
|
|
5487
5735
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -5551,19 +5799,6 @@ class WorkflowEventBridge {
|
|
|
5551
5799
|
}
|
|
5552
5800
|
}
|
|
5553
5801
|
|
|
5554
|
-
// src/task-graph/WorkflowRunContext.ts
|
|
5555
|
-
class WorkflowRunContext {
|
|
5556
|
-
abortController;
|
|
5557
|
-
unsubStreaming;
|
|
5558
|
-
constructor() {
|
|
5559
|
-
this.abortController = new AbortController;
|
|
5560
|
-
}
|
|
5561
|
-
dispose() {
|
|
5562
|
-
this.unsubStreaming?.();
|
|
5563
|
-
this.unsubStreaming = undefined;
|
|
5564
|
-
}
|
|
5565
|
-
}
|
|
5566
|
-
|
|
5567
5802
|
// src/task-graph/WorkflowFactories.ts
|
|
5568
5803
|
function CreateWorkflow(taskClass) {
|
|
5569
5804
|
return Workflow.createWorkflow(taskClass);
|
|
@@ -5594,6 +5829,19 @@ function CreateAdaptiveWorkflow(scalarClass, vectorClass) {
|
|
|
5594
5829
|
};
|
|
5595
5830
|
}
|
|
5596
5831
|
|
|
5832
|
+
// src/task-graph/WorkflowRunContext.ts
|
|
5833
|
+
class WorkflowRunContext {
|
|
5834
|
+
abortController;
|
|
5835
|
+
unsubStreaming;
|
|
5836
|
+
constructor() {
|
|
5837
|
+
this.abortController = new AbortController;
|
|
5838
|
+
}
|
|
5839
|
+
dispose() {
|
|
5840
|
+
this.unsubStreaming?.();
|
|
5841
|
+
this.unsubStreaming = undefined;
|
|
5842
|
+
}
|
|
5843
|
+
}
|
|
5844
|
+
|
|
5597
5845
|
// src/task-graph/WorkflowTask.ts
|
|
5598
5846
|
class WorkflowTask extends GraphAsTask {
|
|
5599
5847
|
static type = "Workflow";
|
|
@@ -6141,112 +6389,11 @@ ${entries.map((e) => `${entryIndent}${e}`).join(`,
|
|
|
6141
6389
|
`)}
|
|
6142
6390
|
${baseIndent}}`;
|
|
6143
6391
|
}
|
|
6144
|
-
return String(value);
|
|
6145
|
-
}
|
|
6146
|
-
function resetMethodNameCache() {
|
|
6147
|
-
methodNameCache = undefined;
|
|
6148
|
-
}
|
|
6149
|
-
// src/task-graph/transforms/pick.ts
|
|
6150
|
-
import { areSemanticallyCompatible as areSemanticallyCompatible2 } from "@workglow/util/schema";
|
|
6151
|
-
function walk(value, path) {
|
|
6152
|
-
if (value == null)
|
|
6153
|
-
return;
|
|
6154
|
-
const parts = path.split(".");
|
|
6155
|
-
let cur = value;
|
|
6156
|
-
for (const p of parts) {
|
|
6157
|
-
if (cur == null)
|
|
6158
|
-
return;
|
|
6159
|
-
cur = cur[p];
|
|
6160
|
-
}
|
|
6161
|
-
return cur;
|
|
6162
|
-
}
|
|
6163
|
-
function walkSchema(schema, path) {
|
|
6164
|
-
const parts = path.split(".");
|
|
6165
|
-
let cur = schema;
|
|
6166
|
-
for (const p of parts) {
|
|
6167
|
-
if (!cur || typeof cur !== "object")
|
|
6168
|
-
return {};
|
|
6169
|
-
if (cur.type !== "object" || !cur.properties || !cur.properties[p]) {
|
|
6170
|
-
return {};
|
|
6171
|
-
}
|
|
6172
|
-
cur = cur.properties[p];
|
|
6173
|
-
}
|
|
6174
|
-
return cur;
|
|
6175
|
-
}
|
|
6176
|
-
var pickTransform = {
|
|
6177
|
-
id: "pick",
|
|
6178
|
-
title: "Pick field",
|
|
6179
|
-
category: "Structural",
|
|
6180
|
-
paramsSchema: {
|
|
6181
|
-
type: "object",
|
|
6182
|
-
properties: {
|
|
6183
|
-
path: { type: "string", description: "Dotted property path" }
|
|
6184
|
-
},
|
|
6185
|
-
required: ["path"]
|
|
6186
|
-
},
|
|
6187
|
-
inferOutputSchema(inputSchema, params) {
|
|
6188
|
-
return walkSchema(inputSchema, params.path);
|
|
6189
|
-
},
|
|
6190
|
-
apply(value, params) {
|
|
6191
|
-
return walk(value, params.path);
|
|
6192
|
-
},
|
|
6193
|
-
suggestFromSchemas(source, target) {
|
|
6194
|
-
if (source.type !== "object" || !source.properties) {
|
|
6195
|
-
return;
|
|
6196
|
-
}
|
|
6197
|
-
const props = source.properties;
|
|
6198
|
-
for (const [name, propSchema] of Object.entries(props)) {
|
|
6199
|
-
const compat = areSemanticallyCompatible2(propSchema, target);
|
|
6200
|
-
if (compat === "static")
|
|
6201
|
-
return { score: 1, params: { path: name } };
|
|
6202
|
-
if (compat === "runtime")
|
|
6203
|
-
return { score: 0.6, params: { path: name } };
|
|
6204
|
-
}
|
|
6205
|
-
return;
|
|
6206
|
-
}
|
|
6207
|
-
};
|
|
6208
|
-
|
|
6209
|
-
// src/task-graph/transforms/index-access.ts
|
|
6210
|
-
import { areSemanticallyCompatible as areSemanticallyCompatible3 } from "@workglow/util/schema";
|
|
6211
|
-
function doIndex(value, idx) {
|
|
6212
|
-
if (!Array.isArray(value))
|
|
6213
|
-
return;
|
|
6214
|
-
const i = idx < 0 ? value.length + idx : idx;
|
|
6215
|
-
return value[i];
|
|
6392
|
+
return String(value);
|
|
6393
|
+
}
|
|
6394
|
+
function resetMethodNameCache() {
|
|
6395
|
+
methodNameCache = undefined;
|
|
6216
6396
|
}
|
|
6217
|
-
var indexTransform = {
|
|
6218
|
-
id: "index",
|
|
6219
|
-
title: "Array index",
|
|
6220
|
-
category: "Structural",
|
|
6221
|
-
paramsSchema: {
|
|
6222
|
-
type: "object",
|
|
6223
|
-
properties: {
|
|
6224
|
-
index: { type: "integer", description: "Array index (negative counts from end)" }
|
|
6225
|
-
},
|
|
6226
|
-
required: ["index"]
|
|
6227
|
-
},
|
|
6228
|
-
inferOutputSchema(inputSchema) {
|
|
6229
|
-
const s = inputSchema;
|
|
6230
|
-
if (s?.type === "array" && s.items)
|
|
6231
|
-
return s.items;
|
|
6232
|
-
return {};
|
|
6233
|
-
},
|
|
6234
|
-
apply(value, params) {
|
|
6235
|
-
return doIndex(value, params.index);
|
|
6236
|
-
},
|
|
6237
|
-
suggestFromSchemas(source, target) {
|
|
6238
|
-
const s = source;
|
|
6239
|
-
if (s?.type !== "array" || !s.items)
|
|
6240
|
-
return;
|
|
6241
|
-
const compat = areSemanticallyCompatible3(s.items, target);
|
|
6242
|
-
if (compat === "static")
|
|
6243
|
-
return { score: 0.9, params: { index: 0 } };
|
|
6244
|
-
if (compat === "runtime")
|
|
6245
|
-
return { score: 0.5, params: { index: 0 } };
|
|
6246
|
-
return;
|
|
6247
|
-
}
|
|
6248
|
-
};
|
|
6249
|
-
|
|
6250
6397
|
// src/task-graph/transforms/coalesce.ts
|
|
6251
6398
|
function stripNullable(schema) {
|
|
6252
6399
|
const s = schema;
|
|
@@ -6273,52 +6420,6 @@ var coalesceTransform = {
|
|
|
6273
6420
|
apply: (v, { defaultValue }) => v == null ? defaultValue : v
|
|
6274
6421
|
};
|
|
6275
6422
|
|
|
6276
|
-
// src/task-graph/transforms/string-casts.ts
|
|
6277
|
-
var stringSchema = { type: "string" };
|
|
6278
|
-
var uppercaseTransform = {
|
|
6279
|
-
id: "uppercase",
|
|
6280
|
-
title: "Uppercase",
|
|
6281
|
-
category: "String",
|
|
6282
|
-
paramsSchema: undefined,
|
|
6283
|
-
inferOutputSchema: () => stringSchema,
|
|
6284
|
-
apply: (v) => String(v ?? "").toUpperCase()
|
|
6285
|
-
};
|
|
6286
|
-
var lowercaseTransform = {
|
|
6287
|
-
id: "lowercase",
|
|
6288
|
-
title: "Lowercase",
|
|
6289
|
-
category: "String",
|
|
6290
|
-
paramsSchema: undefined,
|
|
6291
|
-
inferOutputSchema: () => stringSchema,
|
|
6292
|
-
apply: (v) => String(v ?? "").toLowerCase()
|
|
6293
|
-
};
|
|
6294
|
-
var truncateTransform = {
|
|
6295
|
-
id: "truncate",
|
|
6296
|
-
title: "Truncate",
|
|
6297
|
-
category: "String",
|
|
6298
|
-
paramsSchema: {
|
|
6299
|
-
type: "object",
|
|
6300
|
-
properties: { max: { type: "integer", minimum: 0 } },
|
|
6301
|
-
required: ["max"]
|
|
6302
|
-
},
|
|
6303
|
-
inferOutputSchema: () => stringSchema,
|
|
6304
|
-
apply: (v, { max }) => String(v ?? "").slice(0, max)
|
|
6305
|
-
};
|
|
6306
|
-
var substringTransform = {
|
|
6307
|
-
id: "substring",
|
|
6308
|
-
title: "Substring",
|
|
6309
|
-
category: "String",
|
|
6310
|
-
paramsSchema: {
|
|
6311
|
-
type: "object",
|
|
6312
|
-
properties: {
|
|
6313
|
-
start: { type: "integer" },
|
|
6314
|
-
end: { type: "integer" }
|
|
6315
|
-
},
|
|
6316
|
-
required: ["start"]
|
|
6317
|
-
},
|
|
6318
|
-
inferOutputSchema: () => stringSchema,
|
|
6319
|
-
apply: (v, { start, end }) => String(v ?? "").slice(start, end)
|
|
6320
|
-
};
|
|
6321
|
-
|
|
6322
6423
|
// src/task-graph/transforms/date-conversions.ts
|
|
6323
6424
|
var isoSchema = { type: "string", format: "date-time" };
|
|
6324
6425
|
var numberSchema = { type: "number" };
|
|
@@ -6368,15 +6469,116 @@ var isoDateToUnixTransform = {
|
|
|
6368
6469
|
}
|
|
6369
6470
|
};
|
|
6370
6471
|
|
|
6472
|
+
// src/task-graph/transforms/index-access.ts
|
|
6473
|
+
import { areSemanticallyCompatible as areSemanticallyCompatible2 } from "@workglow/util/schema";
|
|
6474
|
+
function doIndex(value, idx) {
|
|
6475
|
+
if (!Array.isArray(value))
|
|
6476
|
+
return;
|
|
6477
|
+
const i = idx < 0 ? value.length + idx : idx;
|
|
6478
|
+
return value[i];
|
|
6479
|
+
}
|
|
6480
|
+
var indexTransform = {
|
|
6481
|
+
id: "index",
|
|
6482
|
+
title: "Array index",
|
|
6483
|
+
category: "Structural",
|
|
6484
|
+
paramsSchema: {
|
|
6485
|
+
type: "object",
|
|
6486
|
+
properties: {
|
|
6487
|
+
index: { type: "integer", description: "Array index (negative counts from end)" }
|
|
6488
|
+
},
|
|
6489
|
+
required: ["index"]
|
|
6490
|
+
},
|
|
6491
|
+
inferOutputSchema(inputSchema) {
|
|
6492
|
+
const s = inputSchema;
|
|
6493
|
+
if (s?.type === "array" && s.items)
|
|
6494
|
+
return s.items;
|
|
6495
|
+
return {};
|
|
6496
|
+
},
|
|
6497
|
+
apply(value, params) {
|
|
6498
|
+
return doIndex(value, params.index);
|
|
6499
|
+
},
|
|
6500
|
+
suggestFromSchemas(source, target) {
|
|
6501
|
+
const s = source;
|
|
6502
|
+
if (s?.type !== "array" || !s.items)
|
|
6503
|
+
return;
|
|
6504
|
+
const compat = areSemanticallyCompatible2(s.items, target);
|
|
6505
|
+
if (compat === "static")
|
|
6506
|
+
return { score: 0.9, params: { index: 0 } };
|
|
6507
|
+
if (compat === "runtime")
|
|
6508
|
+
return { score: 0.5, params: { index: 0 } };
|
|
6509
|
+
return;
|
|
6510
|
+
}
|
|
6511
|
+
};
|
|
6512
|
+
|
|
6513
|
+
// src/task-graph/transforms/pick.ts
|
|
6514
|
+
import { areSemanticallyCompatible as areSemanticallyCompatible3 } from "@workglow/util/schema";
|
|
6515
|
+
function walk(value, path) {
|
|
6516
|
+
if (value == null)
|
|
6517
|
+
return;
|
|
6518
|
+
const parts = path.split(".");
|
|
6519
|
+
let cur = value;
|
|
6520
|
+
for (const p of parts) {
|
|
6521
|
+
if (cur == null)
|
|
6522
|
+
return;
|
|
6523
|
+
cur = cur[p];
|
|
6524
|
+
}
|
|
6525
|
+
return cur;
|
|
6526
|
+
}
|
|
6527
|
+
function walkSchema(schema, path) {
|
|
6528
|
+
const parts = path.split(".");
|
|
6529
|
+
let cur = schema;
|
|
6530
|
+
for (const p of parts) {
|
|
6531
|
+
if (!cur || typeof cur !== "object")
|
|
6532
|
+
return {};
|
|
6533
|
+
if (cur.type !== "object" || !cur.properties || !cur.properties[p]) {
|
|
6534
|
+
return {};
|
|
6535
|
+
}
|
|
6536
|
+
cur = cur.properties[p];
|
|
6537
|
+
}
|
|
6538
|
+
return cur;
|
|
6539
|
+
}
|
|
6540
|
+
var pickTransform = {
|
|
6541
|
+
id: "pick",
|
|
6542
|
+
title: "Pick field",
|
|
6543
|
+
category: "Structural",
|
|
6544
|
+
paramsSchema: {
|
|
6545
|
+
type: "object",
|
|
6546
|
+
properties: {
|
|
6547
|
+
path: { type: "string", description: "Dotted property path" }
|
|
6548
|
+
},
|
|
6549
|
+
required: ["path"]
|
|
6550
|
+
},
|
|
6551
|
+
inferOutputSchema(inputSchema, params) {
|
|
6552
|
+
return walkSchema(inputSchema, params.path);
|
|
6553
|
+
},
|
|
6554
|
+
apply(value, params) {
|
|
6555
|
+
return walk(value, params.path);
|
|
6556
|
+
},
|
|
6557
|
+
suggestFromSchemas(source, target) {
|
|
6558
|
+
if (source.type !== "object" || !source.properties) {
|
|
6559
|
+
return;
|
|
6560
|
+
}
|
|
6561
|
+
const props = source.properties;
|
|
6562
|
+
for (const [name, propSchema] of Object.entries(props)) {
|
|
6563
|
+
const compat = areSemanticallyCompatible3(propSchema, target);
|
|
6564
|
+
if (compat === "static")
|
|
6565
|
+
return { score: 1, params: { path: name } };
|
|
6566
|
+
if (compat === "runtime")
|
|
6567
|
+
return { score: 0.6, params: { path: name } };
|
|
6568
|
+
}
|
|
6569
|
+
return;
|
|
6570
|
+
}
|
|
6571
|
+
};
|
|
6572
|
+
|
|
6371
6573
|
// src/task-graph/transforms/scalar-conversions.ts
|
|
6372
|
-
var
|
|
6574
|
+
var stringSchema = { type: "string" };
|
|
6373
6575
|
var booleanSchema = { type: "boolean" };
|
|
6374
6576
|
var numberToStringTransform = {
|
|
6375
6577
|
id: "numberToString",
|
|
6376
6578
|
title: "Number \u2192 String",
|
|
6377
6579
|
category: "Conversion",
|
|
6378
6580
|
paramsSchema: undefined,
|
|
6379
|
-
inferOutputSchema: () =>
|
|
6581
|
+
inferOutputSchema: () => stringSchema,
|
|
6380
6582
|
apply: (v) => String(v),
|
|
6381
6583
|
suggestFromSchemas(source, target) {
|
|
6382
6584
|
const s = source;
|
|
@@ -6408,7 +6610,7 @@ var stringifyTransform = {
|
|
|
6408
6610
|
title: "JSON.stringify",
|
|
6409
6611
|
category: "Conversion",
|
|
6410
6612
|
paramsSchema: undefined,
|
|
6411
|
-
inferOutputSchema: () =>
|
|
6613
|
+
inferOutputSchema: () => stringSchema,
|
|
6412
6614
|
apply: (v) => JSON.stringify(v),
|
|
6413
6615
|
suggestFromSchemas(source, target) {
|
|
6414
6616
|
const t = target;
|
|
@@ -6424,6 +6626,52 @@ var parseJsonTransform = {
|
|
|
6424
6626
|
apply: (v) => JSON.parse(String(v))
|
|
6425
6627
|
};
|
|
6426
6628
|
|
|
6629
|
+
// src/task-graph/transforms/string-casts.ts
|
|
6630
|
+
var stringSchema2 = { type: "string" };
|
|
6631
|
+
var uppercaseTransform = {
|
|
6632
|
+
id: "uppercase",
|
|
6633
|
+
title: "Uppercase",
|
|
6634
|
+
category: "String",
|
|
6635
|
+
paramsSchema: undefined,
|
|
6636
|
+
inferOutputSchema: () => stringSchema2,
|
|
6637
|
+
apply: (v) => String(v ?? "").toUpperCase()
|
|
6638
|
+
};
|
|
6639
|
+
var lowercaseTransform = {
|
|
6640
|
+
id: "lowercase",
|
|
6641
|
+
title: "Lowercase",
|
|
6642
|
+
category: "String",
|
|
6643
|
+
paramsSchema: undefined,
|
|
6644
|
+
inferOutputSchema: () => stringSchema2,
|
|
6645
|
+
apply: (v) => String(v ?? "").toLowerCase()
|
|
6646
|
+
};
|
|
6647
|
+
var truncateTransform = {
|
|
6648
|
+
id: "truncate",
|
|
6649
|
+
title: "Truncate",
|
|
6650
|
+
category: "String",
|
|
6651
|
+
paramsSchema: {
|
|
6652
|
+
type: "object",
|
|
6653
|
+
properties: { max: { type: "integer", minimum: 0 } },
|
|
6654
|
+
required: ["max"]
|
|
6655
|
+
},
|
|
6656
|
+
inferOutputSchema: () => stringSchema2,
|
|
6657
|
+
apply: (v, { max }) => String(v ?? "").slice(0, max)
|
|
6658
|
+
};
|
|
6659
|
+
var substringTransform = {
|
|
6660
|
+
id: "substring",
|
|
6661
|
+
title: "Substring",
|
|
6662
|
+
category: "String",
|
|
6663
|
+
paramsSchema: {
|
|
6664
|
+
type: "object",
|
|
6665
|
+
properties: {
|
|
6666
|
+
start: { type: "integer" },
|
|
6667
|
+
end: { type: "integer" }
|
|
6668
|
+
},
|
|
6669
|
+
required: ["start"]
|
|
6670
|
+
},
|
|
6671
|
+
inferOutputSchema: () => stringSchema2,
|
|
6672
|
+
apply: (v, { start, end }) => String(v ?? "").slice(start, end)
|
|
6673
|
+
};
|
|
6674
|
+
|
|
6427
6675
|
// src/task-graph/transforms/index.ts
|
|
6428
6676
|
function registerBuiltInTransforms() {
|
|
6429
6677
|
const all = [
|
|
@@ -6445,7 +6693,7 @@ function registerBuiltInTransforms() {
|
|
|
6445
6693
|
TransformRegistry.registerTransform(t);
|
|
6446
6694
|
}
|
|
6447
6695
|
// src/task/EntitlementProfile.ts
|
|
6448
|
-
import { createServiceToken as
|
|
6696
|
+
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
6449
6697
|
var STATIC_SIGNAL_SOURCE = Object.freeze({
|
|
6450
6698
|
subscribe(_listener) {
|
|
6451
6699
|
return () => {};
|
|
@@ -6548,7 +6796,7 @@ function createPolicyProfile(name, policy, options = {}) {
|
|
|
6548
6796
|
};
|
|
6549
6797
|
return profile;
|
|
6550
6798
|
}
|
|
6551
|
-
var ENTITLEMENT_PROFILE =
|
|
6799
|
+
var ENTITLEMENT_PROFILE = createServiceToken6("workglow.entitlementProfile");
|
|
6552
6800
|
// src/task/EntitlementProfiles.ts
|
|
6553
6801
|
var BROWSER_GRANTS = [
|
|
6554
6802
|
{ id: Entitlements.NETWORK_HTTP },
|
|
@@ -8166,12 +8414,12 @@ function wrapSchemaInArray(schema) {
|
|
|
8166
8414
|
}
|
|
8167
8415
|
// src/task/JobQueueFactory.ts
|
|
8168
8416
|
import {
|
|
8417
|
+
InMemoryQueueStorage,
|
|
8169
8418
|
JobQueueClient,
|
|
8170
8419
|
JobQueueServer
|
|
8171
8420
|
} from "@workglow/job-queue";
|
|
8172
|
-
import {
|
|
8173
|
-
|
|
8174
|
-
var JOB_QUEUE_FACTORY = createServiceToken6("taskgraph.jobQueueFactory");
|
|
8421
|
+
import { createServiceToken as createServiceToken7, globalServiceRegistry as globalServiceRegistry4 } from "@workglow/util";
|
|
8422
|
+
var JOB_QUEUE_FACTORY = createServiceToken7("taskgraph.jobQueueFactory");
|
|
8175
8423
|
var defaultJobQueueFactory = async ({
|
|
8176
8424
|
queueName,
|
|
8177
8425
|
jobClass,
|
|
@@ -8436,8 +8684,8 @@ queueMicrotask(() => {
|
|
|
8436
8684
|
});
|
|
8437
8685
|
// src/task/TaskRegistry.ts
|
|
8438
8686
|
import {
|
|
8439
|
-
createServiceToken as
|
|
8440
|
-
getLogger as
|
|
8687
|
+
createServiceToken as createServiceToken8,
|
|
8688
|
+
getLogger as getLogger10,
|
|
8441
8689
|
globalServiceRegistry as globalServiceRegistry5,
|
|
8442
8690
|
registerInputCompactor,
|
|
8443
8691
|
registerInputResolver
|
|
@@ -8460,7 +8708,7 @@ function registerTask(baseClass) {
|
|
|
8460
8708
|
const result = validateSchema(schema);
|
|
8461
8709
|
if (!result.valid) {
|
|
8462
8710
|
const messages = result.errors.map((e) => `${e.path}: ${e.message}`).join("; ");
|
|
8463
|
-
|
|
8711
|
+
getLogger10().warn(`Task "${baseClass.type}" has invalid ${name}: ${messages}`, {
|
|
8464
8712
|
taskType: baseClass.type,
|
|
8465
8713
|
schemaName: name,
|
|
8466
8714
|
errors: result.errors
|
|
@@ -8476,7 +8724,7 @@ var TaskRegistry = {
|
|
|
8476
8724
|
registerTask,
|
|
8477
8725
|
unregisterTask
|
|
8478
8726
|
};
|
|
8479
|
-
var TASK_CONSTRUCTORS =
|
|
8727
|
+
var TASK_CONSTRUCTORS = createServiceToken8("task.constructors");
|
|
8480
8728
|
function getGlobalTaskConstructors() {
|
|
8481
8729
|
return globalServiceRegistry5.get(TASK_CONSTRUCTORS);
|
|
8482
8730
|
}
|
|
@@ -8645,8 +8893,8 @@ var registerBaseTasks = () => {
|
|
|
8645
8893
|
return tasks;
|
|
8646
8894
|
};
|
|
8647
8895
|
// src/storage/TaskGraphRepository.ts
|
|
8648
|
-
import { createServiceToken as
|
|
8649
|
-
var TASK_GRAPH_REPOSITORY =
|
|
8896
|
+
import { createServiceToken as createServiceToken9, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
8897
|
+
var TASK_GRAPH_REPOSITORY = createServiceToken9("taskgraph.taskGraphRepository");
|
|
8650
8898
|
|
|
8651
8899
|
class TaskGraphRepository {
|
|
8652
8900
|
type = "TaskGraphRepository";
|
|
@@ -8721,8 +8969,8 @@ class TaskGraphTabularRepository extends TaskGraphRepository {
|
|
|
8721
8969
|
}
|
|
8722
8970
|
}
|
|
8723
8971
|
// src/storage/TaskOutputTabularRepository.ts
|
|
8724
|
-
import { compress, decompress } from "@workglow/util/compress";
|
|
8725
8972
|
import { makeFingerprint } from "@workglow/util";
|
|
8973
|
+
import { compress, decompress } from "@workglow/util/compress";
|
|
8726
8974
|
var TaskOutputSchema = {
|
|
8727
8975
|
type: "object",
|
|
8728
8976
|
properties: {
|
|
@@ -8742,6 +8990,10 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8742
8990
|
this.tabularRepository = tabularRepository;
|
|
8743
8991
|
this.outputCompression = outputCompression;
|
|
8744
8992
|
}
|
|
8993
|
+
isDurable() {
|
|
8994
|
+
const backing = this.tabularRepository;
|
|
8995
|
+
return backing.isDurable?.() ?? true;
|
|
8996
|
+
}
|
|
8745
8997
|
async setupDatabase() {
|
|
8746
8998
|
await this.tabularRepository.setupDatabase?.();
|
|
8747
8999
|
}
|
|
@@ -8802,9 +9054,36 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8802
9054
|
await this.tabularRepository.deleteSearch({ createdAt: { value: date, operator: "<" } });
|
|
8803
9055
|
this.emit("output_pruned");
|
|
8804
9056
|
}
|
|
9057
|
+
async deleteByTaskTypePrefix(prefix) {
|
|
9058
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9059
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9060
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9061
|
+
}
|
|
9062
|
+
}
|
|
9063
|
+
}
|
|
9064
|
+
async clearOlderThanWithTaskTypePrefix(prefix, olderThanInMs) {
|
|
9065
|
+
const cutoff = Date.now() - olderThanInMs;
|
|
9066
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9067
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9068
|
+
const ts = typeof row.createdAt === "string" ? new Date(row.createdAt).getTime() : NaN;
|
|
9069
|
+
if (!isNaN(ts) && ts < cutoff) {
|
|
9070
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9071
|
+
}
|
|
9072
|
+
}
|
|
9073
|
+
}
|
|
9074
|
+
}
|
|
9075
|
+
async sizeByTaskTypePrefix(prefix) {
|
|
9076
|
+
let count = 0;
|
|
9077
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9078
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9079
|
+
count++;
|
|
9080
|
+
}
|
|
9081
|
+
}
|
|
9082
|
+
return count;
|
|
9083
|
+
}
|
|
8805
9084
|
}
|
|
8806
9085
|
// src/storage/PortCodecRegistry.ts
|
|
8807
|
-
import {
|
|
9086
|
+
import { _resetPortCodecsForTests, getPortCodec as getPortCodec2, registerPortCodec } from "@workglow/util";
|
|
8808
9087
|
export {
|
|
8809
9088
|
wrapSchemaInArray,
|
|
8810
9089
|
whileTaskConfigSchema,
|
|
@@ -8851,6 +9130,8 @@ export {
|
|
|
8851
9130
|
isoDateToUnixTransform,
|
|
8852
9131
|
isTaskStreamable,
|
|
8853
9132
|
isStrictArraySchema,
|
|
9133
|
+
isPolicyPrivate,
|
|
9134
|
+
isPolicyCached,
|
|
8854
9135
|
isIterationProperty,
|
|
8855
9136
|
isFlexibleSchema,
|
|
8856
9137
|
indexTransform,
|
|
@@ -8964,6 +9245,7 @@ export {
|
|
|
8964
9245
|
STATIC_SIGNAL_SOURCE,
|
|
8965
9246
|
SERVER_GRANTS,
|
|
8966
9247
|
RunScheduler,
|
|
9248
|
+
RunPrivateCacheRepo,
|
|
8967
9249
|
RunContext,
|
|
8968
9250
|
ReduceTask,
|
|
8969
9251
|
PROPERTY_ARRAY,
|
|
@@ -8991,10 +9273,12 @@ export {
|
|
|
8991
9273
|
ENTITLEMENT_ENFORCER,
|
|
8992
9274
|
EMPTY_POLICY,
|
|
8993
9275
|
EMPTY_ENTITLEMENTS,
|
|
9276
|
+
DefaultCacheRegistry,
|
|
8994
9277
|
DataflowArrow,
|
|
8995
9278
|
Dataflow,
|
|
8996
9279
|
DESKTOP_GRANTS,
|
|
8997
9280
|
DENY_ALL_RESOLVER,
|
|
9281
|
+
DEFAULT_CACHE_POLICY,
|
|
8998
9282
|
DATAFLOW_ERROR_PORT,
|
|
8999
9283
|
DATAFLOW_ALL_PORTS,
|
|
9000
9284
|
CreateWorkflow,
|
|
@@ -9002,8 +9286,10 @@ export {
|
|
|
9002
9286
|
CreateEndLoopWorkflow,
|
|
9003
9287
|
CreateAdaptiveWorkflow,
|
|
9004
9288
|
ConditionalTask,
|
|
9289
|
+
CacheJanitor,
|
|
9005
9290
|
CacheCoordinator,
|
|
9291
|
+
CACHE_REGISTRY,
|
|
9006
9292
|
BROWSER_GRANTS
|
|
9007
9293
|
};
|
|
9008
9294
|
|
|
9009
|
-
//# debugId=
|
|
9295
|
+
//# debugId=E2522767BD1162D964756E2164756E21
|