@workglow/task-graph 0.2.36 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -46
- package/dist/browser.js +792 -506
- package/dist/browser.js.map +40 -36
- package/dist/bun.js +792 -506
- package/dist/bun.js.map +40 -36
- package/dist/cache/CacheJanitor.d.ts +27 -0
- package/dist/cache/CacheJanitor.d.ts.map +1 -0
- package/dist/cache/CachePolicy.d.ts +16 -0
- package/dist/cache/CachePolicy.d.ts.map +1 -0
- package/dist/cache/CacheRegistry.d.ts +30 -0
- package/dist/cache/CacheRegistry.d.ts.map +1 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts +56 -0
- package/dist/cache/RunPrivateCacheRepo.d.ts.map +1 -0
- package/dist/cache/index.d.ts +10 -0
- package/dist/cache/index.d.ts.map +1 -0
- package/dist/common.d.ts +1 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +792 -506
- package/dist/node.js.map +40 -36
- package/dist/storage/PortCodecRegistry.d.ts +1 -1
- package/dist/storage/PortCodecRegistry.d.ts.map +1 -1
- package/dist/storage/TaskGraphTabularRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputRepository.d.ts +40 -4
- package/dist/storage/TaskOutputRepository.d.ts.map +1 -1
- package/dist/storage/TaskOutputTabularRepository.d.ts +27 -0
- package/dist/storage/TaskOutputTabularRepository.d.ts.map +1 -1
- package/dist/task/CacheCoordinator.d.ts +18 -1
- package/dist/task/CacheCoordinator.d.ts.map +1 -1
- package/dist/task/ConditionalTask.d.ts +14 -14
- package/dist/task/ConditionalTask.d.ts.map +1 -1
- package/dist/task/EntitlementProfile.d.ts.map +1 -1
- package/dist/task/EntitlementProfiles.d.ts +1 -1
- package/dist/task/EntitlementProfiles.d.ts.map +1 -1
- package/dist/task/FallbackTask.d.ts +8 -9
- package/dist/task/FallbackTask.d.ts.map +1 -1
- package/dist/task/FallbackTaskRunner.d.ts +8 -0
- package/dist/task/FallbackTaskRunner.d.ts.map +1 -1
- package/dist/task/GraphAsTask.d.ts +4 -4
- package/dist/task/GraphAsTask.d.ts.map +1 -1
- package/dist/task/ITask.d.ts +28 -2
- package/dist/task/ITask.d.ts.map +1 -1
- package/dist/task/InputCompactor.d.ts +1 -1
- package/dist/task/InputCompactor.d.ts.map +1 -1
- package/dist/task/InputResolver.d.ts +1 -1
- package/dist/task/InputResolver.d.ts.map +1 -1
- package/dist/task/IteratorTask.d.ts +4 -4
- package/dist/task/JobQueueFactory.d.ts +1 -1
- package/dist/task/JobQueueFactory.d.ts.map +1 -1
- package/dist/task/MapTask.d.ts +4 -4
- package/dist/task/ReduceTask.d.ts +4 -4
- package/dist/task/StreamProcessor.d.ts +1 -1
- package/dist/task/StreamProcessor.d.ts.map +1 -1
- package/dist/task/Task.d.ts +50 -0
- package/dist/task/Task.d.ts.map +1 -1
- package/dist/task/TaskError.d.ts +2 -0
- package/dist/task/TaskError.d.ts.map +1 -1
- package/dist/task/TaskEvents.d.ts +2 -2
- package/dist/task/TaskEvents.d.ts.map +1 -1
- package/dist/task/TaskJSON.d.ts +0 -13
- package/dist/task/TaskJSON.d.ts.map +1 -1
- package/dist/task/TaskQueueRegistry.d.ts +1 -1
- package/dist/task/TaskQueueRegistry.d.ts.map +1 -1
- package/dist/task/TaskRunner.d.ts +28 -0
- package/dist/task/TaskRunner.d.ts.map +1 -1
- package/dist/task/TaskTypes.d.ts +1 -1
- package/dist/task/TaskTypes.d.ts.map +1 -1
- package/dist/task/WhileTask.d.ts +4 -4
- package/dist/task/iterationSchema.d.ts +1 -1
- package/dist/task/iterationSchema.d.ts.map +1 -1
- package/dist/task-graph/Conversions.d.ts.map +1 -1
- package/dist/task-graph/Dataflow.d.ts +1 -1
- package/dist/task-graph/Dataflow.d.ts.map +1 -1
- package/dist/task-graph/IWorkflow.d.ts +1 -1
- package/dist/task-graph/IWorkflow.d.ts.map +1 -1
- package/dist/task-graph/StreamPump.d.ts +8 -0
- package/dist/task-graph/StreamPump.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +10 -1
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +45 -0
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/dist/task-graph/transforms/index.d.ts +6 -6
- package/dist/task-graph/transforms/index.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +91 -2
package/dist/browser.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/task-graph/Dataflow.ts
|
|
2
|
-
import { areSemanticallyCompatible } from "@workglow/util/schema";
|
|
3
2
|
import { EventEmitter } from "@workglow/util";
|
|
3
|
+
import { areSemanticallyCompatible } from "@workglow/util/schema";
|
|
4
4
|
|
|
5
5
|
// src/task/TaskError.ts
|
|
6
6
|
import { BaseError } from "@workglow/util";
|
|
@@ -60,9 +60,13 @@ class TaskFailedError extends TaskError {
|
|
|
60
60
|
class JobTaskFailedError extends TaskFailedError {
|
|
61
61
|
static type = "JobTaskFailedError";
|
|
62
62
|
jobError;
|
|
63
|
+
code;
|
|
63
64
|
constructor(err) {
|
|
64
65
|
super(String(err));
|
|
65
66
|
this.jobError = err;
|
|
67
|
+
if (err.code) {
|
|
68
|
+
this.code = err.code;
|
|
69
|
+
}
|
|
66
70
|
}
|
|
67
71
|
}
|
|
68
72
|
|
|
@@ -1273,14 +1277,14 @@ import { EventEmitter as EventEmitter4, uuid4 as uuid45 } from "@workglow/util";
|
|
|
1273
1277
|
import { DirectedAcyclicGraph } from "@workglow/util/graph";
|
|
1274
1278
|
|
|
1275
1279
|
// src/task/GraphAsTask.ts
|
|
1276
|
-
import { getLogger as
|
|
1280
|
+
import { getLogger as getLogger7 } from "@workglow/util";
|
|
1277
1281
|
import { CycleError } from "@workglow/util/graph";
|
|
1278
1282
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
1279
1283
|
|
|
1280
1284
|
// src/task-graph/TaskGraphRunner.ts
|
|
1281
1285
|
import {
|
|
1282
1286
|
collectPropertyValues,
|
|
1283
|
-
getLogger as
|
|
1287
|
+
getLogger as getLogger6,
|
|
1284
1288
|
getTelemetryProvider as getTelemetryProvider2,
|
|
1285
1289
|
globalServiceRegistry as globalServiceRegistry3,
|
|
1286
1290
|
ResourceScope as ResourceScope2,
|
|
@@ -1289,9 +1293,39 @@ import {
|
|
|
1289
1293
|
uuid4 as uuid44
|
|
1290
1294
|
} from "@workglow/util";
|
|
1291
1295
|
|
|
1296
|
+
// src/cache/CacheJanitor.ts
|
|
1297
|
+
class CacheJanitor {
|
|
1298
|
+
privateBacking;
|
|
1299
|
+
constructor({ privateBacking }) {
|
|
1300
|
+
this.privateBacking = privateBacking;
|
|
1301
|
+
}
|
|
1302
|
+
async sweepStaleRunPrivate(olderThanMs) {
|
|
1303
|
+
await this.privateBacking.clearOlderThanWithTaskTypePrefix("__run:", olderThanMs);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
// src/cache/CachePolicy.ts
|
|
1307
|
+
var DEFAULT_CACHE_POLICY = { kind: "deterministic" };
|
|
1308
|
+
function isPolicyCached(policy) {
|
|
1309
|
+
return policy.kind !== "none";
|
|
1310
|
+
}
|
|
1311
|
+
function isPolicyPrivate(policy) {
|
|
1312
|
+
return policy.kind === "private";
|
|
1313
|
+
}
|
|
1314
|
+
// src/cache/CacheRegistry.ts
|
|
1315
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
1316
|
+
var CACHE_REGISTRY = createServiceToken2("taskgraph.cacheRegistry");
|
|
1317
|
+
|
|
1318
|
+
class DefaultCacheRegistry {
|
|
1319
|
+
deterministic;
|
|
1320
|
+
private;
|
|
1321
|
+
constructor(init = {}) {
|
|
1322
|
+
this.deterministic = init.deterministic;
|
|
1323
|
+
this.private = init.private;
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1292
1326
|
// src/storage/TaskOutputRepository.ts
|
|
1293
|
-
import { createServiceToken as
|
|
1294
|
-
var TASK_OUTPUT_REPOSITORY =
|
|
1327
|
+
import { createServiceToken as createServiceToken3, EventEmitter as EventEmitter2 } from "@workglow/util";
|
|
1328
|
+
var TASK_OUTPUT_REPOSITORY = createServiceToken3("taskgraph.taskOutputRepository");
|
|
1295
1329
|
|
|
1296
1330
|
class TaskOutputRepository {
|
|
1297
1331
|
outputCompression;
|
|
@@ -1317,10 +1351,53 @@ class TaskOutputRepository {
|
|
|
1317
1351
|
emit(name, ...args) {
|
|
1318
1352
|
this._events?.emit(name, ...args);
|
|
1319
1353
|
}
|
|
1354
|
+
async deleteByTaskTypePrefix(_prefix) {
|
|
1355
|
+
throw new Error(`${this.constructor.name}: deleteByTaskTypePrefix is not supported by this repository.`);
|
|
1356
|
+
}
|
|
1357
|
+
async clearOlderThanWithTaskTypePrefix(_prefix, _olderThanMs) {
|
|
1358
|
+
throw new Error(`${this.constructor.name}: clearOlderThanWithTaskTypePrefix is not supported by this repository.`);
|
|
1359
|
+
}
|
|
1360
|
+
async sizeByTaskTypePrefix(_prefix) {
|
|
1361
|
+
throw new Error(`${this.constructor.name}: sizeByTaskTypePrefix is not supported by this repository.`);
|
|
1362
|
+
}
|
|
1320
1363
|
}
|
|
1321
1364
|
|
|
1365
|
+
// src/cache/RunPrivateCacheRepo.ts
|
|
1366
|
+
class RunPrivateCacheRepo extends TaskOutputRepository {
|
|
1367
|
+
backing;
|
|
1368
|
+
runId;
|
|
1369
|
+
constructor({ backing, runId }) {
|
|
1370
|
+
super({ outputCompression: backing.outputCompression });
|
|
1371
|
+
this.backing = backing;
|
|
1372
|
+
this.runId = runId;
|
|
1373
|
+
}
|
|
1374
|
+
ns(taskType) {
|
|
1375
|
+
return `__run:${this.runId}::${taskType}`;
|
|
1376
|
+
}
|
|
1377
|
+
async saveOutput(taskType, inputs, output, createdAt) {
|
|
1378
|
+
await this.backing.saveOutput(this.ns(taskType), inputs, output, createdAt);
|
|
1379
|
+
}
|
|
1380
|
+
async getOutput(taskType, inputs) {
|
|
1381
|
+
return this.backing.getOutput(this.ns(taskType), inputs);
|
|
1382
|
+
}
|
|
1383
|
+
async clear() {
|
|
1384
|
+
await this.clearRun();
|
|
1385
|
+
}
|
|
1386
|
+
async clearRun() {
|
|
1387
|
+
await this.backing.deleteByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1388
|
+
}
|
|
1389
|
+
async size() {
|
|
1390
|
+
return this.backing.sizeByTaskTypePrefix(`__run:${this.runId}::`);
|
|
1391
|
+
}
|
|
1392
|
+
async clearOlderThan(olderThanInMs) {
|
|
1393
|
+
await this.backing.clearOlderThanWithTaskTypePrefix(`__run:${this.runId}::`, olderThanInMs);
|
|
1394
|
+
}
|
|
1395
|
+
isDurable() {
|
|
1396
|
+
return this.backing.isDurable();
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1322
1399
|
// src/task/EntitlementEnforcer.ts
|
|
1323
|
-
import { createServiceToken as
|
|
1400
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1324
1401
|
|
|
1325
1402
|
// src/task/EntitlementPolicy.ts
|
|
1326
1403
|
var EMPTY_POLICY = Object.freeze({
|
|
@@ -1364,7 +1441,7 @@ function can(policy, id, resources) {
|
|
|
1364
1441
|
}
|
|
1365
1442
|
|
|
1366
1443
|
// src/task/EntitlementResolver.ts
|
|
1367
|
-
import { createServiceToken as
|
|
1444
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1368
1445
|
var PERMISSIVE_RESOLVER = {
|
|
1369
1446
|
lookup: () => "grant",
|
|
1370
1447
|
prompt: async () => "grant",
|
|
@@ -1375,7 +1452,7 @@ var DENY_ALL_RESOLVER = {
|
|
|
1375
1452
|
prompt: async () => "deny",
|
|
1376
1453
|
save: () => {}
|
|
1377
1454
|
};
|
|
1378
|
-
var ENTITLEMENT_RESOLVER =
|
|
1455
|
+
var ENTITLEMENT_RESOLVER = createServiceToken4("workglow.entitlementResolver");
|
|
1379
1456
|
|
|
1380
1457
|
// src/task/EntitlementEnforcer.ts
|
|
1381
1458
|
function formatEntitlementDenial(denial) {
|
|
@@ -1448,7 +1525,7 @@ function createScopedEnforcer(grants) {
|
|
|
1448
1525
|
function createGrantListEnforcer(grants) {
|
|
1449
1526
|
return createScopedEnforcer(grants.map((id) => ({ id })));
|
|
1450
1527
|
}
|
|
1451
|
-
var ENTITLEMENT_ENFORCER =
|
|
1528
|
+
var ENTITLEMENT_ENFORCER = createServiceToken5("workglow.entitlementEnforcer");
|
|
1452
1529
|
|
|
1453
1530
|
// src/task/StreamTypes.ts
|
|
1454
1531
|
function getPortStreamMode(schema, portId) {
|
|
@@ -1551,246 +1628,13 @@ function hasStructuredOutput(schema) {
|
|
|
1551
1628
|
return getStructuredOutputSchemas(schema).size > 0;
|
|
1552
1629
|
}
|
|
1553
1630
|
|
|
1554
|
-
// src/task-graph/EdgeMaterializer.ts
|
|
1555
|
-
import { getLogger } from "@workglow/util";
|
|
1556
|
-
import { previewSource } from "@workglow/util/media";
|
|
1557
|
-
class EdgeMaterializer {
|
|
1558
|
-
graph;
|
|
1559
|
-
runner;
|
|
1560
|
-
constructor(graph, runner) {
|
|
1561
|
-
this.graph = graph;
|
|
1562
|
-
this.runner = runner;
|
|
1563
|
-
}
|
|
1564
|
-
filterInputForTask(task, input) {
|
|
1565
|
-
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
1566
|
-
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
1567
|
-
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
1568
|
-
const filteredInput = {};
|
|
1569
|
-
for (const [key, value] of Object.entries(input)) {
|
|
1570
|
-
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
1571
|
-
filteredInput[key] = value;
|
|
1572
|
-
}
|
|
1573
|
-
}
|
|
1574
|
-
return filteredInput;
|
|
1575
|
-
}
|
|
1576
|
-
copyInputFromEdgesToNode(task) {
|
|
1577
|
-
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
1578
|
-
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
1579
|
-
for (const dataflow of dataflows) {
|
|
1580
|
-
const live = dataflow.getCurrentValue();
|
|
1581
|
-
const port = dataflow.targetTaskPortId;
|
|
1582
|
-
let portData;
|
|
1583
|
-
if (port === DATAFLOW_ALL_PORTS) {
|
|
1584
|
-
portData = live;
|
|
1585
|
-
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
1586
|
-
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
1587
|
-
} else {
|
|
1588
|
-
portData = { [port]: live };
|
|
1589
|
-
}
|
|
1590
|
-
this.runner.addInputData(task, portData);
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
async pushOutputFromNodeToEdges(node, results) {
|
|
1594
|
-
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
1595
|
-
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
1596
|
-
for (const port of Object.keys(results)) {
|
|
1597
|
-
const value = results[port];
|
|
1598
|
-
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
1599
|
-
results[port] = await previewSource(value);
|
|
1600
|
-
}
|
|
1601
|
-
}
|
|
1602
|
-
}
|
|
1603
|
-
for (const dataflow of dataflows) {
|
|
1604
|
-
if (dataflow.stream !== undefined)
|
|
1605
|
-
continue;
|
|
1606
|
-
const registry = this.runner["registry"];
|
|
1607
|
-
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
1608
|
-
if (compatibility === "static") {
|
|
1609
|
-
dataflow.setPortData(results);
|
|
1610
|
-
await dataflow.applyTransforms(registry);
|
|
1611
|
-
} else if (compatibility === "runtime") {
|
|
1612
|
-
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
1613
|
-
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
1614
|
-
dataflow.setPortData(narrowed);
|
|
1615
|
-
await dataflow.applyTransforms(registry);
|
|
1616
|
-
} else {
|
|
1617
|
-
const resultsKeys = Object.keys(results);
|
|
1618
|
-
if (resultsKeys.length > 0) {
|
|
1619
|
-
getLogger().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
1620
|
-
dataflowId: dataflow.id,
|
|
1621
|
-
compatibility,
|
|
1622
|
-
resultsKeys
|
|
1623
|
-
});
|
|
1624
|
-
}
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
pushErrorFromNodeToEdges(node) {
|
|
1629
|
-
if (!node?.config?.id)
|
|
1630
|
-
return;
|
|
1631
|
-
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
1632
|
-
dataflow.error = node.error;
|
|
1633
|
-
});
|
|
1634
|
-
}
|
|
1635
|
-
hasErrorOutputEdges(task) {
|
|
1636
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1637
|
-
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
1638
|
-
}
|
|
1639
|
-
pushErrorOutputToEdges(task) {
|
|
1640
|
-
const taskError = task.error;
|
|
1641
|
-
const errorData = {
|
|
1642
|
-
error: taskError?.message ?? "Unknown error",
|
|
1643
|
-
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
1644
|
-
};
|
|
1645
|
-
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
1646
|
-
for (const df of dataflows) {
|
|
1647
|
-
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
1648
|
-
df.value = errorData;
|
|
1649
|
-
df.setStatus(TaskStatus.COMPLETED);
|
|
1650
|
-
} else {
|
|
1651
|
-
df.setStatus(TaskStatus.DISABLED);
|
|
1652
|
-
}
|
|
1653
|
-
}
|
|
1654
|
-
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
1655
|
-
}
|
|
1656
|
-
resetTask(graph, task, runId) {
|
|
1657
|
-
task.status = TaskStatus.PENDING;
|
|
1658
|
-
task.resetInputData();
|
|
1659
|
-
task.runOutputData = {};
|
|
1660
|
-
task.error = undefined;
|
|
1661
|
-
task.progress = 0;
|
|
1662
|
-
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
1663
|
-
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
1664
|
-
if (task?.config?.id) {
|
|
1665
|
-
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
1666
|
-
dataflow.error = task.error;
|
|
1667
|
-
});
|
|
1668
|
-
}
|
|
1669
|
-
task.emit("reset");
|
|
1670
|
-
task.emit("status", task.status);
|
|
1671
|
-
}
|
|
1672
|
-
static isImageValueShape(v) {
|
|
1673
|
-
if (v === null || typeof v !== "object")
|
|
1674
|
-
return false;
|
|
1675
|
-
const o = v;
|
|
1676
|
-
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
// src/task-graph/RunContext.ts
|
|
1681
|
-
import { uuid4 as uuid42 } from "@workglow/util";
|
|
1682
|
-
|
|
1683
|
-
class RunContext {
|
|
1684
|
-
runId;
|
|
1685
|
-
abortController;
|
|
1686
|
-
inProgressTasks = new Map;
|
|
1687
|
-
inProgressFunctions = new Map;
|
|
1688
|
-
failedTaskErrors = new Map;
|
|
1689
|
-
telemetrySpan;
|
|
1690
|
-
graphTimeoutTimer;
|
|
1691
|
-
pendingGraphTimeoutError;
|
|
1692
|
-
activeEnforcer;
|
|
1693
|
-
parentSignalCleanup;
|
|
1694
|
-
constructor(parentSignal) {
|
|
1695
|
-
this.runId = uuid42();
|
|
1696
|
-
this.abortController = new AbortController;
|
|
1697
|
-
if (parentSignal) {
|
|
1698
|
-
const onParentAbort = () => this.abortController.abort();
|
|
1699
|
-
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
1700
|
-
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
1701
|
-
if (parentSignal.aborted) {
|
|
1702
|
-
this.parentSignalCleanup();
|
|
1703
|
-
this.parentSignalCleanup = undefined;
|
|
1704
|
-
this.abortController.abort();
|
|
1705
|
-
}
|
|
1706
|
-
}
|
|
1707
|
-
}
|
|
1708
|
-
dispose() {
|
|
1709
|
-
this.parentSignalCleanup?.();
|
|
1710
|
-
this.parentSignalCleanup = undefined;
|
|
1711
|
-
}
|
|
1712
|
-
}
|
|
1713
|
-
|
|
1714
|
-
// src/task-graph/RunScheduler.ts
|
|
1715
|
-
import { getLogger as getLogger4 } from "@workglow/util";
|
|
1716
|
-
|
|
1717
|
-
// src/task/ConditionalTask.ts
|
|
1718
|
-
import { getLogger as getLogger3 } from "@workglow/util";
|
|
1719
|
-
|
|
1720
|
-
// src/task/ConditionUtils.ts
|
|
1721
|
-
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
1722
|
-
if (fieldValue === null || fieldValue === undefined) {
|
|
1723
|
-
switch (operator) {
|
|
1724
|
-
case "is_empty":
|
|
1725
|
-
return true;
|
|
1726
|
-
case "is_not_empty":
|
|
1727
|
-
return false;
|
|
1728
|
-
case "is_true":
|
|
1729
|
-
return false;
|
|
1730
|
-
case "is_false":
|
|
1731
|
-
return true;
|
|
1732
|
-
default:
|
|
1733
|
-
return false;
|
|
1734
|
-
}
|
|
1735
|
-
}
|
|
1736
|
-
const strValue = String(fieldValue);
|
|
1737
|
-
const numValue = Number(fieldValue);
|
|
1738
|
-
switch (operator) {
|
|
1739
|
-
case "equals":
|
|
1740
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1741
|
-
return numValue === Number(compareValue);
|
|
1742
|
-
}
|
|
1743
|
-
return strValue === compareValue;
|
|
1744
|
-
case "not_equals":
|
|
1745
|
-
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
1746
|
-
return numValue !== Number(compareValue);
|
|
1747
|
-
}
|
|
1748
|
-
return strValue !== compareValue;
|
|
1749
|
-
case "greater_than":
|
|
1750
|
-
return numValue > Number(compareValue);
|
|
1751
|
-
case "greater_or_equal":
|
|
1752
|
-
return numValue >= Number(compareValue);
|
|
1753
|
-
case "less_than":
|
|
1754
|
-
return numValue < Number(compareValue);
|
|
1755
|
-
case "less_or_equal":
|
|
1756
|
-
return numValue <= Number(compareValue);
|
|
1757
|
-
case "contains":
|
|
1758
|
-
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
1759
|
-
case "starts_with":
|
|
1760
|
-
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
1761
|
-
case "ends_with":
|
|
1762
|
-
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
1763
|
-
case "is_empty":
|
|
1764
|
-
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
1765
|
-
case "is_not_empty":
|
|
1766
|
-
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
1767
|
-
case "is_true":
|
|
1768
|
-
return Boolean(fieldValue) === true;
|
|
1769
|
-
case "is_false":
|
|
1770
|
-
return Boolean(fieldValue) === false;
|
|
1771
|
-
default:
|
|
1772
|
-
return false;
|
|
1773
|
-
}
|
|
1774
|
-
}
|
|
1775
|
-
function getNestedValue(obj, path) {
|
|
1776
|
-
const parts = path.split(".");
|
|
1777
|
-
let current = obj;
|
|
1778
|
-
for (const part of parts) {
|
|
1779
|
-
if (current === null || current === undefined || typeof current !== "object") {
|
|
1780
|
-
return;
|
|
1781
|
-
}
|
|
1782
|
-
current = current[part];
|
|
1783
|
-
}
|
|
1784
|
-
return current;
|
|
1785
|
-
}
|
|
1786
|
-
|
|
1787
1631
|
// src/task/Task.ts
|
|
1788
|
-
import { deepEqual, EventEmitter as EventEmitter3, uuid4 as
|
|
1632
|
+
import { deepEqual, EventEmitter as EventEmitter3, getLogger as getLogger2, uuid4 as uuid42 } from "@workglow/util";
|
|
1789
1633
|
import { compileSchema } from "@workglow/util/schema";
|
|
1790
1634
|
|
|
1791
1635
|
// src/task/TaskRunner.ts
|
|
1792
1636
|
import {
|
|
1793
|
-
getLogger
|
|
1637
|
+
getLogger,
|
|
1794
1638
|
getTelemetryProvider,
|
|
1795
1639
|
globalServiceRegistry as globalServiceRegistry2,
|
|
1796
1640
|
ResourceScope,
|
|
@@ -1799,7 +1643,6 @@ import {
|
|
|
1799
1643
|
|
|
1800
1644
|
// src/task/CacheCoordinator.ts
|
|
1801
1645
|
import { getPortCodec } from "@workglow/util";
|
|
1802
|
-
|
|
1803
1646
|
class CacheCoordinator {
|
|
1804
1647
|
task;
|
|
1805
1648
|
constructor(task) {
|
|
@@ -1809,7 +1652,9 @@ class CacheCoordinator {
|
|
|
1809
1652
|
if (!outputCache)
|
|
1810
1653
|
return inputs;
|
|
1811
1654
|
const inputSchema = this.task.constructor.inputSchema();
|
|
1812
|
-
|
|
1655
|
+
const normalized = await CacheCoordinator.normalizeInputsForCacheKey(inputs, inputSchema);
|
|
1656
|
+
normalized.__cv = this.task.getCacheVersion();
|
|
1657
|
+
return normalized;
|
|
1813
1658
|
}
|
|
1814
1659
|
async lookup(keyInputs, outputCache, isStreamable, ctx) {
|
|
1815
1660
|
if (!outputCache || !this.task.cacheable)
|
|
@@ -1837,6 +1682,20 @@ class CacheCoordinator {
|
|
|
1837
1682
|
const wireOutputs = await CacheCoordinator.serializeOutputPorts(output, outputSchema);
|
|
1838
1683
|
await outputCache.saveOutput(this.task.type, keyInputs, wireOutputs);
|
|
1839
1684
|
}
|
|
1685
|
+
repoFor(registry, policy) {
|
|
1686
|
+
if (!registry || !isPolicyCached(policy))
|
|
1687
|
+
return;
|
|
1688
|
+
return isPolicyPrivate(policy) ? registry.private : registry.deterministic;
|
|
1689
|
+
}
|
|
1690
|
+
async buildKeyForPolicy(inputs, registry, policy) {
|
|
1691
|
+
return this.buildKey(inputs, this.repoFor(registry, policy));
|
|
1692
|
+
}
|
|
1693
|
+
async lookupByPolicy(keyInputs, registry, policy, isStreamable, ctx) {
|
|
1694
|
+
return this.lookup(keyInputs, this.repoFor(registry, policy), isStreamable, ctx);
|
|
1695
|
+
}
|
|
1696
|
+
async saveByPolicy(keyInputs, output, registry, policy) {
|
|
1697
|
+
return this.save(keyInputs, output, this.repoFor(registry, policy));
|
|
1698
|
+
}
|
|
1840
1699
|
static async serializeOutputPorts(output, schema) {
|
|
1841
1700
|
if (!schema?.properties)
|
|
1842
1701
|
return output;
|
|
@@ -2072,12 +1931,15 @@ class TaskRunner {
|
|
|
2072
1931
|
task;
|
|
2073
1932
|
currentCtx;
|
|
2074
1933
|
outputCache;
|
|
1934
|
+
cacheRegistry;
|
|
2075
1935
|
cacheCoordinator;
|
|
2076
1936
|
streamProcessor;
|
|
2077
1937
|
registry = globalServiceRegistry2;
|
|
2078
1938
|
resourceScope;
|
|
2079
1939
|
ownsResourceScope = false;
|
|
2080
1940
|
inputStreams;
|
|
1941
|
+
runId;
|
|
1942
|
+
static __privateWithoutRunIdWarned = new Set;
|
|
2081
1943
|
constructor(task) {
|
|
2082
1944
|
this.task = task;
|
|
2083
1945
|
this.own = this.own.bind(this);
|
|
@@ -2090,7 +1952,7 @@ class TaskRunner {
|
|
|
2090
1952
|
throw new TaskConfigurationError(`Task "${this.task.type}" is already running. Concurrent run() calls on the same TaskRunner are not supported.`);
|
|
2091
1953
|
}
|
|
2092
1954
|
const ownsScope = config.resourceScope === undefined;
|
|
2093
|
-
const effectiveConfig = ownsScope ? { ...config, resourceScope: new ResourceScope } : config;
|
|
1955
|
+
const effectiveConfig = ownsScope ? { ...config, resourceScope: new ResourceScope({ strategy: config.disposeStrategy }) } : config;
|
|
2094
1956
|
this.ownsResourceScope = ownsScope;
|
|
2095
1957
|
try {
|
|
2096
1958
|
await this.handleStart(effectiveConfig);
|
|
@@ -2115,11 +1977,23 @@ class TaskRunner {
|
|
|
2115
1977
|
if (!isStreamable && typeof this.task.executeStream !== "function") {
|
|
2116
1978
|
const streamMode = getOutputStreamMode(this.task.outputSchema());
|
|
2117
1979
|
if (streamMode !== "none") {
|
|
2118
|
-
|
|
1980
|
+
getLogger().warn(`Task "${this.task.type}" declares streaming output (x-stream: "${streamMode}") ` + `but does not implement executeStream(). Falling back to non-streaming execute().`);
|
|
2119
1981
|
}
|
|
2120
1982
|
}
|
|
2121
|
-
|
|
2122
|
-
|
|
1983
|
+
let policy = this.task.getCachePolicy(inputs);
|
|
1984
|
+
if (policy.kind === "private" && !this.runId && this.cacheRegistry?.private !== undefined && !(this.cacheRegistry.private instanceof RunPrivateCacheRepo)) {
|
|
1985
|
+
const taskType = this.task.type;
|
|
1986
|
+
if (!TaskRunner.__privateWithoutRunIdWarned.has(taskType)) {
|
|
1987
|
+
TaskRunner.__privateWithoutRunIdWarned.add(taskType);
|
|
1988
|
+
getLogger().warn(`TaskRunner: task "${taskType}" has a private cache policy but no runId was ` + `provided. Private cache writes are skipped for this run — use TaskGraphRunner ` + `with runId for run-namespaced private caching, or provide an already namespaced ` + `private cache repo in CACHE_REGISTRY.`);
|
|
1989
|
+
}
|
|
1990
|
+
policy = { kind: "none" };
|
|
1991
|
+
}
|
|
1992
|
+
this.currentCtx?.telemetrySpan?.setAttributes({
|
|
1993
|
+
"workglow.task.cache_policy": policy.kind
|
|
1994
|
+
});
|
|
1995
|
+
const keyInputs = await this.cacheCoordinator.buildKeyForPolicy(inputs, this.cacheRegistry, policy);
|
|
1996
|
+
let outputs = await this.cacheCoordinator.lookupByPolicy(keyInputs, this.cacheRegistry, policy, isStreamable, ctx);
|
|
2123
1997
|
if (outputs === undefined) {
|
|
2124
1998
|
outputs = isStreamable ? await this.streamProcessor.run(inputs, ctx, {
|
|
2125
1999
|
registry: this.registry,
|
|
@@ -2128,7 +2002,7 @@ class TaskRunner {
|
|
|
2128
2002
|
onProgress: this.handleProgress.bind(this),
|
|
2129
2003
|
own: this.own
|
|
2130
2004
|
}) : await this.executeTask(inputs, ctx);
|
|
2131
|
-
await this.cacheCoordinator.
|
|
2005
|
+
await this.cacheCoordinator.saveByPolicy(keyInputs, outputs, this.cacheRegistry, policy);
|
|
2132
2006
|
this.task.runOutputData = outputs ?? {};
|
|
2133
2007
|
}
|
|
2134
2008
|
await this.handleComplete();
|
|
@@ -2139,7 +2013,7 @@ class TaskRunner {
|
|
|
2139
2013
|
}
|
|
2140
2014
|
} finally {
|
|
2141
2015
|
if (ownsScope) {
|
|
2142
|
-
await effectiveConfig.resourceScope.
|
|
2016
|
+
await effectiveConfig.resourceScope.runComplete();
|
|
2143
2017
|
this.resourceScope = undefined;
|
|
2144
2018
|
}
|
|
2145
2019
|
this.ownsResourceScope = false;
|
|
@@ -2165,7 +2039,7 @@ class TaskRunner {
|
|
|
2165
2039
|
}
|
|
2166
2040
|
await this.handleCompletePreview();
|
|
2167
2041
|
} catch (err) {
|
|
2168
|
-
|
|
2042
|
+
getLogger().debug("runPreview failed", { taskId: this.task.config?.id, error: err });
|
|
2169
2043
|
await this.handleErrorPreview();
|
|
2170
2044
|
} finally {
|
|
2171
2045
|
ctx.dispose();
|
|
@@ -2250,7 +2124,7 @@ class TaskRunner {
|
|
|
2250
2124
|
const out = await this.runPreview(overrides);
|
|
2251
2125
|
yield out;
|
|
2252
2126
|
} catch (err) {
|
|
2253
|
-
|
|
2127
|
+
getLogger().debug("runPreviewStream iteration failed", {
|
|
2254
2128
|
taskId: this.task.config?.id,
|
|
2255
2129
|
error: err
|
|
2256
2130
|
});
|
|
@@ -2293,7 +2167,8 @@ class TaskRunner {
|
|
|
2293
2167
|
updateProgress: this.handleProgress.bind(this),
|
|
2294
2168
|
own: this.own,
|
|
2295
2169
|
registry: this.registry,
|
|
2296
|
-
resourceScope: this.resourceScope
|
|
2170
|
+
resourceScope: this.resourceScope,
|
|
2171
|
+
runId: this.runId
|
|
2297
2172
|
});
|
|
2298
2173
|
return result;
|
|
2299
2174
|
}
|
|
@@ -2307,7 +2182,8 @@ class TaskRunner {
|
|
|
2307
2182
|
const resolved = await resolveSchemaInputs({ ...source }, configSchema, { registry: this.registry });
|
|
2308
2183
|
Object.assign(this.task.config, resolved);
|
|
2309
2184
|
}
|
|
2310
|
-
const
|
|
2185
|
+
const ctor = this.task.constructor;
|
|
2186
|
+
const schema = ctor.hasDynamicSchemas ? this.task.inputSchema() : ctor.inputSchema();
|
|
2311
2187
|
this.task.runInputData = await resolveSchemaInputs(this.task.runInputData, schema, { registry: this.registry });
|
|
2312
2188
|
}
|
|
2313
2189
|
async handleStart(config = {}) {
|
|
@@ -2320,25 +2196,35 @@ class TaskRunner {
|
|
|
2320
2196
|
ctx.abortController.signal.addEventListener("abort", () => {
|
|
2321
2197
|
this.handleAbort();
|
|
2322
2198
|
});
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2199
|
+
if (config.registry) {
|
|
2200
|
+
this.registry = config.registry;
|
|
2201
|
+
}
|
|
2202
|
+
this.runId = config.runId;
|
|
2203
|
+
const legacy = config.outputCache ?? this.task.runConfig?.outputCache;
|
|
2204
|
+
if (legacy === false) {
|
|
2205
|
+
this.cacheRegistry = undefined;
|
|
2206
|
+
this.outputCache = undefined;
|
|
2207
|
+
} else if (legacy instanceof TaskOutputRepository) {
|
|
2208
|
+
this.cacheRegistry = new DefaultCacheRegistry({ deterministic: legacy });
|
|
2209
|
+
this.outputCache = legacy;
|
|
2210
|
+
} else if (legacy === true) {
|
|
2211
|
+
const instance = globalServiceRegistry2.has(TASK_OUTPUT_REPOSITORY) ? globalServiceRegistry2.get(TASK_OUTPUT_REPOSITORY) : undefined;
|
|
2326
2212
|
this.outputCache = instance;
|
|
2327
|
-
|
|
2213
|
+
this.cacheRegistry = instance ? new DefaultCacheRegistry({ deterministic: instance }) : undefined;
|
|
2214
|
+
} else {
|
|
2328
2215
|
this.outputCache = undefined;
|
|
2329
|
-
|
|
2330
|
-
this.outputCache = cache;
|
|
2216
|
+
this.cacheRegistry = this.registry.has(CACHE_REGISTRY) ? this.registry.get(CACHE_REGISTRY) : undefined;
|
|
2331
2217
|
}
|
|
2332
2218
|
ctx.shouldAccumulate = config.shouldAccumulate !== false;
|
|
2333
2219
|
if (config.updateProgress) {
|
|
2334
2220
|
this.updateProgress = config.updateProgress;
|
|
2335
2221
|
}
|
|
2336
|
-
if (config.registry) {
|
|
2337
|
-
this.registry = config.registry;
|
|
2338
|
-
}
|
|
2339
2222
|
if (config.resourceScope) {
|
|
2340
2223
|
this.resourceScope = config.resourceScope;
|
|
2341
2224
|
}
|
|
2225
|
+
if (this.resourceScope) {
|
|
2226
|
+
await this.resourceScope.runStart();
|
|
2227
|
+
}
|
|
2342
2228
|
if (ctx.abortController.signal.aborted)
|
|
2343
2229
|
return;
|
|
2344
2230
|
const timeout = this.task.config.timeout;
|
|
@@ -2354,7 +2240,6 @@ class TaskRunner {
|
|
|
2354
2240
|
attributes: {
|
|
2355
2241
|
"workglow.task.type": this.task.type,
|
|
2356
2242
|
"workglow.task.id": String(this.task.config.id),
|
|
2357
|
-
"workglow.task.cacheable": this.task.cacheable,
|
|
2358
2243
|
"workglow.task.title": this.task.title || undefined
|
|
2359
2244
|
}
|
|
2360
2245
|
});
|
|
@@ -2485,12 +2370,15 @@ class Task {
|
|
|
2485
2370
|
static title = "";
|
|
2486
2371
|
static description = "";
|
|
2487
2372
|
static cacheable = true;
|
|
2373
|
+
static version = 1;
|
|
2374
|
+
static cachePolicy = { kind: "deterministic" };
|
|
2488
2375
|
static hasDynamicSchemas = false;
|
|
2489
2376
|
static passthroughInputsToOutputs = false;
|
|
2490
2377
|
static customizable = false;
|
|
2491
2378
|
static isGraphOutput = false;
|
|
2492
2379
|
static isPassthrough = false;
|
|
2493
2380
|
static hasDynamicEntitlements = false;
|
|
2381
|
+
static __cacheableDeprecationWarned = new Set;
|
|
2494
2382
|
static entitlements() {
|
|
2495
2383
|
return EMPTY_ENTITLEMENTS;
|
|
2496
2384
|
}
|
|
@@ -2569,7 +2457,39 @@ class Task {
|
|
|
2569
2457
|
return this.config?.description ?? this.constructor.description;
|
|
2570
2458
|
}
|
|
2571
2459
|
get cacheable() {
|
|
2572
|
-
|
|
2460
|
+
if (this.runConfig?.cacheable !== undefined)
|
|
2461
|
+
return this.runConfig.cacheable;
|
|
2462
|
+
if (this.config?.cacheable !== undefined)
|
|
2463
|
+
return this.config.cacheable;
|
|
2464
|
+
return this.getCachePolicy(this.runInputData ?? {}).kind !== "none";
|
|
2465
|
+
}
|
|
2466
|
+
getCacheVersion() {
|
|
2467
|
+
const versions = [];
|
|
2468
|
+
let ctor = this.constructor;
|
|
2469
|
+
while (ctor && ctor !== Function.prototype) {
|
|
2470
|
+
if (Object.prototype.hasOwnProperty.call(ctor, "version") && typeof ctor.version === "number") {
|
|
2471
|
+
versions.push(ctor.version);
|
|
2472
|
+
}
|
|
2473
|
+
ctor = Object.getPrototypeOf(ctor);
|
|
2474
|
+
}
|
|
2475
|
+
if (versions.length === 0)
|
|
2476
|
+
versions.push(1);
|
|
2477
|
+
return versions.join(".");
|
|
2478
|
+
}
|
|
2479
|
+
getCachePolicy(_inputs) {
|
|
2480
|
+
const ctor = this.constructor;
|
|
2481
|
+
const hasLegacyOverride = Object.prototype.hasOwnProperty.call(ctor, "cacheable") && ctor.cacheable === false;
|
|
2482
|
+
const hasPolicyOverride = Object.prototype.hasOwnProperty.call(ctor, "cachePolicy");
|
|
2483
|
+
if (hasLegacyOverride && !hasPolicyOverride) {
|
|
2484
|
+
if (!Task.__cacheableDeprecationWarned.has(ctor.type)) {
|
|
2485
|
+
Task.__cacheableDeprecationWarned.add(ctor.type);
|
|
2486
|
+
getLogger2().warn(`Task "${ctor.type}": static \`cacheable = false\` is deprecated. ` + `Use \`static cachePolicy: CachePolicy = { kind: "none" }\` instead.`);
|
|
2487
|
+
}
|
|
2488
|
+
return { kind: "none" };
|
|
2489
|
+
}
|
|
2490
|
+
if (this.runConfig?.cacheable === false || this.config?.cacheable === false)
|
|
2491
|
+
return { kind: "none" };
|
|
2492
|
+
return ctor.cachePolicy ?? DEFAULT_CACHE_POLICY;
|
|
2573
2493
|
}
|
|
2574
2494
|
defaults;
|
|
2575
2495
|
runInputData = {};
|
|
@@ -2604,7 +2524,7 @@ class Task {
|
|
|
2604
2524
|
...title ? { title } : {}
|
|
2605
2525
|
}, restConfig);
|
|
2606
2526
|
if (baseConfig.id === undefined) {
|
|
2607
|
-
baseConfig.id =
|
|
2527
|
+
baseConfig.id = uuid42();
|
|
2608
2528
|
}
|
|
2609
2529
|
this.config = this.validateAndApplyConfigDefaults(baseConfig);
|
|
2610
2530
|
try {
|
|
@@ -2613,6 +2533,10 @@ class Task {
|
|
|
2613
2533
|
this.originalConfig = undefined;
|
|
2614
2534
|
}
|
|
2615
2535
|
this.runConfig = runConfig;
|
|
2536
|
+
const inputSchema = this.constructor.inputSchema();
|
|
2537
|
+
if (inputSchema && typeof inputSchema !== "boolean" && inputSchema.properties && Object.prototype.hasOwnProperty.call(inputSchema.properties, "__cv")) {
|
|
2538
|
+
throw new TaskConfigurationError(`Task "${this.constructor.type}": input port name '__cv' is reserved ` + `for cache versioning. Rename the port to avoid collision with the cache key sentinel.`);
|
|
2539
|
+
}
|
|
2616
2540
|
}
|
|
2617
2541
|
getDefaultInputsFromStaticInputDefinitions() {
|
|
2618
2542
|
const schema = this.inputSchema();
|
|
@@ -2953,48 +2877,277 @@ class Task {
|
|
|
2953
2877
|
if (Object.keys(config).length > 0) {
|
|
2954
2878
|
base.config = config;
|
|
2955
2879
|
}
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2880
|
+
return this.stripSymbols(base);
|
|
2881
|
+
}
|
|
2882
|
+
toDependencyJSON(options) {
|
|
2883
|
+
const json = this.toJSON(options);
|
|
2884
|
+
return json;
|
|
2885
|
+
}
|
|
2886
|
+
hasChildren() {
|
|
2887
|
+
return this._subGraph !== undefined && this._subGraph !== null && this._subGraph.getTasks().length > 0;
|
|
2888
|
+
}
|
|
2889
|
+
_taskAddedListener = () => {
|
|
2890
|
+
this.emit("regenerate");
|
|
2891
|
+
};
|
|
2892
|
+
_subGraph = undefined;
|
|
2893
|
+
set subGraph(subGraph) {
|
|
2894
|
+
if (this._subGraph) {
|
|
2895
|
+
this._subGraph.off("task_added", this._taskAddedListener);
|
|
2896
|
+
}
|
|
2897
|
+
this._subGraph = subGraph;
|
|
2898
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2899
|
+
}
|
|
2900
|
+
get subGraph() {
|
|
2901
|
+
if (!this._subGraph) {
|
|
2902
|
+
this._subGraph = new TaskGraph;
|
|
2903
|
+
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2904
|
+
}
|
|
2905
|
+
return this._subGraph;
|
|
2906
|
+
}
|
|
2907
|
+
regenerateGraph() {
|
|
2908
|
+
if (this.hasChildren()) {
|
|
2909
|
+
for (const dataflow of this.subGraph.getDataflows()) {
|
|
2910
|
+
this.subGraph.removeDataflow(dataflow);
|
|
2911
|
+
}
|
|
2912
|
+
for (const child of this.subGraph.getTasks()) {
|
|
2913
|
+
this.subGraph.removeTask(child.id);
|
|
2914
|
+
}
|
|
2915
|
+
}
|
|
2916
|
+
this.events.emit("regenerate");
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
|
|
2920
|
+
// src/task-graph/EdgeMaterializer.ts
|
|
2921
|
+
import { getLogger as getLogger3 } from "@workglow/util";
|
|
2922
|
+
import { previewSource } from "@workglow/util/media";
|
|
2923
|
+
class EdgeMaterializer {
|
|
2924
|
+
graph;
|
|
2925
|
+
runner;
|
|
2926
|
+
constructor(graph, runner) {
|
|
2927
|
+
this.graph = graph;
|
|
2928
|
+
this.runner = runner;
|
|
2929
|
+
}
|
|
2930
|
+
filterInputForTask(task, input) {
|
|
2931
|
+
const sourceDataflows = this.graph.getSourceDataflows(task.id);
|
|
2932
|
+
const connectedInputs = new Set(sourceDataflows.map((df) => df.targetTaskPortId));
|
|
2933
|
+
const allPortsConnected = connectedInputs.has(DATAFLOW_ALL_PORTS);
|
|
2934
|
+
const filteredInput = {};
|
|
2935
|
+
for (const [key, value] of Object.entries(input)) {
|
|
2936
|
+
if (!connectedInputs.has(key) && !allPortsConnected) {
|
|
2937
|
+
filteredInput[key] = value;
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
return filteredInput;
|
|
2941
|
+
}
|
|
2942
|
+
copyInputFromEdgesToNode(task) {
|
|
2943
|
+
const dataflows = this.graph.getSourceDataflows(task.id);
|
|
2944
|
+
dataflows.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
2945
|
+
for (const dataflow of dataflows) {
|
|
2946
|
+
const live = dataflow.getCurrentValue();
|
|
2947
|
+
const port = dataflow.targetTaskPortId;
|
|
2948
|
+
let portData;
|
|
2949
|
+
if (port === DATAFLOW_ALL_PORTS) {
|
|
2950
|
+
portData = live;
|
|
2951
|
+
} else if (port === DATAFLOW_ERROR_PORT) {
|
|
2952
|
+
portData = { [DATAFLOW_ERROR_PORT]: dataflow.error };
|
|
2953
|
+
} else {
|
|
2954
|
+
portData = { [port]: live };
|
|
2955
|
+
}
|
|
2956
|
+
this.runner.addInputData(task, portData);
|
|
2957
|
+
}
|
|
2958
|
+
}
|
|
2959
|
+
async pushOutputFromNodeToEdges(node, results) {
|
|
2960
|
+
const dataflows = this.graph.getTargetDataflows(node.id);
|
|
2961
|
+
if (this.runner["previewRunning"] && Object.keys(results).length > 0) {
|
|
2962
|
+
for (const port of Object.keys(results)) {
|
|
2963
|
+
const value = results[port];
|
|
2964
|
+
if (EdgeMaterializer.isImageValueShape(value)) {
|
|
2965
|
+
results[port] = await previewSource(value);
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
}
|
|
2969
|
+
for (const dataflow of dataflows) {
|
|
2970
|
+
if (dataflow.stream !== undefined)
|
|
2971
|
+
continue;
|
|
2972
|
+
const registry = this.runner["registry"];
|
|
2973
|
+
const compatibility = dataflow.semanticallyCompatible(this.graph, dataflow, registry);
|
|
2974
|
+
if (compatibility === "static") {
|
|
2975
|
+
dataflow.setPortData(results);
|
|
2976
|
+
await dataflow.applyTransforms(registry);
|
|
2977
|
+
} else if (compatibility === "runtime") {
|
|
2978
|
+
const task = this.graph.getTask(dataflow.targetTaskId);
|
|
2979
|
+
const narrowed = await task.narrowInput({ ...results }, registry);
|
|
2980
|
+
dataflow.setPortData(narrowed);
|
|
2981
|
+
await dataflow.applyTransforms(registry);
|
|
2982
|
+
} else {
|
|
2983
|
+
const resultsKeys = Object.keys(results);
|
|
2984
|
+
if (resultsKeys.length > 0) {
|
|
2985
|
+
getLogger3().warn("pushOutputFromNodeToEdge not compatible, not setting port data", {
|
|
2986
|
+
dataflowId: dataflow.id,
|
|
2987
|
+
compatibility,
|
|
2988
|
+
resultsKeys
|
|
2989
|
+
});
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
}
|
|
2994
|
+
pushErrorFromNodeToEdges(node) {
|
|
2995
|
+
if (!node?.config?.id)
|
|
2996
|
+
return;
|
|
2997
|
+
this.graph.getTargetDataflows(node.id).forEach((dataflow) => {
|
|
2998
|
+
dataflow.error = node.error;
|
|
2999
|
+
});
|
|
3000
|
+
}
|
|
3001
|
+
hasErrorOutputEdges(task) {
|
|
3002
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3003
|
+
return dataflows.some((df) => df.sourceTaskPortId === DATAFLOW_ERROR_PORT);
|
|
3004
|
+
}
|
|
3005
|
+
pushErrorOutputToEdges(task) {
|
|
3006
|
+
const taskError = task.error;
|
|
3007
|
+
const errorData = {
|
|
3008
|
+
error: taskError?.message ?? "Unknown error",
|
|
3009
|
+
errorType: taskError?.constructor?.type ?? "TaskError"
|
|
3010
|
+
};
|
|
3011
|
+
const dataflows = this.graph.getTargetDataflows(task.id);
|
|
3012
|
+
for (const df of dataflows) {
|
|
3013
|
+
if (df.sourceTaskPortId === DATAFLOW_ERROR_PORT) {
|
|
3014
|
+
df.value = errorData;
|
|
3015
|
+
df.setStatus(TaskStatus.COMPLETED);
|
|
3016
|
+
} else {
|
|
3017
|
+
df.setStatus(TaskStatus.DISABLED);
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
this.runner["runScheduler"].propagateDisabledStatus(this.runner["currentCtx"]);
|
|
3021
|
+
}
|
|
3022
|
+
resetTask(graph, task, runId) {
|
|
3023
|
+
task.status = TaskStatus.PENDING;
|
|
3024
|
+
task.resetInputData();
|
|
3025
|
+
task.runOutputData = {};
|
|
3026
|
+
task.error = undefined;
|
|
3027
|
+
task.progress = 0;
|
|
3028
|
+
task.runConfig = { ...task.runConfig, runnerId: runId };
|
|
3029
|
+
this.runner["runScheduler"].pushStatusFromNodeToEdges(task, this.runner["currentCtx"], undefined, graph);
|
|
3030
|
+
if (task?.config?.id) {
|
|
3031
|
+
graph.getTargetDataflows(task.id).forEach((dataflow) => {
|
|
3032
|
+
dataflow.error = task.error;
|
|
3033
|
+
});
|
|
2959
3034
|
}
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
toDependencyJSON(options) {
|
|
2963
|
-
const json = this.toJSON(options);
|
|
2964
|
-
return json;
|
|
3035
|
+
task.emit("reset");
|
|
3036
|
+
task.emit("status", task.status);
|
|
2965
3037
|
}
|
|
2966
|
-
|
|
2967
|
-
|
|
3038
|
+
static isImageValueShape(v) {
|
|
3039
|
+
if (v === null || typeof v !== "object")
|
|
3040
|
+
return false;
|
|
3041
|
+
const o = v;
|
|
3042
|
+
return typeof o.width === "number" && typeof o.height === "number" && typeof o.previewScale === "number";
|
|
2968
3043
|
}
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
3044
|
+
}
|
|
3045
|
+
|
|
3046
|
+
// src/task-graph/RunContext.ts
|
|
3047
|
+
import { uuid4 as uuid43 } from "@workglow/util";
|
|
3048
|
+
|
|
3049
|
+
class RunContext {
|
|
3050
|
+
runId;
|
|
3051
|
+
abortController;
|
|
3052
|
+
inProgressTasks = new Map;
|
|
3053
|
+
inProgressFunctions = new Map;
|
|
3054
|
+
failedTaskErrors = new Map;
|
|
3055
|
+
telemetrySpan;
|
|
3056
|
+
graphTimeoutTimer;
|
|
3057
|
+
pendingGraphTimeoutError;
|
|
3058
|
+
activeEnforcer;
|
|
3059
|
+
parentSignalCleanup;
|
|
3060
|
+
constructor(parentSignal) {
|
|
3061
|
+
this.runId = uuid43();
|
|
3062
|
+
this.abortController = new AbortController;
|
|
3063
|
+
if (parentSignal) {
|
|
3064
|
+
const onParentAbort = () => this.abortController.abort();
|
|
3065
|
+
parentSignal.addEventListener("abort", onParentAbort, { once: true });
|
|
3066
|
+
this.parentSignalCleanup = () => parentSignal.removeEventListener("abort", onParentAbort);
|
|
3067
|
+
if (parentSignal.aborted) {
|
|
3068
|
+
this.parentSignalCleanup();
|
|
3069
|
+
this.parentSignalCleanup = undefined;
|
|
3070
|
+
this.abortController.abort();
|
|
3071
|
+
}
|
|
2976
3072
|
}
|
|
2977
|
-
this._subGraph = subGraph;
|
|
2978
|
-
this._subGraph.on("task_added", this._taskAddedListener);
|
|
2979
3073
|
}
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
3074
|
+
dispose() {
|
|
3075
|
+
this.parentSignalCleanup?.();
|
|
3076
|
+
this.parentSignalCleanup = undefined;
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
|
|
3080
|
+
// src/task-graph/RunScheduler.ts
|
|
3081
|
+
import { getLogger as getLogger5 } from "@workglow/util";
|
|
3082
|
+
|
|
3083
|
+
// src/task/ConditionalTask.ts
|
|
3084
|
+
import { getLogger as getLogger4 } from "@workglow/util";
|
|
3085
|
+
|
|
3086
|
+
// src/task/ConditionUtils.ts
|
|
3087
|
+
function evaluateCondition(fieldValue, operator, compareValue) {
|
|
3088
|
+
if (fieldValue === null || fieldValue === undefined) {
|
|
3089
|
+
switch (operator) {
|
|
3090
|
+
case "is_empty":
|
|
3091
|
+
return true;
|
|
3092
|
+
case "is_not_empty":
|
|
3093
|
+
return false;
|
|
3094
|
+
case "is_true":
|
|
3095
|
+
return false;
|
|
3096
|
+
case "is_false":
|
|
3097
|
+
return true;
|
|
3098
|
+
default:
|
|
3099
|
+
return false;
|
|
2984
3100
|
}
|
|
2985
|
-
return this._subGraph;
|
|
2986
3101
|
}
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
3102
|
+
const strValue = String(fieldValue);
|
|
3103
|
+
const numValue = Number(fieldValue);
|
|
3104
|
+
switch (operator) {
|
|
3105
|
+
case "equals":
|
|
3106
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3107
|
+
return numValue === Number(compareValue);
|
|
2991
3108
|
}
|
|
2992
|
-
|
|
2993
|
-
|
|
3109
|
+
return strValue === compareValue;
|
|
3110
|
+
case "not_equals":
|
|
3111
|
+
if (!isNaN(numValue) && !isNaN(Number(compareValue))) {
|
|
3112
|
+
return numValue !== Number(compareValue);
|
|
2994
3113
|
}
|
|
3114
|
+
return strValue !== compareValue;
|
|
3115
|
+
case "greater_than":
|
|
3116
|
+
return numValue > Number(compareValue);
|
|
3117
|
+
case "greater_or_equal":
|
|
3118
|
+
return numValue >= Number(compareValue);
|
|
3119
|
+
case "less_than":
|
|
3120
|
+
return numValue < Number(compareValue);
|
|
3121
|
+
case "less_or_equal":
|
|
3122
|
+
return numValue <= Number(compareValue);
|
|
3123
|
+
case "contains":
|
|
3124
|
+
return strValue.toLowerCase().includes(compareValue.toLowerCase());
|
|
3125
|
+
case "starts_with":
|
|
3126
|
+
return strValue.toLowerCase().startsWith(compareValue.toLowerCase());
|
|
3127
|
+
case "ends_with":
|
|
3128
|
+
return strValue.toLowerCase().endsWith(compareValue.toLowerCase());
|
|
3129
|
+
case "is_empty":
|
|
3130
|
+
return strValue === "" || Array.isArray(fieldValue) && fieldValue.length === 0;
|
|
3131
|
+
case "is_not_empty":
|
|
3132
|
+
return strValue !== "" && !(Array.isArray(fieldValue) && fieldValue.length === 0);
|
|
3133
|
+
case "is_true":
|
|
3134
|
+
return Boolean(fieldValue) === true;
|
|
3135
|
+
case "is_false":
|
|
3136
|
+
return Boolean(fieldValue) === false;
|
|
3137
|
+
default:
|
|
3138
|
+
return false;
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
function getNestedValue(obj, path) {
|
|
3142
|
+
const parts = path.split(".");
|
|
3143
|
+
let current = obj;
|
|
3144
|
+
for (const part of parts) {
|
|
3145
|
+
if (current === null || current === undefined || typeof current !== "object") {
|
|
3146
|
+
return;
|
|
2995
3147
|
}
|
|
2996
|
-
|
|
3148
|
+
current = current[part];
|
|
2997
3149
|
}
|
|
3150
|
+
return current;
|
|
2998
3151
|
}
|
|
2999
3152
|
|
|
3000
3153
|
// src/task/ConditionalTask.ts
|
|
@@ -3086,7 +3239,7 @@ class ConditionalTask extends Task {
|
|
|
3086
3239
|
}
|
|
3087
3240
|
}
|
|
3088
3241
|
} catch (error) {
|
|
3089
|
-
|
|
3242
|
+
getLogger4().error(`Condition evaluation failed for branch "${branch.id}":`, { error });
|
|
3090
3243
|
}
|
|
3091
3244
|
}
|
|
3092
3245
|
if (this.activeBranches.size === 0 && defaultBranch) {
|
|
@@ -3358,7 +3511,7 @@ class RunScheduler {
|
|
|
3358
3511
|
ctx.inProgressFunctions.set(Symbol(task.id), runAsync());
|
|
3359
3512
|
}
|
|
3360
3513
|
} catch (err) {
|
|
3361
|
-
|
|
3514
|
+
getLogger5().error("Error running graph", { error: err });
|
|
3362
3515
|
}
|
|
3363
3516
|
await Promise.allSettled(Array.from(ctx.inProgressTasks.values()));
|
|
3364
3517
|
await Promise.allSettled(Array.from(ctx.inProgressFunctions.values()));
|
|
@@ -3434,11 +3587,12 @@ class StreamPump {
|
|
|
3434
3587
|
task.on("stream_end", onStreamEnd);
|
|
3435
3588
|
try {
|
|
3436
3589
|
const results = await task.runner.run(input, {
|
|
3437
|
-
outputCache: options.
|
|
3590
|
+
outputCache: options.legacyCacheExplicitlyDisabled ? false : options.outputCache,
|
|
3438
3591
|
shouldAccumulate,
|
|
3439
3592
|
updateProgress: options.updateProgress,
|
|
3440
3593
|
registry: options.registry,
|
|
3441
|
-
resourceScope: options.resourceScope
|
|
3594
|
+
resourceScope: options.resourceScope,
|
|
3595
|
+
runId: options.runId
|
|
3442
3596
|
});
|
|
3443
3597
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3444
3598
|
return {
|
|
@@ -3707,10 +3861,14 @@ class TaskGraphRunner {
|
|
|
3707
3861
|
previewRunning = false;
|
|
3708
3862
|
graph;
|
|
3709
3863
|
outputCache;
|
|
3864
|
+
legacyCacheExplicitlyDisabled = false;
|
|
3710
3865
|
accumulateLeafOutputs = true;
|
|
3711
3866
|
registry = globalServiceRegistry3;
|
|
3712
3867
|
resourceScope;
|
|
3713
3868
|
currentCtx;
|
|
3869
|
+
runId;
|
|
3870
|
+
currentRunPrivate;
|
|
3871
|
+
baseRegistryForRun;
|
|
3714
3872
|
edgeMaterializer;
|
|
3715
3873
|
streamPump;
|
|
3716
3874
|
runScheduler;
|
|
@@ -3727,7 +3885,10 @@ class TaskGraphRunner {
|
|
|
3727
3885
|
}
|
|
3728
3886
|
async runGraph(input = {}, config) {
|
|
3729
3887
|
const ownsScope = config?.resourceScope === undefined;
|
|
3730
|
-
const effectiveConfig = ownsScope ? {
|
|
3888
|
+
const effectiveConfig = ownsScope ? {
|
|
3889
|
+
...config,
|
|
3890
|
+
resourceScope: new ResourceScope2({ strategy: config?.disposeStrategy })
|
|
3891
|
+
} : config;
|
|
3731
3892
|
try {
|
|
3732
3893
|
await this.handleStart(effectiveConfig);
|
|
3733
3894
|
const ctx = this.currentCtx;
|
|
@@ -3747,10 +3908,19 @@ class TaskGraphRunner {
|
|
|
3747
3908
|
throw new TaskAbortedError;
|
|
3748
3909
|
}
|
|
3749
3910
|
await this.handleComplete();
|
|
3911
|
+
const runPrivateToClean = this.currentRunPrivate;
|
|
3912
|
+
this.currentRunPrivate = undefined;
|
|
3913
|
+
if (runPrivateToClean) {
|
|
3914
|
+
try {
|
|
3915
|
+
await runPrivateToClean.clearRun();
|
|
3916
|
+
} catch (e) {
|
|
3917
|
+
getLogger6().warn("RunPrivateCacheRepo.clearRun failed", { error: e });
|
|
3918
|
+
}
|
|
3919
|
+
}
|
|
3750
3920
|
return this.filterLeafResults(results);
|
|
3751
3921
|
} finally {
|
|
3752
3922
|
if (ownsScope) {
|
|
3753
|
-
await effectiveConfig.resourceScope.
|
|
3923
|
+
await effectiveConfig.resourceScope.runComplete();
|
|
3754
3924
|
this.resourceScope = undefined;
|
|
3755
3925
|
}
|
|
3756
3926
|
}
|
|
@@ -3818,7 +3988,7 @@ class TaskGraphRunner {
|
|
|
3818
3988
|
});
|
|
3819
3989
|
previewSpan.setStatus(SpanStatusCode2.OK);
|
|
3820
3990
|
previewSpan.end();
|
|
3821
|
-
|
|
3991
|
+
getLogger6().debug("task graph runPreview timings", {
|
|
3822
3992
|
previewRunId,
|
|
3823
3993
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3824
3994
|
taskTimings
|
|
@@ -3836,7 +4006,7 @@ class TaskGraphRunner {
|
|
|
3836
4006
|
});
|
|
3837
4007
|
previewSpan.setStatus(SpanStatusCode2.ERROR, message);
|
|
3838
4008
|
previewSpan.end();
|
|
3839
|
-
|
|
4009
|
+
getLogger6().debug("task graph runPreview failed", {
|
|
3840
4010
|
previewRunId,
|
|
3841
4011
|
totalMs: Math.round(totalMs * 1000) / 1000,
|
|
3842
4012
|
taskTimings,
|
|
@@ -3907,14 +4077,17 @@ class TaskGraphRunner {
|
|
|
3907
4077
|
outputCache: this.outputCache,
|
|
3908
4078
|
resourceScope: this.resourceScope,
|
|
3909
4079
|
accumulateLeafOutputs: this.accumulateLeafOutputs,
|
|
3910
|
-
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a)
|
|
4080
|
+
updateProgress: (t, p, m, ...a) => this.runScheduler.handleProgress(this.currentCtx, t, p, m, ...a),
|
|
4081
|
+
runId: this.runId,
|
|
4082
|
+
legacyCacheExplicitlyDisabled: this.legacyCacheExplicitlyDisabled
|
|
3911
4083
|
});
|
|
3912
4084
|
}
|
|
3913
4085
|
const results = await task.runner.run(input, {
|
|
3914
|
-
outputCache: this.
|
|
4086
|
+
outputCache: this.legacyCacheExplicitlyDisabled ? false : this.outputCache,
|
|
3915
4087
|
updateProgress: async (task2, progress, message, ...args) => await this.runScheduler.handleProgress(this.currentCtx, task2, progress, message, ...args),
|
|
3916
4088
|
registry: this.registry,
|
|
3917
|
-
resourceScope: this.resourceScope
|
|
4089
|
+
resourceScope: this.resourceScope,
|
|
4090
|
+
runId: this.runId
|
|
3918
4091
|
});
|
|
3919
4092
|
await this.edgeMaterializer.pushOutputFromNodeToEdges(task, results);
|
|
3920
4093
|
return {
|
|
@@ -3935,6 +4108,19 @@ class TaskGraphRunner {
|
|
|
3935
4108
|
dataflow.reset();
|
|
3936
4109
|
});
|
|
3937
4110
|
}
|
|
4111
|
+
static __durabilityWarnedRepos = new WeakSet;
|
|
4112
|
+
static graphUsesPrivatePolicy(graph) {
|
|
4113
|
+
return graph.getTasks().some((t) => {
|
|
4114
|
+
const ctor = t.constructor;
|
|
4115
|
+
if (ctor.cachePolicy?.kind === "private")
|
|
4116
|
+
return true;
|
|
4117
|
+
const proto = ctor.prototype;
|
|
4118
|
+
if (typeof proto.getCachePolicy === "function" && proto.getCachePolicy !== Task.prototype.getCachePolicy) {
|
|
4119
|
+
return true;
|
|
4120
|
+
}
|
|
4121
|
+
return false;
|
|
4122
|
+
});
|
|
4123
|
+
}
|
|
3938
4124
|
async handleStart(config) {
|
|
3939
4125
|
if (config?.registry !== undefined) {
|
|
3940
4126
|
this.registry = config.registry;
|
|
@@ -3944,18 +4130,61 @@ class TaskGraphRunner {
|
|
|
3944
4130
|
if (config?.resourceScope !== undefined) {
|
|
3945
4131
|
this.resourceScope = config.resourceScope;
|
|
3946
4132
|
}
|
|
4133
|
+
this.baseRegistryForRun = this.registry;
|
|
4134
|
+
this.runId = config?.runId;
|
|
4135
|
+
if (this.registry.has(CACHE_REGISTRY)) {
|
|
4136
|
+
const checkRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4137
|
+
if (checkRegistry.private && !checkRegistry.private.isDurable()) {
|
|
4138
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4139
|
+
const repo = checkRegistry.private;
|
|
4140
|
+
if (!TaskGraphRunner.__durabilityWarnedRepos.has(repo)) {
|
|
4141
|
+
TaskGraphRunner.__durabilityWarnedRepos.add(repo);
|
|
4142
|
+
getLogger6().warn("TaskGraphRunner: private cache repo may be used but is non-durable — " + "restart-survival will not work. Ensure the CacheRegistry 'private' " + "slot is backed by a durable storage backend.");
|
|
4143
|
+
}
|
|
4144
|
+
}
|
|
4145
|
+
}
|
|
4146
|
+
if (!this.runId && checkRegistry.private) {
|
|
4147
|
+
if (TaskGraphRunner.graphUsesPrivatePolicy(this.graph)) {
|
|
4148
|
+
throw new TaskConfigurationError("TaskGraphRunner: graph contains a private-policy task but no runId was provided. " + "Provide `runId` in TaskGraphRunConfig so private cache entries can be namespaced.");
|
|
4149
|
+
}
|
|
4150
|
+
}
|
|
4151
|
+
}
|
|
4152
|
+
if (this.runId && this.registry.has(CACHE_REGISTRY)) {
|
|
4153
|
+
const baseRegistry = this.registry.get(CACHE_REGISTRY);
|
|
4154
|
+
if (baseRegistry.private) {
|
|
4155
|
+
const wrappedPrivate = new RunPrivateCacheRepo({
|
|
4156
|
+
backing: baseRegistry.private,
|
|
4157
|
+
runId: this.runId
|
|
4158
|
+
});
|
|
4159
|
+
this.currentRunPrivate = wrappedPrivate;
|
|
4160
|
+
const wrappedRegistry = new DefaultCacheRegistry({
|
|
4161
|
+
deterministic: baseRegistry.deterministic,
|
|
4162
|
+
private: wrappedPrivate
|
|
4163
|
+
});
|
|
4164
|
+
const childContainer = this.registry.container.createChildContainer();
|
|
4165
|
+
const runtimeServices = new ServiceRegistry3(childContainer);
|
|
4166
|
+
runtimeServices.registerInstance(CACHE_REGISTRY, wrappedRegistry);
|
|
4167
|
+
this.registry = runtimeServices;
|
|
4168
|
+
this.resourceScope?.register(`cache:private:${this.runId}`, async () => {});
|
|
4169
|
+
}
|
|
4170
|
+
}
|
|
3947
4171
|
this.accumulateLeafOutputs = config?.accumulateLeafOutputs !== false;
|
|
3948
4172
|
if (config?.outputCache !== undefined) {
|
|
3949
4173
|
if (typeof config.outputCache === "boolean") {
|
|
3950
4174
|
if (config.outputCache === true) {
|
|
3951
4175
|
this.outputCache = this.registry.get(TASK_OUTPUT_REPOSITORY);
|
|
4176
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3952
4177
|
} else {
|
|
3953
4178
|
this.outputCache = undefined;
|
|
4179
|
+
this.legacyCacheExplicitlyDisabled = true;
|
|
3954
4180
|
}
|
|
3955
4181
|
} else {
|
|
3956
4182
|
this.outputCache = config.outputCache;
|
|
4183
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3957
4184
|
}
|
|
3958
4185
|
this.graph.outputCache = this.outputCache;
|
|
4186
|
+
} else {
|
|
4187
|
+
this.legacyCacheExplicitlyDisabled = false;
|
|
3959
4188
|
}
|
|
3960
4189
|
if (this.running || this.previewRunning) {
|
|
3961
4190
|
throw new TaskConfigurationError("Graph is already running");
|
|
@@ -3969,6 +4198,9 @@ class TaskGraphRunner {
|
|
|
3969
4198
|
if (config?.timeout !== undefined) {
|
|
3970
4199
|
this.runScheduler.armGraphTimeout(config.timeout, ctx);
|
|
3971
4200
|
}
|
|
4201
|
+
if (this.resourceScope) {
|
|
4202
|
+
await this.resourceScope.runStart();
|
|
4203
|
+
}
|
|
3972
4204
|
if (ctx.abortController.signal.aborted)
|
|
3973
4205
|
return;
|
|
3974
4206
|
this.resetGraph(this.graph, ctx.runId);
|
|
@@ -4041,6 +4273,10 @@ class TaskGraphRunner {
|
|
|
4041
4273
|
}
|
|
4042
4274
|
ctx?.dispose();
|
|
4043
4275
|
this.currentCtx = undefined;
|
|
4276
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4277
|
+
this.registry = this.baseRegistryForRun;
|
|
4278
|
+
this.baseRegistryForRun = undefined;
|
|
4279
|
+
}
|
|
4044
4280
|
this.graph.emit("complete");
|
|
4045
4281
|
}
|
|
4046
4282
|
async handleCompletePreview() {
|
|
@@ -4053,6 +4289,7 @@ class TaskGraphRunner {
|
|
|
4053
4289
|
return task.abort();
|
|
4054
4290
|
}
|
|
4055
4291
|
}));
|
|
4292
|
+
this.currentRunPrivate = undefined;
|
|
4056
4293
|
const ctx = this.currentCtx;
|
|
4057
4294
|
this.running = false;
|
|
4058
4295
|
if (ctx?.telemetrySpan) {
|
|
@@ -4062,6 +4299,10 @@ class TaskGraphRunner {
|
|
|
4062
4299
|
}
|
|
4063
4300
|
ctx?.dispose();
|
|
4064
4301
|
this.currentCtx = undefined;
|
|
4302
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4303
|
+
this.registry = this.baseRegistryForRun;
|
|
4304
|
+
this.baseRegistryForRun = undefined;
|
|
4305
|
+
}
|
|
4065
4306
|
this.graph.emit("error", error);
|
|
4066
4307
|
}
|
|
4067
4308
|
async handleErrorPreview() {
|
|
@@ -4077,6 +4318,7 @@ class TaskGraphRunner {
|
|
|
4077
4318
|
return task.abort();
|
|
4078
4319
|
}
|
|
4079
4320
|
}));
|
|
4321
|
+
this.currentRunPrivate = undefined;
|
|
4080
4322
|
const ctx = this.currentCtx;
|
|
4081
4323
|
if (ctx?.telemetrySpan) {
|
|
4082
4324
|
ctx.telemetrySpan.setStatus(SpanStatusCode2.ERROR, "aborted");
|
|
@@ -4085,6 +4327,10 @@ class TaskGraphRunner {
|
|
|
4085
4327
|
}
|
|
4086
4328
|
ctx?.dispose();
|
|
4087
4329
|
this.currentCtx = undefined;
|
|
4330
|
+
if (this.baseRegistryForRun !== undefined) {
|
|
4331
|
+
this.registry = this.baseRegistryForRun;
|
|
4332
|
+
this.baseRegistryForRun = undefined;
|
|
4333
|
+
}
|
|
4088
4334
|
this.graph.emit("abort");
|
|
4089
4335
|
}
|
|
4090
4336
|
async handleAbortPreview() {
|
|
@@ -4208,7 +4454,7 @@ class GraphAsTask extends Task {
|
|
|
4208
4454
|
const schemaNode = Task.generateInputSchemaNode(dataPortSchema);
|
|
4209
4455
|
this._inputSchemaNode = schemaNode;
|
|
4210
4456
|
} catch (error) {
|
|
4211
|
-
|
|
4457
|
+
getLogger7().warn(`GraphAsTask "${this.type}" (${this.id}): Failed to compile input schema, ` + `falling back to permissive validation. Inputs will NOT be validated.`, { error, taskType: this.type, taskId: this.id });
|
|
4212
4458
|
this._inputSchemaNode = compileSchema2({});
|
|
4213
4459
|
}
|
|
4214
4460
|
}
|
|
@@ -4492,7 +4738,7 @@ function convertPipeFunctionToTask(fn, config) {
|
|
|
4492
4738
|
additionalProperties: true
|
|
4493
4739
|
};
|
|
4494
4740
|
};
|
|
4495
|
-
static
|
|
4741
|
+
static cachePolicy = { kind: "none" };
|
|
4496
4742
|
async execute(input, context) {
|
|
4497
4743
|
return fn(input, context);
|
|
4498
4744
|
}
|
|
@@ -4574,7 +4820,9 @@ class TaskGraph {
|
|
|
4574
4820
|
registry: config?.registry,
|
|
4575
4821
|
timeout: config?.timeout,
|
|
4576
4822
|
maxTasks: config?.maxTasks,
|
|
4577
|
-
resourceScope: config?.resourceScope
|
|
4823
|
+
resourceScope: config?.resourceScope,
|
|
4824
|
+
disposeStrategy: config?.disposeStrategy,
|
|
4825
|
+
runId: config?.runId
|
|
4578
4826
|
});
|
|
4579
4827
|
}
|
|
4580
4828
|
runPreview(input = {}, config = {}) {
|
|
@@ -5123,7 +5371,7 @@ function autoConnect(graph, sourceTask, targetTask, options) {
|
|
|
5123
5371
|
}
|
|
5124
5372
|
|
|
5125
5373
|
// src/task-graph/LoopBuilderContext.ts
|
|
5126
|
-
import { getLogger as
|
|
5374
|
+
import { getLogger as getLogger8 } from "@workglow/util";
|
|
5127
5375
|
function runLoopAutoConnect(parentGraph, pending) {
|
|
5128
5376
|
const { parent, iteratorTask } = pending;
|
|
5129
5377
|
if (parentGraph.getTargetDataflows(parent.id).length !== 0)
|
|
@@ -5137,7 +5385,7 @@ function runLoopAutoConnect(parentGraph, pending) {
|
|
|
5137
5385
|
const result = autoConnect(parentGraph, parent, iteratorTask, { earlierTasks });
|
|
5138
5386
|
if (result.error) {
|
|
5139
5387
|
const message = result.error + " Task not added.";
|
|
5140
|
-
|
|
5388
|
+
getLogger8().error(message);
|
|
5141
5389
|
parentGraph.removeTask(iteratorTask.id);
|
|
5142
5390
|
return message;
|
|
5143
5391
|
}
|
|
@@ -5176,7 +5424,7 @@ class LoopBuilderContext {
|
|
|
5176
5424
|
}
|
|
5177
5425
|
|
|
5178
5426
|
// src/task-graph/WorkflowBuilder.ts
|
|
5179
|
-
import { getLogger as
|
|
5427
|
+
import { getLogger as getLogger9, uuid4 as uuid47 } from "@workglow/util";
|
|
5180
5428
|
|
|
5181
5429
|
// src/task-graph/ConditionalBuilder.ts
|
|
5182
5430
|
import { uuid4 as uuid46 } from "@workglow/util";
|
|
@@ -5335,7 +5583,7 @@ class WorkflowBuilder {
|
|
|
5335
5583
|
const taskSchema = task.inputSchema();
|
|
5336
5584
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5337
5585
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5338
|
-
|
|
5586
|
+
getLogger9().error(this._error);
|
|
5339
5587
|
return;
|
|
5340
5588
|
}
|
|
5341
5589
|
dataflow.targetTaskId = task.id;
|
|
@@ -5360,10 +5608,10 @@ class WorkflowBuilder {
|
|
|
5360
5608
|
if (result.error) {
|
|
5361
5609
|
if (this._loopContext !== undefined) {
|
|
5362
5610
|
this._error = result.error;
|
|
5363
|
-
|
|
5611
|
+
getLogger9().warn(this._error);
|
|
5364
5612
|
} else {
|
|
5365
5613
|
this._error = result.error + " Task not added.";
|
|
5366
|
-
|
|
5614
|
+
getLogger9().error(this._error);
|
|
5367
5615
|
this._facade.graph.removeTask(task.id);
|
|
5368
5616
|
}
|
|
5369
5617
|
}
|
|
@@ -5386,7 +5634,7 @@ class WorkflowBuilder {
|
|
|
5386
5634
|
const taskSchema = task.inputSchema();
|
|
5387
5635
|
if (typeof taskSchema !== "boolean" && taskSchema.properties?.[dataflow.targetTaskPortId] === undefined && taskSchema.additionalProperties !== true || taskSchema === true && dataflow.targetTaskPortId !== DATAFLOW_ALL_PORTS) {
|
|
5388
5636
|
this._error = `Input ${dataflow.targetTaskPortId} not found on task ${task.id}`;
|
|
5389
|
-
|
|
5637
|
+
getLogger9().error(this._error);
|
|
5390
5638
|
return;
|
|
5391
5639
|
}
|
|
5392
5640
|
dataflow.targetTaskId = task.id;
|
|
@@ -5438,7 +5686,7 @@ class WorkflowBuilder {
|
|
|
5438
5686
|
if (-index > nodes.length) {
|
|
5439
5687
|
const errorMsg = `Back index greater than number of tasks`;
|
|
5440
5688
|
this._error = errorMsg;
|
|
5441
|
-
|
|
5689
|
+
getLogger9().error(this._error);
|
|
5442
5690
|
throw new WorkflowError(errorMsg);
|
|
5443
5691
|
}
|
|
5444
5692
|
const lastNode = nodes[nodes.length + index];
|
|
@@ -5447,13 +5695,13 @@ class WorkflowBuilder {
|
|
|
5447
5695
|
if (outputSchema === false && source !== DATAFLOW_ALL_PORTS) {
|
|
5448
5696
|
const errorMsg = `Task ${lastNode.id} has schema 'false' and outputs nothing`;
|
|
5449
5697
|
this._error = errorMsg;
|
|
5450
|
-
|
|
5698
|
+
getLogger9().error(this._error);
|
|
5451
5699
|
throw new WorkflowError(errorMsg);
|
|
5452
5700
|
}
|
|
5453
5701
|
} else if (!outputSchema.properties?.[source] && source !== DATAFLOW_ALL_PORTS) {
|
|
5454
5702
|
const errorMsg = `Output ${source} not found on task ${lastNode.id}`;
|
|
5455
5703
|
this._error = errorMsg;
|
|
5456
|
-
|
|
5704
|
+
getLogger9().error(this._error);
|
|
5457
5705
|
throw new WorkflowError(errorMsg);
|
|
5458
5706
|
}
|
|
5459
5707
|
const df = new Dataflow(lastNode.id, source, undefined, target);
|
|
@@ -5466,7 +5714,7 @@ class WorkflowBuilder {
|
|
|
5466
5714
|
const parent = getLastTask(this._facade);
|
|
5467
5715
|
if (!parent) {
|
|
5468
5716
|
this._error = "onError() requires a preceding task in the workflow";
|
|
5469
|
-
|
|
5717
|
+
getLogger9().error(this._error);
|
|
5470
5718
|
throw new WorkflowError(this._error);
|
|
5471
5719
|
}
|
|
5472
5720
|
const handlerTask = ensureTask(handler);
|
|
@@ -5480,7 +5728,7 @@ class WorkflowBuilder {
|
|
|
5480
5728
|
const nodes = this._facade.graph.getTasks();
|
|
5481
5729
|
if (nodes.length === 0) {
|
|
5482
5730
|
this._error = "No tasks to remove";
|
|
5483
|
-
|
|
5731
|
+
getLogger9().error(this._error);
|
|
5484
5732
|
return;
|
|
5485
5733
|
}
|
|
5486
5734
|
const lastNode = nodes[nodes.length - 1];
|
|
@@ -5550,19 +5798,6 @@ class WorkflowEventBridge {
|
|
|
5550
5798
|
}
|
|
5551
5799
|
}
|
|
5552
5800
|
|
|
5553
|
-
// src/task-graph/WorkflowRunContext.ts
|
|
5554
|
-
class WorkflowRunContext {
|
|
5555
|
-
abortController;
|
|
5556
|
-
unsubStreaming;
|
|
5557
|
-
constructor() {
|
|
5558
|
-
this.abortController = new AbortController;
|
|
5559
|
-
}
|
|
5560
|
-
dispose() {
|
|
5561
|
-
this.unsubStreaming?.();
|
|
5562
|
-
this.unsubStreaming = undefined;
|
|
5563
|
-
}
|
|
5564
|
-
}
|
|
5565
|
-
|
|
5566
5801
|
// src/task-graph/WorkflowFactories.ts
|
|
5567
5802
|
function CreateWorkflow(taskClass) {
|
|
5568
5803
|
return Workflow.createWorkflow(taskClass);
|
|
@@ -5593,6 +5828,19 @@ function CreateAdaptiveWorkflow(scalarClass, vectorClass) {
|
|
|
5593
5828
|
};
|
|
5594
5829
|
}
|
|
5595
5830
|
|
|
5831
|
+
// src/task-graph/WorkflowRunContext.ts
|
|
5832
|
+
class WorkflowRunContext {
|
|
5833
|
+
abortController;
|
|
5834
|
+
unsubStreaming;
|
|
5835
|
+
constructor() {
|
|
5836
|
+
this.abortController = new AbortController;
|
|
5837
|
+
}
|
|
5838
|
+
dispose() {
|
|
5839
|
+
this.unsubStreaming?.();
|
|
5840
|
+
this.unsubStreaming = undefined;
|
|
5841
|
+
}
|
|
5842
|
+
}
|
|
5843
|
+
|
|
5596
5844
|
// src/task-graph/WorkflowTask.ts
|
|
5597
5845
|
class WorkflowTask extends GraphAsTask {
|
|
5598
5846
|
static type = "Workflow";
|
|
@@ -6140,112 +6388,11 @@ ${entries.map((e) => `${entryIndent}${e}`).join(`,
|
|
|
6140
6388
|
`)}
|
|
6141
6389
|
${baseIndent}}`;
|
|
6142
6390
|
}
|
|
6143
|
-
return String(value);
|
|
6144
|
-
}
|
|
6145
|
-
function resetMethodNameCache() {
|
|
6146
|
-
methodNameCache = undefined;
|
|
6147
|
-
}
|
|
6148
|
-
// src/task-graph/transforms/pick.ts
|
|
6149
|
-
import { areSemanticallyCompatible as areSemanticallyCompatible2 } from "@workglow/util/schema";
|
|
6150
|
-
function walk(value, path) {
|
|
6151
|
-
if (value == null)
|
|
6152
|
-
return;
|
|
6153
|
-
const parts = path.split(".");
|
|
6154
|
-
let cur = value;
|
|
6155
|
-
for (const p of parts) {
|
|
6156
|
-
if (cur == null)
|
|
6157
|
-
return;
|
|
6158
|
-
cur = cur[p];
|
|
6159
|
-
}
|
|
6160
|
-
return cur;
|
|
6161
|
-
}
|
|
6162
|
-
function walkSchema(schema, path) {
|
|
6163
|
-
const parts = path.split(".");
|
|
6164
|
-
let cur = schema;
|
|
6165
|
-
for (const p of parts) {
|
|
6166
|
-
if (!cur || typeof cur !== "object")
|
|
6167
|
-
return {};
|
|
6168
|
-
if (cur.type !== "object" || !cur.properties || !cur.properties[p]) {
|
|
6169
|
-
return {};
|
|
6170
|
-
}
|
|
6171
|
-
cur = cur.properties[p];
|
|
6172
|
-
}
|
|
6173
|
-
return cur;
|
|
6174
|
-
}
|
|
6175
|
-
var pickTransform = {
|
|
6176
|
-
id: "pick",
|
|
6177
|
-
title: "Pick field",
|
|
6178
|
-
category: "Structural",
|
|
6179
|
-
paramsSchema: {
|
|
6180
|
-
type: "object",
|
|
6181
|
-
properties: {
|
|
6182
|
-
path: { type: "string", description: "Dotted property path" }
|
|
6183
|
-
},
|
|
6184
|
-
required: ["path"]
|
|
6185
|
-
},
|
|
6186
|
-
inferOutputSchema(inputSchema, params) {
|
|
6187
|
-
return walkSchema(inputSchema, params.path);
|
|
6188
|
-
},
|
|
6189
|
-
apply(value, params) {
|
|
6190
|
-
return walk(value, params.path);
|
|
6191
|
-
},
|
|
6192
|
-
suggestFromSchemas(source, target) {
|
|
6193
|
-
if (source.type !== "object" || !source.properties) {
|
|
6194
|
-
return;
|
|
6195
|
-
}
|
|
6196
|
-
const props = source.properties;
|
|
6197
|
-
for (const [name, propSchema] of Object.entries(props)) {
|
|
6198
|
-
const compat = areSemanticallyCompatible2(propSchema, target);
|
|
6199
|
-
if (compat === "static")
|
|
6200
|
-
return { score: 1, params: { path: name } };
|
|
6201
|
-
if (compat === "runtime")
|
|
6202
|
-
return { score: 0.6, params: { path: name } };
|
|
6203
|
-
}
|
|
6204
|
-
return;
|
|
6205
|
-
}
|
|
6206
|
-
};
|
|
6207
|
-
|
|
6208
|
-
// src/task-graph/transforms/index-access.ts
|
|
6209
|
-
import { areSemanticallyCompatible as areSemanticallyCompatible3 } from "@workglow/util/schema";
|
|
6210
|
-
function doIndex(value, idx) {
|
|
6211
|
-
if (!Array.isArray(value))
|
|
6212
|
-
return;
|
|
6213
|
-
const i = idx < 0 ? value.length + idx : idx;
|
|
6214
|
-
return value[i];
|
|
6391
|
+
return String(value);
|
|
6392
|
+
}
|
|
6393
|
+
function resetMethodNameCache() {
|
|
6394
|
+
methodNameCache = undefined;
|
|
6215
6395
|
}
|
|
6216
|
-
var indexTransform = {
|
|
6217
|
-
id: "index",
|
|
6218
|
-
title: "Array index",
|
|
6219
|
-
category: "Structural",
|
|
6220
|
-
paramsSchema: {
|
|
6221
|
-
type: "object",
|
|
6222
|
-
properties: {
|
|
6223
|
-
index: { type: "integer", description: "Array index (negative counts from end)" }
|
|
6224
|
-
},
|
|
6225
|
-
required: ["index"]
|
|
6226
|
-
},
|
|
6227
|
-
inferOutputSchema(inputSchema) {
|
|
6228
|
-
const s = inputSchema;
|
|
6229
|
-
if (s?.type === "array" && s.items)
|
|
6230
|
-
return s.items;
|
|
6231
|
-
return {};
|
|
6232
|
-
},
|
|
6233
|
-
apply(value, params) {
|
|
6234
|
-
return doIndex(value, params.index);
|
|
6235
|
-
},
|
|
6236
|
-
suggestFromSchemas(source, target) {
|
|
6237
|
-
const s = source;
|
|
6238
|
-
if (s?.type !== "array" || !s.items)
|
|
6239
|
-
return;
|
|
6240
|
-
const compat = areSemanticallyCompatible3(s.items, target);
|
|
6241
|
-
if (compat === "static")
|
|
6242
|
-
return { score: 0.9, params: { index: 0 } };
|
|
6243
|
-
if (compat === "runtime")
|
|
6244
|
-
return { score: 0.5, params: { index: 0 } };
|
|
6245
|
-
return;
|
|
6246
|
-
}
|
|
6247
|
-
};
|
|
6248
|
-
|
|
6249
6396
|
// src/task-graph/transforms/coalesce.ts
|
|
6250
6397
|
function stripNullable(schema) {
|
|
6251
6398
|
const s = schema;
|
|
@@ -6272,52 +6419,6 @@ var coalesceTransform = {
|
|
|
6272
6419
|
apply: (v, { defaultValue }) => v == null ? defaultValue : v
|
|
6273
6420
|
};
|
|
6274
6421
|
|
|
6275
|
-
// src/task-graph/transforms/string-casts.ts
|
|
6276
|
-
var stringSchema = { type: "string" };
|
|
6277
|
-
var uppercaseTransform = {
|
|
6278
|
-
id: "uppercase",
|
|
6279
|
-
title: "Uppercase",
|
|
6280
|
-
category: "String",
|
|
6281
|
-
paramsSchema: undefined,
|
|
6282
|
-
inferOutputSchema: () => stringSchema,
|
|
6283
|
-
apply: (v) => String(v ?? "").toUpperCase()
|
|
6284
|
-
};
|
|
6285
|
-
var lowercaseTransform = {
|
|
6286
|
-
id: "lowercase",
|
|
6287
|
-
title: "Lowercase",
|
|
6288
|
-
category: "String",
|
|
6289
|
-
paramsSchema: undefined,
|
|
6290
|
-
inferOutputSchema: () => stringSchema,
|
|
6291
|
-
apply: (v) => String(v ?? "").toLowerCase()
|
|
6292
|
-
};
|
|
6293
|
-
var truncateTransform = {
|
|
6294
|
-
id: "truncate",
|
|
6295
|
-
title: "Truncate",
|
|
6296
|
-
category: "String",
|
|
6297
|
-
paramsSchema: {
|
|
6298
|
-
type: "object",
|
|
6299
|
-
properties: { max: { type: "integer", minimum: 0 } },
|
|
6300
|
-
required: ["max"]
|
|
6301
|
-
},
|
|
6302
|
-
inferOutputSchema: () => stringSchema,
|
|
6303
|
-
apply: (v, { max }) => String(v ?? "").slice(0, max)
|
|
6304
|
-
};
|
|
6305
|
-
var substringTransform = {
|
|
6306
|
-
id: "substring",
|
|
6307
|
-
title: "Substring",
|
|
6308
|
-
category: "String",
|
|
6309
|
-
paramsSchema: {
|
|
6310
|
-
type: "object",
|
|
6311
|
-
properties: {
|
|
6312
|
-
start: { type: "integer" },
|
|
6313
|
-
end: { type: "integer" }
|
|
6314
|
-
},
|
|
6315
|
-
required: ["start"]
|
|
6316
|
-
},
|
|
6317
|
-
inferOutputSchema: () => stringSchema,
|
|
6318
|
-
apply: (v, { start, end }) => String(v ?? "").slice(start, end)
|
|
6319
|
-
};
|
|
6320
|
-
|
|
6321
6422
|
// src/task-graph/transforms/date-conversions.ts
|
|
6322
6423
|
var isoSchema = { type: "string", format: "date-time" };
|
|
6323
6424
|
var numberSchema = { type: "number" };
|
|
@@ -6367,15 +6468,116 @@ var isoDateToUnixTransform = {
|
|
|
6367
6468
|
}
|
|
6368
6469
|
};
|
|
6369
6470
|
|
|
6471
|
+
// src/task-graph/transforms/index-access.ts
|
|
6472
|
+
import { areSemanticallyCompatible as areSemanticallyCompatible2 } from "@workglow/util/schema";
|
|
6473
|
+
function doIndex(value, idx) {
|
|
6474
|
+
if (!Array.isArray(value))
|
|
6475
|
+
return;
|
|
6476
|
+
const i = idx < 0 ? value.length + idx : idx;
|
|
6477
|
+
return value[i];
|
|
6478
|
+
}
|
|
6479
|
+
var indexTransform = {
|
|
6480
|
+
id: "index",
|
|
6481
|
+
title: "Array index",
|
|
6482
|
+
category: "Structural",
|
|
6483
|
+
paramsSchema: {
|
|
6484
|
+
type: "object",
|
|
6485
|
+
properties: {
|
|
6486
|
+
index: { type: "integer", description: "Array index (negative counts from end)" }
|
|
6487
|
+
},
|
|
6488
|
+
required: ["index"]
|
|
6489
|
+
},
|
|
6490
|
+
inferOutputSchema(inputSchema) {
|
|
6491
|
+
const s = inputSchema;
|
|
6492
|
+
if (s?.type === "array" && s.items)
|
|
6493
|
+
return s.items;
|
|
6494
|
+
return {};
|
|
6495
|
+
},
|
|
6496
|
+
apply(value, params) {
|
|
6497
|
+
return doIndex(value, params.index);
|
|
6498
|
+
},
|
|
6499
|
+
suggestFromSchemas(source, target) {
|
|
6500
|
+
const s = source;
|
|
6501
|
+
if (s?.type !== "array" || !s.items)
|
|
6502
|
+
return;
|
|
6503
|
+
const compat = areSemanticallyCompatible2(s.items, target);
|
|
6504
|
+
if (compat === "static")
|
|
6505
|
+
return { score: 0.9, params: { index: 0 } };
|
|
6506
|
+
if (compat === "runtime")
|
|
6507
|
+
return { score: 0.5, params: { index: 0 } };
|
|
6508
|
+
return;
|
|
6509
|
+
}
|
|
6510
|
+
};
|
|
6511
|
+
|
|
6512
|
+
// src/task-graph/transforms/pick.ts
|
|
6513
|
+
import { areSemanticallyCompatible as areSemanticallyCompatible3 } from "@workglow/util/schema";
|
|
6514
|
+
function walk(value, path) {
|
|
6515
|
+
if (value == null)
|
|
6516
|
+
return;
|
|
6517
|
+
const parts = path.split(".");
|
|
6518
|
+
let cur = value;
|
|
6519
|
+
for (const p of parts) {
|
|
6520
|
+
if (cur == null)
|
|
6521
|
+
return;
|
|
6522
|
+
cur = cur[p];
|
|
6523
|
+
}
|
|
6524
|
+
return cur;
|
|
6525
|
+
}
|
|
6526
|
+
function walkSchema(schema, path) {
|
|
6527
|
+
const parts = path.split(".");
|
|
6528
|
+
let cur = schema;
|
|
6529
|
+
for (const p of parts) {
|
|
6530
|
+
if (!cur || typeof cur !== "object")
|
|
6531
|
+
return {};
|
|
6532
|
+
if (cur.type !== "object" || !cur.properties || !cur.properties[p]) {
|
|
6533
|
+
return {};
|
|
6534
|
+
}
|
|
6535
|
+
cur = cur.properties[p];
|
|
6536
|
+
}
|
|
6537
|
+
return cur;
|
|
6538
|
+
}
|
|
6539
|
+
var pickTransform = {
|
|
6540
|
+
id: "pick",
|
|
6541
|
+
title: "Pick field",
|
|
6542
|
+
category: "Structural",
|
|
6543
|
+
paramsSchema: {
|
|
6544
|
+
type: "object",
|
|
6545
|
+
properties: {
|
|
6546
|
+
path: { type: "string", description: "Dotted property path" }
|
|
6547
|
+
},
|
|
6548
|
+
required: ["path"]
|
|
6549
|
+
},
|
|
6550
|
+
inferOutputSchema(inputSchema, params) {
|
|
6551
|
+
return walkSchema(inputSchema, params.path);
|
|
6552
|
+
},
|
|
6553
|
+
apply(value, params) {
|
|
6554
|
+
return walk(value, params.path);
|
|
6555
|
+
},
|
|
6556
|
+
suggestFromSchemas(source, target) {
|
|
6557
|
+
if (source.type !== "object" || !source.properties) {
|
|
6558
|
+
return;
|
|
6559
|
+
}
|
|
6560
|
+
const props = source.properties;
|
|
6561
|
+
for (const [name, propSchema] of Object.entries(props)) {
|
|
6562
|
+
const compat = areSemanticallyCompatible3(propSchema, target);
|
|
6563
|
+
if (compat === "static")
|
|
6564
|
+
return { score: 1, params: { path: name } };
|
|
6565
|
+
if (compat === "runtime")
|
|
6566
|
+
return { score: 0.6, params: { path: name } };
|
|
6567
|
+
}
|
|
6568
|
+
return;
|
|
6569
|
+
}
|
|
6570
|
+
};
|
|
6571
|
+
|
|
6370
6572
|
// src/task-graph/transforms/scalar-conversions.ts
|
|
6371
|
-
var
|
|
6573
|
+
var stringSchema = { type: "string" };
|
|
6372
6574
|
var booleanSchema = { type: "boolean" };
|
|
6373
6575
|
var numberToStringTransform = {
|
|
6374
6576
|
id: "numberToString",
|
|
6375
6577
|
title: "Number → String",
|
|
6376
6578
|
category: "Conversion",
|
|
6377
6579
|
paramsSchema: undefined,
|
|
6378
|
-
inferOutputSchema: () =>
|
|
6580
|
+
inferOutputSchema: () => stringSchema,
|
|
6379
6581
|
apply: (v) => String(v),
|
|
6380
6582
|
suggestFromSchemas(source, target) {
|
|
6381
6583
|
const s = source;
|
|
@@ -6407,7 +6609,7 @@ var stringifyTransform = {
|
|
|
6407
6609
|
title: "JSON.stringify",
|
|
6408
6610
|
category: "Conversion",
|
|
6409
6611
|
paramsSchema: undefined,
|
|
6410
|
-
inferOutputSchema: () =>
|
|
6612
|
+
inferOutputSchema: () => stringSchema,
|
|
6411
6613
|
apply: (v) => JSON.stringify(v),
|
|
6412
6614
|
suggestFromSchemas(source, target) {
|
|
6413
6615
|
const t = target;
|
|
@@ -6423,6 +6625,52 @@ var parseJsonTransform = {
|
|
|
6423
6625
|
apply: (v) => JSON.parse(String(v))
|
|
6424
6626
|
};
|
|
6425
6627
|
|
|
6628
|
+
// src/task-graph/transforms/string-casts.ts
|
|
6629
|
+
var stringSchema2 = { type: "string" };
|
|
6630
|
+
var uppercaseTransform = {
|
|
6631
|
+
id: "uppercase",
|
|
6632
|
+
title: "Uppercase",
|
|
6633
|
+
category: "String",
|
|
6634
|
+
paramsSchema: undefined,
|
|
6635
|
+
inferOutputSchema: () => stringSchema2,
|
|
6636
|
+
apply: (v) => String(v ?? "").toUpperCase()
|
|
6637
|
+
};
|
|
6638
|
+
var lowercaseTransform = {
|
|
6639
|
+
id: "lowercase",
|
|
6640
|
+
title: "Lowercase",
|
|
6641
|
+
category: "String",
|
|
6642
|
+
paramsSchema: undefined,
|
|
6643
|
+
inferOutputSchema: () => stringSchema2,
|
|
6644
|
+
apply: (v) => String(v ?? "").toLowerCase()
|
|
6645
|
+
};
|
|
6646
|
+
var truncateTransform = {
|
|
6647
|
+
id: "truncate",
|
|
6648
|
+
title: "Truncate",
|
|
6649
|
+
category: "String",
|
|
6650
|
+
paramsSchema: {
|
|
6651
|
+
type: "object",
|
|
6652
|
+
properties: { max: { type: "integer", minimum: 0 } },
|
|
6653
|
+
required: ["max"]
|
|
6654
|
+
},
|
|
6655
|
+
inferOutputSchema: () => stringSchema2,
|
|
6656
|
+
apply: (v, { max }) => String(v ?? "").slice(0, max)
|
|
6657
|
+
};
|
|
6658
|
+
var substringTransform = {
|
|
6659
|
+
id: "substring",
|
|
6660
|
+
title: "Substring",
|
|
6661
|
+
category: "String",
|
|
6662
|
+
paramsSchema: {
|
|
6663
|
+
type: "object",
|
|
6664
|
+
properties: {
|
|
6665
|
+
start: { type: "integer" },
|
|
6666
|
+
end: { type: "integer" }
|
|
6667
|
+
},
|
|
6668
|
+
required: ["start"]
|
|
6669
|
+
},
|
|
6670
|
+
inferOutputSchema: () => stringSchema2,
|
|
6671
|
+
apply: (v, { start, end }) => String(v ?? "").slice(start, end)
|
|
6672
|
+
};
|
|
6673
|
+
|
|
6426
6674
|
// src/task-graph/transforms/index.ts
|
|
6427
6675
|
function registerBuiltInTransforms() {
|
|
6428
6676
|
const all = [
|
|
@@ -6444,7 +6692,7 @@ function registerBuiltInTransforms() {
|
|
|
6444
6692
|
TransformRegistry.registerTransform(t);
|
|
6445
6693
|
}
|
|
6446
6694
|
// src/task/EntitlementProfile.ts
|
|
6447
|
-
import { createServiceToken as
|
|
6695
|
+
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
6448
6696
|
var STATIC_SIGNAL_SOURCE = Object.freeze({
|
|
6449
6697
|
subscribe(_listener) {
|
|
6450
6698
|
return () => {};
|
|
@@ -6547,7 +6795,7 @@ function createPolicyProfile(name, policy, options = {}) {
|
|
|
6547
6795
|
};
|
|
6548
6796
|
return profile;
|
|
6549
6797
|
}
|
|
6550
|
-
var ENTITLEMENT_PROFILE =
|
|
6798
|
+
var ENTITLEMENT_PROFILE = createServiceToken6("workglow.entitlementProfile");
|
|
6551
6799
|
// src/task/EntitlementProfiles.ts
|
|
6552
6800
|
var BROWSER_GRANTS = [
|
|
6553
6801
|
{ id: Entitlements.NETWORK_HTTP },
|
|
@@ -8165,12 +8413,12 @@ function wrapSchemaInArray(schema) {
|
|
|
8165
8413
|
}
|
|
8166
8414
|
// src/task/JobQueueFactory.ts
|
|
8167
8415
|
import {
|
|
8416
|
+
InMemoryQueueStorage,
|
|
8168
8417
|
JobQueueClient,
|
|
8169
8418
|
JobQueueServer
|
|
8170
8419
|
} from "@workglow/job-queue";
|
|
8171
|
-
import {
|
|
8172
|
-
|
|
8173
|
-
var JOB_QUEUE_FACTORY = createServiceToken6("taskgraph.jobQueueFactory");
|
|
8420
|
+
import { createServiceToken as createServiceToken7, globalServiceRegistry as globalServiceRegistry4 } from "@workglow/util";
|
|
8421
|
+
var JOB_QUEUE_FACTORY = createServiceToken7("taskgraph.jobQueueFactory");
|
|
8174
8422
|
var defaultJobQueueFactory = async ({
|
|
8175
8423
|
queueName,
|
|
8176
8424
|
jobClass,
|
|
@@ -8435,8 +8683,8 @@ queueMicrotask(() => {
|
|
|
8435
8683
|
});
|
|
8436
8684
|
// src/task/TaskRegistry.ts
|
|
8437
8685
|
import {
|
|
8438
|
-
createServiceToken as
|
|
8439
|
-
getLogger as
|
|
8686
|
+
createServiceToken as createServiceToken8,
|
|
8687
|
+
getLogger as getLogger10,
|
|
8440
8688
|
globalServiceRegistry as globalServiceRegistry5,
|
|
8441
8689
|
registerInputCompactor,
|
|
8442
8690
|
registerInputResolver
|
|
@@ -8459,7 +8707,7 @@ function registerTask(baseClass) {
|
|
|
8459
8707
|
const result = validateSchema(schema);
|
|
8460
8708
|
if (!result.valid) {
|
|
8461
8709
|
const messages = result.errors.map((e) => `${e.path}: ${e.message}`).join("; ");
|
|
8462
|
-
|
|
8710
|
+
getLogger10().warn(`Task "${baseClass.type}" has invalid ${name}: ${messages}`, {
|
|
8463
8711
|
taskType: baseClass.type,
|
|
8464
8712
|
schemaName: name,
|
|
8465
8713
|
errors: result.errors
|
|
@@ -8475,7 +8723,7 @@ var TaskRegistry = {
|
|
|
8475
8723
|
registerTask,
|
|
8476
8724
|
unregisterTask
|
|
8477
8725
|
};
|
|
8478
|
-
var TASK_CONSTRUCTORS =
|
|
8726
|
+
var TASK_CONSTRUCTORS = createServiceToken8("task.constructors");
|
|
8479
8727
|
function getGlobalTaskConstructors() {
|
|
8480
8728
|
return globalServiceRegistry5.get(TASK_CONSTRUCTORS);
|
|
8481
8729
|
}
|
|
@@ -8644,8 +8892,8 @@ var registerBaseTasks = () => {
|
|
|
8644
8892
|
return tasks;
|
|
8645
8893
|
};
|
|
8646
8894
|
// src/storage/TaskGraphRepository.ts
|
|
8647
|
-
import { createServiceToken as
|
|
8648
|
-
var TASK_GRAPH_REPOSITORY =
|
|
8895
|
+
import { createServiceToken as createServiceToken9, EventEmitter as EventEmitter7 } from "@workglow/util";
|
|
8896
|
+
var TASK_GRAPH_REPOSITORY = createServiceToken9("taskgraph.taskGraphRepository");
|
|
8649
8897
|
|
|
8650
8898
|
class TaskGraphRepository {
|
|
8651
8899
|
type = "TaskGraphRepository";
|
|
@@ -8720,8 +8968,8 @@ class TaskGraphTabularRepository extends TaskGraphRepository {
|
|
|
8720
8968
|
}
|
|
8721
8969
|
}
|
|
8722
8970
|
// src/storage/TaskOutputTabularRepository.ts
|
|
8723
|
-
import { compress, decompress } from "@workglow/util/compress";
|
|
8724
8971
|
import { makeFingerprint } from "@workglow/util";
|
|
8972
|
+
import { compress, decompress } from "@workglow/util/compress";
|
|
8725
8973
|
var TaskOutputSchema = {
|
|
8726
8974
|
type: "object",
|
|
8727
8975
|
properties: {
|
|
@@ -8741,6 +8989,10 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8741
8989
|
this.tabularRepository = tabularRepository;
|
|
8742
8990
|
this.outputCompression = outputCompression;
|
|
8743
8991
|
}
|
|
8992
|
+
isDurable() {
|
|
8993
|
+
const backing = this.tabularRepository;
|
|
8994
|
+
return backing.isDurable?.() ?? true;
|
|
8995
|
+
}
|
|
8744
8996
|
async setupDatabase() {
|
|
8745
8997
|
await this.tabularRepository.setupDatabase?.();
|
|
8746
8998
|
}
|
|
@@ -8801,9 +9053,36 @@ class TaskOutputTabularRepository extends TaskOutputRepository {
|
|
|
8801
9053
|
await this.tabularRepository.deleteSearch({ createdAt: { value: date, operator: "<" } });
|
|
8802
9054
|
this.emit("output_pruned");
|
|
8803
9055
|
}
|
|
9056
|
+
async deleteByTaskTypePrefix(prefix) {
|
|
9057
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9058
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9059
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9060
|
+
}
|
|
9061
|
+
}
|
|
9062
|
+
}
|
|
9063
|
+
async clearOlderThanWithTaskTypePrefix(prefix, olderThanInMs) {
|
|
9064
|
+
const cutoff = Date.now() - olderThanInMs;
|
|
9065
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9066
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9067
|
+
const ts = typeof row.createdAt === "string" ? new Date(row.createdAt).getTime() : NaN;
|
|
9068
|
+
if (!isNaN(ts) && ts < cutoff) {
|
|
9069
|
+
await this.tabularRepository.delete({ key: row.key, taskType: row.taskType });
|
|
9070
|
+
}
|
|
9071
|
+
}
|
|
9072
|
+
}
|
|
9073
|
+
}
|
|
9074
|
+
async sizeByTaskTypePrefix(prefix) {
|
|
9075
|
+
let count = 0;
|
|
9076
|
+
for await (const row of this.tabularRepository.records()) {
|
|
9077
|
+
if (typeof row.taskType === "string" && row.taskType.startsWith(prefix)) {
|
|
9078
|
+
count++;
|
|
9079
|
+
}
|
|
9080
|
+
}
|
|
9081
|
+
return count;
|
|
9082
|
+
}
|
|
8804
9083
|
}
|
|
8805
9084
|
// src/storage/PortCodecRegistry.ts
|
|
8806
|
-
import {
|
|
9085
|
+
import { _resetPortCodecsForTests, getPortCodec as getPortCodec2, registerPortCodec } from "@workglow/util";
|
|
8807
9086
|
// src/debug/console/ConsoleFormatters.ts
|
|
8808
9087
|
import { DirectedAcyclicGraph as DirectedAcyclicGraph2 } from "@workglow/util/graph";
|
|
8809
9088
|
function formatDuration(ms) {
|
|
@@ -9485,6 +9764,8 @@ export {
|
|
|
9485
9764
|
isoDateToUnixTransform,
|
|
9486
9765
|
isTaskStreamable,
|
|
9487
9766
|
isStrictArraySchema,
|
|
9767
|
+
isPolicyPrivate,
|
|
9768
|
+
isPolicyCached,
|
|
9488
9769
|
isIterationProperty,
|
|
9489
9770
|
isFlexibleSchema,
|
|
9490
9771
|
isDarkMode,
|
|
@@ -9600,6 +9881,7 @@ export {
|
|
|
9600
9881
|
STATIC_SIGNAL_SOURCE,
|
|
9601
9882
|
SERVER_GRANTS,
|
|
9602
9883
|
RunScheduler,
|
|
9884
|
+
RunPrivateCacheRepo,
|
|
9603
9885
|
RunContext,
|
|
9604
9886
|
ReduceTask,
|
|
9605
9887
|
PROPERTY_ARRAY,
|
|
@@ -9627,10 +9909,12 @@ export {
|
|
|
9627
9909
|
ENTITLEMENT_ENFORCER,
|
|
9628
9910
|
EMPTY_POLICY,
|
|
9629
9911
|
EMPTY_ENTITLEMENTS,
|
|
9912
|
+
DefaultCacheRegistry,
|
|
9630
9913
|
DataflowArrow,
|
|
9631
9914
|
Dataflow,
|
|
9632
9915
|
DESKTOP_GRANTS,
|
|
9633
9916
|
DENY_ALL_RESOLVER,
|
|
9917
|
+
DEFAULT_CACHE_POLICY,
|
|
9634
9918
|
DATAFLOW_ERROR_PORT,
|
|
9635
9919
|
DATAFLOW_ALL_PORTS,
|
|
9636
9920
|
CreateWorkflow,
|
|
@@ -9638,8 +9922,10 @@ export {
|
|
|
9638
9922
|
CreateEndLoopWorkflow,
|
|
9639
9923
|
CreateAdaptiveWorkflow,
|
|
9640
9924
|
ConditionalTask,
|
|
9925
|
+
CacheJanitor,
|
|
9641
9926
|
CacheCoordinator,
|
|
9927
|
+
CACHE_REGISTRY,
|
|
9642
9928
|
BROWSER_GRANTS
|
|
9643
9929
|
};
|
|
9644
9930
|
|
|
9645
|
-
//# debugId=
|
|
9931
|
+
//# debugId=4C6784BFF95D87B164756E2164756E21
|