@fevex/core 0.1.0-alpha.4 → 0.1.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -2
- package/dist/agents/index.d.mts +1 -1
- package/dist/artifacts/index.d.mts +2 -0
- package/dist/artifacts/index.mjs +2 -0
- package/dist/artifacts-gUzemET7.mjs +440 -0
- package/dist/channels/index.d.mts +1 -1
- package/dist/evals/index.d.mts +4 -4
- package/dist/{fevex-BoPbZ-Tw.d.mts → fevex-B5NxFKcV.d.mts} +41 -11
- package/dist/http/index.d.mts +65 -5
- package/dist/http/index.mjs +236 -16
- package/dist/{index-DyyNLuwO.d.mts → index--_LdSEuF.d.mts} +3 -3
- package/dist/{index-CrOId2px.d.mts → index-BFnWUu7j.d.mts} +1 -1
- package/dist/{index-CTIwrVf9.d.mts → index-BPwkgdo3.d.mts} +1 -1
- package/dist/{index-CknU-glw.d.mts → index-Br-zPxZH.d.mts} +2 -2
- package/dist/{index-CaE864Kz.d.mts → index-CGmHXV31.d.mts} +35 -3
- package/dist/{index-CqOjvtQ9.d.mts → index-CKQCNvJz.d.mts} +4 -2
- package/dist/{index-dkFyTNh4.d.mts → index-Cb3AB98n.d.mts} +2 -2
- package/dist/{index-voIZhQXB.d.mts → index-D6lnI92z.d.mts} +3 -3
- package/dist/{index-C-IFfCNw.d.mts → index-Dsulk-n6.d.mts} +3 -3
- package/dist/{index-hhbgvfIt.d.mts → index-aayvtf0J.d.mts} +2 -2
- package/dist/index.d.mts +13 -13
- package/dist/index.mjs +444 -95
- package/dist/knowledge/index.d.mts +1 -1
- package/dist/model-stream-JQdHDDcj.mjs +131 -0
- package/dist/models/index.d.mts +1 -1
- package/dist/observability/index.d.mts +1 -1
- package/dist/policies/index.d.mts +1 -1
- package/dist/{model-stream-u2xMsEaf.mjs → run-support-CmZ6Xo6H.mjs} +41 -130
- package/dist/runtime/index.d.mts +2 -2
- package/dist/runtime/index.mjs +43 -1
- package/dist/sandbox/index.d.mts +1 -1
- package/dist/sandbox/index.mjs +1 -1
- package/dist/teams/index.d.mts +1 -1
- package/dist/testing/index.d.mts +8 -7
- package/dist/testing/index.mjs +94 -2
- package/dist/tools/index.d.mts +2 -2
- package/dist/types-BDVUoAU6.d.mts +559 -0
- package/dist/workflows/index.d.mts +1 -1
- package/package.json +6 -2
- package/dist/types-CuoRx12Q.d.mts +0 -333
package/dist/index.mjs
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { defineAgent } from "./agents/index.mjs";
|
|
2
|
+
import { a as CatalogedArtifactStore, c as assertArtifactId, i as FileSystemArtifactStore, n as InMemoryArtifactStore, o as ArtifactError, r as InMemoryArtifactCatalog, s as artifactTenantId, t as S3ArtifactStore } from "./artifacts-gUzemET7.mjs";
|
|
2
3
|
import { n as RunPausedError, t as FevexRunError } from "./run-error-Cr17wPGH.mjs";
|
|
3
4
|
import { ChannelError, deliverChannelRun, handleChannelInput } from "./channels/index.mjs";
|
|
4
5
|
import { InMemoryMemoryStore, createMemoryContextProvider, defineContextProvider, defineSkill } from "./knowledge/index.mjs";
|
|
5
6
|
import { n as teamAsWorkflow, t as defineTeam } from "./teams-Ctz676G4.mjs";
|
|
6
|
-
import { InMemoryRunStore, isDurableRunStore, validateListEventsOptions } from "./runtime/index.mjs";
|
|
7
|
+
import { InMemoryRunStore, isDurableRunStore, isRetainableRunStore, validateListEventsOptions, validatePurgeRunsOptions } from "./runtime/index.mjs";
|
|
7
8
|
import { FEVEX_JSON_SCHEMA_PROFILE_VERSION, IntegrationError, compileFevexJsonSchema, defineConnection, defineTool, toToolSpec, validateFevexJsonSchemaProfile } from "./tools/index.mjs";
|
|
8
|
-
import { a as assertTokenBudget, c as
|
|
9
|
+
import { _ as serializeValue, a as assertTokenBudget, c as eventUsage, d as remainingDeadlineMs, f as remainingOutputTokens, g as serializeJsonValue, h as toErrorMessage, i as assertContinuationBudget, l as isTimeoutError, m as runSignal, n as abortable, o as cancellationReason, p as resumeDeadlineSignal, r as addUsage, s as deadlineAtFromMs, u as minDeadline, v as toJsonValue } from "./run-support-CmZ6Xo6H.mjs";
|
|
10
|
+
import { t as readModelStream } from "./model-stream-JQdHDDcj.mjs";
|
|
9
11
|
import "./models/index.mjs";
|
|
10
12
|
import { definePolicy } from "./policies/index.mjs";
|
|
11
13
|
import { SandboxError, createLocalSandbox, sandboxResultToJson } from "./sandbox/index.mjs";
|
|
@@ -103,6 +105,9 @@ function isRunStore(value) {
|
|
|
103
105
|
function isSandbox(value) {
|
|
104
106
|
return isRecord(value) && typeof value.run === "function";
|
|
105
107
|
}
|
|
108
|
+
function isArtifactStore(value) {
|
|
109
|
+
return (typeof value === "object" && value !== null || typeof value === "function") && typeof value.put === "function" && typeof value.get === "function";
|
|
110
|
+
}
|
|
106
111
|
function isJsonObject(value) {
|
|
107
112
|
return isRecord(value);
|
|
108
113
|
}
|
|
@@ -300,7 +305,21 @@ function createComposition(config) {
|
|
|
300
305
|
assertConfiguration(config.runStore === void 0 || isRunStore(config.runStore), "INVALID_CONFIG", "Fevex config \"runStore\" must implement RunStore");
|
|
301
306
|
assertConfiguration(config.credentialStore === void 0 || typeof config.credentialStore?.resolve === "function", "INVALID_CONFIG", "Fevex config \"credentialStore\" must implement resolve");
|
|
302
307
|
assertConfiguration(config.sandbox === void 0 || isSandbox(config.sandbox), "INVALID_CONFIG", "Fevex config \"sandbox\" must implement Sandbox");
|
|
308
|
+
assertConfiguration(config.artifacts === void 0 || isArtifactStore(config.artifacts), "INVALID_CONFIG", "Fevex config \"artifacts\" must implement ArtifactStore");
|
|
303
309
|
assertConfiguration(config.policies === void 0 || Array.isArray(config.policies), "INVALID_CONFIG", "Fevex config \"policies\" must be an array");
|
|
310
|
+
if (config.admission !== void 0) {
|
|
311
|
+
assertConfiguration(isRecord(config.admission), "INVALID_CONFIG", "Fevex config \"admission\" must be an object");
|
|
312
|
+
for (const name of ["maxActiveRuns", "maxActiveRunsPerTenant"]) {
|
|
313
|
+
const value = config.admission[name];
|
|
314
|
+
assertConfiguration(value === void 0 || typeof value === "number" && Number.isSafeInteger(value) && value > 0, "INVALID_CONFIG", `Fevex config "admission.${name}" must be a positive safe integer`);
|
|
315
|
+
}
|
|
316
|
+
assertConfiguration(config.admission.tenantKey === void 0 || typeof config.admission.tenantKey === "function", "INVALID_CONFIG", "Fevex config \"admission.tenantKey\" must be a function");
|
|
317
|
+
assertConfiguration(config.admission.authorize === void 0 || typeof config.admission.authorize === "function", "INVALID_CONFIG", "Fevex config \"admission.authorize\" must be a function");
|
|
318
|
+
}
|
|
319
|
+
if (config.limits !== void 0) {
|
|
320
|
+
assertConfiguration(isRecord(config.limits), "INVALID_CONFIG", "Fevex config \"limits\" must be an object");
|
|
321
|
+
assertConfiguration(config.limits.maxEventsPerRun === void 0 || typeof config.limits.maxEventsPerRun === "number" && Number.isSafeInteger(config.limits.maxEventsPerRun) && config.limits.maxEventsPerRun > 0, "INVALID_CONFIG", "Fevex config \"limits.maxEventsPerRun\" must be a positive safe integer");
|
|
322
|
+
}
|
|
304
323
|
if (config.observability !== void 0) {
|
|
305
324
|
assertConfiguration(isRecord(config.observability), "INVALID_CONFIG", "Fevex config \"observability\" must be an object");
|
|
306
325
|
assertConfiguration(Array.isArray(config.observability.exporters) && config.observability.exporters.every((exporter) => isRecord(exporter) && typeof exporter.export === "function"), "INVALID_CONFIG", "Fevex config \"observability.exporters\" must contain TraceExporters");
|
|
@@ -594,8 +613,11 @@ function createComposition(config) {
|
|
|
594
613
|
runStore: config.runStore ?? new InMemoryRunStore(),
|
|
595
614
|
credentialStore: config.credentialStore,
|
|
596
615
|
sandbox: config.sandbox,
|
|
616
|
+
artifacts: config.artifacts,
|
|
597
617
|
policies,
|
|
598
618
|
onEvent: config.onEvent,
|
|
619
|
+
admission: config.admission === void 0 ? void 0 : { ...config.admission },
|
|
620
|
+
limits: config.limits === void 0 ? void 0 : { ...config.limits },
|
|
599
621
|
observability: config.observability === void 0 ? void 0 : {
|
|
600
622
|
exporters: [...config.observability.exporters],
|
|
601
623
|
...config.observability.calculateCost ? { calculateCost: config.observability.calculateCost } : {},
|
|
@@ -732,6 +754,7 @@ const toolErrorPayload = (tool, error) => ({
|
|
|
732
754
|
} : {}
|
|
733
755
|
});
|
|
734
756
|
const runErrorPayload = (error) => {
|
|
757
|
+
if (error instanceof FevexRunError) return { errorCode: error.code };
|
|
735
758
|
if (!(error instanceof IntegrationError)) return {};
|
|
736
759
|
return {
|
|
737
760
|
errorCode: error.code,
|
|
@@ -866,9 +889,9 @@ function addWorkflowBudget(first, second) {
|
|
|
866
889
|
* engine — that direction is what keeps the two separable.
|
|
867
890
|
*/
|
|
868
891
|
function createAgentEngine(composition, core) {
|
|
869
|
-
const { models, agents, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, policies } = composition;
|
|
870
|
-
const { activeRuns, activeSessions, runtimeOwner, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
|
|
871
|
-
const
|
|
892
|
+
const { models, agents, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, artifacts, policies } = composition;
|
|
893
|
+
const { activeRuns, activeSessions, runtimeOwner, admitStart, markInternalRun, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
|
|
894
|
+
const prepareExecutionBody = async (name, request, internal = false) => {
|
|
872
895
|
const agent = agents.get(name);
|
|
873
896
|
if (!agent) throw new FevexRunError("AGENT_NOT_FOUND", `Agent "${name}" is not registered`);
|
|
874
897
|
const { modelName, model } = resolveModel(agent, request.model);
|
|
@@ -887,9 +910,10 @@ function createAgentEngine(composition, core) {
|
|
|
887
910
|
session = stored;
|
|
888
911
|
}
|
|
889
912
|
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" already has an active run`);
|
|
890
|
-
activeSessions.add(session.id);
|
|
891
913
|
const controller = new AbortController();
|
|
892
|
-
const signal =
|
|
914
|
+
const signal = runSignal(controller, request);
|
|
915
|
+
const deadlineAt = deadlineAtFromMs(request.deadlineMs);
|
|
916
|
+
activeSessions.add(session.id);
|
|
893
917
|
const run = {
|
|
894
918
|
id: crypto.randomUUID(),
|
|
895
919
|
sessionId: session.id,
|
|
@@ -899,8 +923,10 @@ function createAgentEngine(composition, core) {
|
|
|
899
923
|
status: "running",
|
|
900
924
|
revision: 0,
|
|
901
925
|
createdAt: now,
|
|
902
|
-
updatedAt: now
|
|
926
|
+
updatedAt: now,
|
|
927
|
+
...deadlineAt === void 0 ? {} : { deadlineAt }
|
|
903
928
|
};
|
|
929
|
+
if (internal) markInternalRun(run.id);
|
|
904
930
|
const state = {
|
|
905
931
|
run,
|
|
906
932
|
session,
|
|
@@ -981,6 +1007,14 @@ function createAgentEngine(composition, core) {
|
|
|
981
1007
|
activeRuns.set(run.id, state);
|
|
982
1008
|
return state;
|
|
983
1009
|
};
|
|
1010
|
+
const prepareExecution = async (name, request, options = {}) => {
|
|
1011
|
+
const release = options.admit === false ? void 0 : await admitStart("agent", name, request);
|
|
1012
|
+
try {
|
|
1013
|
+
return await prepareExecutionBody(name, request, options.admit === false);
|
|
1014
|
+
} finally {
|
|
1015
|
+
release?.();
|
|
1016
|
+
}
|
|
1017
|
+
};
|
|
984
1018
|
const durableCheckpoint = async (state, model, modelName, messages, inputContent, usage, providerState, step, toolCallCount, seenToolCallIds, pendingTools, pendingIndex) => {
|
|
985
1019
|
const agent = agents.get(state.run.agentName);
|
|
986
1020
|
if (agent.model !== void 0 && typeof agent.model !== "string") throw new FevexRunError("RUN_NOT_RESUMABLE", "Durable continuation requires a named model", state.run.id);
|
|
@@ -1519,6 +1553,20 @@ function createAgentEngine(composition, core) {
|
|
|
1519
1553
|
}
|
|
1520
1554
|
events.push(started);
|
|
1521
1555
|
yield started;
|
|
1556
|
+
const createdArtifacts = [];
|
|
1557
|
+
const toolArtifacts = artifacts === void 0 ? void 0 : scopedArtifacts(artifacts, {
|
|
1558
|
+
runId: state.run.id,
|
|
1559
|
+
sessionId: state.session.id,
|
|
1560
|
+
request,
|
|
1561
|
+
emit: async (descriptor) => {
|
|
1562
|
+
createdArtifacts.push(await emit("artifact.created", {
|
|
1563
|
+
descriptor,
|
|
1564
|
+
toolCallId: pending.call.id,
|
|
1565
|
+
toolName: pending.call.name,
|
|
1566
|
+
...toolSourcePayload(tool)
|
|
1567
|
+
}));
|
|
1568
|
+
}
|
|
1569
|
+
});
|
|
1522
1570
|
try {
|
|
1523
1571
|
const raw = await abortable(() => tool.execute(pending.input, {
|
|
1524
1572
|
runId: state.run.id,
|
|
@@ -1527,9 +1575,11 @@ function createAgentEngine(composition, core) {
|
|
|
1527
1575
|
idempotencyKey: pending.idempotencyKey,
|
|
1528
1576
|
getCredential,
|
|
1529
1577
|
...scopedSandbox === void 0 ? {} : { sandbox: scopedSandbox },
|
|
1578
|
+
...toolArtifacts === void 0 ? {} : { artifacts: toolArtifacts },
|
|
1530
1579
|
context: request.context,
|
|
1531
1580
|
signal: request.signal
|
|
1532
1581
|
}), request.signal);
|
|
1582
|
+
for (const event of createdArtifacts.splice(0)) yield event;
|
|
1533
1583
|
output = toJsonValue(tool.outputSchema === void 0 ? raw : await validateSchema(tool.outputSchema, raw, `Output from tool "${tool.name}" does not match outputSchema`), `Output from tool "${tool.name}" must be JSON-serializable`);
|
|
1534
1584
|
if (containsSecret(output, secrets)) throw new Error(`Output from tool "${tool.name}" contains a credential`);
|
|
1535
1585
|
record = {
|
|
@@ -1557,6 +1607,7 @@ function createAgentEngine(composition, core) {
|
|
|
1557
1607
|
yield completed;
|
|
1558
1608
|
break;
|
|
1559
1609
|
} catch (error) {
|
|
1610
|
+
for (const event of createdArtifacts.splice(0)) yield event;
|
|
1560
1611
|
if (request.signal.aborted) throw error;
|
|
1561
1612
|
const safeError = redact(toErrorMessage(error), secrets);
|
|
1562
1613
|
record = {
|
|
@@ -1664,7 +1715,7 @@ function createAgentEngine(composition, core) {
|
|
|
1664
1715
|
}
|
|
1665
1716
|
}
|
|
1666
1717
|
};
|
|
1667
|
-
const
|
|
1718
|
+
const resumeAgentBody = async (runId, resolution, waitForCompletion = false, recoveryActor) => {
|
|
1668
1719
|
if (resolution?.type === "timer" || resolution?.type === "event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
1669
1720
|
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "resumeRun requires DurableRunStore", runId);
|
|
1670
1721
|
const run = await runStore.getRun(runId);
|
|
@@ -1672,6 +1723,11 @@ function createAgentEngine(composition, core) {
|
|
|
1672
1723
|
if (!run || !checkpoint) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be ${resolution ? "resumed" : "recovered"}`, runId);
|
|
1673
1724
|
if (checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${runId}" is unsupported`, runId);
|
|
1674
1725
|
if (resolution && run.status !== "paused" && run.status !== "running" || !resolution && run.status !== "running") throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" is not ${resolution ? "resumable" : "recoverable"}`, runId);
|
|
1726
|
+
if (run.deadlineAt !== void 0 && Date.parse(run.deadlineAt) <= Date.now()) {
|
|
1727
|
+
if (!await cancelStoredAgent(runId, "timeout")) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" deadline has passed`, runId);
|
|
1728
|
+
return await runStore.getRun(runId);
|
|
1729
|
+
}
|
|
1730
|
+
const deadlineSignal = resumeDeadlineSignal(run.deadlineAt, runId, resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE");
|
|
1675
1731
|
const agent = agents.get(run.agentName);
|
|
1676
1732
|
if (!agent) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Agent "${run.agentName}" is unavailable`, runId);
|
|
1677
1733
|
const checkpointModel = await resolveCheckpointModel(run.agentName, agent, checkpoint);
|
|
@@ -1695,6 +1751,7 @@ function createAgentEngine(composition, core) {
|
|
|
1695
1751
|
if (!await runStore.acquireLease(lease)) throw new FevexRunError("RUN_CONFLICT", `Run "${runId}" is leased`, runId);
|
|
1696
1752
|
try {
|
|
1697
1753
|
const controller = new AbortController();
|
|
1754
|
+
const signal = deadlineSignal ? AbortSignal.any([controller.signal, deadlineSignal]) : controller.signal;
|
|
1698
1755
|
const state = {
|
|
1699
1756
|
run,
|
|
1700
1757
|
session,
|
|
@@ -1705,7 +1762,7 @@ function createAgentEngine(composition, core) {
|
|
|
1705
1762
|
sessionId: session.id,
|
|
1706
1763
|
context: checkpoint.context,
|
|
1707
1764
|
limits: checkpoint.limits,
|
|
1708
|
-
signal
|
|
1765
|
+
signal
|
|
1709
1766
|
},
|
|
1710
1767
|
controller,
|
|
1711
1768
|
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
@@ -1874,10 +1931,27 @@ function createAgentEngine(composition, core) {
|
|
|
1874
1931
|
throw error;
|
|
1875
1932
|
}
|
|
1876
1933
|
};
|
|
1877
|
-
const
|
|
1934
|
+
const resumeAgent = async (runId, resolution, waitForCompletion = false, recoveryActor, options = {}) => {
|
|
1935
|
+
let release;
|
|
1936
|
+
if (options.admit !== false) {
|
|
1937
|
+
const run = await runStore.getRun(runId);
|
|
1938
|
+
const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(runId) : void 0;
|
|
1939
|
+
if (run && checkpoint && (run.status === "paused" || run.status === "running")) release = await admitStart("agent", run.agentName, {
|
|
1940
|
+
input: "",
|
|
1941
|
+
...checkpoint.context === void 0 ? {} : { context: checkpoint.context }
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
try {
|
|
1945
|
+
if (options.internal) markInternalRun(runId);
|
|
1946
|
+
return await resumeAgentBody(runId, resolution, waitForCompletion, recoveryActor);
|
|
1947
|
+
} finally {
|
|
1948
|
+
release?.();
|
|
1949
|
+
}
|
|
1950
|
+
};
|
|
1951
|
+
const cancelStoredAgent = async (runId, reason = "aborted") => {
|
|
1878
1952
|
const active = activeRuns.get(runId);
|
|
1879
1953
|
if (active && !active.request.signal.aborted) {
|
|
1880
|
-
active.controller.abort(new DOMException("Run cancelled", "AbortError"));
|
|
1954
|
+
active.controller.abort(new DOMException(reason === "timeout" ? "Run deadline exceeded" : "Run cancelled", reason === "timeout" ? "TimeoutError" : "AbortError"));
|
|
1881
1955
|
if (!active.advancing) {
|
|
1882
1956
|
await cancelExecution(active);
|
|
1883
1957
|
return true;
|
|
@@ -1920,8 +1994,8 @@ function createAgentEngine(composition, core) {
|
|
|
1920
1994
|
};
|
|
1921
1995
|
run.status = "cancelled";
|
|
1922
1996
|
run.pause = void 0;
|
|
1923
|
-
run.error =
|
|
1924
|
-
const cancelled = createEvent(state, "run.cancelled", { reason
|
|
1997
|
+
run.error = reason;
|
|
1998
|
+
const cancelled = createEvent(state, "run.cancelled", { reason });
|
|
1925
1999
|
await commit(state, {
|
|
1926
2000
|
checkpoint: null,
|
|
1927
2001
|
events: [cancelled]
|
|
@@ -1943,6 +2017,37 @@ function createAgentEngine(composition, core) {
|
|
|
1943
2017
|
cancelStoredAgent
|
|
1944
2018
|
};
|
|
1945
2019
|
}
|
|
2020
|
+
function scopedArtifacts(store, options) {
|
|
2021
|
+
const scopedContext = (value) => {
|
|
2022
|
+
const contextTenant = options.request.context?.attributes?.tenantId;
|
|
2023
|
+
const tenantId = typeof contextTenant === "string" ? contextTenant : value?.tenantId;
|
|
2024
|
+
const actor = options.request.context?.actor;
|
|
2025
|
+
return {
|
|
2026
|
+
...value,
|
|
2027
|
+
runId: options.runId,
|
|
2028
|
+
sessionId: options.sessionId,
|
|
2029
|
+
...tenantId === void 0 ? {} : { tenantId },
|
|
2030
|
+
...actor === void 0 ? {} : { actor },
|
|
2031
|
+
signal: options.request.signal
|
|
2032
|
+
};
|
|
2033
|
+
};
|
|
2034
|
+
return {
|
|
2035
|
+
async put(input, context) {
|
|
2036
|
+
const descriptor = await store.put(input, scopedContext(context));
|
|
2037
|
+
await options.emit(descriptor);
|
|
2038
|
+
return descriptor;
|
|
2039
|
+
},
|
|
2040
|
+
get(artifactId, context) {
|
|
2041
|
+
return store.get(artifactId, scopedContext(context));
|
|
2042
|
+
},
|
|
2043
|
+
...store.head === void 0 ? {} : { head(artifactId, context) {
|
|
2044
|
+
return store.head(artifactId, scopedContext(context));
|
|
2045
|
+
} },
|
|
2046
|
+
...store.delete === void 0 ? {} : { delete(artifactId, context) {
|
|
2047
|
+
return store.delete(artifactId, scopedContext(context));
|
|
2048
|
+
} }
|
|
2049
|
+
};
|
|
2050
|
+
}
|
|
1946
2051
|
//#endregion
|
|
1947
2052
|
//#region src/internal/observability.ts
|
|
1948
2053
|
const TERMINAL_TYPES = /* @__PURE__ */ new Set([
|
|
@@ -2236,14 +2341,103 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
|
|
|
2236
2341
|
* the lease table, so a second instance would corrupt event ordering and lease
|
|
2237
2342
|
* ownership without raising an error.
|
|
2238
2343
|
*/
|
|
2239
|
-
function createRunCore({ models, agents, tools, runStore, policies, onEvent, observability }) {
|
|
2344
|
+
function createRunCore({ models, agents, tools, runStore, policies, onEvent, observability, admission, limits }) {
|
|
2240
2345
|
const activeRuns = /* @__PURE__ */ new Map();
|
|
2241
2346
|
const activeWorkflowRuns = /* @__PURE__ */ new Map();
|
|
2242
2347
|
const activeSessions = /* @__PURE__ */ new Set();
|
|
2348
|
+
const internalRuns = /* @__PURE__ */ new Set();
|
|
2349
|
+
const pendingTenantStarts = /* @__PURE__ */ new Map();
|
|
2350
|
+
let pendingStarts = 0;
|
|
2243
2351
|
const pendingExports = /* @__PURE__ */ new Set();
|
|
2244
2352
|
const exportFailures = [];
|
|
2245
2353
|
const exportedRuns = /* @__PURE__ */ new Set();
|
|
2246
2354
|
const runtimeOwner = crypto.randomUUID();
|
|
2355
|
+
const unboundedEvents = /* @__PURE__ */ new Set([
|
|
2356
|
+
"run.started",
|
|
2357
|
+
"run.recovered",
|
|
2358
|
+
"run.paused",
|
|
2359
|
+
"run.resumed",
|
|
2360
|
+
"elicitation.requested",
|
|
2361
|
+
"elicitation.resolved",
|
|
2362
|
+
"approval.requested",
|
|
2363
|
+
"approval.resolved",
|
|
2364
|
+
"tool.execution_unknown",
|
|
2365
|
+
"run.completed",
|
|
2366
|
+
"run.failed",
|
|
2367
|
+
"run.cancelled",
|
|
2368
|
+
"workflow.run.started",
|
|
2369
|
+
"workflow.run.recovered",
|
|
2370
|
+
"workflow.run.paused",
|
|
2371
|
+
"workflow.run.resumed",
|
|
2372
|
+
"workflow.wait.started",
|
|
2373
|
+
"workflow.wait.completed",
|
|
2374
|
+
"workflow.run.completed",
|
|
2375
|
+
"workflow.run.failed",
|
|
2376
|
+
"workflow.run.cancelled",
|
|
2377
|
+
"team.run.started",
|
|
2378
|
+
"team.run.recovered",
|
|
2379
|
+
"team.run.paused",
|
|
2380
|
+
"team.run.resumed",
|
|
2381
|
+
"team.run.completed",
|
|
2382
|
+
"team.run.failed",
|
|
2383
|
+
"team.run.cancelled"
|
|
2384
|
+
]);
|
|
2385
|
+
const tenantOf = (context) => {
|
|
2386
|
+
const configured = admission?.tenantKey?.(context);
|
|
2387
|
+
if (configured !== void 0) return configured;
|
|
2388
|
+
const tenantId = context?.attributes?.tenantId;
|
|
2389
|
+
if (typeof tenantId !== "string") return void 0;
|
|
2390
|
+
return tenantId.trim() || void 0;
|
|
2391
|
+
};
|
|
2392
|
+
const countsForAdmission = (runId, state) => !internalRuns.has(runId) && !state.request.signal?.aborted;
|
|
2393
|
+
const activeTopLevelRuns = () => {
|
|
2394
|
+
let count = 0;
|
|
2395
|
+
for (const [runId, state] of activeRuns) if (countsForAdmission(runId, state)) count += 1;
|
|
2396
|
+
for (const [runId, state] of activeWorkflowRuns) if (countsForAdmission(runId, state)) count += 1;
|
|
2397
|
+
return count;
|
|
2398
|
+
};
|
|
2399
|
+
const activeTenantRuns = (tenant) => {
|
|
2400
|
+
let count = 0;
|
|
2401
|
+
for (const [runId, state] of activeRuns) if (countsForAdmission(runId, state) && tenantOf(state.request.context) === tenant) count += 1;
|
|
2402
|
+
for (const [runId, state] of activeWorkflowRuns) if (countsForAdmission(runId, state) && tenantOf(state.request.context) === tenant) count += 1;
|
|
2403
|
+
return count;
|
|
2404
|
+
};
|
|
2405
|
+
const admitStart = async (kind, name, request) => {
|
|
2406
|
+
if (!admission) return () => {};
|
|
2407
|
+
const running = activeTopLevelRuns() + pendingStarts;
|
|
2408
|
+
const tenant = tenantOf(request.context);
|
|
2409
|
+
const tenantRuns = tenant === void 0 ? 0 : activeTenantRuns(tenant) + (pendingTenantStarts.get(tenant) ?? 0);
|
|
2410
|
+
if (admission.maxActiveRuns !== void 0 && running >= admission.maxActiveRuns) throw new FevexRunError("ADMISSION_DENIED", `Runtime reached maxActiveRuns limit of ${admission.maxActiveRuns}`);
|
|
2411
|
+
if (admission.maxActiveRunsPerTenant !== void 0 && tenant !== void 0 && tenantRuns >= admission.maxActiveRunsPerTenant) throw new FevexRunError("ADMISSION_DENIED", `Tenant "${tenant}" reached maxActiveRunsPerTenant limit of ${admission.maxActiveRunsPerTenant}`);
|
|
2412
|
+
pendingStarts += 1;
|
|
2413
|
+
if (tenant !== void 0) pendingTenantStarts.set(tenant, (pendingTenantStarts.get(tenant) ?? 0) + 1);
|
|
2414
|
+
let released = false;
|
|
2415
|
+
const release = () => {
|
|
2416
|
+
if (released) return;
|
|
2417
|
+
released = true;
|
|
2418
|
+
pendingStarts = Math.max(0, pendingStarts - 1);
|
|
2419
|
+
if (tenant === void 0) return;
|
|
2420
|
+
const next = (pendingTenantStarts.get(tenant) ?? 1) - 1;
|
|
2421
|
+
if (next <= 0) pendingTenantStarts.delete(tenant);
|
|
2422
|
+
else pendingTenantStarts.set(tenant, next);
|
|
2423
|
+
};
|
|
2424
|
+
try {
|
|
2425
|
+
if (admission.authorize && !await admission.authorize({
|
|
2426
|
+
kind,
|
|
2427
|
+
name,
|
|
2428
|
+
request,
|
|
2429
|
+
activeRuns: running,
|
|
2430
|
+
activeTenantRuns: tenantRuns
|
|
2431
|
+
})) throw new FevexRunError("ADMISSION_DENIED", "Run admission was denied by the host");
|
|
2432
|
+
} catch (error) {
|
|
2433
|
+
release();
|
|
2434
|
+
throw error;
|
|
2435
|
+
}
|
|
2436
|
+
return release;
|
|
2437
|
+
};
|
|
2438
|
+
const markInternalRun = (runId) => {
|
|
2439
|
+
internalRuns.add(runId);
|
|
2440
|
+
};
|
|
2247
2441
|
const resolveModel = (agent, requestedModel) => {
|
|
2248
2442
|
const modelName = requestedModel ?? (typeof agent.model === "string" ? agent.model : "default");
|
|
2249
2443
|
if (!modelName.trim()) throw new FevexRunError("MODEL_NOT_FOUND", "Model name cannot be empty");
|
|
@@ -2302,14 +2496,17 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
|
|
|
2302
2496
|
Promise.resolve(onEvent(event)).catch(() => {});
|
|
2303
2497
|
} catch {}
|
|
2304
2498
|
};
|
|
2305
|
-
const createEvent = (state, type, payload) =>
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2499
|
+
const createEvent = (state, type, payload) => {
|
|
2500
|
+
if (limits?.maxEventsPerRun !== void 0 && !unboundedEvents.has(type) && state.eventSequence >= limits.maxEventsPerRun) throw new FevexRunError("EVENT_LIMIT_EXCEEDED", `Run exceeded maxEventsPerRun limit of ${limits.maxEventsPerRun}`, state.run.id);
|
|
2501
|
+
return {
|
|
2502
|
+
id: crypto.randomUUID(),
|
|
2503
|
+
sequence: state.eventSequence += 1,
|
|
2504
|
+
type,
|
|
2505
|
+
runId: state.run.id,
|
|
2506
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2507
|
+
...payload === void 0 ? {} : { payload }
|
|
2508
|
+
};
|
|
2509
|
+
};
|
|
2313
2510
|
const createCoordinatorEvent = (state, workflowType, workflowPayload, teamType, teamPayload = workflowPayload) => createEvent(state, isTeamRun(state.run) ? teamType : workflowType, isTeamRun(state.run) ? teamPayload : workflowPayload);
|
|
2314
2511
|
const emitEvent = async (state, type, payload) => {
|
|
2315
2512
|
const event = createEvent(state, type, payload);
|
|
@@ -2323,6 +2520,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
|
|
|
2323
2520
|
const releaseExecution = async (state) => {
|
|
2324
2521
|
activeRuns.delete(state.run.id);
|
|
2325
2522
|
activeSessions.delete(state.session.id);
|
|
2523
|
+
internalRuns.delete(state.run.id);
|
|
2326
2524
|
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
2327
2525
|
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2328
2526
|
state.leaseTimer = void 0;
|
|
@@ -2411,6 +2609,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
|
|
|
2411
2609
|
const releaseWorkflowExecution = async (state) => {
|
|
2412
2610
|
activeWorkflowRuns.delete(state.run.id);
|
|
2413
2611
|
activeSessions.delete(state.session.id);
|
|
2612
|
+
internalRuns.delete(state.run.id);
|
|
2414
2613
|
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
2415
2614
|
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2416
2615
|
state.leaseTimer = void 0;
|
|
@@ -2452,6 +2651,8 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
|
|
|
2452
2651
|
pendingExports,
|
|
2453
2652
|
exportFailures,
|
|
2454
2653
|
runtimeOwner,
|
|
2654
|
+
admitStart,
|
|
2655
|
+
markInternalRun,
|
|
2455
2656
|
resolveModel,
|
|
2456
2657
|
resolveCheckpointModel,
|
|
2457
2658
|
notifyObserver,
|
|
@@ -2479,7 +2680,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
|
|
|
2479
2680
|
*/
|
|
2480
2681
|
function createWorkflowEngine(composition, core, agentEngine) {
|
|
2481
2682
|
const { models, agents, workflows, teams, runStore } = composition;
|
|
2482
|
-
const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
|
|
2683
|
+
const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, admitStart, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
|
|
2483
2684
|
const { prepareExecution, executeAgent, nextEvent, resumeAgent, cancelStoredAgent } = agentEngine;
|
|
2484
2685
|
const cancelWorkflowExecution = async (state, reason = cancellationReason(state.request.signal)) => {
|
|
2485
2686
|
if (state.leaseLost) {
|
|
@@ -2488,7 +2689,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2488
2689
|
}
|
|
2489
2690
|
if (state.run.status === "completed" || state.run.status === "failed" || state.run.status === "cancelled") return;
|
|
2490
2691
|
const childRunIds = Object.values(state.checkpoint.steps).filter((record) => record.type === "agent" && record.status === "running").map(({ childRunId }) => childRunId);
|
|
2491
|
-
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId)) throw new FevexRunError("RUN_CONFLICT", `Workflow child "${childRunId}" could not be cancelled`, state.run.id);
|
|
2692
|
+
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId, reason === "timeout" ? "timeout" : "aborted")) throw new FevexRunError("RUN_CONFLICT", `Workflow child "${childRunId}" could not be cancelled`, state.run.id);
|
|
2492
2693
|
state.run.status = "cancelled";
|
|
2493
2694
|
state.run.pause = void 0;
|
|
2494
2695
|
state.run.error = reason;
|
|
@@ -2501,7 +2702,76 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2501
2702
|
await releaseWorkflowExecution(state);
|
|
2502
2703
|
return event;
|
|
2503
2704
|
};
|
|
2504
|
-
const
|
|
2705
|
+
const cancelStoredWorkflow = async (runId, reason = "aborted") => {
|
|
2706
|
+
const active = activeWorkflowRuns.get(runId);
|
|
2707
|
+
if (active && !active.request.signal.aborted) {
|
|
2708
|
+
active.controller.abort(new DOMException(reason === "timeout" ? "Run deadline exceeded" : "Run cancelled", reason === "timeout" ? "TimeoutError" : "AbortError"));
|
|
2709
|
+
if (!active.advancing) {
|
|
2710
|
+
await cancelWorkflowExecution(active);
|
|
2711
|
+
return true;
|
|
2712
|
+
}
|
|
2713
|
+
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
2714
|
+
const current = await runStore.getRun(runId);
|
|
2715
|
+
if (current?.status === "cancelled" || current?.status === "completed" || current?.status === "failed") return true;
|
|
2716
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
2717
|
+
}
|
|
2718
|
+
return false;
|
|
2719
|
+
}
|
|
2720
|
+
if (!isDurableRunStore(runStore)) return false;
|
|
2721
|
+
const run = await runStore.getRun(runId);
|
|
2722
|
+
if (!run) return false;
|
|
2723
|
+
if (run.status === "completed" || run.status === "failed" || run.status === "cancelled") return true;
|
|
2724
|
+
const session = await runStore.getSession(run.sessionId);
|
|
2725
|
+
const checkpoint = await runStore.getCheckpoint(runId);
|
|
2726
|
+
if (!session || !checkpoint) return false;
|
|
2727
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
2728
|
+
const lease = {
|
|
2729
|
+
generation: 0,
|
|
2730
|
+
runId,
|
|
2731
|
+
ownerId,
|
|
2732
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
2733
|
+
};
|
|
2734
|
+
if (!await runStore.acquireLease(lease)) return false;
|
|
2735
|
+
try {
|
|
2736
|
+
const childRunIds = Object.values(checkpoint.steps).filter((record) => record.type === "agent" && record.status === "running").map(({ childRunId }) => childRunId);
|
|
2737
|
+
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId, reason)) return false;
|
|
2738
|
+
const state = {
|
|
2739
|
+
run,
|
|
2740
|
+
session,
|
|
2741
|
+
request: {
|
|
2742
|
+
input: checkpoint.input,
|
|
2743
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
2744
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
2745
|
+
sessionId: session.id,
|
|
2746
|
+
context: checkpoint.context,
|
|
2747
|
+
limits: checkpoint.limits,
|
|
2748
|
+
signal: new AbortController().signal
|
|
2749
|
+
},
|
|
2750
|
+
controller: new AbortController(),
|
|
2751
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
2752
|
+
checkpoint,
|
|
2753
|
+
advancing: false,
|
|
2754
|
+
lease,
|
|
2755
|
+
commitQueue: Promise.resolve()
|
|
2756
|
+
};
|
|
2757
|
+
run.status = "cancelled";
|
|
2758
|
+
run.pause = void 0;
|
|
2759
|
+
run.error = reason;
|
|
2760
|
+
run.usage = checkpoint.budget?.usage;
|
|
2761
|
+
const cancelled = createCoordinatorEvent(state, "workflow.run.cancelled", { reason }, "team.run.cancelled");
|
|
2762
|
+
await commit(state, {
|
|
2763
|
+
checkpoint: null,
|
|
2764
|
+
events: [cancelled]
|
|
2765
|
+
});
|
|
2766
|
+
return true;
|
|
2767
|
+
} catch (error) {
|
|
2768
|
+
if (error instanceof FevexRunError && error.code === "RUN_CONFLICT" && (await runStore.getRun(runId))?.status !== "paused") return false;
|
|
2769
|
+
throw error;
|
|
2770
|
+
} finally {
|
|
2771
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
2772
|
+
}
|
|
2773
|
+
};
|
|
2774
|
+
const prepareWorkflowExecutionBody = async (name, request, kind = "workflow") => {
|
|
2505
2775
|
const workflow = workflows.get(name);
|
|
2506
2776
|
const team = kind === "team" ? teams.get(name) : void 0;
|
|
2507
2777
|
if (!workflow || kind === "team" && !team || kind === "workflow" && teams.has(name)) throw new FevexRunError(kind === "team" ? "TEAM_NOT_FOUND" : "WORKFLOW_NOT_FOUND", `${kind === "team" ? "Team" : "Workflow"} "${name}" is not registered`);
|
|
@@ -2520,9 +2790,9 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2520
2790
|
session = stored;
|
|
2521
2791
|
}
|
|
2522
2792
|
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" already has an active run`);
|
|
2523
|
-
activeSessions.add(session.id);
|
|
2524
2793
|
const controller = new AbortController();
|
|
2525
|
-
const signal =
|
|
2794
|
+
const signal = runSignal(controller, request);
|
|
2795
|
+
const deadlineAt = deadlineAtFromMs(request.deadlineMs);
|
|
2526
2796
|
const input = toJsonValue(workflow.inputSchema === void 0 ? request.input : await abortable(() => validateSchema(workflow.inputSchema, request.input, `Input for ${kind} "${name}" does not match inputSchema`), signal), `${kind === "team" ? "Team" : "Workflow"} input must be a string or JSON-serializable value`);
|
|
2527
2797
|
const id = crypto.randomUUID();
|
|
2528
2798
|
const run = kind === "team" ? {
|
|
@@ -2533,7 +2803,8 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2533
2803
|
status: "running",
|
|
2534
2804
|
revision: 0,
|
|
2535
2805
|
createdAt: now,
|
|
2536
|
-
updatedAt: now
|
|
2806
|
+
updatedAt: now,
|
|
2807
|
+
...deadlineAt === void 0 ? {} : { deadlineAt }
|
|
2537
2808
|
} : {
|
|
2538
2809
|
kind: "workflow",
|
|
2539
2810
|
id,
|
|
@@ -2542,7 +2813,8 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2542
2813
|
status: "running",
|
|
2543
2814
|
revision: 0,
|
|
2544
2815
|
createdAt: now,
|
|
2545
|
-
updatedAt: now
|
|
2816
|
+
updatedAt: now,
|
|
2817
|
+
...deadlineAt === void 0 ? {} : { deadlineAt }
|
|
2546
2818
|
};
|
|
2547
2819
|
const checkpoint = kind === "team" ? {
|
|
2548
2820
|
version: 2,
|
|
@@ -2569,6 +2841,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2569
2841
|
steps: {},
|
|
2570
2842
|
limits: combineLimits(workflow.limits, request.limits)
|
|
2571
2843
|
};
|
|
2844
|
+
activeSessions.add(session.id);
|
|
2572
2845
|
const state = {
|
|
2573
2846
|
run,
|
|
2574
2847
|
session,
|
|
@@ -2610,6 +2883,14 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2610
2883
|
activeWorkflowRuns.set(run.id, state);
|
|
2611
2884
|
return state;
|
|
2612
2885
|
};
|
|
2886
|
+
const prepareWorkflowExecution = async (name, request, kind = "workflow") => {
|
|
2887
|
+
const release = await admitStart(kind, name, request);
|
|
2888
|
+
try {
|
|
2889
|
+
return await prepareWorkflowExecutionBody(name, request, kind);
|
|
2890
|
+
} finally {
|
|
2891
|
+
release();
|
|
2892
|
+
}
|
|
2893
|
+
};
|
|
2613
2894
|
async function* executeWorkflow(name, state) {
|
|
2614
2895
|
const workflow = workflows.get(name);
|
|
2615
2896
|
const events = [];
|
|
@@ -2771,7 +3052,10 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2771
3052
|
if (!child) throw new Error(`Child run "${existing.childRunId}" does not exist`);
|
|
2772
3053
|
if (child.status === "running") {
|
|
2773
3054
|
if (!state.recoveryActor) throw new FevexRunError("RUN_NOT_RECOVERABLE", `Child run "${child.id}" is still running`, state.run.id);
|
|
2774
|
-
child = await resumeAgent(child.id, void 0, true, state.recoveryActor
|
|
3055
|
+
child = await resumeAgent(child.id, void 0, true, state.recoveryActor, {
|
|
3056
|
+
admit: false,
|
|
3057
|
+
internal: true
|
|
3058
|
+
});
|
|
2775
3059
|
}
|
|
2776
3060
|
if (child.status === "completed") return completeAgentStep(stepId, existing.agentName, existing.childRunId, toJsonValue(child.output, "Child agent output must be JSON-serializable"), options);
|
|
2777
3061
|
if (child.status === "paused" && child.pause) return pauseForChild(stepId, new RunPausedError(child.id, child.pause));
|
|
@@ -2782,12 +3066,13 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
2782
3066
|
const childSignal = request.signal ? AbortSignal.any([state.request.signal, request.signal]) : state.request.signal;
|
|
2783
3067
|
const childState = await prepareExecution(agentName, {
|
|
2784
3068
|
...request,
|
|
3069
|
+
deadlineMs: minDeadline(remainingDeadlineMs(state.run.deadlineAt, state.run.id), request.deadlineMs),
|
|
2785
3070
|
model: request.model ?? state.request.model,
|
|
2786
3071
|
reasoning: request.reasoning ?? state.request.reasoning,
|
|
2787
3072
|
context: mergeExecutionContext(state.request.context, request.context),
|
|
2788
3073
|
limits: combineLimits(request.limits, remainingWorkflowLimits(state.checkpoint.limits, state.checkpoint.budget)),
|
|
2789
3074
|
signal: childSignal
|
|
2790
|
-
});
|
|
3075
|
+
}, { admit: false });
|
|
2791
3076
|
const action = options?.metadata?.teamAction === "handoff" ? "handoff" : "delegate";
|
|
2792
3077
|
const lifecycleEvents = [];
|
|
2793
3078
|
if (isTeamRun(state.run) && action === "handoff") lifecycleEvents.push(createEvent(state, "team.handoff.created", {
|
|
@@ -3121,8 +3406,10 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3121
3406
|
};
|
|
3122
3407
|
} catch (error) {
|
|
3123
3408
|
let thrown = error;
|
|
3124
|
-
|
|
3125
|
-
|
|
3409
|
+
const parentDeadline = state.run.deadlineAt === void 0 ? void 0 : Date.parse(state.run.deadlineAt);
|
|
3410
|
+
const parentTimedOut = isTimeoutError(error) && parentDeadline !== void 0 && Date.now() >= parentDeadline - 10;
|
|
3411
|
+
if (state.request.signal.aborted || parentTimedOut) try {
|
|
3412
|
+
const event = await cancelWorkflowExecution(state, parentTimedOut ? "timeout" : void 0);
|
|
3126
3413
|
if (event) {
|
|
3127
3414
|
events.push(event);
|
|
3128
3415
|
yield event;
|
|
@@ -3184,13 +3471,18 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3184
3471
|
}
|
|
3185
3472
|
}
|
|
3186
3473
|
};
|
|
3187
|
-
const
|
|
3474
|
+
const resumeWorkflowBody = async (runId, resolution, recoveryActor) => {
|
|
3188
3475
|
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "resumeRun requires DurableRunStore", runId);
|
|
3189
3476
|
const run = await runStore.getRun(runId);
|
|
3190
3477
|
const checkpoint = await runStore.getCheckpoint(runId);
|
|
3191
3478
|
if (!run || !checkpoint) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be ${resolution ? "resumed" : "recovered"}`, runId);
|
|
3192
3479
|
if (checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${runId}" is unsupported`, runId);
|
|
3193
3480
|
if (resolution && run.status !== "paused" && run.status !== "running" || !resolution && run.status !== "running") throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" is not ${resolution ? "resumable" : "recoverable"}`, runId);
|
|
3481
|
+
if (run.deadlineAt !== void 0 && Date.parse(run.deadlineAt) <= Date.now()) {
|
|
3482
|
+
if (!await cancelStoredWorkflow(runId, "timeout")) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" deadline has passed`, runId);
|
|
3483
|
+
return await runStore.getRun(runId);
|
|
3484
|
+
}
|
|
3485
|
+
const deadlineSignal = resumeDeadlineSignal(run.deadlineAt, runId, resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE");
|
|
3194
3486
|
const name = coordinatorName(run);
|
|
3195
3487
|
const workflow = workflows.get(name);
|
|
3196
3488
|
const team = isTeamRun(run) ? teams.get(name) : void 0;
|
|
@@ -3213,6 +3505,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3213
3505
|
if (!resolution) {
|
|
3214
3506
|
if (!recoveryActor?.id.trim()) throw new FevexRunError("RUN_NOT_RECOVERABLE", "Recovery actor is required", runId);
|
|
3215
3507
|
const controller = new AbortController();
|
|
3508
|
+
const signal = deadlineSignal ? AbortSignal.any([controller.signal, deadlineSignal]) : controller.signal;
|
|
3216
3509
|
const state = {
|
|
3217
3510
|
run,
|
|
3218
3511
|
session,
|
|
@@ -3223,7 +3516,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3223
3516
|
sessionId: session.id,
|
|
3224
3517
|
context: checkpoint.context,
|
|
3225
3518
|
limits: checkpoint.limits,
|
|
3226
|
-
signal
|
|
3519
|
+
signal
|
|
3227
3520
|
},
|
|
3228
3521
|
controller,
|
|
3229
3522
|
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
@@ -3266,6 +3559,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3266
3559
|
if (validatedPayload !== void 0) eventPayload = toJsonValue(validatedPayload, "Workflow event payload must be JSON-serializable");
|
|
3267
3560
|
}
|
|
3268
3561
|
const controller = new AbortController();
|
|
3562
|
+
const signal = deadlineSignal ? AbortSignal.any([controller.signal, deadlineSignal]) : controller.signal;
|
|
3269
3563
|
const state = {
|
|
3270
3564
|
run,
|
|
3271
3565
|
session,
|
|
@@ -3276,7 +3570,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3276
3570
|
sessionId: session.id,
|
|
3277
3571
|
context: checkpoint.context,
|
|
3278
3572
|
limits: checkpoint.limits,
|
|
3279
|
-
signal
|
|
3573
|
+
signal
|
|
3280
3574
|
},
|
|
3281
3575
|
controller,
|
|
3282
3576
|
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
@@ -3314,8 +3608,93 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3314
3608
|
if (resolution.type === "timer" || resolution.type === "event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
3315
3609
|
const childRunId = workflowPause.type === "workflow_children" ? workflowPause.children.find(({ childPause }) => pauseMatchesResolution(childPause, resolution))?.childRunId : workflowPause.childRunId;
|
|
3316
3610
|
if (!childRunId) throw new FevexRunError("RUN_NOT_RESUMABLE", "Resolution does not match a paused child run", runId);
|
|
3317
|
-
|
|
3611
|
+
let childRun;
|
|
3612
|
+
try {
|
|
3613
|
+
childRun = await resumeAgent(childRunId, resolution, true, void 0, {
|
|
3614
|
+
admit: false,
|
|
3615
|
+
internal: true
|
|
3616
|
+
});
|
|
3617
|
+
} catch (error) {
|
|
3618
|
+
const parentDeadline = run.deadlineAt === void 0 ? void 0 : Date.parse(run.deadlineAt);
|
|
3619
|
+
const child = await runStore.getRun(childRunId);
|
|
3620
|
+
const childDeadline = child?.deadlineAt === void 0 ? void 0 : Date.parse(child.deadlineAt);
|
|
3621
|
+
if (isTimeoutError(error) && parentDeadline !== void 0 && childDeadline !== void 0 && Math.abs(childDeadline - parentDeadline) <= 5) {
|
|
3622
|
+
const pausedChildIds = workflowPause.type === "workflow_children" ? workflowPause.children.map(({ childRunId }) => childRunId) : [workflowPause.childRunId];
|
|
3623
|
+
for (const pausedChildId of pausedChildIds) if (!await cancelStoredAgent(pausedChildId, "timeout")) throw error;
|
|
3624
|
+
const cancelController = new AbortController();
|
|
3625
|
+
const cancelState = {
|
|
3626
|
+
run,
|
|
3627
|
+
session,
|
|
3628
|
+
request: {
|
|
3629
|
+
input: checkpoint.input,
|
|
3630
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
3631
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3632
|
+
sessionId: session.id,
|
|
3633
|
+
context: checkpoint.context,
|
|
3634
|
+
limits: checkpoint.limits,
|
|
3635
|
+
signal: cancelController.signal
|
|
3636
|
+
},
|
|
3637
|
+
controller: cancelController,
|
|
3638
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3639
|
+
checkpoint,
|
|
3640
|
+
advancing: false,
|
|
3641
|
+
lease,
|
|
3642
|
+
commitQueue: Promise.resolve()
|
|
3643
|
+
};
|
|
3644
|
+
run.status = "cancelled";
|
|
3645
|
+
run.pause = void 0;
|
|
3646
|
+
run.error = "timeout";
|
|
3647
|
+
run.usage = checkpoint.budget?.usage;
|
|
3648
|
+
const cancelled = createCoordinatorEvent(cancelState, "workflow.run.cancelled", { reason: "timeout" }, "team.run.cancelled");
|
|
3649
|
+
await commit(cancelState, {
|
|
3650
|
+
checkpoint: null,
|
|
3651
|
+
events: [cancelled]
|
|
3652
|
+
});
|
|
3653
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
3654
|
+
return structuredClone(run);
|
|
3655
|
+
}
|
|
3656
|
+
if (isTimeoutError(error)) {
|
|
3657
|
+
const pausedChildIds = workflowPause.type === "workflow_children" ? workflowPause.children.map(({ childRunId }) => childRunId) : [workflowPause.childRunId];
|
|
3658
|
+
for (const pausedChildId of pausedChildIds) if (!await cancelStoredAgent(pausedChildId, "aborted")) throw error;
|
|
3659
|
+
const failController = new AbortController();
|
|
3660
|
+
const failState = {
|
|
3661
|
+
run,
|
|
3662
|
+
session,
|
|
3663
|
+
request: {
|
|
3664
|
+
input: checkpoint.input,
|
|
3665
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
3666
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3667
|
+
sessionId: session.id,
|
|
3668
|
+
context: checkpoint.context,
|
|
3669
|
+
limits: checkpoint.limits,
|
|
3670
|
+
signal: failController.signal
|
|
3671
|
+
},
|
|
3672
|
+
controller: failController,
|
|
3673
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3674
|
+
checkpoint,
|
|
3675
|
+
advancing: false,
|
|
3676
|
+
lease,
|
|
3677
|
+
commitQueue: Promise.resolve()
|
|
3678
|
+
};
|
|
3679
|
+
run.status = "failed";
|
|
3680
|
+
run.pause = void 0;
|
|
3681
|
+
run.error = toErrorMessage(error);
|
|
3682
|
+
run.usage = checkpoint.budget?.usage;
|
|
3683
|
+
const failed = createCoordinatorEvent(failState, "workflow.run.failed", {
|
|
3684
|
+
error: toErrorMessage(error),
|
|
3685
|
+
...runErrorPayload(error)
|
|
3686
|
+
}, "team.run.failed");
|
|
3687
|
+
await commit(failState, {
|
|
3688
|
+
checkpoint: null,
|
|
3689
|
+
events: [failed]
|
|
3690
|
+
});
|
|
3691
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
3692
|
+
return structuredClone(run);
|
|
3693
|
+
}
|
|
3694
|
+
throw error;
|
|
3695
|
+
}
|
|
3318
3696
|
const controller = new AbortController();
|
|
3697
|
+
const signal = deadlineSignal ? AbortSignal.any([controller.signal, deadlineSignal]) : controller.signal;
|
|
3319
3698
|
const state = {
|
|
3320
3699
|
run,
|
|
3321
3700
|
session,
|
|
@@ -3326,7 +3705,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3326
3705
|
sessionId: session.id,
|
|
3327
3706
|
context: checkpoint.context,
|
|
3328
3707
|
limits: checkpoint.limits,
|
|
3329
|
-
signal
|
|
3708
|
+
signal
|
|
3330
3709
|
},
|
|
3331
3710
|
controller,
|
|
3332
3711
|
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
@@ -3339,7 +3718,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3339
3718
|
run.status = "cancelled";
|
|
3340
3719
|
run.pause = void 0;
|
|
3341
3720
|
run.error = childRun.error ?? "aborted";
|
|
3342
|
-
const reason = childRun.error === "approval_rejected" ? "approval_rejected" : "aborted";
|
|
3721
|
+
const reason = childRun.error === "approval_rejected" ? "approval_rejected" : childRun.error === "timeout" ? "timeout" : "aborted";
|
|
3343
3722
|
const cancelled = createCoordinatorEvent(state, "workflow.run.cancelled", { reason }, "team.run.cancelled");
|
|
3344
3723
|
await commit(state, {
|
|
3345
3724
|
checkpoint: null,
|
|
@@ -3365,13 +3744,30 @@ function createWorkflowEngine(composition, core, agentEngine) {
|
|
|
3365
3744
|
throw error;
|
|
3366
3745
|
}
|
|
3367
3746
|
};
|
|
3747
|
+
const resumeWorkflow = async (runId, resolution, recoveryActor, options = {}) => {
|
|
3748
|
+
let release;
|
|
3749
|
+
if (options.admit !== false) {
|
|
3750
|
+
const run = await runStore.getRun(runId);
|
|
3751
|
+
const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(runId) : void 0;
|
|
3752
|
+
if (run && checkpoint && (run.status === "paused" || run.status === "running")) release = await admitStart(isTeamRun(run) ? "team" : "workflow", coordinatorName(run), {
|
|
3753
|
+
input: "",
|
|
3754
|
+
...checkpoint.context === void 0 ? {} : { context: checkpoint.context }
|
|
3755
|
+
});
|
|
3756
|
+
}
|
|
3757
|
+
try {
|
|
3758
|
+
return await resumeWorkflowBody(runId, resolution, recoveryActor);
|
|
3759
|
+
} finally {
|
|
3760
|
+
release?.();
|
|
3761
|
+
}
|
|
3762
|
+
};
|
|
3368
3763
|
return {
|
|
3369
3764
|
prepareWorkflowExecution,
|
|
3370
3765
|
executeWorkflow,
|
|
3371
3766
|
nextWorkflowEvent,
|
|
3372
3767
|
drainWorkflowExecution,
|
|
3373
3768
|
resumeWorkflow,
|
|
3374
|
-
cancelWorkflowExecution
|
|
3769
|
+
cancelWorkflowExecution,
|
|
3770
|
+
cancelStoredWorkflow
|
|
3375
3771
|
};
|
|
3376
3772
|
}
|
|
3377
3773
|
//#endregion
|
|
@@ -3382,7 +3778,7 @@ function createRuntime(composition) {
|
|
|
3382
3778
|
const { activeRuns, activeWorkflowRuns, activeSessions, pendingExports, exportFailures, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
|
|
3383
3779
|
const agentEngine = createAgentEngine(composition, core);
|
|
3384
3780
|
const { prepareExecution, executeAgent, nextEvent, drainExecution, resumeAgent, cancelStoredAgent } = agentEngine;
|
|
3385
|
-
const { prepareWorkflowExecution, executeWorkflow, nextWorkflowEvent, drainWorkflowExecution, resumeWorkflow, cancelWorkflowExecution } = createWorkflowEngine(composition, core, agentEngine);
|
|
3781
|
+
const { prepareWorkflowExecution, executeWorkflow, nextWorkflowEvent, drainWorkflowExecution, resumeWorkflow, cancelWorkflowExecution, cancelStoredWorkflow } = createWorkflowEngine(composition, core, agentEngine);
|
|
3386
3782
|
const resumeRun = async (runId, resolution) => {
|
|
3387
3783
|
const run = await runStore.getRun(runId);
|
|
3388
3784
|
if (run && isCoordinatorRun(run)) return resumeWorkflow(runId, resolution);
|
|
@@ -3499,58 +3895,7 @@ function createRuntime(composition) {
|
|
|
3499
3895
|
}
|
|
3500
3896
|
const run = await runStore.getRun(runId);
|
|
3501
3897
|
if (!run || run.status !== "paused" && run.status !== "running") return false;
|
|
3502
|
-
if (
|
|
3503
|
-
const session = await runStore.getSession(run.sessionId);
|
|
3504
|
-
if (!session) return false;
|
|
3505
|
-
if (isCoordinatorRun(run)) {
|
|
3506
|
-
const checkpoint = await runStore.getCheckpoint(runId);
|
|
3507
|
-
if (!checkpoint) return false;
|
|
3508
|
-
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
3509
|
-
const lease = {
|
|
3510
|
-
generation: 0,
|
|
3511
|
-
runId,
|
|
3512
|
-
ownerId,
|
|
3513
|
-
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
3514
|
-
};
|
|
3515
|
-
if (!await runStore.acquireLease(lease)) return false;
|
|
3516
|
-
try {
|
|
3517
|
-
const childRunIds = Object.values(checkpoint.steps).filter((record) => record.type === "agent" && record.status === "running").map(({ childRunId }) => childRunId);
|
|
3518
|
-
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId)) return false;
|
|
3519
|
-
const state = {
|
|
3520
|
-
run,
|
|
3521
|
-
session,
|
|
3522
|
-
request: {
|
|
3523
|
-
input: checkpoint.input,
|
|
3524
|
-
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3525
|
-
sessionId: session.id,
|
|
3526
|
-
context: checkpoint.context,
|
|
3527
|
-
limits: checkpoint.limits,
|
|
3528
|
-
signal: new AbortController().signal
|
|
3529
|
-
},
|
|
3530
|
-
controller: new AbortController(),
|
|
3531
|
-
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3532
|
-
checkpoint,
|
|
3533
|
-
advancing: false,
|
|
3534
|
-
lease,
|
|
3535
|
-
commitQueue: Promise.resolve()
|
|
3536
|
-
};
|
|
3537
|
-
run.status = "cancelled";
|
|
3538
|
-
run.pause = void 0;
|
|
3539
|
-
run.error = "aborted";
|
|
3540
|
-
run.usage = checkpoint.budget?.usage;
|
|
3541
|
-
const cancelled = createCoordinatorEvent(state, "workflow.run.cancelled", { reason: "aborted" }, "team.run.cancelled");
|
|
3542
|
-
await commit(state, {
|
|
3543
|
-
checkpoint: null,
|
|
3544
|
-
events: [cancelled]
|
|
3545
|
-
});
|
|
3546
|
-
return true;
|
|
3547
|
-
} catch (error) {
|
|
3548
|
-
if (error instanceof FevexRunError && error.code === "RUN_CONFLICT" && (await runStore.getRun(runId))?.status !== "paused") return false;
|
|
3549
|
-
throw error;
|
|
3550
|
-
} finally {
|
|
3551
|
-
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
3552
|
-
}
|
|
3553
|
-
}
|
|
3898
|
+
if (isCoordinatorRun(run)) return cancelStoredWorkflow(runId);
|
|
3554
3899
|
return cancelStoredAgent(runId);
|
|
3555
3900
|
},
|
|
3556
3901
|
resumeRun(runId, resolution) {
|
|
@@ -3559,6 +3904,10 @@ function createRuntime(composition) {
|
|
|
3559
3904
|
recoverRun(runId, options) {
|
|
3560
3905
|
return recoverRun(runId, options.actor);
|
|
3561
3906
|
},
|
|
3907
|
+
async purgeRuns(options) {
|
|
3908
|
+
if (!isRetainableRunStore(runStore)) throw new FevexRunError("RETENTION_UNSUPPORTED", "The configured run store does not support purgeRuns");
|
|
3909
|
+
return runStore.purgeRuns(options);
|
|
3910
|
+
},
|
|
3562
3911
|
async compactSession(sessionId, summary) {
|
|
3563
3912
|
if (typeof summary !== "string" || !summary.trim()) throw new Error("Session summary must be a non-empty string");
|
|
3564
3913
|
if (activeSessions.has(sessionId)) throw new Error(`Session "${sessionId}" has an active run`);
|
|
@@ -3593,4 +3942,4 @@ function createFevex(config) {
|
|
|
3593
3942
|
return createRuntime(createComposition(config));
|
|
3594
3943
|
}
|
|
3595
3944
|
//#endregion
|
|
3596
|
-
export { ChannelError, FEVEX_JSON_SCHEMA_PROFILE_VERSION, FevexConfigurationError, FevexRunError, InMemoryMemoryStore, InMemoryRunStore, IntegrationError, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, SCHEMA_NOT_TRANSPORTABLE, SandboxError, compileFevexJsonSchema, createFevex, createLocalSandbox, createMemoryContextProvider, defineAgent, defineConnection, defineContextProvider, definePolicy, defineSkill, defineTeam, defineTool, defineWorkflow, deliverChannelRun, handleChannelInput, isDurableRunStore, sandboxResultToJson, toToolSpec, validateFevexJsonSchemaProfile, validateListEventsOptions };
|
|
3945
|
+
export { ArtifactError, CatalogedArtifactStore, ChannelError, FEVEX_JSON_SCHEMA_PROFILE_VERSION, FevexConfigurationError, FevexRunError, FileSystemArtifactStore, InMemoryArtifactCatalog, InMemoryArtifactStore, InMemoryMemoryStore, InMemoryRunStore, IntegrationError, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, S3ArtifactStore, SCHEMA_NOT_TRANSPORTABLE, SandboxError, artifactTenantId, assertArtifactId, compileFevexJsonSchema, createFevex, createLocalSandbox, createMemoryContextProvider, defineAgent, defineConnection, defineContextProvider, definePolicy, defineSkill, defineTeam, defineTool, defineWorkflow, deliverChannelRun, handleChannelInput, isDurableRunStore, isRetainableRunStore, sandboxResultToJson, toToolSpec, validateFevexJsonSchemaProfile, validateListEventsOptions, validatePurgeRunsOptions };
|