@fevex/core 0.1.0-alpha.1 → 0.1.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +424 -6
- package/dist/agents/index.d.mts +1 -1
- package/dist/agents/index.mjs +1 -1
- package/dist/channels/index.d.mts +2 -0
- package/dist/channels/index.mjs +150 -0
- package/dist/evals/index.d.mts +19 -11
- package/dist/evals/index.mjs +125 -122
- package/dist/fevex-Ddyi6z6L.d.mts +60 -0
- package/dist/http/index.d.mts +90 -0
- package/dist/http/index.mjs +470 -0
- package/dist/{index-ojCY3lwm.d.mts → index-B4PIkqOU.d.mts} +5 -4
- package/dist/index-Bg9rr8Qe.d.mts +50 -0
- package/dist/index-Bgcxgy66.d.mts +82 -0
- package/dist/index-CTIwrVf9.d.mts +51 -0
- package/dist/index-CqOjvtQ9.d.mts +132 -0
- package/dist/index-CrOId2px.d.mts +66 -0
- package/dist/index-DeXUtyd9.d.mts +70 -0
- package/dist/index-DviHtsLo.d.mts +58 -0
- package/dist/index-Dx_LotWZ.d.mts +425 -0
- package/dist/{index-QTkd6a7C.d.mts → index-dkFyTNh4.d.mts} +4 -3
- package/dist/index.d.mts +22 -10
- package/dist/index.mjs +2784 -720
- package/dist/knowledge/index.d.mts +2 -0
- package/dist/knowledge/index.mjs +89 -0
- package/dist/models/index.d.mts +2 -46
- package/dist/observability/index.d.mts +1 -1
- package/dist/policies/index.d.mts +1 -1
- package/dist/policies/index.mjs +1 -1
- package/dist/run-error-Cr17wPGH.mjs +23 -0
- package/dist/runtime/index.d.mts +2 -2
- package/dist/runtime/index.mjs +58 -9
- package/dist/sandbox/index.d.mts +2 -0
- package/dist/sandbox/index.mjs +122 -0
- package/dist/teams/index.d.mts +2 -0
- package/dist/teams/index.mjs +2 -0
- package/dist/teams-Ctz676G4.mjs +82 -0
- package/dist/testing/index.d.mts +33 -9
- package/dist/testing/index.mjs +415 -27
- package/dist/tools/index.d.mts +3 -3
- package/dist/tools/index.mjs +222 -2
- package/dist/types-CuoRx12Q.d.mts +333 -0
- package/dist/workflows/index.d.mts +2 -0
- package/dist/workflows/index.mjs +6 -0
- package/package.json +27 -3
- package/dist/fevex-Dz6GBz9o.d.mts +0 -32
- package/dist/index-BzxHfyYV.d.mts +0 -23
- package/dist/index-CZGyhNIQ.d.mts +0 -47
- package/dist/index-C_Zpr13f.d.mts +0 -124
- package/dist/index-DGWOGXzc.d.mts +0 -181
package/dist/index.mjs
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { defineAgent } from "./agents/index.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { ChannelError, handleChannelInput } from "./channels/index.mjs";
|
|
3
|
+
import { InMemoryMemoryStore, createMemoryContextProvider, defineContextProvider, defineSkill } from "./knowledge/index.mjs";
|
|
4
|
+
import { n as teamAsWorkflow, t as defineTeam } from "./teams-Ctz676G4.mjs";
|
|
5
|
+
import { n as RunPausedError, t as FevexRunError } from "./run-error-Cr17wPGH.mjs";
|
|
6
|
+
import { InMemoryRunStore, isDurableRunStore, validateListEventsOptions } from "./runtime/index.mjs";
|
|
7
|
+
import { FEVEX_JSON_SCHEMA_PROFILE_VERSION, IntegrationError, compileFevexJsonSchema, defineConnection, defineTool, toToolSpec, validateFevexJsonSchemaProfile } from "./tools/index.mjs";
|
|
3
8
|
import { a as assertTokenBudget, c as remainingOutputTokens, d as serializeValue, f as toJsonValue, i as assertContinuationBudget, l as toErrorMessage, n as abortable, o as cancellationReason, r as addUsage, s as eventUsage, t as readModelStream, u as serializeJsonValue } from "./model-stream-u2xMsEaf.mjs";
|
|
4
|
-
import { defineTool, toToolSpec } from "./tools/index.mjs";
|
|
5
9
|
import "./models/index.mjs";
|
|
6
10
|
import { definePolicy } from "./policies/index.mjs";
|
|
11
|
+
import { SandboxError, createLocalSandbox, sandboxResultToJson } from "./sandbox/index.mjs";
|
|
12
|
+
import { defineWorkflow } from "./workflows/index.mjs";
|
|
7
13
|
//#region src/configuration-error.ts
|
|
8
14
|
var FevexConfigurationError = class extends Error {
|
|
9
15
|
code;
|
|
@@ -14,7 +20,7 @@ var FevexConfigurationError = class extends Error {
|
|
|
14
20
|
}
|
|
15
21
|
};
|
|
16
22
|
//#endregion
|
|
17
|
-
//#region src/core/
|
|
23
|
+
//#region src/core/types.ts
|
|
18
24
|
const SCHEMA_NOT_TRANSPORTABLE = "SCHEMA_NOT_TRANSPORTABLE";
|
|
19
25
|
const PROVIDER_SCHEMA_UNSUPPORTED = "PROVIDER_SCHEMA_UNSUPPORTED";
|
|
20
26
|
const PROVIDER_REASONING_UNSUPPORTED = "PROVIDER_REASONING_UNSUPPORTED";
|
|
@@ -25,9 +31,6 @@ function isStandardSchema(value) {
|
|
|
25
31
|
const standard = value["~standard"];
|
|
26
32
|
return typeof standard === "object" && standard !== null && standard.version === 1 && typeof standard.validate === "function";
|
|
27
33
|
}
|
|
28
|
-
function assertStandardSchema(value, errorMessage) {
|
|
29
|
-
if (!isStandardSchema(value)) throw new TypeError(errorMessage);
|
|
30
|
-
}
|
|
31
34
|
function isStandardJsonSchema(value) {
|
|
32
35
|
if (!isStandardSchema(value)) return false;
|
|
33
36
|
const jsonSchema = value["~standard"].jsonSchema;
|
|
@@ -69,8 +72,12 @@ const REASONING_EFFORTS = /* @__PURE__ */ new Set([
|
|
|
69
72
|
"minimal",
|
|
70
73
|
"low",
|
|
71
74
|
"medium",
|
|
72
|
-
"high"
|
|
75
|
+
"high",
|
|
76
|
+
"xhigh",
|
|
77
|
+
"max"
|
|
73
78
|
]);
|
|
79
|
+
const DEFAULT_CONNECTION_TIMEOUT_MS = 3e4;
|
|
80
|
+
const INTERNAL_TOOL_PREFIX = "fevex__";
|
|
74
81
|
function isRecord(value) {
|
|
75
82
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
76
83
|
}
|
|
@@ -93,18 +100,206 @@ function isRunStore(value) {
|
|
|
93
100
|
"listEvents"
|
|
94
101
|
].every((method) => typeof value[method] === "function");
|
|
95
102
|
}
|
|
96
|
-
function
|
|
103
|
+
function isSandbox(value) {
|
|
104
|
+
return isRecord(value) && typeof value.run === "function";
|
|
105
|
+
}
|
|
106
|
+
function isJsonObject(value) {
|
|
107
|
+
return isRecord(value);
|
|
108
|
+
}
|
|
109
|
+
function isToolProvider(value) {
|
|
110
|
+
return (typeof value === "object" && value !== null || typeof value === "function") && typeof value.listTools === "function" && typeof value.callTool === "function";
|
|
111
|
+
}
|
|
112
|
+
function isContextProvider(value) {
|
|
113
|
+
return (typeof value === "object" && value !== null || typeof value === "function") && typeof value.name === "string" && typeof value.read === "function";
|
|
114
|
+
}
|
|
115
|
+
function isMemoryStore(value) {
|
|
116
|
+
return (typeof value === "object" && value !== null || typeof value === "function") && typeof value.search === "function" && typeof value.write === "function";
|
|
117
|
+
}
|
|
118
|
+
function connectionAllowlist(connection) {
|
|
119
|
+
if (connection.allowlist) return connection.allowlist;
|
|
120
|
+
if (connection.tools && Array.isArray(connection.tools.allow)) return connection.tools.allow;
|
|
121
|
+
}
|
|
122
|
+
function connectionMetadata(connection, remoteName) {
|
|
123
|
+
if (!connection.tools || "allow" in connection.tools) return void 0;
|
|
124
|
+
return connection.tools[remoteName];
|
|
125
|
+
}
|
|
126
|
+
function namespacedToolName(connectionName, toolName) {
|
|
127
|
+
return `${connectionName}__${toolName}`;
|
|
128
|
+
}
|
|
129
|
+
function timeoutSignal(parent, timeoutMs) {
|
|
130
|
+
const controller = new AbortController();
|
|
131
|
+
const onAbort = () => controller.abort(parent?.reason);
|
|
132
|
+
const timer = setTimeout(() => {
|
|
133
|
+
controller.abort(new IntegrationError("CONNECTION_TIMEOUT", "timeout", true, "Connection tool call timed out"));
|
|
134
|
+
}, timeoutMs);
|
|
135
|
+
parent?.addEventListener("abort", onAbort, { once: true });
|
|
136
|
+
return {
|
|
137
|
+
signal: controller.signal,
|
|
138
|
+
dispose() {
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
parent?.removeEventListener("abort", onAbort);
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
function connectionError(error, source) {
|
|
145
|
+
if (error instanceof IntegrationError) {
|
|
146
|
+
if (!source || error.source) return error;
|
|
147
|
+
return new IntegrationError(error.code, error.category, error.retryable, error.safeMessage, {
|
|
148
|
+
cause: error,
|
|
149
|
+
source
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
if (error instanceof DOMException && error.name === "AbortError") return new IntegrationError("CONNECTION_TIMEOUT", "timeout", true, "Connection tool call timed out", {
|
|
153
|
+
cause: error,
|
|
154
|
+
source
|
|
155
|
+
});
|
|
156
|
+
return new IntegrationError("CONNECTION_REMOTE_ERROR", "remote", false, "Connection tool call failed", {
|
|
157
|
+
cause: error,
|
|
158
|
+
source
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
function expandConnection(connection) {
|
|
162
|
+
assertConfiguration(isRecord(connection), "INVALID_CONNECTION", "Connection must be an object");
|
|
163
|
+
const allowlist = connectionAllowlist(connection);
|
|
164
|
+
assertConfiguration(typeof connection.name === "string" && Boolean(connection.name.trim()), "INVALID_CONNECTION", "Connection name must be a non-empty string");
|
|
165
|
+
assertConfiguration(!connection.name.includes("__"), "INVALID_CONNECTION", `Connection "${connection.name}" name cannot contain "__"`);
|
|
166
|
+
assertConfiguration(isToolProvider(connection.provider), "INVALID_CONNECTION", `Connection "${connection.name}" provider must implement listTools and callTool`);
|
|
167
|
+
assertConfiguration(Array.isArray(allowlist) && allowlist.length > 0, "INVALID_CONNECTION", `Connection "${connection.name}" allowlist must be a non-empty array`);
|
|
168
|
+
assertConfiguration(connection.timeoutMs === void 0 || Number.isInteger(connection.timeoutMs) && connection.timeoutMs > 0, "INVALID_CONNECTION", `Connection "${connection.name}" timeoutMs must be a positive integer`);
|
|
169
|
+
const seen = /* @__PURE__ */ new Set();
|
|
170
|
+
const listRemoteTools = async (context) => {
|
|
171
|
+
const tools = await connection.provider.listTools(context);
|
|
172
|
+
return new Map(tools.map((tool) => [tool.name, tool]));
|
|
173
|
+
};
|
|
174
|
+
return allowlist.map((remoteName) => {
|
|
175
|
+
assertConfiguration(typeof remoteName === "string" && Boolean(remoteName.trim()), "INVALID_TOOL", `Connection "${connection.name}" tool names must be non-empty strings`);
|
|
176
|
+
assertConfiguration(!remoteName.includes("__"), "INVALID_TOOL", `Connection "${connection.name}" tool "${remoteName}" cannot contain "__"`);
|
|
177
|
+
assertConfiguration(!seen.has(remoteName), "DUPLICATE_TOOL", `Connection "${connection.name}" tool "${remoteName}" is duplicated`);
|
|
178
|
+
seen.add(remoteName);
|
|
179
|
+
const policy = connectionMetadata(connection, remoteName);
|
|
180
|
+
const source = {
|
|
181
|
+
kind: "connection",
|
|
182
|
+
provider: typeof connection.provider.kind === "string" && connection.provider.kind.trim() ? connection.provider.kind : "custom",
|
|
183
|
+
connectionName: connection.name,
|
|
184
|
+
remoteToolName: remoteName
|
|
185
|
+
};
|
|
186
|
+
assertConfiguration(policy?.inputSchema === void 0 || isJsonObject(policy.inputSchema), "INVALID_TOOL", `Input schema for connection tool "${remoteName}" must be a JSON object`);
|
|
187
|
+
assertConfiguration(policy?.outputSchema === void 0 || isJsonObject(policy.outputSchema), "INVALID_TOOL", `Output schema for connection tool "${remoteName}" must be a JSON object`);
|
|
188
|
+
return {
|
|
189
|
+
name: namespacedToolName(connection.name, remoteName),
|
|
190
|
+
...policy?.label === void 0 ? {} : { label: policy.label },
|
|
191
|
+
...policy?.description === void 0 ? {} : { description: policy.description },
|
|
192
|
+
...policy?.inputSchema === void 0 ? {} : { inputJsonSchema: policy.inputSchema },
|
|
193
|
+
...policy?.outputSchema === void 0 ? {} : { outputJsonSchema: policy.outputSchema },
|
|
194
|
+
...policy?.risk === void 0 ? {} : { risk: policy.risk },
|
|
195
|
+
...policy?.approval === void 0 ? {} : { approval: policy.approval },
|
|
196
|
+
...policy?.idempotency === void 0 ? {} : { idempotency: policy.idempotency },
|
|
197
|
+
...policy?.retry === void 0 ? {} : { retry: policy.retry },
|
|
198
|
+
...policy?.credentials === void 0 ? {} : { credentials: policy.credentials },
|
|
199
|
+
source,
|
|
200
|
+
async resolve(context) {
|
|
201
|
+
try {
|
|
202
|
+
return (await listRemoteTools(context)).get(remoteName);
|
|
203
|
+
} catch (error) {
|
|
204
|
+
throw connectionError(error, source);
|
|
205
|
+
}
|
|
206
|
+
},
|
|
207
|
+
async execute(input, context) {
|
|
208
|
+
const timeout = timeoutSignal(context.signal, connection.timeoutMs ?? DEFAULT_CONNECTION_TIMEOUT_MS);
|
|
209
|
+
try {
|
|
210
|
+
return await Promise.race([connection.provider.callTool(remoteName, input, {
|
|
211
|
+
runId: context.runId,
|
|
212
|
+
toolCallId: context.toolCallId,
|
|
213
|
+
attempt: context.attempt,
|
|
214
|
+
idempotencyKey: context.idempotencyKey,
|
|
215
|
+
context: context.context,
|
|
216
|
+
signal: timeout.signal
|
|
217
|
+
}), new Promise((_, reject) => {
|
|
218
|
+
timeout.signal.addEventListener("abort", () => {
|
|
219
|
+
reject(timeout.signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
220
|
+
}, { once: true });
|
|
221
|
+
})]);
|
|
222
|
+
} catch (error) {
|
|
223
|
+
throw connectionError(error, source);
|
|
224
|
+
} finally {
|
|
225
|
+
timeout.dispose();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
function assertLimit(value, name, minimum, ownerName, allowFalse = false, owner = "Agent", code = "INVALID_AGENT") {
|
|
97
232
|
if (value === void 0 || allowFalse && value === false) return;
|
|
98
|
-
if (typeof value !== "number" || !Number.isInteger(value) || value < minimum) throw configurationError(
|
|
233
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < minimum) throw configurationError(code, `${owner} "${ownerName}" limit "${name}" must be ${minimum === 0 ? "a non-negative" : "a positive"} integer${allowFalse ? " or false" : ""}`);
|
|
234
|
+
}
|
|
235
|
+
function assertStringArray(value, code, message) {
|
|
236
|
+
assertConfiguration(value === void 0 || Array.isArray(value) && value.every((item) => typeof item === "string" && item.trim()), code, message);
|
|
237
|
+
}
|
|
238
|
+
function assertPositiveInteger(value, code, message) {
|
|
239
|
+
assertConfiguration(value === void 0 || Number.isInteger(value) && value > 0, code, message);
|
|
240
|
+
}
|
|
241
|
+
function assertSandboxCapabilities(value, ownerName) {
|
|
242
|
+
if (value === void 0) return;
|
|
243
|
+
assertConfiguration(isRecord(value), "INVALID_TOOL", `Tool "${ownerName}" sandbox must be an object`);
|
|
244
|
+
const capabilities = value;
|
|
245
|
+
const process = capabilities.process;
|
|
246
|
+
if (process !== void 0) {
|
|
247
|
+
assertConfiguration(isRecord(process), "INVALID_TOOL", `Tool "${ownerName}" sandbox.process must be an object`);
|
|
248
|
+
assertStringArray(process.commands, "INVALID_TOOL", `Tool "${ownerName}" sandbox.process.commands must contain non-empty strings`);
|
|
249
|
+
assertPositiveInteger(process.timeoutMs, "INVALID_TOOL", `Tool "${ownerName}" sandbox.process.timeoutMs must be a positive integer`);
|
|
250
|
+
}
|
|
251
|
+
const filesystem = capabilities.filesystem;
|
|
252
|
+
if (filesystem !== void 0) {
|
|
253
|
+
assertConfiguration(isRecord(filesystem), "INVALID_TOOL", `Tool "${ownerName}" sandbox.filesystem must be an object`);
|
|
254
|
+
assertConfiguration(filesystem.cwd === void 0 || typeof filesystem.cwd === "string", "INVALID_TOOL", `Tool "${ownerName}" sandbox.filesystem.cwd must be a string`);
|
|
255
|
+
assertStringArray(filesystem.read, "INVALID_TOOL", `Tool "${ownerName}" sandbox.filesystem.read must contain non-empty strings`);
|
|
256
|
+
assertStringArray(filesystem.write, "INVALID_TOOL", `Tool "${ownerName}" sandbox.filesystem.write must contain non-empty strings`);
|
|
257
|
+
}
|
|
258
|
+
const network = capabilities.network;
|
|
259
|
+
if (network !== void 0 && network !== false) {
|
|
260
|
+
assertConfiguration(isRecord(network), "INVALID_TOOL", `Tool "${ownerName}" sandbox.network must be false or an object`);
|
|
261
|
+
assertStringArray(network.allow, "INVALID_TOOL", `Tool "${ownerName}" sandbox.network.allow must contain non-empty strings`);
|
|
262
|
+
}
|
|
263
|
+
const resources = capabilities.resources;
|
|
264
|
+
if (resources !== void 0) {
|
|
265
|
+
assertConfiguration(isRecord(resources), "INVALID_TOOL", `Tool "${ownerName}" sandbox.resources must be an object`);
|
|
266
|
+
assertPositiveInteger(resources.timeoutMs, "INVALID_TOOL", `Tool "${ownerName}" sandbox.resources.timeoutMs must be a positive integer`);
|
|
267
|
+
assertPositiveInteger(resources.maxOutputBytes, "INVALID_TOOL", `Tool "${ownerName}" sandbox.resources.maxOutputBytes must be a positive integer`);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
function copySandboxCapabilities(value) {
|
|
271
|
+
if (!value) return void 0;
|
|
272
|
+
return {
|
|
273
|
+
...value.process === void 0 ? {} : { process: {
|
|
274
|
+
...value.process.commands === void 0 ? {} : { commands: [...value.process.commands] },
|
|
275
|
+
...value.process.timeoutMs === void 0 ? {} : { timeoutMs: value.process.timeoutMs }
|
|
276
|
+
} },
|
|
277
|
+
...value.filesystem === void 0 ? {} : { filesystem: {
|
|
278
|
+
...value.filesystem.cwd === void 0 ? {} : { cwd: value.filesystem.cwd },
|
|
279
|
+
...value.filesystem.read === void 0 ? {} : { read: [...value.filesystem.read] },
|
|
280
|
+
...value.filesystem.write === void 0 ? {} : { write: [...value.filesystem.write] }
|
|
281
|
+
} },
|
|
282
|
+
...value.network === void 0 ? {} : { network: value.network === false ? false : { allow: [...value.network.allow ?? []] } },
|
|
283
|
+
...value.resources === void 0 ? {} : { resources: {
|
|
284
|
+
...value.resources.timeoutMs === void 0 ? {} : { timeoutMs: value.resources.timeoutMs },
|
|
285
|
+
...value.resources.maxOutputBytes === void 0 ? {} : { maxOutputBytes: value.resources.maxOutputBytes }
|
|
286
|
+
} }
|
|
287
|
+
};
|
|
99
288
|
}
|
|
100
289
|
function createComposition(config) {
|
|
101
290
|
assertConfiguration(isRecord(config), "INVALID_CONFIG", "Fevex config must be an object");
|
|
102
291
|
assertConfiguration(isRecord(config.models), "INVALID_CONFIG", "Fevex config \"models\" must be an object");
|
|
103
292
|
assertConfiguration(Array.isArray(config.agents), "INVALID_CONFIG", "Fevex config \"agents\" must be an array");
|
|
293
|
+
assertConfiguration(config.workflows === void 0 || Array.isArray(config.workflows), "INVALID_CONFIG", "Fevex config \"workflows\" must be an array");
|
|
294
|
+
assertConfiguration(config.teams === void 0 || Array.isArray(config.teams), "INVALID_CONFIG", "Fevex config \"teams\" must be an array");
|
|
104
295
|
assertConfiguration(config.tools === void 0 || Array.isArray(config.tools), "INVALID_CONFIG", "Fevex config \"tools\" must be an array");
|
|
296
|
+
assertConfiguration(config.connections === void 0 || Array.isArray(config.connections), "INVALID_CONFIG", "Fevex config \"connections\" must be an array");
|
|
297
|
+
assertConfiguration(config.contextProviders === void 0 || Array.isArray(config.contextProviders), "INVALID_CONFIG", "Fevex config \"contextProviders\" must be an array");
|
|
298
|
+
assertConfiguration(config.memoryStore === void 0 || isMemoryStore(config.memoryStore), "INVALID_CONFIG", "Fevex config \"memoryStore\" must implement MemoryStore");
|
|
105
299
|
assertConfiguration(config.onEvent === void 0 || typeof config.onEvent === "function", "INVALID_CONFIG", "Fevex config \"onEvent\" must be a function");
|
|
106
300
|
assertConfiguration(config.runStore === void 0 || isRunStore(config.runStore), "INVALID_CONFIG", "Fevex config \"runStore\" must implement RunStore");
|
|
107
301
|
assertConfiguration(config.credentialStore === void 0 || typeof config.credentialStore?.resolve === "function", "INVALID_CONFIG", "Fevex config \"credentialStore\" must implement resolve");
|
|
302
|
+
assertConfiguration(config.sandbox === void 0 || isSandbox(config.sandbox), "INVALID_CONFIG", "Fevex config \"sandbox\" must implement Sandbox");
|
|
108
303
|
assertConfiguration(config.policies === void 0 || Array.isArray(config.policies), "INVALID_CONFIG", "Fevex config \"policies\" must be an array");
|
|
109
304
|
if (config.observability !== void 0) {
|
|
110
305
|
assertConfiguration(isRecord(config.observability), "INVALID_CONFIG", "Fevex config \"observability\" must be an object");
|
|
@@ -121,18 +316,30 @@ function createComposition(config) {
|
|
|
121
316
|
}
|
|
122
317
|
const models = /* @__PURE__ */ new Map();
|
|
123
318
|
const agents = /* @__PURE__ */ new Map();
|
|
319
|
+
const workflows = /* @__PURE__ */ new Map();
|
|
320
|
+
const teams = /* @__PURE__ */ new Map();
|
|
124
321
|
const tools = /* @__PURE__ */ new Map();
|
|
322
|
+
const contextProviders = /* @__PURE__ */ new Map();
|
|
125
323
|
for (const [name, model] of Object.entries(config.models)) {
|
|
126
324
|
assertConfiguration(name.trim(), "INVALID_MODEL", "Model name cannot be empty");
|
|
127
325
|
assertConfiguration(isModelGateway(model), "INVALID_MODEL", `Model "${name}" must implement stream`);
|
|
128
326
|
assertConfiguration(model.stateCodec === void 0 || isRecord(model.stateCodec) && typeof model.stateCodec.serialize === "function" && typeof model.stateCodec.restore === "function", "INVALID_MODEL", `Model "${name}" stateCodec must implement serialize and restore`);
|
|
129
327
|
models.set(name, model);
|
|
130
328
|
}
|
|
131
|
-
for (const [index,
|
|
329
|
+
for (const [index, provider] of (config.contextProviders ?? []).entries()) {
|
|
330
|
+
assertConfiguration(isContextProvider(provider), "INVALID_CONTEXT_PROVIDER", `Context provider at index ${index} must have a name and read`);
|
|
331
|
+
const name = provider.name;
|
|
332
|
+
assertConfiguration(name.trim(), "INVALID_CONTEXT_PROVIDER", "Context provider name cannot be empty");
|
|
333
|
+
assertConfiguration(!contextProviders.has(name), "DUPLICATE_CONTEXT_PROVIDER", `Context provider "${name}" is duplicated`);
|
|
334
|
+
contextProviders.set(name, provider);
|
|
335
|
+
}
|
|
336
|
+
const expandedTools = [...config.tools ?? [], ...(config.connections ?? []).flatMap((connection) => expandConnection(connection))];
|
|
337
|
+
for (const [index, definition] of expandedTools.entries()) {
|
|
132
338
|
assertConfiguration(isRecord(definition), "INVALID_TOOL", `Tool at index ${index} must be an object`);
|
|
133
339
|
assertConfiguration(typeof definition.name === "string", "INVALID_TOOL", "Tool name must be a string");
|
|
134
340
|
const name = definition.name;
|
|
135
341
|
assertConfiguration(name.trim(), "INVALID_TOOL", "Tool name cannot be empty");
|
|
342
|
+
assertConfiguration(!name.startsWith(INTERNAL_TOOL_PREFIX), "INVALID_TOOL", `Tool "${name}" uses reserved prefix "${INTERNAL_TOOL_PREFIX}"`);
|
|
136
343
|
assertConfiguration(!tools.has(name), "DUPLICATE_TOOL", `Tool "${name}" is duplicated`);
|
|
137
344
|
assertConfiguration(definition.description === void 0 || typeof definition.description === "string", "INVALID_TOOL", `Tool "${name}" description must be a string`);
|
|
138
345
|
assertConfiguration(typeof definition.execute === "function", "INVALID_TOOL", `Tool "${name}" must implement execute`);
|
|
@@ -145,25 +352,36 @@ function createComposition(config) {
|
|
|
145
352
|
assertConfiguration(definition.approval === void 0 || definition.approval === "never" || definition.approval === "required", "INVALID_TOOL", `Tool "${name}" approval is invalid`);
|
|
146
353
|
assertConfiguration(definition.idempotency === void 0 || definition.idempotency === "none" || definition.idempotency === "keyed", "INVALID_TOOL", `Tool "${name}" idempotency is invalid`);
|
|
147
354
|
assertConfiguration(definition.credentials === void 0 || Array.isArray(definition.credentials) && definition.credentials.every((value) => typeof value === "string" && value.trim()), "INVALID_TOOL", `Tool "${name}" credentials must be non-empty strings`);
|
|
355
|
+
assertSandboxCapabilities(definition.sandbox, name);
|
|
356
|
+
assertConfiguration(definition.sandbox === void 0 || config.sandbox !== void 0, "INVALID_TOOL", `Tool "${name}" requires Fevex config "sandbox"`);
|
|
148
357
|
if (definition.retry !== void 0) assertConfiguration(isRecord(definition.retry) && Number.isInteger(definition.retry.maxAttempts) && definition.retry.maxAttempts >= 1 && Number.isInteger(definition.retry.backoffMs) && definition.retry.backoffMs >= 0 && (definition.retry.maxBackoffMs === void 0 || Number.isInteger(definition.retry.maxBackoffMs) && definition.retry.maxBackoffMs >= 0), "INVALID_TOOL", `Tool "${name}" retry is invalid`);
|
|
149
358
|
if (definition.inputSchema !== void 0) assertConfiguration(isStandardSchema(definition.inputSchema), "INVALID_TOOL", `Input schema for tool "${name}" must implement Standard Schema`);
|
|
359
|
+
if (definition.inputSchema !== void 0) assertConfiguration(isStandardSchema(definition.inputSchema), "INVALID_AGENT", `Input schema for agent "${name}" must implement Standard Schema`);
|
|
150
360
|
if (definition.outputSchema !== void 0) assertConfiguration(isStandardSchema(definition.outputSchema), "INVALID_TOOL", `Output schema for tool "${name}" must implement Standard Schema`);
|
|
361
|
+
if (definition.inputJsonSchema !== void 0) assertConfiguration(isJsonObject(definition.inputJsonSchema), "INVALID_TOOL", `Input JSON schema for tool "${name}" must be an object`);
|
|
362
|
+
if (definition.outputJsonSchema !== void 0) assertConfiguration(isJsonObject(definition.outputJsonSchema), "INVALID_TOOL", `Output JSON schema for tool "${name}" must be an object`);
|
|
151
363
|
tools.set(name, {
|
|
152
364
|
name,
|
|
365
|
+
...definition.label === void 0 ? {} : { label: definition.label },
|
|
153
366
|
...definition.description === void 0 ? {} : { description: definition.description },
|
|
154
367
|
...definition.inputSchema === void 0 ? {} : { inputSchema: definition.inputSchema },
|
|
155
368
|
...definition.outputSchema === void 0 ? {} : { outputSchema: definition.outputSchema },
|
|
369
|
+
...definition.inputJsonSchema === void 0 ? {} : { inputJsonSchema: definition.inputJsonSchema },
|
|
370
|
+
...definition.outputJsonSchema === void 0 ? {} : { outputJsonSchema: definition.outputJsonSchema },
|
|
156
371
|
...definition.risk === void 0 ? {} : { risk: definition.risk },
|
|
157
372
|
...definition.approval === void 0 ? {} : { approval: definition.approval },
|
|
158
373
|
...definition.idempotency === void 0 ? {} : { idempotency: definition.idempotency },
|
|
159
374
|
...definition.retry === void 0 ? {} : { retry: { ...definition.retry } },
|
|
160
375
|
...definition.credentials === void 0 ? {} : { credentials: [...definition.credentials] },
|
|
376
|
+
...definition.sandbox === void 0 ? {} : { sandbox: copySandboxCapabilities(definition.sandbox) },
|
|
377
|
+
...definition.source === void 0 ? {} : { source: { ...definition.source } },
|
|
378
|
+
...definition.resolve === void 0 ? {} : { resolve: definition.resolve },
|
|
161
379
|
execute: definition.execute
|
|
162
380
|
});
|
|
163
381
|
}
|
|
164
382
|
const policies = [];
|
|
165
383
|
for (const [index, policy] of (config.policies ?? []).entries()) {
|
|
166
|
-
assertConfiguration(isRecord(policy) && typeof policy.name === "string" && Boolean(policy.name.trim()) && typeof policy.authorize === "function", "
|
|
384
|
+
assertConfiguration(isRecord(policy) && typeof policy.name === "string" && Boolean(policy.name.trim()) && typeof policy.authorize === "function", "INVALID_POLICY", `Policy at index ${index} must have a name and authorize`);
|
|
167
385
|
policies.push({
|
|
168
386
|
name: policy.name,
|
|
169
387
|
authorize: policy.authorize
|
|
@@ -192,8 +410,39 @@ function createComposition(config) {
|
|
|
192
410
|
agentTools.push(toolName);
|
|
193
411
|
}
|
|
194
412
|
}
|
|
413
|
+
let agentContext;
|
|
414
|
+
if (definition.context !== void 0) {
|
|
415
|
+
assertConfiguration(Array.isArray(definition.context), "INVALID_AGENT", `Agent "${name}" context must be an array`);
|
|
416
|
+
agentContext = [];
|
|
417
|
+
const seenContext = /* @__PURE__ */ new Set();
|
|
418
|
+
for (const [contextIndex, providerName] of definition.context.entries()) {
|
|
419
|
+
assertConfiguration(typeof providerName === "string" && Boolean(providerName.trim()), "INVALID_AGENT", `Context provider name at index ${contextIndex} for agent "${name}" must be a non-empty string`);
|
|
420
|
+
assertConfiguration(!seenContext.has(providerName), "INVALID_AGENT", `Context provider "${providerName}" is duplicated in agent "${name}"`);
|
|
421
|
+
seenContext.add(providerName);
|
|
422
|
+
agentContext.push(providerName);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
let agentSkills;
|
|
426
|
+
if (definition.skills !== void 0) {
|
|
427
|
+
assertConfiguration(Array.isArray(definition.skills), "INVALID_AGENT", `Agent "${name}" skills must be an array`);
|
|
428
|
+
agentSkills = [];
|
|
429
|
+
const seenSkills = /* @__PURE__ */ new Set();
|
|
430
|
+
for (const [skillIndex, skillName] of definition.skills.entries()) {
|
|
431
|
+
assertConfiguration(typeof skillName === "string" && Boolean(skillName.trim()), "INVALID_AGENT", `Skill name at index ${skillIndex} for agent "${name}" must be a non-empty string`);
|
|
432
|
+
assertConfiguration(!seenSkills.has(skillName), "INVALID_AGENT", `Skill "${skillName}" is duplicated in agent "${name}"`);
|
|
433
|
+
seenSkills.add(skillName);
|
|
434
|
+
agentSkills.push(skillName);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
assertConfiguration(definition.memory === void 0 || isRecord(definition.memory), "INVALID_AGENT", `Agent "${name}" memory must be an object`);
|
|
438
|
+
assertConfiguration(definition.memory?.read === void 0 || typeof definition.memory.read === "boolean", "INVALID_AGENT", `Agent "${name}" memory.read must be a boolean`);
|
|
439
|
+
assertConfiguration(definition.memory?.write === void 0 || typeof definition.memory.write === "boolean", "INVALID_AGENT", `Agent "${name}" memory.write must be a boolean`);
|
|
440
|
+
assertConfiguration(definition.memory?.limit === void 0 || Number.isInteger(definition.memory.limit) && definition.memory.limit > 0, "INVALID_AGENT", `Agent "${name}" memory.limit must be a positive integer`);
|
|
195
441
|
assertConfiguration(definition.reasoning === void 0 || typeof definition.reasoning === "string" && REASONING_EFFORTS.has(definition.reasoning), "INVALID_AGENT", `Agent "${name}" reasoning is invalid`);
|
|
196
442
|
assertConfiguration(definition.modelOptions === void 0 || isRecord(definition.modelOptions), "INVALID_AGENT", `Agent "${name}" modelOptions must be an object`);
|
|
443
|
+
assertConfiguration(definition.elicitation === void 0 || definition.elicitation === "pause" || definition.elicitation === "forbid", "INVALID_AGENT", `Agent "${name}" elicitation is invalid`);
|
|
444
|
+
assertConfiguration(definition.approvalMode === void 0 || definition.approvalMode === "pause" || definition.approvalMode === "deny", "INVALID_AGENT", `Agent "${name}" approvalMode is invalid`);
|
|
445
|
+
assertConfiguration(definition.toolChoice === void 0 || definition.toolChoice === "auto" || definition.toolChoice === "required" || definition.toolChoice === "none" || isRecord(definition.toolChoice) && typeof definition.toolChoice.name === "string" && Boolean(definition.toolChoice.name.trim()), "INVALID_AGENT", `Agent "${name}" toolChoice is invalid`);
|
|
197
446
|
assertConfiguration(definition.limits === void 0 || isRecord(definition.limits), "INVALID_AGENT", `Agent "${name}" limits must be an object`);
|
|
198
447
|
if (definition.outputSchema !== void 0) assertConfiguration(isStandardSchema(definition.outputSchema), "INVALID_AGENT", `Output schema for agent "${name}" must implement Standard Schema`);
|
|
199
448
|
const rawLimits = definition.limits;
|
|
@@ -206,8 +455,19 @@ function createComposition(config) {
|
|
|
206
455
|
instructions: definition.instructions,
|
|
207
456
|
...model === void 0 ? {} : { model },
|
|
208
457
|
...agentTools === void 0 ? {} : { tools: agentTools },
|
|
458
|
+
...agentContext === void 0 ? {} : { context: agentContext },
|
|
459
|
+
...agentSkills === void 0 ? {} : { skills: agentSkills },
|
|
460
|
+
...definition.memory === void 0 ? {} : { memory: {
|
|
461
|
+
...definition.memory.read === void 0 ? {} : { read: definition.memory.read },
|
|
462
|
+
...definition.memory.write === void 0 ? {} : { write: definition.memory.write },
|
|
463
|
+
...definition.memory.limit === void 0 ? {} : { limit: definition.memory.limit }
|
|
464
|
+
} },
|
|
209
465
|
...definition.reasoning === void 0 ? {} : { reasoning: definition.reasoning },
|
|
210
466
|
...definition.modelOptions === void 0 ? {} : { modelOptions: { ...definition.modelOptions } },
|
|
467
|
+
...definition.toolChoice === void 0 ? {} : { toolChoice: typeof definition.toolChoice === "string" ? definition.toolChoice : { name: definition.toolChoice.name } },
|
|
468
|
+
...definition.elicitation === void 0 ? {} : { elicitation: definition.elicitation },
|
|
469
|
+
...definition.approvalMode === void 0 ? {} : { approvalMode: definition.approvalMode },
|
|
470
|
+
...definition.inputSchema === void 0 ? {} : { inputSchema: definition.inputSchema },
|
|
211
471
|
...definition.outputSchema === void 0 ? {} : { outputSchema: definition.outputSchema },
|
|
212
472
|
...rawLimits === void 0 ? {} : { limits: {
|
|
213
473
|
...rawLimits.maxSteps === void 0 ? {} : { maxSteps: rawLimits.maxSteps },
|
|
@@ -221,13 +481,119 @@ function createComposition(config) {
|
|
|
221
481
|
if (typeof agent.model === "string" && !models.has(agent.model)) throw configurationError("MISSING_MODEL", `Model "${agent.model}" required by agent "${agent.name}" is not registered`);
|
|
222
482
|
if (agent.model === void 0 && !models.has("default")) throw configurationError("MISSING_MODEL", `Default model "default" required by agent "${agent.name}" is not registered`);
|
|
223
483
|
for (const toolName of agent.tools ?? []) if (!tools.has(toolName)) throw configurationError("MISSING_TOOL", `Tool "${toolName}" required by agent "${agent.name}" is not registered`);
|
|
484
|
+
if (typeof agent.toolChoice === "object" && !agent.tools?.includes(agent.toolChoice.name)) throw configurationError("MISSING_TOOL", `Tool "${agent.toolChoice.name}" required by agent "${agent.name}" toolChoice is not registered`);
|
|
485
|
+
for (const providerName of [...agent.context ?? [], ...agent.skills ?? []]) if (!contextProviders.has(providerName)) throw configurationError("INVALID_AGENT", `Context provider "${providerName}" required by agent "${agent.name}" is not registered`);
|
|
486
|
+
}
|
|
487
|
+
for (const [index, definition] of (config.workflows ?? []).entries()) {
|
|
488
|
+
assertConfiguration(isRecord(definition), "INVALID_WORKFLOW", `Workflow at index ${index} must be an object`);
|
|
489
|
+
assertConfiguration(typeof definition.name === "string", "INVALID_WORKFLOW", "Workflow name must be a string");
|
|
490
|
+
const name = definition.name;
|
|
491
|
+
assertConfiguration(name.trim(), "INVALID_WORKFLOW", "Workflow name cannot be empty");
|
|
492
|
+
assertConfiguration(typeof definition.run === "function", "INVALID_WORKFLOW", `Workflow "${name}" must implement run`);
|
|
493
|
+
assertConfiguration(definition.version === void 0 || typeof definition.version === "string" && Boolean(definition.version.trim()), "INVALID_WORKFLOW", `Workflow "${name}" version must be a non-empty string`);
|
|
494
|
+
assertConfiguration(definition.limits === void 0 || isRecord(definition.limits), "INVALID_WORKFLOW", `Workflow "${name}" limits must be an object`);
|
|
495
|
+
if (definition.inputSchema !== void 0) assertConfiguration(isStandardSchema(definition.inputSchema), "INVALID_WORKFLOW", `Input schema for workflow "${name}" must implement Standard Schema`);
|
|
496
|
+
if (definition.outputSchema !== void 0) assertConfiguration(isStandardSchema(definition.outputSchema), "INVALID_WORKFLOW", `Output schema for workflow "${name}" must implement Standard Schema`);
|
|
497
|
+
assertConfiguration(definition.events === void 0 || isRecord(definition.events), "INVALID_WORKFLOW", `Workflow "${name}" events must be an object`);
|
|
498
|
+
const workflowEvents = {};
|
|
499
|
+
for (const [eventName, rawEvent] of Object.entries(definition.events ?? {})) {
|
|
500
|
+
assertConfiguration(Boolean(eventName.trim()) && isRecord(rawEvent), "INVALID_WORKFLOW", `Workflow "${name}" event definitions must be named objects`);
|
|
501
|
+
assertConfiguration(rawEvent.requireActor === void 0 || typeof rawEvent.requireActor === "boolean", "INVALID_WORKFLOW", `Workflow "${name}" event "${eventName}" requireActor must be a boolean`);
|
|
502
|
+
assertConfiguration(rawEvent.payloadSchema === void 0 || isStandardSchema(rawEvent.payloadSchema), "INVALID_WORKFLOW", `Payload schema for workflow "${name}" event "${eventName}" must implement Standard Schema`);
|
|
503
|
+
workflowEvents[eventName] = {
|
|
504
|
+
...rawEvent.payloadSchema === void 0 ? {} : { payloadSchema: rawEvent.payloadSchema },
|
|
505
|
+
...rawEvent.requireActor === void 0 ? {} : { requireActor: rawEvent.requireActor }
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
const rawLimits = definition.limits;
|
|
509
|
+
assertLimit(rawLimits?.maxSteps, "maxSteps", 1, name, false, "Workflow", "INVALID_WORKFLOW");
|
|
510
|
+
assertLimit(rawLimits?.maxToolCalls, "maxToolCalls", 0, name, false, "Workflow", "INVALID_WORKFLOW");
|
|
511
|
+
assertLimit(rawLimits?.maxInputTokens, "maxInputTokens", 1, name, true, "Workflow", "INVALID_WORKFLOW");
|
|
512
|
+
assertLimit(rawLimits?.maxOutputTokens, "maxOutputTokens", 1, name, true, "Workflow", "INVALID_WORKFLOW");
|
|
513
|
+
assertConfiguration(!workflows.has(name), "DUPLICATE_WORKFLOW", `Workflow "${name}" is duplicated`);
|
|
514
|
+
workflows.set(name, {
|
|
515
|
+
name,
|
|
516
|
+
version: definition.version ?? "1",
|
|
517
|
+
...definition.inputSchema === void 0 ? {} : { inputSchema: definition.inputSchema },
|
|
518
|
+
...definition.outputSchema === void 0 ? {} : { outputSchema: definition.outputSchema },
|
|
519
|
+
...definition.events === void 0 ? {} : { events: workflowEvents },
|
|
520
|
+
...rawLimits === void 0 ? {} : { limits: {
|
|
521
|
+
...rawLimits.maxSteps === void 0 ? {} : { maxSteps: rawLimits.maxSteps },
|
|
522
|
+
...rawLimits.maxToolCalls === void 0 ? {} : { maxToolCalls: rawLimits.maxToolCalls },
|
|
523
|
+
...rawLimits.maxInputTokens === void 0 ? {} : { maxInputTokens: rawLimits.maxInputTokens },
|
|
524
|
+
...rawLimits.maxOutputTokens === void 0 ? {} : { maxOutputTokens: rawLimits.maxOutputTokens }
|
|
525
|
+
} },
|
|
526
|
+
run: definition.run
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
for (const [index, definition] of (config.teams ?? []).entries()) {
|
|
530
|
+
assertConfiguration(isRecord(definition), "INVALID_TEAM", `Team at index ${index} must be an object`);
|
|
531
|
+
assertConfiguration(typeof definition.name === "string", "INVALID_TEAM", "Team name must be a string");
|
|
532
|
+
const name = definition.name;
|
|
533
|
+
assertConfiguration(name.trim(), "INVALID_TEAM", "Team name cannot be empty");
|
|
534
|
+
assertConfiguration(typeof definition.supervisor === "string" && Boolean(definition.supervisor.trim()), "INVALID_TEAM", `Team "${name}" supervisor must be a non-empty string`);
|
|
535
|
+
assertConfiguration(typeof definition.run === "function", "INVALID_TEAM", `Team "${name}" must implement run`);
|
|
536
|
+
assertConfiguration(definition.version === void 0 || typeof definition.version === "string" && Boolean(definition.version.trim()), "INVALID_TEAM", `Team "${name}" version must be a non-empty string`);
|
|
537
|
+
assertConfiguration(Array.isArray(definition.members), "INVALID_TEAM", `Team "${name}" members must be an array`);
|
|
538
|
+
const members = [];
|
|
539
|
+
const memberNames = /* @__PURE__ */ new Set();
|
|
540
|
+
for (const [memberIndex, rawMember] of definition.members.entries()) {
|
|
541
|
+
assertConfiguration(isRecord(rawMember) && typeof rawMember.agent === "string" && Boolean(rawMember.agent.trim()) && typeof rawMember.role === "string" && Boolean(rawMember.role.trim()) && (rawMember.description === void 0 || typeof rawMember.description === "string"), "INVALID_TEAM", `Team "${name}" member at index ${memberIndex} is invalid`);
|
|
542
|
+
assertConfiguration(!memberNames.has(rawMember.agent), "INVALID_TEAM", `Team "${name}" member "${rawMember.agent}" is duplicated`);
|
|
543
|
+
memberNames.add(rawMember.agent);
|
|
544
|
+
members.push({
|
|
545
|
+
agent: rawMember.agent,
|
|
546
|
+
role: rawMember.role,
|
|
547
|
+
...rawMember.description === void 0 ? {} : { description: rawMember.description }
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
const supervisor = definition.supervisor;
|
|
551
|
+
assertConfiguration(agents.has(supervisor), "INVALID_TEAM", `Supervisor "${supervisor}" required by team "${name}" is not registered`);
|
|
552
|
+
for (const member of members) assertConfiguration(agents.has(member.agent), "INVALID_TEAM", `Agent "${member.agent}" required by team "${name}" is not registered`);
|
|
553
|
+
if (definition.inputSchema !== void 0) assertConfiguration(isStandardSchema(definition.inputSchema), "INVALID_TEAM", `Input schema for team "${name}" must implement Standard Schema`);
|
|
554
|
+
if (definition.outputSchema !== void 0) assertConfiguration(isStandardSchema(definition.outputSchema), "INVALID_TEAM", `Output schema for team "${name}" must implement Standard Schema`);
|
|
555
|
+
assertConfiguration(definition.limits === void 0 || isRecord(definition.limits), "INVALID_TEAM", `Team "${name}" limits must be an object`);
|
|
556
|
+
const rawLimits = definition.limits;
|
|
557
|
+
assertLimit(rawLimits?.maxDelegations, "maxDelegations", 1, name, false, "Team", "INVALID_TEAM");
|
|
558
|
+
assertLimit(rawLimits?.maxParallel, "maxParallel", 1, name, false, "Team", "INVALID_TEAM");
|
|
559
|
+
assertLimit(rawLimits?.maxSteps, "maxSteps", 1, name, false, "Team", "INVALID_TEAM");
|
|
560
|
+
assertLimit(rawLimits?.maxToolCalls, "maxToolCalls", 0, name, false, "Team", "INVALID_TEAM");
|
|
561
|
+
assertLimit(rawLimits?.maxInputTokens, "maxInputTokens", 1, name, true, "Team", "INVALID_TEAM");
|
|
562
|
+
assertLimit(rawLimits?.maxOutputTokens, "maxOutputTokens", 1, name, true, "Team", "INVALID_TEAM");
|
|
563
|
+
assertConfiguration(!teams.has(name), "DUPLICATE_TEAM", `Team "${name}" is duplicated`);
|
|
564
|
+
assertConfiguration(!workflows.has(name), "INVALID_TEAM", `Team "${name}" conflicts with a workflow of the same name`);
|
|
565
|
+
const team = {
|
|
566
|
+
name,
|
|
567
|
+
version: definition.version ?? "1",
|
|
568
|
+
supervisor,
|
|
569
|
+
members,
|
|
570
|
+
...definition.inputSchema === void 0 ? {} : { inputSchema: definition.inputSchema },
|
|
571
|
+
...definition.outputSchema === void 0 ? {} : { outputSchema: definition.outputSchema },
|
|
572
|
+
...rawLimits === void 0 ? {} : { limits: {
|
|
573
|
+
...rawLimits.maxDelegations === void 0 ? {} : { maxDelegations: rawLimits.maxDelegations },
|
|
574
|
+
...rawLimits.maxParallel === void 0 ? {} : { maxParallel: rawLimits.maxParallel },
|
|
575
|
+
...rawLimits.maxSteps === void 0 ? {} : { maxSteps: rawLimits.maxSteps },
|
|
576
|
+
...rawLimits.maxToolCalls === void 0 ? {} : { maxToolCalls: rawLimits.maxToolCalls },
|
|
577
|
+
...rawLimits.maxInputTokens === void 0 ? {} : { maxInputTokens: rawLimits.maxInputTokens },
|
|
578
|
+
...rawLimits.maxOutputTokens === void 0 ? {} : { maxOutputTokens: rawLimits.maxOutputTokens }
|
|
579
|
+
} },
|
|
580
|
+
run: definition.run
|
|
581
|
+
};
|
|
582
|
+
teams.set(name, team);
|
|
583
|
+
workflows.set(name, teamAsWorkflow(team));
|
|
224
584
|
}
|
|
585
|
+
const usesMemory = [...agents.values()].some((agent) => agent.memory && (agent.memory.read !== false || agent.memory.write === true));
|
|
225
586
|
return {
|
|
226
587
|
models,
|
|
227
588
|
agents,
|
|
589
|
+
workflows,
|
|
590
|
+
teams,
|
|
228
591
|
tools,
|
|
592
|
+
contextProviders,
|
|
593
|
+
memoryStore: config.memoryStore ?? (usesMemory ? new InMemoryMemoryStore() : void 0),
|
|
229
594
|
runStore: config.runStore ?? new InMemoryRunStore(),
|
|
230
595
|
credentialStore: config.credentialStore,
|
|
596
|
+
sandbox: config.sandbox,
|
|
231
597
|
policies,
|
|
232
598
|
onEvent: config.onEvent,
|
|
233
599
|
observability: config.observability === void 0 ? void 0 : {
|
|
@@ -241,552 +607,454 @@ function createComposition(config) {
|
|
|
241
607
|
};
|
|
242
608
|
}
|
|
243
609
|
//#endregion
|
|
244
|
-
//#region src/
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
610
|
+
//#region src/internal/runtime-constants.ts
|
|
611
|
+
/** How long a run lease stays valid before it must be renewed. */
|
|
612
|
+
const LEASE_MS = 3e4;
|
|
613
|
+
/** How often a running execution renews its lease. */
|
|
614
|
+
const LEASE_RENEW_MS = 1e4;
|
|
615
|
+
/** Reserved name of the internal elicitation tool exposed to the model. */
|
|
616
|
+
const ELICIT_TOOL_NAME = "fevex__elicit";
|
|
617
|
+
//#endregion
|
|
618
|
+
//#region src/internal/run-helpers.ts
|
|
619
|
+
function redact(message, secrets) {
|
|
620
|
+
return secrets.reduce((safe, secret) => secret ? safe.split(secret).join("[REDACTED]") : safe, message);
|
|
621
|
+
}
|
|
622
|
+
function containsSecret(value, secrets) {
|
|
623
|
+
if (typeof value === "string") return secrets.some((secret) => secret && value.includes(secret));
|
|
624
|
+
if (Array.isArray(value)) return value.some((item) => containsSecret(item, secrets));
|
|
625
|
+
if (value && typeof value === "object") return Object.values(value).some((item) => containsSecret(item, secrets));
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
function effectiveElicitationMode(agent, request) {
|
|
629
|
+
return request.elicitation ?? agent.elicitation ?? "forbid";
|
|
630
|
+
}
|
|
631
|
+
function effectiveApprovalMode(agent, request) {
|
|
632
|
+
return request.approvalMode ?? agent.approvalMode ?? "pause";
|
|
633
|
+
}
|
|
634
|
+
function effectiveToolChoice(agent, request) {
|
|
635
|
+
return request.toolChoice ?? agent.toolChoice ?? "auto";
|
|
636
|
+
}
|
|
637
|
+
function isElicitationToolCall(call) {
|
|
638
|
+
return call.name === ELICIT_TOOL_NAME;
|
|
639
|
+
}
|
|
640
|
+
function readElicitationInput(input) {
|
|
641
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) throw new TypeError("Elicitation input must be an object");
|
|
642
|
+
const prompt = input.prompt;
|
|
643
|
+
const responseSchema = input.responseSchema;
|
|
644
|
+
const expiresAt = input.expiresAt;
|
|
645
|
+
if (typeof prompt !== "string" || !prompt.trim()) throw new TypeError("Elicitation prompt must be a non-empty string");
|
|
646
|
+
if (typeof responseSchema !== "object" || responseSchema === null || Array.isArray(responseSchema)) throw new TypeError("Elicitation responseSchema must be an object");
|
|
647
|
+
validateFevexJsonSchemaProfile(responseSchema);
|
|
648
|
+
if (expiresAt !== void 0) {
|
|
649
|
+
if (typeof expiresAt !== "string" || !Number.isFinite(Date.parse(expiresAt))) throw new TypeError("Elicitation expiresAt must be a valid ISO date string");
|
|
650
|
+
}
|
|
651
|
+
return {
|
|
652
|
+
prompt,
|
|
653
|
+
responseSchema,
|
|
654
|
+
...expiresAt === void 0 ? {} : { expiresAt }
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
function isCoordinatorRun(run) {
|
|
658
|
+
return run.kind === "workflow" || run.kind === "team";
|
|
659
|
+
}
|
|
660
|
+
function isTeamRun(run) {
|
|
661
|
+
return run.kind === "team";
|
|
662
|
+
}
|
|
663
|
+
function coordinatorName(run) {
|
|
664
|
+
return run.kind === "team" ? run.teamName : run.workflowName;
|
|
665
|
+
}
|
|
666
|
+
var WorkflowChildPausedError = class extends Error {
|
|
667
|
+
stepId;
|
|
668
|
+
paused;
|
|
669
|
+
constructor(stepId, paused) {
|
|
670
|
+
super(paused.message, { cause: paused });
|
|
671
|
+
this.stepId = stepId;
|
|
672
|
+
this.paused = paused;
|
|
253
673
|
}
|
|
254
674
|
};
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
pause;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
675
|
+
function pauseMatchesResolution(pause, resolution) {
|
|
676
|
+
if (pause.type === "elicitation" && resolution.type === "elicitation") return pause.request.id === resolution.requestId;
|
|
677
|
+
if (pause.type === "approval" && resolution.type === "approval") return pause.approval.id === resolution.approvalId;
|
|
678
|
+
if (pause.type === "tool_execution_unknown" && resolution.type === "tool_execution") return pause.toolCallId === resolution.toolCallId;
|
|
679
|
+
return false;
|
|
680
|
+
}
|
|
681
|
+
function mergeExecutionContext(parent, child) {
|
|
682
|
+
if (!parent) return child;
|
|
683
|
+
if (!child) return parent;
|
|
684
|
+
return {
|
|
685
|
+
...parent,
|
|
686
|
+
...child,
|
|
687
|
+
actor: parent.actor ?? child.actor,
|
|
688
|
+
...parent.attributes || child.attributes ? { attributes: {
|
|
689
|
+
...parent.attributes,
|
|
690
|
+
...child.attributes
|
|
691
|
+
} } : {},
|
|
692
|
+
...parent.prompt || child.prompt ? { prompt: {
|
|
693
|
+
...parent.prompt,
|
|
694
|
+
...child.prompt
|
|
695
|
+
} } : {}
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
function systemBlock(label, provider, block) {
|
|
699
|
+
return {
|
|
700
|
+
role: "system",
|
|
701
|
+
content: `[${label}: ${provider}/${block.id}]\n${block.content}`
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
async function readMemoryMessages(agent, context, memoryStore) {
|
|
705
|
+
if (!agent.memory || agent.memory.read === false || !memoryStore) return [];
|
|
706
|
+
try {
|
|
707
|
+
return (await abortable(() => memoryStore.search({
|
|
708
|
+
query: context.input,
|
|
709
|
+
limit: agent.memory?.limit ?? 5,
|
|
710
|
+
agentName: context.agentName,
|
|
711
|
+
sessionId: context.sessionId,
|
|
712
|
+
namespace: context.context?.namespace,
|
|
713
|
+
actor: context.context?.actor
|
|
714
|
+
}, context), context.signal)).filter((record) => record.content.trim()).map((record) => ({
|
|
715
|
+
role: "system",
|
|
716
|
+
content: `[Memory: ${record.id}]\n${record.content}`
|
|
717
|
+
}));
|
|
718
|
+
} catch (cause) {
|
|
719
|
+
throw new Error("Memory search failed", { cause });
|
|
263
720
|
}
|
|
721
|
+
}
|
|
722
|
+
const toolSourcePayload = (tool, error) => {
|
|
723
|
+
const source = error instanceof IntegrationError ? error.source ?? tool.source : tool.source;
|
|
724
|
+
return source === void 0 ? {} : { source };
|
|
725
|
+
};
|
|
726
|
+
const toolLabelPayload = (tool) => tool.label === void 0 ? {} : { toolLabel: tool.label };
|
|
727
|
+
const toolErrorPayload = (tool, error) => ({
|
|
728
|
+
...toolSourcePayload(tool, error),
|
|
729
|
+
...error instanceof IntegrationError ? {
|
|
730
|
+
errorCode: error.code,
|
|
731
|
+
retryable: error.retryable
|
|
732
|
+
} : {}
|
|
733
|
+
});
|
|
734
|
+
const runErrorPayload = (error) => {
|
|
735
|
+
if (!(error instanceof IntegrationError)) return {};
|
|
736
|
+
return {
|
|
737
|
+
errorCode: error.code,
|
|
738
|
+
retryable: error.retryable,
|
|
739
|
+
...error.source === void 0 ? {} : { source: error.source }
|
|
740
|
+
};
|
|
264
741
|
};
|
|
265
742
|
//#endregion
|
|
266
|
-
//#region src/internal/
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
"
|
|
270
|
-
|
|
271
|
-
]);
|
|
272
|
-
const SENSITIVE_KEY = /api[-_]?key|authorization|cookie|credential|password|secret|token/i;
|
|
273
|
-
function durationMs(startedAt, endedAt) {
|
|
274
|
-
return Math.max(0, Date.parse(endedAt) - Date.parse(startedAt));
|
|
743
|
+
//#region src/internal/definition-hash.ts
|
|
744
|
+
function canonicalize(value) {
|
|
745
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
746
|
+
if (typeof value !== "object" || value === null) return value;
|
|
747
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
|
|
275
748
|
}
|
|
276
|
-
function
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
749
|
+
async function transportableTool(tool) {
|
|
750
|
+
return {
|
|
751
|
+
name: tool.name,
|
|
752
|
+
description: tool.description,
|
|
753
|
+
inputSchema: tool.inputJsonSchema ?? (tool.inputSchema === void 0 ? void 0 : toTransportableSchema(tool.inputSchema, "input", `Input schema for tool "${tool.name}" is not transportable`)),
|
|
754
|
+
outputSchema: tool.outputJsonSchema ?? (tool.outputSchema === void 0 ? void 0 : toTransportableSchema(tool.outputSchema, "output", `Output schema for tool "${tool.name}" is not transportable`)),
|
|
755
|
+
risk: tool.risk ?? "read",
|
|
756
|
+
approval: tool.approval ?? "never",
|
|
757
|
+
idempotency: tool.idempotency ?? "none",
|
|
758
|
+
retry: tool.retry,
|
|
759
|
+
credentials: tool.credentials ?? [],
|
|
760
|
+
sandbox: tool.sandbox,
|
|
761
|
+
source: tool.source
|
|
762
|
+
};
|
|
280
763
|
}
|
|
281
|
-
function
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
for (const [key, item] of Object.entries(value)) result[key] = SENSITIVE_KEY.test(key) ? "[REDACTED]" : redactKeys(item);
|
|
286
|
-
return result;
|
|
764
|
+
async function sha256(value) {
|
|
765
|
+
const encoded = new TextEncoder().encode(JSON.stringify(canonicalize(value)));
|
|
766
|
+
const digest = await crypto.subtle.digest("SHA-256", encoded);
|
|
767
|
+
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
287
768
|
}
|
|
288
|
-
function
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
769
|
+
async function definitionHash(name, modelName, agent, tools) {
|
|
770
|
+
const toolDefinitions = await Promise.all((agent.tools ?? []).map((toolName) => transportableTool(tools.get(toolName))));
|
|
771
|
+
return sha256({
|
|
772
|
+
name,
|
|
773
|
+
modelName,
|
|
774
|
+
instructions: agent.instructions,
|
|
775
|
+
context: agent.context,
|
|
776
|
+
memory: agent.memory,
|
|
777
|
+
skills: agent.skills,
|
|
778
|
+
tools: toolDefinitions,
|
|
779
|
+
reasoning: agent.reasoning,
|
|
780
|
+
modelOptions: agent.modelOptions,
|
|
781
|
+
toolChoice: agent.toolChoice,
|
|
782
|
+
elicitation: agent.elicitation,
|
|
783
|
+
approvalMode: agent.approvalMode,
|
|
784
|
+
limits: agent.limits,
|
|
785
|
+
inputSchema: agent.inputSchema === void 0 ? void 0 : toTransportableSchema(agent.inputSchema, "input", `Input schema for agent "${name}" is not transportable`),
|
|
786
|
+
outputSchema: agent.outputSchema === void 0 ? void 0 : toTransportableSchema(agent.outputSchema, "output", `Output schema for agent "${name}" is not transportable`)
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
function workflowDefinitionHash(name, workflow) {
|
|
790
|
+
return sha256({
|
|
791
|
+
name,
|
|
792
|
+
version: workflow.version ?? "1",
|
|
793
|
+
limits: workflow.limits,
|
|
794
|
+
inputSchema: workflow.inputSchema === void 0 ? void 0 : toTransportableSchema(workflow.inputSchema, "input", `Input schema for workflow "${name}" is not transportable`),
|
|
795
|
+
outputSchema: workflow.outputSchema === void 0 ? void 0 : toTransportableSchema(workflow.outputSchema, "output", `Output schema for workflow "${name}" is not transportable`),
|
|
796
|
+
events: Object.fromEntries(Object.entries(workflow.events ?? {}).map(([eventName, event]) => [eventName, {
|
|
797
|
+
requireActor: event.requireActor ?? false,
|
|
798
|
+
payloadSchema: event.payloadSchema === void 0 ? void 0 : toTransportableSchema(event.payloadSchema, "input", `Payload schema for workflow "${name}" event "${eventName}" is not transportable`)
|
|
799
|
+
}]))
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
function teamDefinitionHash(name, team) {
|
|
803
|
+
return sha256({
|
|
804
|
+
name,
|
|
805
|
+
version: team.version ?? "1",
|
|
806
|
+
supervisor: team.supervisor,
|
|
807
|
+
members: team.members,
|
|
808
|
+
limits: team.limits,
|
|
809
|
+
inputSchema: team.inputSchema === void 0 ? void 0 : toTransportableSchema(team.inputSchema, "input", `Input schema for team "${name}" is not transportable`),
|
|
810
|
+
outputSchema: team.outputSchema === void 0 ? void 0 : toTransportableSchema(team.outputSchema, "output", `Output schema for team "${name}" is not transportable`)
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
//#endregion
|
|
814
|
+
//#region src/internal/workflow-budget.ts
|
|
815
|
+
function minLimit(first, second) {
|
|
816
|
+
if (first === void 0) return second;
|
|
817
|
+
if (second === void 0) return first;
|
|
818
|
+
if (first === false) return second;
|
|
819
|
+
if (second === false) return first;
|
|
820
|
+
return Math.min(first, second);
|
|
821
|
+
}
|
|
822
|
+
function combineLimits(first, second) {
|
|
823
|
+
if (!first) return second;
|
|
824
|
+
if (!second) return first;
|
|
825
|
+
return {
|
|
826
|
+
...minLimit(first.maxSteps, second.maxSteps) === void 0 ? {} : { maxSteps: minLimit(first.maxSteps, second.maxSteps) },
|
|
827
|
+
...minLimit(first.maxToolCalls, second.maxToolCalls) === void 0 ? {} : { maxToolCalls: minLimit(first.maxToolCalls, second.maxToolCalls) },
|
|
828
|
+
...minLimit(first.maxInputTokens, second.maxInputTokens) === void 0 ? {} : { maxInputTokens: minLimit(first.maxInputTokens, second.maxInputTokens) },
|
|
829
|
+
...minLimit(first.maxOutputTokens, second.maxOutputTokens) === void 0 ? {} : { maxOutputTokens: minLimit(first.maxOutputTokens, second.maxOutputTokens) }
|
|
302
830
|
};
|
|
303
831
|
}
|
|
304
|
-
function
|
|
305
|
-
if (
|
|
306
|
-
|
|
307
|
-
if (typeof value.inputTokens === "number") usage.inputTokens = value.inputTokens;
|
|
308
|
-
if (typeof value.outputTokens === "number") usage.outputTokens = value.outputTokens;
|
|
309
|
-
if (typeof value.totalTokens === "number") usage.totalTokens = value.totalTokens;
|
|
310
|
-
return Object.keys(usage).length ? usage : void 0;
|
|
832
|
+
function subtractLimit(limit, used) {
|
|
833
|
+
if (limit === void 0 || limit === false) return limit;
|
|
834
|
+
return Math.max(0, limit - (used ?? 0));
|
|
311
835
|
}
|
|
312
|
-
function
|
|
313
|
-
if (!
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
return Object.keys(usage).length ? usage : void 0;
|
|
836
|
+
function remainingWorkflowLimits(limits, budget) {
|
|
837
|
+
if (!limits) return void 0;
|
|
838
|
+
return {
|
|
839
|
+
...limits.maxSteps === void 0 ? {} : { maxSteps: subtractLimit(limits.maxSteps, budget?.steps) },
|
|
840
|
+
...limits.maxToolCalls === void 0 ? {} : { maxToolCalls: subtractLimit(limits.maxToolCalls, budget?.toolCalls) },
|
|
841
|
+
...limits.maxInputTokens === void 0 ? {} : { maxInputTokens: subtractLimit(limits.maxInputTokens, budget?.usage?.inputTokens) },
|
|
842
|
+
...limits.maxOutputTokens === void 0 ? {} : { maxOutputTokens: subtractLimit(limits.maxOutputTokens, budget?.usage?.outputTokens) }
|
|
843
|
+
};
|
|
321
844
|
}
|
|
322
|
-
function
|
|
323
|
-
if (!
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
if (!Number.isFinite(cost.amount) || cost.amount < 0 || !cost.currency.trim()) throw new TypeError("CostCalculator must return a non-negative amount and currency");
|
|
845
|
+
function assertWorkflowBudget(workflowName, limits, budget) {
|
|
846
|
+
if (!limits || !budget) return;
|
|
847
|
+
if (limits.maxSteps !== void 0 && budget.steps > limits.maxSteps) throw new Error(`Workflow "${workflowName}" exceeded maxSteps limit of ${limits.maxSteps}`);
|
|
848
|
+
if (limits.maxToolCalls !== void 0 && budget.toolCalls > limits.maxToolCalls) throw new Error(`Workflow "${workflowName}" exceeded maxToolCalls limit of ${limits.maxToolCalls}`);
|
|
849
|
+
assertTokenBudget(`workflow:${workflowName}`, "maxInputTokens", "inputTokens", limits.maxInputTokens, budget.usage, budget.usage);
|
|
850
|
+
assertTokenBudget(`workflow:${workflowName}`, "maxOutputTokens", "outputTokens", limits.maxOutputTokens, budget.usage, budget.usage);
|
|
851
|
+
}
|
|
852
|
+
function addWorkflowBudget(first, second) {
|
|
331
853
|
return {
|
|
332
|
-
|
|
333
|
-
|
|
854
|
+
usage: addUsage(first?.usage, second.usage),
|
|
855
|
+
steps: (first?.steps ?? 0) + second.steps,
|
|
856
|
+
toolCalls: (first?.toolCalls ?? 0) + second.toolCalls
|
|
334
857
|
};
|
|
335
858
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
859
|
+
//#endregion
|
|
860
|
+
//#region src/internal/agent-engine.ts
|
|
861
|
+
/**
|
|
862
|
+
* Everything that runs a single agent: preparation, the model/tool loop,
|
|
863
|
+
* checkpointing, knowledge and memory, resume and out-of-band cancellation.
|
|
864
|
+
*
|
|
865
|
+
* Depends only on {@link RunCore}. It must never reach into the workflow
|
|
866
|
+
* engine — that direction is what keeps the two separable.
|
|
867
|
+
*/
|
|
868
|
+
function createAgentEngine(composition, core) {
|
|
869
|
+
const { models, agents, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, policies } = composition;
|
|
870
|
+
const { activeRuns, activeSessions, runtimeOwner, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
|
|
871
|
+
const prepareExecution = async (name, request) => {
|
|
872
|
+
const agent = agents.get(name);
|
|
873
|
+
if (!agent) throw new FevexRunError("AGENT_NOT_FOUND", `Agent "${name}" is not registered`);
|
|
874
|
+
const { modelName } = resolveModel(agent, request.model);
|
|
875
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
876
|
+
let session;
|
|
877
|
+
const newSession = request.sessionId === void 0;
|
|
878
|
+
if (newSession) session = {
|
|
879
|
+
id: crypto.randomUUID(),
|
|
880
|
+
history: [],
|
|
881
|
+
createdAt: now,
|
|
882
|
+
updatedAt: now
|
|
358
883
|
};
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
884
|
+
else {
|
|
885
|
+
const stored = await runStore.getSession(request.sessionId);
|
|
886
|
+
if (!stored) throw new FevexRunError("SESSION_NOT_FOUND", `Session "${request.sessionId}" does not exist`);
|
|
887
|
+
session = stored;
|
|
888
|
+
}
|
|
889
|
+
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" already has an active run`);
|
|
890
|
+
activeSessions.add(session.id);
|
|
891
|
+
const controller = new AbortController();
|
|
892
|
+
const signal = request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal;
|
|
893
|
+
const run = {
|
|
894
|
+
id: crypto.randomUUID(),
|
|
895
|
+
sessionId: session.id,
|
|
896
|
+
agentName: name,
|
|
897
|
+
status: "running",
|
|
898
|
+
revision: 0,
|
|
899
|
+
createdAt: now,
|
|
900
|
+
updatedAt: now
|
|
367
901
|
};
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
902
|
+
const state = {
|
|
903
|
+
run,
|
|
904
|
+
session,
|
|
905
|
+
request: {
|
|
906
|
+
...request,
|
|
907
|
+
sessionId: session.id,
|
|
908
|
+
signal
|
|
909
|
+
},
|
|
910
|
+
controller,
|
|
911
|
+
eventSequence: 0,
|
|
912
|
+
advancing: false
|
|
377
913
|
};
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
914
|
+
try {
|
|
915
|
+
if (isDurableRunStore(runStore)) {
|
|
916
|
+
const inputContent = serializeValue(agent.inputSchema === void 0 ? request.input : await abortable(() => validateSchema(agent.inputSchema, request.input, `Input for agent "${name}" does not match inputSchema`), signal), "Run input must be a string or JSON-serializable value");
|
|
917
|
+
const knowledge = await readKnowledgeMessages(state, agent, inputContent);
|
|
918
|
+
const messages = [
|
|
919
|
+
{
|
|
920
|
+
role: "system",
|
|
921
|
+
content: agent.instructions
|
|
922
|
+
},
|
|
923
|
+
...knowledge.skills,
|
|
924
|
+
...session.history,
|
|
925
|
+
...knowledge.context,
|
|
926
|
+
...knowledge.memory,
|
|
927
|
+
{
|
|
928
|
+
role: "user",
|
|
929
|
+
content: inputContent
|
|
930
|
+
}
|
|
931
|
+
];
|
|
932
|
+
const checkpoint = {
|
|
933
|
+
version: 2,
|
|
934
|
+
runId: run.id,
|
|
935
|
+
...request.model !== void 0 || typeof agent.model === "string" ? { modelName } : {},
|
|
936
|
+
...request.reasoning === void 0 ? {} : { reasoning: request.reasoning },
|
|
937
|
+
definitionHash: await definitionHash(name, modelName, agent, tools),
|
|
938
|
+
limits: combineLimits(agent.limits, request.limits),
|
|
939
|
+
messages,
|
|
940
|
+
inputContent,
|
|
941
|
+
context: request.context,
|
|
942
|
+
step: 1,
|
|
943
|
+
toolCallCount: 0,
|
|
944
|
+
seenToolCallIds: [],
|
|
945
|
+
pendingTools: [],
|
|
946
|
+
pendingIndex: 0,
|
|
947
|
+
effectiveElicitationMode: effectiveElicitationMode(agent, request),
|
|
948
|
+
effectiveApprovalMode: effectiveApprovalMode(agent, request),
|
|
949
|
+
effectiveToolChoice: effectiveToolChoice(agent, request)
|
|
950
|
+
};
|
|
951
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
952
|
+
const lease = {
|
|
953
|
+
generation: 0,
|
|
954
|
+
runId: run.id,
|
|
955
|
+
ownerId,
|
|
956
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
957
|
+
};
|
|
958
|
+
const started = createEvent(state, "run.started", void 0);
|
|
959
|
+
if (!await runStore.createExecution({
|
|
960
|
+
run,
|
|
961
|
+
checkpoint,
|
|
962
|
+
session,
|
|
963
|
+
lease,
|
|
964
|
+
events: [started]
|
|
965
|
+
})) throw new FevexRunError("RUN_CONFLICT", `Run or session "${session.id}" is active or was modified`, run.id);
|
|
966
|
+
state.checkpoint = checkpoint;
|
|
967
|
+
state.lease = lease;
|
|
968
|
+
state.initialEvents = [started];
|
|
969
|
+
notifyObserver(started);
|
|
970
|
+
startLease(state, runStore);
|
|
971
|
+
} else {
|
|
972
|
+
if (newSession) await runStore.saveSession(session);
|
|
973
|
+
await runStore.saveRun(run);
|
|
384
974
|
}
|
|
975
|
+
} catch (error) {
|
|
976
|
+
activeSessions.delete(session.id);
|
|
977
|
+
throw error;
|
|
978
|
+
}
|
|
979
|
+
activeRuns.set(run.id, state);
|
|
980
|
+
return state;
|
|
981
|
+
};
|
|
982
|
+
const durableCheckpoint = async (state, model, modelName, messages, inputContent, usage, providerState, step, toolCallCount, seenToolCallIds, pendingTools, pendingIndex) => {
|
|
983
|
+
const agent = agents.get(state.run.agentName);
|
|
984
|
+
if (agent.model !== void 0 && typeof agent.model !== "string") throw new FevexRunError("RUN_NOT_RESUMABLE", "Durable continuation requires a named model", state.run.id);
|
|
985
|
+
let serializedState;
|
|
986
|
+
if (providerState !== void 0) {
|
|
987
|
+
if (!model.stateCodec) throw new FevexRunError("RUN_NOT_RESUMABLE", "ModelGateway must provide stateCodec for durable continuation", state.run.id);
|
|
988
|
+
serializedState = toJsonValue(model.stateCodec.serialize(providerState), "Model provider state must be JSON-serializable");
|
|
989
|
+
}
|
|
990
|
+
return {
|
|
991
|
+
version: 2,
|
|
992
|
+
runId: state.run.id,
|
|
993
|
+
modelName,
|
|
994
|
+
...state.request.reasoning === void 0 ? {} : { reasoning: state.request.reasoning },
|
|
995
|
+
definitionHash: await definitionHash(state.run.agentName, modelName, agent, tools),
|
|
996
|
+
limits: state.checkpoint?.limits ?? combineLimits(agent.limits, state.request.limits),
|
|
997
|
+
messages: structuredClone(messages),
|
|
998
|
+
inputContent,
|
|
999
|
+
context: state.request.context,
|
|
1000
|
+
usage,
|
|
1001
|
+
providerState: serializedState,
|
|
1002
|
+
step,
|
|
1003
|
+
toolCallCount,
|
|
1004
|
+
seenToolCallIds: [...seenToolCallIds],
|
|
1005
|
+
pendingTools: structuredClone(pendingTools),
|
|
1006
|
+
pendingIndex,
|
|
1007
|
+
effectiveElicitationMode: state.checkpoint?.effectiveElicitationMode ?? effectiveElicitationMode(agent, state.request),
|
|
1008
|
+
effectiveApprovalMode: state.checkpoint?.effectiveApprovalMode ?? effectiveApprovalMode(agent, state.request),
|
|
1009
|
+
effectiveToolChoice: state.checkpoint?.effectiveToolChoice ?? effectiveToolChoice(agent, state.request)
|
|
385
1010
|
};
|
|
386
|
-
default: return;
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
|
|
390
|
-
const events = [...rawEvents].sort((left, right) => left.sequence - right.sequence);
|
|
391
|
-
const terminal = [...events].reverse().find((event) => TERMINAL_TYPES.has(event.type));
|
|
392
|
-
if (!terminal) throw new TypeError(`Run "${run.id}" has no terminal event`);
|
|
393
|
-
const started = events.find((event) => event.type === "run.started") ?? events[0];
|
|
394
|
-
if (!started) throw new TypeError(`Run "${run.id}" has no events`);
|
|
395
|
-
const status = traceStatus(terminal);
|
|
396
|
-
const annotations = [];
|
|
397
|
-
const spans = [];
|
|
398
|
-
const openModels = /* @__PURE__ */ new Map();
|
|
399
|
-
const openTools = /* @__PURE__ */ new Map();
|
|
400
|
-
const previousUsage = {};
|
|
401
|
-
const closeModel = (step, endedAt, spanStatus) => {
|
|
402
|
-
const open = openModels.get(step);
|
|
403
|
-
if (!open) return;
|
|
404
|
-
openModels.delete(step);
|
|
405
|
-
const completed = events.find((event) => event.type === "model.completed" && event.payload.step === step && event.sequence >= open.event.sequence);
|
|
406
|
-
const cumulative = completed?.type === "model.completed" ? usageFrom(completed.payload.usage) : void 0;
|
|
407
|
-
const usage = subtractUsage(cumulative, previousUsage);
|
|
408
|
-
if (cumulative) Object.assign(previousUsage, cumulative);
|
|
409
|
-
const spanContent = open.output.length ? content(options, "model.output", open.output.join("")) : void 0;
|
|
410
|
-
const span = {
|
|
411
|
-
id: open.event.id,
|
|
412
|
-
parentId: run.id,
|
|
413
|
-
kind: "model",
|
|
414
|
-
name: `model step ${step}`,
|
|
415
|
-
startedAt: open.event.timestamp,
|
|
416
|
-
endedAt,
|
|
417
|
-
durationMs: durationMs(open.event.timestamp, endedAt),
|
|
418
|
-
status: spanStatus,
|
|
419
|
-
attributes: {
|
|
420
|
-
step,
|
|
421
|
-
modelRegistryName
|
|
422
|
-
},
|
|
423
|
-
annotations: open.annotations,
|
|
424
|
-
...usage ? { usage } : {},
|
|
425
|
-
...spanContent ? { content: [spanContent] } : {}
|
|
426
|
-
};
|
|
427
|
-
const spanCost = costFor(options, modelRegistryName, model, usage);
|
|
428
|
-
if (spanCost) span.cost = spanCost;
|
|
429
|
-
spans.push(span);
|
|
430
1011
|
};
|
|
431
|
-
const
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
1012
|
+
const knowledgeContext = (state, input) => ({
|
|
1013
|
+
agentName: state.run.agentName,
|
|
1014
|
+
input,
|
|
1015
|
+
sessionId: state.session.id,
|
|
1016
|
+
...state.request.context === void 0 ? {} : { context: state.request.context },
|
|
1017
|
+
signal: state.request.signal
|
|
1018
|
+
});
|
|
1019
|
+
const readProvider = async (providerName, label, context) => {
|
|
1020
|
+
try {
|
|
1021
|
+
const provider = contextProviders.get(providerName);
|
|
1022
|
+
return (await abortable(() => provider.read(context), context.signal)).filter((block) => block.content.trim()).map((block) => systemBlock(label, providerName, block));
|
|
1023
|
+
} catch (cause) {
|
|
1024
|
+
throw new Error(`${label} provider "${providerName}" failed`, { cause });
|
|
437
1025
|
}
|
|
438
1026
|
};
|
|
439
|
-
const
|
|
440
|
-
const
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
const errorContent = error ? content(options, "error.message", error) : void 0;
|
|
447
|
-
spans.push({
|
|
448
|
-
id: event.id,
|
|
449
|
-
parentId: run.id,
|
|
450
|
-
kind: "tool",
|
|
451
|
-
name: event.payload.toolName,
|
|
452
|
-
startedAt: event.timestamp,
|
|
453
|
-
endedAt,
|
|
454
|
-
durationMs: durationMs(event.timestamp, endedAt),
|
|
455
|
-
status: spanStatus,
|
|
456
|
-
attributes: {
|
|
457
|
-
step: event.payload.step,
|
|
458
|
-
toolCallId,
|
|
459
|
-
toolName: event.payload.toolName,
|
|
460
|
-
attempt: event.payload.attempt ?? 1
|
|
461
|
-
},
|
|
462
|
-
annotations: open.annotations,
|
|
463
|
-
...errorContent ? { content: [errorContent] } : {}
|
|
464
|
-
});
|
|
1027
|
+
const readKnowledgeMessages = async (state, agent, input) => {
|
|
1028
|
+
const context = knowledgeContext(state, input);
|
|
1029
|
+
return {
|
|
1030
|
+
skills: (await Promise.all((agent.skills ?? []).map((name) => readProvider(name, "Skill", context)))).flat(),
|
|
1031
|
+
context: (await Promise.all((agent.context ?? []).map((name) => readProvider(name, "Context", context)))).flat(),
|
|
1032
|
+
memory: await readMemoryMessages(agent, context, memoryStore)
|
|
1033
|
+
};
|
|
465
1034
|
};
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
case "model.completed":
|
|
481
|
-
closeModel(event.payload.step, event.timestamp, "completed");
|
|
482
|
-
break;
|
|
483
|
-
case "tool.started": {
|
|
484
|
-
const attempt = event.payload.attempt ?? 1;
|
|
485
|
-
openTools.set(`${event.payload.toolCallId}:${attempt}`, {
|
|
486
|
-
event,
|
|
487
|
-
annotations: [],
|
|
488
|
-
output: []
|
|
489
|
-
});
|
|
490
|
-
break;
|
|
491
|
-
}
|
|
492
|
-
case "tool.retrying": {
|
|
493
|
-
const open = findOpenTool(event.payload.toolCallId)?.[1];
|
|
494
|
-
const retry = annotation(event);
|
|
495
|
-
if (open && retry) open.annotations.push(retry);
|
|
496
|
-
closeTool(event.payload.toolCallId, event.timestamp, "failed", event.payload.error);
|
|
497
|
-
break;
|
|
498
|
-
}
|
|
499
|
-
case "tool.completed":
|
|
500
|
-
closeTool(event.payload.toolCallId, event.timestamp, "completed");
|
|
501
|
-
break;
|
|
502
|
-
case "tool.failed":
|
|
503
|
-
closeTool(event.payload.toolCallId, event.timestamp, "failed", event.payload.error);
|
|
504
|
-
break;
|
|
505
|
-
case "tool.execution_unknown":
|
|
506
|
-
closeTool(event.payload.toolCallId, event.timestamp, "failed");
|
|
507
|
-
break;
|
|
1035
|
+
const writeMemory = async (state, agent, input, output) => {
|
|
1036
|
+
if (!agent.memory?.write || !memoryStore) return;
|
|
1037
|
+
const context = knowledgeContext(state, input);
|
|
1038
|
+
try {
|
|
1039
|
+
await abortable(() => memoryStore.write({
|
|
1040
|
+
content: `User: ${input}\nAssistant: ${serializeJsonValue(output)}`,
|
|
1041
|
+
agentName: state.run.agentName,
|
|
1042
|
+
sessionId: state.session.id,
|
|
1043
|
+
namespace: state.request.context?.namespace,
|
|
1044
|
+
actor: state.request.context?.actor,
|
|
1045
|
+
metadata: { runId: state.run.id }
|
|
1046
|
+
}, context), state.request.signal);
|
|
1047
|
+
} catch (cause) {
|
|
1048
|
+
throw new Error("Memory write failed", { cause });
|
|
508
1049
|
}
|
|
509
|
-
}
|
|
510
|
-
for (const step of [...openModels.keys()]) closeModel(step, terminal.timestamp, status);
|
|
511
|
-
for (const open of [...openTools.values()]) if (open.event.type === "tool.started") closeTool(open.event.payload.toolCallId, terminal.timestamp, status);
|
|
512
|
-
const finalUsage = run.usage ?? previousUsage;
|
|
513
|
-
const totalCost = costFor(options, modelRegistryName, model, finalUsage);
|
|
514
|
-
const traceContent = [];
|
|
515
|
-
if (terminal.type === "run.completed") {
|
|
516
|
-
const output = content(options, "run.output", terminal.payload.output);
|
|
517
|
-
if (output) traceContent.push(output);
|
|
518
|
-
} else if (terminal.type === "run.failed") {
|
|
519
|
-
const error = content(options, "error.message", terminal.payload.error);
|
|
520
|
-
if (error) traceContent.push(error);
|
|
521
|
-
}
|
|
522
|
-
return {
|
|
523
|
-
schemaVersion: "1",
|
|
524
|
-
traceId: run.id,
|
|
525
|
-
runId: run.id,
|
|
526
|
-
sessionId: run.sessionId,
|
|
527
|
-
agentName: run.agentName,
|
|
528
|
-
modelRegistryName,
|
|
529
|
-
...model ? { model: { ...model } } : {},
|
|
530
|
-
startedAt: started.timestamp,
|
|
531
|
-
endedAt: terminal.timestamp,
|
|
532
|
-
durationMs: durationMs(started.timestamp, terminal.timestamp),
|
|
533
|
-
status,
|
|
534
|
-
attributes: {
|
|
535
|
-
eventCount: events.length,
|
|
536
|
-
modelCallCount: spans.filter(({ kind }) => kind === "model").length,
|
|
537
|
-
toolCallCount: spans.filter(({ kind }) => kind === "tool").length
|
|
538
|
-
},
|
|
539
|
-
annotations,
|
|
540
|
-
spans,
|
|
541
|
-
...Object.keys(finalUsage).length ? { usage: { ...finalUsage } } : {},
|
|
542
|
-
...totalCost ? { cost: totalCost } : {},
|
|
543
|
-
...traceContent.length ? { content: traceContent } : {}
|
|
544
|
-
};
|
|
545
|
-
}
|
|
546
|
-
//#endregion
|
|
547
|
-
//#region src/internal/runtime.ts
|
|
548
|
-
const LEASE_MS = 3e4;
|
|
549
|
-
const LEASE_RENEW_MS = 1e4;
|
|
550
|
-
function redact(message, secrets) {
|
|
551
|
-
return secrets.reduce((safe, secret) => secret ? safe.split(secret).join("[REDACTED]") : safe, message);
|
|
552
|
-
}
|
|
553
|
-
function containsSecret(value, secrets) {
|
|
554
|
-
if (typeof value === "string") return secrets.some((secret) => secret && value.includes(secret));
|
|
555
|
-
if (Array.isArray(value)) return value.some((item) => containsSecret(item, secrets));
|
|
556
|
-
if (value && typeof value === "object") return Object.values(value).some((item) => containsSecret(item, secrets));
|
|
557
|
-
return false;
|
|
558
|
-
}
|
|
559
|
-
function canonicalize(value) {
|
|
560
|
-
if (Array.isArray(value)) return value.map(canonicalize);
|
|
561
|
-
if (typeof value !== "object" || value === null) return value;
|
|
562
|
-
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== void 0).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalize(item)]));
|
|
563
|
-
}
|
|
564
|
-
async function definitionHash(name, modelName, agent, tools) {
|
|
565
|
-
const portable = JSON.stringify(canonicalize({
|
|
566
|
-
name,
|
|
567
|
-
modelName,
|
|
568
|
-
instructions: agent.instructions,
|
|
569
|
-
tools: (agent.tools ?? []).map((toolName) => {
|
|
570
|
-
const tool = tools.get(toolName);
|
|
571
|
-
return {
|
|
572
|
-
name: tool.name,
|
|
573
|
-
description: tool.description,
|
|
574
|
-
inputSchema: tool.inputSchema === void 0 ? void 0 : toTransportableSchema(tool.inputSchema, "input", `Input schema for tool "${tool.name}" is not transportable`),
|
|
575
|
-
outputSchema: tool.outputSchema === void 0 ? void 0 : toTransportableSchema(tool.outputSchema, "output", `Output schema for tool "${tool.name}" is not transportable`),
|
|
576
|
-
risk: tool.risk ?? "read",
|
|
577
|
-
approval: tool.approval ?? "never",
|
|
578
|
-
idempotency: tool.idempotency ?? "none",
|
|
579
|
-
retry: tool.retry,
|
|
580
|
-
credentials: tool.credentials ?? []
|
|
581
|
-
};
|
|
582
|
-
}),
|
|
583
|
-
reasoning: agent.reasoning,
|
|
584
|
-
modelOptions: agent.modelOptions,
|
|
585
|
-
limits: agent.limits,
|
|
586
|
-
outputSchema: agent.outputSchema === void 0 ? void 0 : toTransportableSchema(agent.outputSchema, "output", `Output schema for agent "${name}" is not transportable`)
|
|
587
|
-
}));
|
|
588
|
-
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(portable));
|
|
589
|
-
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
590
|
-
}
|
|
591
|
-
function createRuntime({ models, agents, tools, runStore, credentialStore, policies, onEvent, observability }) {
|
|
592
|
-
const activeRuns = /* @__PURE__ */ new Map();
|
|
593
|
-
const activeSessions = /* @__PURE__ */ new Set();
|
|
594
|
-
const pendingExports = /* @__PURE__ */ new Set();
|
|
595
|
-
const exportFailures = [];
|
|
596
|
-
const exportedRuns = /* @__PURE__ */ new Set();
|
|
597
|
-
const runtimeOwner = crypto.randomUUID();
|
|
598
|
-
const scheduleTraceExport = (event) => {
|
|
599
|
-
if (!observability || ![
|
|
600
|
-
"run.completed",
|
|
601
|
-
"run.failed",
|
|
602
|
-
"run.cancelled"
|
|
603
|
-
].includes(event.type) || exportedRuns.has(event.runId)) return;
|
|
604
|
-
exportedRuns.add(event.runId);
|
|
605
|
-
let task;
|
|
606
|
-
task = (async () => {
|
|
607
|
-
const run = await runStore.getRun(event.runId);
|
|
608
|
-
if (!run) throw new Error(`Run "${event.runId}" does not exist`);
|
|
609
|
-
const agent = agents.get(run.agentName);
|
|
610
|
-
if (!agent) throw new Error(`Agent "${run.agentName}" is not registered`);
|
|
611
|
-
const modelRegistryName = typeof agent.model === "string" ? agent.model : "default";
|
|
612
|
-
const gateway = typeof agent.model === "string" ? models.get(agent.model) : agent.model ?? models.get("default");
|
|
613
|
-
const trace = buildRunTrace(run, await runStore.listEvents(run.id), modelRegistryName, gateway?.metadata, observability);
|
|
614
|
-
const failures = (await Promise.allSettled(observability.exporters.map((exporter) => Promise.resolve().then(() => exporter.export(trace))))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
|
|
615
|
-
if (failures.length) throw new AggregateError(failures, "Trace export failed");
|
|
616
|
-
})().catch((error) => {
|
|
617
|
-
exportFailures.push(error);
|
|
618
|
-
}).finally(() => pendingExports.delete(task));
|
|
619
|
-
pendingExports.add(task);
|
|
620
|
-
};
|
|
621
|
-
const notifyObserver = (event) => {
|
|
622
|
-
scheduleTraceExport(event);
|
|
623
|
-
if (!onEvent) return;
|
|
624
|
-
try {
|
|
625
|
-
Promise.resolve(onEvent(event)).catch(() => {});
|
|
626
|
-
} catch {}
|
|
627
|
-
};
|
|
628
|
-
const createEvent = (state, type, payload) => ({
|
|
629
|
-
id: crypto.randomUUID(),
|
|
630
|
-
sequence: state.eventSequence += 1,
|
|
631
|
-
type,
|
|
632
|
-
runId: state.run.id,
|
|
633
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
634
|
-
...payload === void 0 ? {} : { payload }
|
|
635
|
-
});
|
|
636
|
-
const emitEvent = async (state, type, payload) => {
|
|
637
|
-
const event = createEvent(state, type, payload);
|
|
638
|
-
await runStore.appendEvent(event);
|
|
639
|
-
notifyObserver(event);
|
|
640
|
-
return event;
|
|
641
|
-
};
|
|
642
|
-
const releaseExecution = async (state) => {
|
|
643
|
-
activeRuns.delete(state.run.id);
|
|
644
|
-
activeSessions.delete(state.session.id);
|
|
645
|
-
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
646
|
-
if (state.leaseOwner && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.leaseOwner).catch(() => {});
|
|
647
|
-
};
|
|
648
|
-
const commit = async (state, options = {}) => {
|
|
649
|
-
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "This run requires a DurableRunStore", state.run.id);
|
|
650
|
-
state.run.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
651
|
-
if (!await runStore.commitExecution({
|
|
652
|
-
expectedRevision: state.run.revision,
|
|
653
|
-
run: state.run,
|
|
654
|
-
...options
|
|
655
|
-
})) throw new FevexRunError("RUN_CONFLICT", `Run "${state.run.id}" was modified`, state.run.id);
|
|
656
|
-
for (const event of options.events ?? []) notifyObserver(event);
|
|
657
|
-
};
|
|
658
|
-
const startLease = (state, store) => {
|
|
659
|
-
state.leaseTimer = setInterval(() => {
|
|
660
|
-
if (!state.leaseOwner || state.request.signal.aborted) return;
|
|
661
|
-
const expiresAt = new Date(Date.now() + LEASE_MS).toISOString();
|
|
662
|
-
store.renewLease({
|
|
663
|
-
runId: state.run.id,
|
|
664
|
-
ownerId: state.leaseOwner,
|
|
665
|
-
expiresAt
|
|
666
|
-
}).then((renewed) => {
|
|
667
|
-
if (!renewed) state.controller.abort(/* @__PURE__ */ new Error("Run lease was lost"));
|
|
668
|
-
});
|
|
669
|
-
}, LEASE_RENEW_MS);
|
|
670
|
-
};
|
|
671
|
-
const cancelExecution = async (state, reason = cancellationReason(state.request.signal)) => {
|
|
672
|
-
if (state.run.status === "completed" || state.run.status === "failed" || state.run.status === "cancelled") return;
|
|
673
|
-
Object.assign(state.run, {
|
|
674
|
-
status: "cancelled",
|
|
675
|
-
pause: void 0,
|
|
676
|
-
error: reason,
|
|
677
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
678
|
-
});
|
|
679
|
-
const event = createEvent(state, "run.cancelled", { reason });
|
|
680
|
-
if (isDurableRunStore(runStore)) await commit(state, {
|
|
681
|
-
checkpoint: null,
|
|
682
|
-
events: [event]
|
|
683
|
-
});
|
|
684
|
-
else {
|
|
685
|
-
await runStore.saveRun(state.run);
|
|
686
|
-
await runStore.appendEvent(event);
|
|
687
|
-
notifyObserver(event);
|
|
688
|
-
}
|
|
689
|
-
await releaseExecution(state);
|
|
690
|
-
return event;
|
|
691
|
-
};
|
|
692
|
-
const prepareExecution = async (name, request) => {
|
|
693
|
-
if (!agents.has(name)) throw new Error(`Agent "${name}" is not registered`);
|
|
694
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
695
|
-
let session;
|
|
696
|
-
if (request.sessionId === void 0) {
|
|
697
|
-
session = {
|
|
698
|
-
id: crypto.randomUUID(),
|
|
699
|
-
history: [],
|
|
700
|
-
createdAt: now,
|
|
701
|
-
updatedAt: now
|
|
702
|
-
};
|
|
703
|
-
await runStore.saveSession(session);
|
|
704
|
-
} else {
|
|
705
|
-
const stored = await runStore.getSession(request.sessionId);
|
|
706
|
-
if (!stored) throw new Error(`Session "${request.sessionId}" does not exist`);
|
|
707
|
-
session = stored;
|
|
708
|
-
}
|
|
709
|
-
if (activeSessions.has(session.id)) throw new Error(`Session "${session.id}" already has an active run`);
|
|
710
|
-
activeSessions.add(session.id);
|
|
711
|
-
const controller = new AbortController();
|
|
712
|
-
const signal = request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal;
|
|
713
|
-
const run = {
|
|
714
|
-
id: crypto.randomUUID(),
|
|
715
|
-
sessionId: session.id,
|
|
716
|
-
agentName: name,
|
|
717
|
-
status: "running",
|
|
718
|
-
revision: 0,
|
|
719
|
-
createdAt: now,
|
|
720
|
-
updatedAt: now
|
|
721
|
-
};
|
|
722
|
-
const state = {
|
|
723
|
-
run,
|
|
724
|
-
session,
|
|
725
|
-
request: {
|
|
726
|
-
...request,
|
|
727
|
-
sessionId: session.id,
|
|
728
|
-
signal
|
|
729
|
-
},
|
|
730
|
-
controller,
|
|
731
|
-
eventSequence: 0,
|
|
732
|
-
advancing: false
|
|
733
|
-
};
|
|
734
|
-
try {
|
|
735
|
-
await runStore.saveRun(run);
|
|
736
|
-
} catch (error) {
|
|
737
|
-
activeSessions.delete(session.id);
|
|
738
|
-
throw error;
|
|
739
|
-
}
|
|
740
|
-
activeRuns.set(run.id, state);
|
|
741
|
-
return state;
|
|
742
|
-
};
|
|
743
|
-
const authorize = async (state, tool, input, action, context = state.request.context) => {
|
|
744
|
-
let approval = false;
|
|
745
|
-
for (const policy of policies) {
|
|
746
|
-
const decision = await policy.authorize({
|
|
747
|
-
runId: state.run.id,
|
|
748
|
-
agentName: state.run.agentName,
|
|
749
|
-
toolName: tool.name,
|
|
750
|
-
risk: tool.risk ?? "read",
|
|
751
|
-
input,
|
|
752
|
-
action,
|
|
753
|
-
context
|
|
754
|
-
});
|
|
755
|
-
if (decision === "deny") throw new FevexRunError("POLICY_DENIED", `Policy "${policy.name}" denied tool "${tool.name}"`, state.run.id);
|
|
756
|
-
if (decision === "require_approval") approval = true;
|
|
757
|
-
}
|
|
758
|
-
return approval ? "require_approval" : "allow";
|
|
759
|
-
};
|
|
760
|
-
const durableCheckpoint = async (state, model, modelName, messages, inputContent, usage, providerState, step, toolCallCount, seenToolCallIds, pendingTools, pendingIndex) => {
|
|
761
|
-
const agent = agents.get(state.run.agentName);
|
|
762
|
-
if (agent.model !== void 0 && typeof agent.model !== "string") throw new FevexRunError("RUN_NOT_RESUMABLE", "Durable continuation requires a named model", state.run.id);
|
|
763
|
-
if (state.request.outputSchema !== void 0) throw new FevexRunError("RUN_NOT_RESUMABLE", "Durable runs must declare outputSchema on the agent", state.run.id);
|
|
764
|
-
let serializedState;
|
|
765
|
-
if (providerState !== void 0) {
|
|
766
|
-
if (!model.stateCodec) throw new FevexRunError("RUN_NOT_RESUMABLE", "ModelGateway must provide stateCodec for durable continuation", state.run.id);
|
|
767
|
-
serializedState = toJsonValue(model.stateCodec.serialize(providerState), "Model provider state must be JSON-serializable");
|
|
768
|
-
}
|
|
769
|
-
return {
|
|
770
|
-
runId: state.run.id,
|
|
771
|
-
definitionHash: await definitionHash(state.run.agentName, modelName, agent, tools),
|
|
772
|
-
messages: structuredClone(messages),
|
|
773
|
-
inputContent,
|
|
774
|
-
context: state.request.context,
|
|
775
|
-
usage,
|
|
776
|
-
providerState: serializedState,
|
|
777
|
-
step,
|
|
778
|
-
toolCallCount,
|
|
779
|
-
seenToolCallIds: [...seenToolCallIds],
|
|
780
|
-
pendingTools: structuredClone(pendingTools),
|
|
781
|
-
pendingIndex
|
|
782
|
-
};
|
|
783
1050
|
};
|
|
784
1051
|
async function* executeAgent(name, state) {
|
|
785
1052
|
const { request } = state;
|
|
786
1053
|
const agent = agents.get(name);
|
|
787
|
-
const modelName =
|
|
788
|
-
const
|
|
789
|
-
const
|
|
1054
|
+
const { modelName, model } = resolveModel(agent, state.checkpoint?.modelName ?? request.model);
|
|
1055
|
+
const events = [...state.initialEvents ?? []];
|
|
1056
|
+
const firstEvents = state.initialEvents ?? [];
|
|
1057
|
+
state.initialEvents = void 0;
|
|
790
1058
|
const emit = async (type, payload) => {
|
|
791
1059
|
const event = await emitEvent(state, type, payload);
|
|
792
1060
|
events.push(event);
|
|
@@ -802,8 +1070,10 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
802
1070
|
let pendingTools;
|
|
803
1071
|
let pendingIndex;
|
|
804
1072
|
try {
|
|
1073
|
+
for (const event of firstEvents) yield event;
|
|
805
1074
|
if (state.checkpoint) {
|
|
806
1075
|
const checkpoint = state.checkpoint;
|
|
1076
|
+
if (checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${state.run.id}" is unsupported`, state.run.id);
|
|
807
1077
|
const hash = await definitionHash(name, modelName, agent, tools);
|
|
808
1078
|
if (checkpoint.definitionHash !== hash) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Definition for agent "${name}" changed`, state.run.id);
|
|
809
1079
|
inputContent = checkpoint.inputContent;
|
|
@@ -818,14 +1088,17 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
818
1088
|
} else {
|
|
819
1089
|
yield await emit("run.started", void 0);
|
|
820
1090
|
request.signal.throwIfAborted();
|
|
821
|
-
|
|
822
|
-
|
|
1091
|
+
inputContent = serializeValue(agent.inputSchema === void 0 ? request.input : await abortable(() => validateSchema(agent.inputSchema, request.input, `Input for agent "${name}" does not match inputSchema`), request.signal), "Run input must be a string or JSON-serializable value");
|
|
1092
|
+
const knowledge = await readKnowledgeMessages(state, agent, inputContent);
|
|
823
1093
|
messages = [
|
|
824
1094
|
{
|
|
825
1095
|
role: "system",
|
|
826
1096
|
content: agent.instructions
|
|
827
1097
|
},
|
|
1098
|
+
...knowledge.skills,
|
|
828
1099
|
...state.session.history,
|
|
1100
|
+
...knowledge.context,
|
|
1101
|
+
...knowledge.memory,
|
|
829
1102
|
{
|
|
830
1103
|
role: "user",
|
|
831
1104
|
content: inputContent
|
|
@@ -837,26 +1110,58 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
837
1110
|
pendingTools = [];
|
|
838
1111
|
pendingIndex = 0;
|
|
839
1112
|
}
|
|
840
|
-
const activeOutputSchema =
|
|
1113
|
+
const activeOutputSchema = agent.outputSchema;
|
|
841
1114
|
const outputSchema = activeOutputSchema === void 0 ? void 0 : toTransportableSchema(activeOutputSchema, "output", `Output schema for agent "${name}" is not transportable`);
|
|
842
|
-
const
|
|
1115
|
+
const currentElicitationMode = state.checkpoint?.effectiveElicitationMode ?? effectiveElicitationMode(agent, request);
|
|
1116
|
+
const currentApprovalMode = state.checkpoint?.effectiveApprovalMode ?? effectiveApprovalMode(agent, request);
|
|
1117
|
+
const currentToolChoice = state.checkpoint?.effectiveToolChoice ?? effectiveToolChoice(agent, request);
|
|
1118
|
+
const agentTools = await Promise.all((agent.tools ?? []).map(async (toolName) => {
|
|
843
1119
|
const tool = tools.get(toolName);
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1120
|
+
const remote = await tool.resolve?.({ context: request.context });
|
|
1121
|
+
const inputSchema = tool.inputJsonSchema ?? remote?.inputSchema ?? (tool.inputSchema === void 0 ? void 0 : toTransportableSchema(tool.inputSchema, "input", `Input schema for tool "${tool.name}" is not transportable`));
|
|
1122
|
+
return toToolSpec(tool.description === void 0 && remote?.description !== void 0 ? {
|
|
1123
|
+
...tool,
|
|
1124
|
+
description: remote.description
|
|
1125
|
+
} : tool, inputSchema);
|
|
1126
|
+
}));
|
|
1127
|
+
const allTools = currentElicitationMode === "pause" ? [...agentTools, {
|
|
1128
|
+
name: ELICIT_TOOL_NAME,
|
|
1129
|
+
description: "Request external information required to continue this run. Use prompt for explanatory text shown outside the form. Put short field labels in responseSchema property titles and optional field help in property descriptions.",
|
|
1130
|
+
inputSchema: {
|
|
1131
|
+
type: "object",
|
|
1132
|
+
properties: {
|
|
1133
|
+
prompt: {
|
|
1134
|
+
type: "string",
|
|
1135
|
+
description: "Explanatory text for the user; do not duplicate field labels here."
|
|
1136
|
+
},
|
|
1137
|
+
responseSchema: {
|
|
1138
|
+
type: "object",
|
|
1139
|
+
description: "JSON Schema for the expected response. Each property should include a short title for the form label and may include a description for field help."
|
|
1140
|
+
},
|
|
1141
|
+
expiresAt: { type: "string" }
|
|
1142
|
+
},
|
|
1143
|
+
required: ["prompt", "responseSchema"],
|
|
1144
|
+
additionalProperties: false
|
|
1145
|
+
}
|
|
1146
|
+
}] : agentTools;
|
|
1147
|
+
const effectiveLimits = state.checkpoint?.limits ?? combineLimits(agent.limits, request.limits);
|
|
1148
|
+
const maxSteps = effectiveLimits?.maxSteps ?? 8;
|
|
1149
|
+
const maxToolCalls = effectiveLimits?.maxToolCalls ?? 16;
|
|
1150
|
+
if (typeof currentToolChoice === "object" && !allTools.some((tool) => tool.name === currentToolChoice.name)) throw new Error(`Tool "${currentToolChoice.name}" is not available to agent "${name}"`);
|
|
848
1151
|
while (step <= maxSteps) {
|
|
849
1152
|
request.signal.throwIfAborted();
|
|
850
1153
|
if (pendingTools.length === 0) {
|
|
851
1154
|
const hasToolBudget = step < maxSteps && toolCallCount < maxToolCalls;
|
|
1155
|
+
const modelToolChoice = hasToolBudget && allTools.length ? currentToolChoice === "auto" ? void 0 : currentToolChoice : void 0;
|
|
852
1156
|
yield await emit("model.started", { step });
|
|
853
1157
|
const modelStream = readModelStream(model, {
|
|
854
1158
|
messages: [...messages],
|
|
855
|
-
tools: hasToolBudget &&
|
|
856
|
-
|
|
1159
|
+
tools: hasToolBudget && allTools.length ? allTools : void 0,
|
|
1160
|
+
...modelToolChoice === void 0 ? {} : { toolChoice: modelToolChoice },
|
|
1161
|
+
reasoning: state.request.reasoning ?? agent.reasoning,
|
|
857
1162
|
modelOptions: agent.modelOptions,
|
|
858
1163
|
outputSchema,
|
|
859
|
-
...remainingOutputTokens(
|
|
1164
|
+
...remainingOutputTokens(effectiveLimits?.maxOutputTokens, usage) === void 0 ? {} : { maxOutputTokens: remainingOutputTokens(effectiveLimits?.maxOutputTokens, usage) },
|
|
860
1165
|
...providerState === void 0 ? {} : { providerState },
|
|
861
1166
|
signal: request.signal
|
|
862
1167
|
});
|
|
@@ -876,11 +1181,12 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
876
1181
|
step,
|
|
877
1182
|
...usagePayload === void 0 ? {} : { usage: usagePayload }
|
|
878
1183
|
});
|
|
879
|
-
assertTokenBudget(name, "maxInputTokens", "inputTokens",
|
|
880
|
-
assertTokenBudget(name, "maxOutputTokens", "outputTokens",
|
|
1184
|
+
assertTokenBudget(name, "maxInputTokens", "inputTokens", effectiveLimits?.maxInputTokens, result.usage, usage);
|
|
1185
|
+
assertTokenBudget(name, "maxOutputTokens", "outputTokens", effectiveLimits?.maxOutputTokens, result.usage, usage);
|
|
881
1186
|
if (!result.toolCalls?.length) {
|
|
882
1187
|
if (result.output === void 0) throw new Error(`Model for agent "${name}" returned no output`);
|
|
883
1188
|
const output = toJsonValue(activeOutputSchema === void 0 ? result.output : await abortable(() => validateSchema(activeOutputSchema, result.output, `Output from agent "${name}" does not match outputSchema`), request.signal), `Output from agent "${name}" must be JSON-serializable`);
|
|
1189
|
+
await writeMemory(state, agent, inputContent, output);
|
|
884
1190
|
state.session.history.push({
|
|
885
1191
|
role: "user",
|
|
886
1192
|
content: inputContent
|
|
@@ -923,9 +1229,56 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
923
1229
|
};
|
|
924
1230
|
}
|
|
925
1231
|
if (!hasToolBudget) throw new Error(step >= maxSteps ? `Agent "${name}" reached maxSteps limit of ${maxSteps}` : `Agent "${name}" reached maxToolCalls limit of ${maxToolCalls}`);
|
|
926
|
-
assertContinuationBudget(name, "maxInputTokens", "inputTokens",
|
|
927
|
-
assertContinuationBudget(name, "maxOutputTokens", "outputTokens",
|
|
1232
|
+
assertContinuationBudget(name, "maxInputTokens", "inputTokens", effectiveLimits?.maxInputTokens, usage);
|
|
1233
|
+
assertContinuationBudget(name, "maxOutputTokens", "outputTokens", effectiveLimits?.maxOutputTokens, usage);
|
|
928
1234
|
if (result.toolCalls.length > maxToolCalls - toolCallCount) throw new Error(`Agent "${name}" exceeded maxToolCalls limit of ${maxToolCalls}`);
|
|
1235
|
+
const elicitationCall = result.toolCalls.find(isElicitationToolCall);
|
|
1236
|
+
if (elicitationCall) {
|
|
1237
|
+
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "Elicitation requires a DurableRunStore", state.run.id);
|
|
1238
|
+
if (currentElicitationMode !== "pause") throw new Error(`Tool "${ELICIT_TOOL_NAME}" is not available to agent "${name}"`);
|
|
1239
|
+
if (result.toolCalls.length !== 1) throw new Error(`${ELICIT_TOOL_NAME} must be the only tool call in a model turn`);
|
|
1240
|
+
if (!elicitationCall.id?.trim()) throw new Error("Tool call id cannot be empty");
|
|
1241
|
+
if (seenToolCallIds.has(elicitationCall.id)) throw new Error(`Tool call id "${elicitationCall.id}" is duplicated in run "${state.run.id}"`);
|
|
1242
|
+
const rawInput = toJsonValue(elicitationCall.input, `Input for tool "${ELICIT_TOOL_NAME}" must be JSON-serializable`);
|
|
1243
|
+
const input = readElicitationInput(rawInput);
|
|
1244
|
+
seenToolCallIds.add(elicitationCall.id);
|
|
1245
|
+
messages.push({
|
|
1246
|
+
role: "assistant",
|
|
1247
|
+
content: result.output === void 0 ? "" : serializeValue(result.output, `Model output for agent "${name}" must be serializable`),
|
|
1248
|
+
toolCalls: [{
|
|
1249
|
+
...elicitationCall,
|
|
1250
|
+
input: rawInput
|
|
1251
|
+
}]
|
|
1252
|
+
});
|
|
1253
|
+
const checkpoint = await durableCheckpoint(state, model, modelName, messages, inputContent, usage, providerState, step + 1, toolCallCount, seenToolCallIds, [], 0);
|
|
1254
|
+
const request = {
|
|
1255
|
+
id: crypto.randomUUID(),
|
|
1256
|
+
toolCallId: elicitationCall.id,
|
|
1257
|
+
prompt: input.prompt,
|
|
1258
|
+
responseSchema: input.responseSchema,
|
|
1259
|
+
requestedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1260
|
+
...input.expiresAt === void 0 ? {} : { expiresAt: input.expiresAt }
|
|
1261
|
+
};
|
|
1262
|
+
state.run.status = "paused";
|
|
1263
|
+
state.run.pause = {
|
|
1264
|
+
type: "elicitation",
|
|
1265
|
+
request
|
|
1266
|
+
};
|
|
1267
|
+
const requested = createEvent(state, "elicitation.requested", { request });
|
|
1268
|
+
const paused = createEvent(state, "run.paused", {
|
|
1269
|
+
reason: "elicitation",
|
|
1270
|
+
toolCallId: elicitationCall.id
|
|
1271
|
+
});
|
|
1272
|
+
await commit(state, {
|
|
1273
|
+
checkpoint,
|
|
1274
|
+
events: [requested, paused]
|
|
1275
|
+
});
|
|
1276
|
+
events.push(requested, paused);
|
|
1277
|
+
await releaseExecution(state);
|
|
1278
|
+
yield requested;
|
|
1279
|
+
yield paused;
|
|
1280
|
+
return void 0;
|
|
1281
|
+
}
|
|
929
1282
|
const batch = /* @__PURE__ */ new Set();
|
|
930
1283
|
pendingTools = [];
|
|
931
1284
|
for (const call of result.toolCalls) {
|
|
@@ -944,7 +1297,8 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
944
1297
|
step,
|
|
945
1298
|
toolCallId: call.id,
|
|
946
1299
|
toolName: call.name,
|
|
947
|
-
error: toErrorMessage(error)
|
|
1300
|
+
error: toErrorMessage(error),
|
|
1301
|
+
...toolErrorPayload(tool, error)
|
|
948
1302
|
});
|
|
949
1303
|
throw error;
|
|
950
1304
|
}
|
|
@@ -972,13 +1326,16 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
972
1326
|
const tool = tools.get(pending.call.name);
|
|
973
1327
|
const durable = isDurableRunStore(runStore) ? runStore : void 0;
|
|
974
1328
|
if (!durable && (tool.approval === "required" || tool.idempotency === "keyed")) throw new FevexRunError("DURABLE_STORE_REQUIRED", `Tool "${tool.name}" requires a DurableRunStore`, state.run.id);
|
|
1329
|
+
let record = durable ? await durable.getToolExecution(state.run.id, pending.call.id) : void 0;
|
|
975
1330
|
const policy = await authorize(state, tool, pending.input, "tool.execute");
|
|
976
|
-
if ((tool.approval === "required" || policy === "require_approval") && state.approvedToolCallId !== pending.call.id) {
|
|
1331
|
+
if ((tool.approval === "required" || policy === "require_approval") && state.approvedToolCallId !== pending.call.id && record?.status !== "started" && record?.status !== "completed") {
|
|
1332
|
+
if (currentApprovalMode === "deny") throw new FevexRunError("POLICY_DENIED", `Tool "${tool.name}" requires approval but approvalMode is "deny"`, state.run.id);
|
|
977
1333
|
const checkpoint = await durableCheckpoint(state, model, modelName, messages, inputContent, usage, providerState, step, toolCallCount, seenToolCallIds, pendingTools, pendingIndex);
|
|
978
1334
|
const approval = {
|
|
979
1335
|
id: crypto.randomUUID(),
|
|
980
1336
|
toolCallId: pending.call.id,
|
|
981
1337
|
toolName: pending.call.name,
|
|
1338
|
+
...tool.label === void 0 ? {} : { toolLabel: tool.label },
|
|
982
1339
|
input: pending.input,
|
|
983
1340
|
risk: tool.risk ?? "read",
|
|
984
1341
|
requestedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -991,7 +1348,9 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
991
1348
|
const requested = createEvent(state, "approval.requested", {
|
|
992
1349
|
approvalId: approval.id,
|
|
993
1350
|
toolCallId: approval.toolCallId,
|
|
994
|
-
toolName: approval.toolName
|
|
1351
|
+
toolName: approval.toolName,
|
|
1352
|
+
...toolLabelPayload(tool),
|
|
1353
|
+
...toolSourcePayload(tool)
|
|
995
1354
|
});
|
|
996
1355
|
const paused = createEvent(state, "run.paused", {
|
|
997
1356
|
reason: "approval",
|
|
@@ -1013,7 +1372,6 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1013
1372
|
await commit(state, { checkpoint });
|
|
1014
1373
|
state.checkpoint = checkpoint;
|
|
1015
1374
|
}
|
|
1016
|
-
let record = durable ? await durable.getToolExecution(state.run.id, pending.call.id) : void 0;
|
|
1017
1375
|
if (record?.status === "completed") {
|
|
1018
1376
|
messages.push({
|
|
1019
1377
|
role: "tool",
|
|
@@ -1037,7 +1395,8 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1037
1395
|
const unknown = createEvent(state, "tool.execution_unknown", {
|
|
1038
1396
|
step,
|
|
1039
1397
|
toolCallId: pending.call.id,
|
|
1040
|
-
toolName: pending.call.name
|
|
1398
|
+
toolName: pending.call.name,
|
|
1399
|
+
...toolSourcePayload(tool)
|
|
1041
1400
|
});
|
|
1042
1401
|
const paused = createEvent(state, "run.paused", {
|
|
1043
1402
|
reason: "tool_execution_unknown",
|
|
@@ -1053,18 +1412,22 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1053
1412
|
yield paused;
|
|
1054
1413
|
return void 0;
|
|
1055
1414
|
}
|
|
1415
|
+
const forcedRetry = state.forcedRetryToolCallId === pending.call.id;
|
|
1056
1416
|
state.forcedRetryToolCallId = void 0;
|
|
1057
|
-
const
|
|
1417
|
+
const configuredMaxAttempts = tool.retry?.maxAttempts ?? 1;
|
|
1418
|
+
const maxAttempts = forcedRetry ? Math.max(configuredMaxAttempts, (record?.attempt ?? 0) + 1) : configuredMaxAttempts;
|
|
1058
1419
|
let output;
|
|
1059
|
-
let attempt = record?.attempt ?? 0;
|
|
1420
|
+
let attempt = record?.status === "started" && tool.idempotency === "keyed" ? Math.max(0, record.attempt - 1) : record?.attempt ?? 0;
|
|
1060
1421
|
while (attempt < maxAttempts) {
|
|
1061
1422
|
if (attempt > 0) {
|
|
1062
1423
|
if (await authorize(state, tool, pending.input, "tool.execute") === "require_approval") {
|
|
1424
|
+
if (currentApprovalMode === "deny") throw new FevexRunError("POLICY_DENIED", `Tool "${tool.name}" requires approval but approvalMode is "deny"`, state.run.id);
|
|
1063
1425
|
const checkpoint = await durableCheckpoint(state, model, modelName, messages, inputContent, usage, providerState, step, toolCallCount, seenToolCallIds, pendingTools, pendingIndex);
|
|
1064
1426
|
const approval = {
|
|
1065
1427
|
id: crypto.randomUUID(),
|
|
1066
1428
|
toolCallId: pending.call.id,
|
|
1067
1429
|
toolName: pending.call.name,
|
|
1430
|
+
...tool.label === void 0 ? {} : { toolLabel: tool.label },
|
|
1068
1431
|
input: pending.input,
|
|
1069
1432
|
risk: tool.risk ?? "read",
|
|
1070
1433
|
requestedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -1077,7 +1440,9 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1077
1440
|
const requested = createEvent(state, "approval.requested", {
|
|
1078
1441
|
approvalId: approval.id,
|
|
1079
1442
|
toolCallId: approval.toolCallId,
|
|
1080
|
-
toolName: approval.toolName
|
|
1443
|
+
toolName: approval.toolName,
|
|
1444
|
+
...toolLabelPayload(tool),
|
|
1445
|
+
...toolSourcePayload(tool)
|
|
1081
1446
|
});
|
|
1082
1447
|
const paused = createEvent(state, "run.paused", {
|
|
1083
1448
|
reason: "approval",
|
|
@@ -1111,6 +1476,20 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1111
1476
|
if (!tool.credentials?.includes(credentialName)) throw new FevexRunError("CREDENTIAL_NOT_FOUND", `Credential "${credentialName}" is not declared for tool "${tool.name}"`, state.run.id);
|
|
1112
1477
|
return resolvedCredentials.get(credentialName);
|
|
1113
1478
|
};
|
|
1479
|
+
const scopedSandbox = tool.sandbox === void 0 ? void 0 : { run: (sandboxRequest) => {
|
|
1480
|
+
if (!sandbox) throw new FevexRunError("SANDBOX_REQUIRED", `Tool "${tool.name}" requires a sandbox`, state.run.id);
|
|
1481
|
+
const signal = sandboxRequest.signal ? AbortSignal.any([request.signal, sandboxRequest.signal]) : request.signal;
|
|
1482
|
+
return sandbox.run({
|
|
1483
|
+
...sandboxRequest,
|
|
1484
|
+
capabilities: tool.sandbox,
|
|
1485
|
+
runId: state.run.id,
|
|
1486
|
+
toolCallId: pending.call.id,
|
|
1487
|
+
attempt,
|
|
1488
|
+
idempotencyKey: pending.idempotencyKey,
|
|
1489
|
+
context: request.context,
|
|
1490
|
+
signal
|
|
1491
|
+
});
|
|
1492
|
+
} };
|
|
1114
1493
|
record = {
|
|
1115
1494
|
runId: state.run.id,
|
|
1116
1495
|
toolCallId: pending.call.id,
|
|
@@ -1125,6 +1504,7 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1125
1504
|
step,
|
|
1126
1505
|
toolCallId: pending.call.id,
|
|
1127
1506
|
toolName: pending.call.name,
|
|
1507
|
+
...toolSourcePayload(tool),
|
|
1128
1508
|
...attempt > 1 ? { attempt } : {}
|
|
1129
1509
|
});
|
|
1130
1510
|
if (durable) await commit(state, {
|
|
@@ -1144,6 +1524,7 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1144
1524
|
attempt,
|
|
1145
1525
|
idempotencyKey: pending.idempotencyKey,
|
|
1146
1526
|
getCredential,
|
|
1527
|
+
...scopedSandbox === void 0 ? {} : { sandbox: scopedSandbox },
|
|
1147
1528
|
context: request.context,
|
|
1148
1529
|
signal: request.signal
|
|
1149
1530
|
}), request.signal);
|
|
@@ -1159,6 +1540,7 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1159
1540
|
step,
|
|
1160
1541
|
toolCallId: pending.call.id,
|
|
1161
1542
|
toolName: pending.call.name,
|
|
1543
|
+
...toolSourcePayload(tool),
|
|
1162
1544
|
...attempt > 1 ? { attempt } : {}
|
|
1163
1545
|
});
|
|
1164
1546
|
if (durable) await commit(state, {
|
|
@@ -1187,69 +1569,1589 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1187
1569
|
step,
|
|
1188
1570
|
toolCallId: pending.call.id,
|
|
1189
1571
|
toolName: pending.call.name,
|
|
1190
|
-
error: safeError
|
|
1572
|
+
error: safeError,
|
|
1573
|
+
...toolErrorPayload(tool, error)
|
|
1191
1574
|
});
|
|
1192
1575
|
throw secrets.length ? new Error(safeError, { cause: error }) : error;
|
|
1193
1576
|
}
|
|
1194
|
-
const delayMs = Math.min(tool.retry.backoffMs * 2 ** (attempt - 1), tool.retry.maxBackoffMs ?? Number.MAX_SAFE_INTEGER);
|
|
1195
|
-
yield await emit("tool.retrying", {
|
|
1196
|
-
step,
|
|
1197
|
-
toolCallId: pending.call.id,
|
|
1198
|
-
toolName: pending.call.name,
|
|
1199
|
-
attempt: attempt + 1,
|
|
1200
|
-
delayMs,
|
|
1201
|
-
error: safeError
|
|
1577
|
+
const delayMs = Math.min(tool.retry.backoffMs * 2 ** (attempt - 1), tool.retry.maxBackoffMs ?? Number.MAX_SAFE_INTEGER);
|
|
1578
|
+
yield await emit("tool.retrying", {
|
|
1579
|
+
step,
|
|
1580
|
+
toolCallId: pending.call.id,
|
|
1581
|
+
toolName: pending.call.name,
|
|
1582
|
+
attempt: attempt + 1,
|
|
1583
|
+
delayMs,
|
|
1584
|
+
error: safeError,
|
|
1585
|
+
...toolErrorPayload(tool, error)
|
|
1586
|
+
});
|
|
1587
|
+
if (delayMs) await abortable(() => new Promise((resolve) => setTimeout(resolve, delayMs)), request.signal);
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
if (output === void 0) throw new FevexRunError("TOOL_EXECUTION_UNKNOWN", `Tool "${tool.name}" exhausted its configured attempts`, state.run.id);
|
|
1591
|
+
messages.push({
|
|
1592
|
+
role: "tool",
|
|
1593
|
+
name: pending.call.name,
|
|
1594
|
+
toolCallId: pending.call.id,
|
|
1595
|
+
content: serializeJsonValue(output)
|
|
1596
|
+
});
|
|
1597
|
+
toolCallCount += 1;
|
|
1598
|
+
pendingIndex += 1;
|
|
1599
|
+
}
|
|
1600
|
+
pendingTools = [];
|
|
1601
|
+
pendingIndex = 0;
|
|
1602
|
+
step += 1;
|
|
1603
|
+
}
|
|
1604
|
+
throw new Error(`Agent "${name}" reached maxSteps limit of ${maxSteps}`);
|
|
1605
|
+
} catch (error) {
|
|
1606
|
+
if (request.signal.aborted) {
|
|
1607
|
+
const event = await cancelExecution(state);
|
|
1608
|
+
if (event) {
|
|
1609
|
+
events.push(event);
|
|
1610
|
+
yield event;
|
|
1611
|
+
}
|
|
1612
|
+
} else if (state.run.status === "running") {
|
|
1613
|
+
Object.assign(state.run, {
|
|
1614
|
+
status: "failed",
|
|
1615
|
+
error: toErrorMessage(error),
|
|
1616
|
+
...usage === void 0 ? {} : { usage },
|
|
1617
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1618
|
+
});
|
|
1619
|
+
const failed = createEvent(state, "run.failed", {
|
|
1620
|
+
error: toErrorMessage(error),
|
|
1621
|
+
...runErrorPayload(error)
|
|
1622
|
+
});
|
|
1623
|
+
if (isDurableRunStore(runStore)) try {
|
|
1624
|
+
await commit(state, {
|
|
1625
|
+
checkpoint: null,
|
|
1626
|
+
events: [failed]
|
|
1627
|
+
});
|
|
1628
|
+
} catch (commitError) {
|
|
1629
|
+
await releaseExecution(state);
|
|
1630
|
+
throw commitError;
|
|
1631
|
+
}
|
|
1632
|
+
else {
|
|
1633
|
+
await runStore.saveRun(state.run);
|
|
1634
|
+
await runStore.appendEvent(failed);
|
|
1635
|
+
notifyObserver(failed);
|
|
1636
|
+
}
|
|
1637
|
+
events.push(failed);
|
|
1638
|
+
await releaseExecution(state);
|
|
1639
|
+
yield failed;
|
|
1640
|
+
} else await releaseExecution(state);
|
|
1641
|
+
throw error;
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
const nextEvent = async (state, execution) => {
|
|
1645
|
+
state.advancing = true;
|
|
1646
|
+
try {
|
|
1647
|
+
return await execution.next();
|
|
1648
|
+
} finally {
|
|
1649
|
+
state.advancing = false;
|
|
1650
|
+
}
|
|
1651
|
+
};
|
|
1652
|
+
const drainExecution = async (name, state, throwOnPause) => {
|
|
1653
|
+
const execution = executeAgent(name, state);
|
|
1654
|
+
while (true) {
|
|
1655
|
+
const next = await nextEvent(state, execution);
|
|
1656
|
+
if (next.done) {
|
|
1657
|
+
if (next.value === void 0 && throwOnPause) {
|
|
1658
|
+
const run = await runStore.getRun(state.run.id);
|
|
1659
|
+
if (run?.pause) throw new RunPausedError(run.id, run.pause);
|
|
1660
|
+
}
|
|
1661
|
+
return next.value;
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
};
|
|
1665
|
+
const resumeAgent = async (runId, resolution, waitForCompletion = false, recoveryActor) => {
|
|
1666
|
+
if (resolution?.type === "timer" || resolution?.type === "event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
1667
|
+
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "resumeRun requires DurableRunStore", runId);
|
|
1668
|
+
const run = await runStore.getRun(runId);
|
|
1669
|
+
const checkpoint = await runStore.getCheckpoint(runId);
|
|
1670
|
+
if (!run || !checkpoint) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be ${resolution ? "resumed" : "recovered"}`, runId);
|
|
1671
|
+
if (checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${runId}" is unsupported`, runId);
|
|
1672
|
+
if (resolution && run.status !== "paused" && run.status !== "running" || !resolution && run.status !== "running") throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" is not ${resolution ? "resumable" : "recoverable"}`, runId);
|
|
1673
|
+
const agent = agents.get(run.agentName);
|
|
1674
|
+
if (!agent) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Agent "${run.agentName}" is unavailable`, runId);
|
|
1675
|
+
const checkpointModel = await resolveCheckpointModel(run.agentName, agent, checkpoint);
|
|
1676
|
+
if (!checkpointModel) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Definition for agent "${run.agentName}" changed`, runId);
|
|
1677
|
+
const { modelName, model } = checkpointModel;
|
|
1678
|
+
if (checkpoint.providerState !== void 0 && !model.stateCodec) throw new FevexRunError("RUN_NOT_RESUMABLE", "ModelGateway no longer provides its stateCodec", runId);
|
|
1679
|
+
const session = await runStore.getSession(run.sessionId);
|
|
1680
|
+
if (!session) throw new Error(`Session "${run.sessionId}" does not exist`);
|
|
1681
|
+
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" is active`, runId);
|
|
1682
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
1683
|
+
const lease = {
|
|
1684
|
+
generation: 0,
|
|
1685
|
+
runId,
|
|
1686
|
+
ownerId,
|
|
1687
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
1688
|
+
};
|
|
1689
|
+
if (!await runStore.acquireLease(lease)) throw new FevexRunError("RUN_CONFLICT", `Run "${runId}" is leased`, runId);
|
|
1690
|
+
try {
|
|
1691
|
+
const controller = new AbortController();
|
|
1692
|
+
const state = {
|
|
1693
|
+
run,
|
|
1694
|
+
session,
|
|
1695
|
+
request: {
|
|
1696
|
+
input: "",
|
|
1697
|
+
model: modelName,
|
|
1698
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
1699
|
+
sessionId: session.id,
|
|
1700
|
+
context: checkpoint.context,
|
|
1701
|
+
limits: checkpoint.limits,
|
|
1702
|
+
signal: controller.signal
|
|
1703
|
+
},
|
|
1704
|
+
controller,
|
|
1705
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
1706
|
+
advancing: false,
|
|
1707
|
+
checkpoint,
|
|
1708
|
+
lease,
|
|
1709
|
+
...resolution?.type === "approval" ? { approvedToolCallId: run.pause?.type === "approval" ? run.pause.approval.toolCallId : void 0 } : resolution?.type === "tool_execution" ? {
|
|
1710
|
+
approvedToolCallId: resolution.toolCallId,
|
|
1711
|
+
...resolution.decision === "retry" ? { forcedRetryToolCallId: resolution.toolCallId } : {}
|
|
1712
|
+
} : {}
|
|
1713
|
+
};
|
|
1714
|
+
if (!resolution) {
|
|
1715
|
+
if (!recoveryActor?.id.trim()) throw new FevexRunError("RUN_NOT_RECOVERABLE", "Recovery actor is required", runId);
|
|
1716
|
+
const recovered = createEvent(state, "run.recovered", { actorId: recoveryActor.id });
|
|
1717
|
+
await commit(state, {
|
|
1718
|
+
checkpoint,
|
|
1719
|
+
events: [recovered]
|
|
1720
|
+
});
|
|
1721
|
+
state.initialEvents = [recovered];
|
|
1722
|
+
activeRuns.set(runId, state);
|
|
1723
|
+
activeSessions.add(session.id);
|
|
1724
|
+
startLease(state, runStore);
|
|
1725
|
+
if (waitForCompletion) await drainExecution(run.agentName, state, false);
|
|
1726
|
+
else drainExecution(run.agentName, state, false).catch(() => {});
|
|
1727
|
+
return structuredClone(run);
|
|
1728
|
+
}
|
|
1729
|
+
if (!resolution.actor?.id?.trim()) {
|
|
1730
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1731
|
+
throw new FevexRunError(resolution.type === "elicitation" ? "ELICITATION_INVALID" : "APPROVAL_INVALID", "Resolution actor is required", runId);
|
|
1732
|
+
}
|
|
1733
|
+
if (resolution.type === "elicitation") {
|
|
1734
|
+
const pause = run.pause;
|
|
1735
|
+
if (pause?.type !== "elicitation" || pause.request.id !== resolution.requestId) {
|
|
1736
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1737
|
+
throw new FevexRunError("ELICITATION_INVALID", "Elicitation does not match the run", runId);
|
|
1738
|
+
}
|
|
1739
|
+
if (pause.request.expiresAt && Date.now() > Date.parse(pause.request.expiresAt)) {
|
|
1740
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1741
|
+
throw new FevexRunError("ELICITATION_INVALID", "Elicitation has expired", runId);
|
|
1742
|
+
}
|
|
1743
|
+
let value;
|
|
1744
|
+
try {
|
|
1745
|
+
value = compileFevexJsonSchema(pause.request.responseSchema).validate(resolution.value);
|
|
1746
|
+
} catch (error) {
|
|
1747
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1748
|
+
throw new FevexRunError("ELICITATION_INVALID", `Elicitation value does not match responseSchema: ${toErrorMessage(error)}`, runId, { cause: error });
|
|
1749
|
+
}
|
|
1750
|
+
checkpoint.messages.push({
|
|
1751
|
+
role: "tool",
|
|
1752
|
+
name: ELICIT_TOOL_NAME,
|
|
1753
|
+
toolCallId: pause.request.toolCallId,
|
|
1754
|
+
content: serializeJsonValue(value)
|
|
1755
|
+
});
|
|
1756
|
+
run.status = "running";
|
|
1757
|
+
run.pause = void 0;
|
|
1758
|
+
const resolved = createEvent(state, "elicitation.resolved", {
|
|
1759
|
+
requestId: pause.request.id,
|
|
1760
|
+
toolCallId: pause.request.toolCallId,
|
|
1761
|
+
actorId: resolution.actor.id
|
|
1762
|
+
});
|
|
1763
|
+
const resumed = createEvent(state, "run.resumed", void 0);
|
|
1764
|
+
await commit(state, {
|
|
1765
|
+
checkpoint,
|
|
1766
|
+
events: [resolved, resumed]
|
|
1767
|
+
});
|
|
1768
|
+
activeRuns.set(runId, state);
|
|
1769
|
+
activeSessions.add(session.id);
|
|
1770
|
+
startLease(state, runStore);
|
|
1771
|
+
if (waitForCompletion) await drainExecution(run.agentName, state, false);
|
|
1772
|
+
else drainExecution(run.agentName, state, false).catch(() => {});
|
|
1773
|
+
return structuredClone(run);
|
|
1774
|
+
}
|
|
1775
|
+
if (resolution.type === "approval") {
|
|
1776
|
+
if (run.pause?.type !== "approval" || run.pause.approval.id !== resolution.approvalId) {
|
|
1777
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1778
|
+
throw new FevexRunError("APPROVAL_INVALID", "Approval does not match the run", runId);
|
|
1779
|
+
}
|
|
1780
|
+
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1781
|
+
const tool = pending && tools.get(pending.call.name);
|
|
1782
|
+
if (!pending || !tool) {
|
|
1783
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1784
|
+
throw new FevexRunError("APPROVAL_INVALID", "Approval tool is unavailable", runId);
|
|
1785
|
+
}
|
|
1786
|
+
if (resolution.decision === "approve") await authorize(state, tool, pending.input, "approval.resolve", {
|
|
1787
|
+
...state.request.context,
|
|
1788
|
+
actor: resolution.actor
|
|
1789
|
+
});
|
|
1790
|
+
if (resolution.decision === "reject") {
|
|
1791
|
+
const approval = run.pause.approval;
|
|
1792
|
+
const toolCallId = approval.toolCallId;
|
|
1793
|
+
run.status = "cancelled";
|
|
1794
|
+
run.pause = void 0;
|
|
1795
|
+
run.error = "approval_rejected";
|
|
1796
|
+
const resolved = createEvent(state, "approval.resolved", {
|
|
1797
|
+
approvalId: resolution.approvalId,
|
|
1798
|
+
toolCallId,
|
|
1799
|
+
decision: "reject",
|
|
1800
|
+
actorId: resolution.actor.id,
|
|
1801
|
+
...approval.toolLabel === void 0 ? {} : { toolLabel: approval.toolLabel },
|
|
1802
|
+
...toolSourcePayload(tool)
|
|
1803
|
+
});
|
|
1804
|
+
const cancelled = createEvent(state, "run.cancelled", { reason: "approval_rejected" });
|
|
1805
|
+
await commit(state, {
|
|
1806
|
+
checkpoint: null,
|
|
1807
|
+
events: [resolved, cancelled]
|
|
1808
|
+
});
|
|
1809
|
+
await releaseExecution(state);
|
|
1810
|
+
return structuredClone(run);
|
|
1811
|
+
}
|
|
1812
|
+
} else {
|
|
1813
|
+
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1814
|
+
const record = await runStore.getToolExecution(runId, resolution.toolCallId);
|
|
1815
|
+
const matchesUnknownPause = run.pause?.type === "tool_execution_unknown" && run.pause.toolCallId === resolution.toolCallId;
|
|
1816
|
+
const matchesInterruptedAttempt = run.status === "running" && pending?.call.id === resolution.toolCallId && (record?.status === "started" || record?.status === "completed");
|
|
1817
|
+
if (!matchesUnknownPause && !matchesInterruptedAttempt) {
|
|
1818
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1819
|
+
throw new FevexRunError("RUN_NOT_RESUMABLE", "Tool execution resolution does not match a pending attempt", runId);
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
if (resolution.type === "tool_execution" && resolution.decision === "use_output") {
|
|
1823
|
+
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1824
|
+
const tool = tools.get(pending.call.name);
|
|
1825
|
+
if (!tool) {
|
|
1826
|
+
await runStore.releaseLease(runId, ownerId, lease.generation);
|
|
1827
|
+
throw new FevexRunError("RUN_DEFINITION_CHANGED", "Pending tool is unavailable", runId);
|
|
1828
|
+
}
|
|
1829
|
+
const output = toJsonValue(tool.outputSchema === void 0 ? resolution.output : await validateSchema(tool.outputSchema, resolution.output, `Manual output for tool "${tool.name}" does not match outputSchema`), "Manual tool output must be JSON-serializable");
|
|
1830
|
+
const current = await runStore.getToolExecution(runId, resolution.toolCallId);
|
|
1831
|
+
await commit(state, { toolExecution: {
|
|
1832
|
+
runId,
|
|
1833
|
+
toolCallId: resolution.toolCallId,
|
|
1834
|
+
toolName: pending.call.name,
|
|
1835
|
+
input: pending.input,
|
|
1836
|
+
status: "completed",
|
|
1837
|
+
attempt: current?.attempt ?? 1,
|
|
1838
|
+
idempotencyKey: current?.idempotencyKey ?? pending.idempotencyKey,
|
|
1839
|
+
output,
|
|
1840
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1841
|
+
} });
|
|
1842
|
+
}
|
|
1843
|
+
const approval = run.pause?.type === "approval" ? run.pause.approval : void 0;
|
|
1844
|
+
const approvalTool = approval ? tools.get(approval.toolName) : void 0;
|
|
1845
|
+
run.status = "running";
|
|
1846
|
+
run.pause = void 0;
|
|
1847
|
+
const resumeEvents = [createEvent(state, "run.resumed", void 0)];
|
|
1848
|
+
if (approval && resolution.type === "approval") resumeEvents.push(createEvent(state, "approval.resolved", {
|
|
1849
|
+
approvalId: approval.id,
|
|
1850
|
+
toolCallId: approval.toolCallId,
|
|
1851
|
+
decision: "approve",
|
|
1852
|
+
actorId: resolution.actor.id,
|
|
1853
|
+
...approval.toolLabel === void 0 ? {} : { toolLabel: approval.toolLabel },
|
|
1854
|
+
...approvalTool === void 0 ? {} : toolSourcePayload(approvalTool)
|
|
1855
|
+
}));
|
|
1856
|
+
await commit(state, {
|
|
1857
|
+
checkpoint,
|
|
1858
|
+
events: resumeEvents
|
|
1859
|
+
});
|
|
1860
|
+
activeRuns.set(runId, state);
|
|
1861
|
+
activeSessions.add(session.id);
|
|
1862
|
+
startLease(state, runStore);
|
|
1863
|
+
if (waitForCompletion) await drainExecution(run.agentName, state, false);
|
|
1864
|
+
else drainExecution(run.agentName, state, false).catch(() => {});
|
|
1865
|
+
return structuredClone(run);
|
|
1866
|
+
} catch (error) {
|
|
1867
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
1868
|
+
throw error;
|
|
1869
|
+
}
|
|
1870
|
+
};
|
|
1871
|
+
const cancelStoredAgent = async (runId) => {
|
|
1872
|
+
const active = activeRuns.get(runId);
|
|
1873
|
+
if (active && !active.request.signal.aborted) {
|
|
1874
|
+
active.controller.abort(new DOMException("Run cancelled", "AbortError"));
|
|
1875
|
+
if (!active.advancing) {
|
|
1876
|
+
await cancelExecution(active);
|
|
1877
|
+
return true;
|
|
1878
|
+
}
|
|
1879
|
+
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
1880
|
+
const current = await runStore.getRun(runId);
|
|
1881
|
+
if (current?.status === "cancelled" || current?.status === "completed" || current?.status === "failed") return true;
|
|
1882
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1883
|
+
}
|
|
1884
|
+
return false;
|
|
1885
|
+
}
|
|
1886
|
+
if (!isDurableRunStore(runStore)) return false;
|
|
1887
|
+
const run = await runStore.getRun(runId);
|
|
1888
|
+
if (!run) return false;
|
|
1889
|
+
if (run.status === "completed" || run.status === "failed" || run.status === "cancelled") return true;
|
|
1890
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
1891
|
+
const lease = {
|
|
1892
|
+
generation: 0,
|
|
1893
|
+
runId,
|
|
1894
|
+
ownerId,
|
|
1895
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
1896
|
+
};
|
|
1897
|
+
if (!await runStore.acquireLease(lease)) return false;
|
|
1898
|
+
try {
|
|
1899
|
+
const session = await runStore.getSession(run.sessionId);
|
|
1900
|
+
if (!session) return false;
|
|
1901
|
+
const controller = new AbortController();
|
|
1902
|
+
const state = {
|
|
1903
|
+
run,
|
|
1904
|
+
session,
|
|
1905
|
+
request: {
|
|
1906
|
+
input: "",
|
|
1907
|
+
sessionId: session.id,
|
|
1908
|
+
signal: controller.signal
|
|
1909
|
+
},
|
|
1910
|
+
controller,
|
|
1911
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
1912
|
+
advancing: false,
|
|
1913
|
+
lease
|
|
1914
|
+
};
|
|
1915
|
+
run.status = "cancelled";
|
|
1916
|
+
run.pause = void 0;
|
|
1917
|
+
run.error = "aborted";
|
|
1918
|
+
const cancelled = createEvent(state, "run.cancelled", { reason: "aborted" });
|
|
1919
|
+
await commit(state, {
|
|
1920
|
+
checkpoint: null,
|
|
1921
|
+
events: [cancelled]
|
|
1922
|
+
});
|
|
1923
|
+
return true;
|
|
1924
|
+
} catch (error) {
|
|
1925
|
+
if (error instanceof FevexRunError && error.code === "RUN_CONFLICT") return false;
|
|
1926
|
+
throw error;
|
|
1927
|
+
} finally {
|
|
1928
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
1929
|
+
}
|
|
1930
|
+
};
|
|
1931
|
+
return {
|
|
1932
|
+
prepareExecution,
|
|
1933
|
+
executeAgent,
|
|
1934
|
+
nextEvent,
|
|
1935
|
+
drainExecution,
|
|
1936
|
+
resumeAgent,
|
|
1937
|
+
cancelStoredAgent
|
|
1938
|
+
};
|
|
1939
|
+
}
|
|
1940
|
+
//#endregion
|
|
1941
|
+
//#region src/internal/observability.ts
|
|
1942
|
+
const TERMINAL_TYPES = /* @__PURE__ */ new Set([
|
|
1943
|
+
"run.completed",
|
|
1944
|
+
"run.failed",
|
|
1945
|
+
"run.cancelled"
|
|
1946
|
+
]);
|
|
1947
|
+
const SENSITIVE_KEY = /api[-_]?key|authorization|cookie|credential|password|secret|token/i;
|
|
1948
|
+
function durationMs(startedAt, endedAt) {
|
|
1949
|
+
return Math.max(0, Date.parse(endedAt) - Date.parse(startedAt));
|
|
1950
|
+
}
|
|
1951
|
+
function traceStatus(event) {
|
|
1952
|
+
if (event.type === "run.completed") return "completed";
|
|
1953
|
+
if (event.type === "run.cancelled") return "cancelled";
|
|
1954
|
+
return "failed";
|
|
1955
|
+
}
|
|
1956
|
+
function redactKeys(value) {
|
|
1957
|
+
if (Array.isArray(value)) return value.map(redactKeys);
|
|
1958
|
+
if (value === null || typeof value !== "object") return value;
|
|
1959
|
+
const result = {};
|
|
1960
|
+
for (const [key, item] of Object.entries(value)) result[key] = SENSITIVE_KEY.test(key) ? "[REDACTED]" : redactKeys(item);
|
|
1961
|
+
return result;
|
|
1962
|
+
}
|
|
1963
|
+
function content(options, kind, value) {
|
|
1964
|
+
if (!options.content?.include.includes(kind)) return void 0;
|
|
1965
|
+
let parsed = value;
|
|
1966
|
+
if (typeof value === "string") try {
|
|
1967
|
+
parsed = JSON.parse(value);
|
|
1968
|
+
} catch {}
|
|
1969
|
+
const redacted = redactKeys(parsed);
|
|
1970
|
+
const safe = options.content.redact ? options.content.redact({
|
|
1971
|
+
kind,
|
|
1972
|
+
value: redacted
|
|
1973
|
+
}) : redacted;
|
|
1974
|
+
return safe === void 0 ? void 0 : {
|
|
1975
|
+
kind,
|
|
1976
|
+
value: toJsonValue(safe, `TraceRedactor output for "${kind}" must be JSON-serializable`)
|
|
1977
|
+
};
|
|
1978
|
+
}
|
|
1979
|
+
function usageFrom(value) {
|
|
1980
|
+
if (!value) return void 0;
|
|
1981
|
+
const usage = {};
|
|
1982
|
+
if (typeof value.inputTokens === "number") usage.inputTokens = value.inputTokens;
|
|
1983
|
+
if (typeof value.outputTokens === "number") usage.outputTokens = value.outputTokens;
|
|
1984
|
+
if (typeof value.totalTokens === "number") usage.totalTokens = value.totalTokens;
|
|
1985
|
+
return Object.keys(usage).length ? usage : void 0;
|
|
1986
|
+
}
|
|
1987
|
+
function subtractUsage(current, previous) {
|
|
1988
|
+
if (!current) return void 0;
|
|
1989
|
+
const usage = {};
|
|
1990
|
+
for (const key of [
|
|
1991
|
+
"inputTokens",
|
|
1992
|
+
"outputTokens",
|
|
1993
|
+
"totalTokens"
|
|
1994
|
+
]) if (current[key] !== void 0) usage[key] = Math.max(0, current[key] - (previous[key] ?? 0));
|
|
1995
|
+
return Object.keys(usage).length ? usage : void 0;
|
|
1996
|
+
}
|
|
1997
|
+
function costFor(options, modelRegistryName, model, usage) {
|
|
1998
|
+
if (!options.calculateCost || !usage) return void 0;
|
|
1999
|
+
const cost = options.calculateCost({
|
|
2000
|
+
modelRegistryName,
|
|
2001
|
+
model,
|
|
2002
|
+
usage
|
|
2003
|
+
});
|
|
2004
|
+
if (!cost) return void 0;
|
|
2005
|
+
if (!Number.isFinite(cost.amount) || cost.amount < 0 || !cost.currency.trim()) throw new TypeError("CostCalculator must return a non-negative amount and currency");
|
|
2006
|
+
return {
|
|
2007
|
+
amount: cost.amount,
|
|
2008
|
+
currency: cost.currency
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
function annotation(event) {
|
|
2012
|
+
switch (event.type) {
|
|
2013
|
+
case "run.paused": return {
|
|
2014
|
+
name: event.type,
|
|
2015
|
+
timestamp: event.timestamp,
|
|
2016
|
+
attributes: {
|
|
2017
|
+
reason: event.payload.reason,
|
|
2018
|
+
toolCallId: event.payload.toolCallId
|
|
2019
|
+
}
|
|
2020
|
+
};
|
|
2021
|
+
case "run.resumed": return {
|
|
2022
|
+
name: event.type,
|
|
2023
|
+
timestamp: event.timestamp
|
|
2024
|
+
};
|
|
2025
|
+
case "approval.requested": return {
|
|
2026
|
+
name: event.type,
|
|
2027
|
+
timestamp: event.timestamp,
|
|
2028
|
+
attributes: {
|
|
2029
|
+
approvalId: event.payload.approvalId,
|
|
2030
|
+
toolCallId: event.payload.toolCallId,
|
|
2031
|
+
toolName: event.payload.toolName
|
|
2032
|
+
}
|
|
2033
|
+
};
|
|
2034
|
+
case "approval.resolved": return {
|
|
2035
|
+
name: event.type,
|
|
2036
|
+
timestamp: event.timestamp,
|
|
2037
|
+
attributes: {
|
|
2038
|
+
approvalId: event.payload.approvalId,
|
|
2039
|
+
toolCallId: event.payload.toolCallId,
|
|
2040
|
+
decision: event.payload.decision
|
|
2041
|
+
}
|
|
2042
|
+
};
|
|
2043
|
+
case "tool.retrying": return {
|
|
2044
|
+
name: event.type,
|
|
2045
|
+
timestamp: event.timestamp,
|
|
2046
|
+
attributes: {
|
|
2047
|
+
toolCallId: event.payload.toolCallId,
|
|
2048
|
+
toolName: event.payload.toolName,
|
|
2049
|
+
attempt: event.payload.attempt,
|
|
2050
|
+
delayMs: event.payload.delayMs
|
|
2051
|
+
}
|
|
2052
|
+
};
|
|
2053
|
+
case "tool.execution_unknown": return {
|
|
2054
|
+
name: event.type,
|
|
2055
|
+
timestamp: event.timestamp,
|
|
2056
|
+
attributes: {
|
|
2057
|
+
toolCallId: event.payload.toolCallId,
|
|
2058
|
+
toolName: event.payload.toolName
|
|
2059
|
+
}
|
|
2060
|
+
};
|
|
2061
|
+
default: return;
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
|
|
2065
|
+
const events = [...rawEvents].sort((left, right) => left.sequence - right.sequence);
|
|
2066
|
+
const terminal = [...events].reverse().find((event) => TERMINAL_TYPES.has(event.type));
|
|
2067
|
+
if (!terminal) throw new TypeError(`Run "${run.id}" has no terminal event`);
|
|
2068
|
+
const started = events.find((event) => event.type === "run.started") ?? events[0];
|
|
2069
|
+
if (!started) throw new TypeError(`Run "${run.id}" has no events`);
|
|
2070
|
+
const status = traceStatus(terminal);
|
|
2071
|
+
const annotations = [];
|
|
2072
|
+
const spans = [];
|
|
2073
|
+
const openModels = /* @__PURE__ */ new Map();
|
|
2074
|
+
const openTools = /* @__PURE__ */ new Map();
|
|
2075
|
+
const previousUsage = {};
|
|
2076
|
+
const closeModel = (step, endedAt, spanStatus) => {
|
|
2077
|
+
const open = openModels.get(step);
|
|
2078
|
+
if (!open) return;
|
|
2079
|
+
openModels.delete(step);
|
|
2080
|
+
const completed = events.find((event) => event.type === "model.completed" && event.payload.step === step && event.sequence >= open.event.sequence);
|
|
2081
|
+
const cumulative = completed?.type === "model.completed" ? usageFrom(completed.payload.usage) : void 0;
|
|
2082
|
+
const usage = subtractUsage(cumulative, previousUsage);
|
|
2083
|
+
if (cumulative) Object.assign(previousUsage, cumulative);
|
|
2084
|
+
const spanContent = open.output.length ? content(options, "model.output", open.output.join("")) : void 0;
|
|
2085
|
+
const span = {
|
|
2086
|
+
id: open.event.id,
|
|
2087
|
+
parentId: run.id,
|
|
2088
|
+
kind: "model",
|
|
2089
|
+
name: `model step ${step}`,
|
|
2090
|
+
startedAt: open.event.timestamp,
|
|
2091
|
+
endedAt,
|
|
2092
|
+
durationMs: durationMs(open.event.timestamp, endedAt),
|
|
2093
|
+
status: spanStatus,
|
|
2094
|
+
attributes: {
|
|
2095
|
+
step,
|
|
2096
|
+
modelRegistryName
|
|
2097
|
+
},
|
|
2098
|
+
annotations: open.annotations,
|
|
2099
|
+
...usage ? { usage } : {},
|
|
2100
|
+
...spanContent ? { content: [spanContent] } : {}
|
|
2101
|
+
};
|
|
2102
|
+
const spanCost = costFor(options, modelRegistryName, model, usage);
|
|
2103
|
+
if (spanCost) span.cost = spanCost;
|
|
2104
|
+
spans.push(span);
|
|
2105
|
+
};
|
|
2106
|
+
const findOpenTool = (toolCallId) => {
|
|
2107
|
+
const entries = [...openTools.entries()];
|
|
2108
|
+
for (let index = entries.length - 1; index >= 0; index -= 1) {
|
|
2109
|
+
const entry = entries[index];
|
|
2110
|
+
const event = entry[1].event;
|
|
2111
|
+
if (event.type === "tool.started" && event.payload.toolCallId === toolCallId) return entry;
|
|
2112
|
+
}
|
|
2113
|
+
};
|
|
2114
|
+
const closeTool = (toolCallId, endedAt, spanStatus, error) => {
|
|
2115
|
+
const entry = findOpenTool(toolCallId);
|
|
2116
|
+
if (!entry) return;
|
|
2117
|
+
const [key, open] = entry;
|
|
2118
|
+
openTools.delete(key);
|
|
2119
|
+
const event = open.event;
|
|
2120
|
+
if (event.type !== "tool.started") return;
|
|
2121
|
+
const errorContent = error ? content(options, "error.message", error) : void 0;
|
|
2122
|
+
spans.push({
|
|
2123
|
+
id: event.id,
|
|
2124
|
+
parentId: run.id,
|
|
2125
|
+
kind: "tool",
|
|
2126
|
+
name: event.payload.toolName,
|
|
2127
|
+
startedAt: event.timestamp,
|
|
2128
|
+
endedAt,
|
|
2129
|
+
durationMs: durationMs(event.timestamp, endedAt),
|
|
2130
|
+
status: spanStatus,
|
|
2131
|
+
attributes: {
|
|
2132
|
+
step: event.payload.step,
|
|
2133
|
+
toolCallId,
|
|
2134
|
+
toolName: event.payload.toolName,
|
|
2135
|
+
attempt: event.payload.attempt ?? 1
|
|
2136
|
+
},
|
|
2137
|
+
annotations: open.annotations,
|
|
2138
|
+
...errorContent ? { content: [errorContent] } : {}
|
|
2139
|
+
});
|
|
2140
|
+
};
|
|
2141
|
+
for (const event of events) {
|
|
2142
|
+
const item = annotation(event);
|
|
2143
|
+
if (item) annotations.push(item);
|
|
2144
|
+
switch (event.type) {
|
|
2145
|
+
case "model.started":
|
|
2146
|
+
openModels.set(event.payload.step, {
|
|
2147
|
+
event,
|
|
2148
|
+
annotations: [],
|
|
2149
|
+
output: []
|
|
2150
|
+
});
|
|
2151
|
+
break;
|
|
2152
|
+
case "model.output.delta":
|
|
2153
|
+
openModels.get(event.payload.step)?.output.push(event.payload.delta);
|
|
2154
|
+
break;
|
|
2155
|
+
case "model.completed":
|
|
2156
|
+
closeModel(event.payload.step, event.timestamp, "completed");
|
|
2157
|
+
break;
|
|
2158
|
+
case "tool.started": {
|
|
2159
|
+
const attempt = event.payload.attempt ?? 1;
|
|
2160
|
+
openTools.set(`${event.payload.toolCallId}:${attempt}`, {
|
|
2161
|
+
event,
|
|
2162
|
+
annotations: [],
|
|
2163
|
+
output: []
|
|
2164
|
+
});
|
|
2165
|
+
break;
|
|
2166
|
+
}
|
|
2167
|
+
case "tool.retrying": {
|
|
2168
|
+
const open = findOpenTool(event.payload.toolCallId)?.[1];
|
|
2169
|
+
const retry = annotation(event);
|
|
2170
|
+
if (open && retry) open.annotations.push(retry);
|
|
2171
|
+
closeTool(event.payload.toolCallId, event.timestamp, "failed", event.payload.error);
|
|
2172
|
+
break;
|
|
2173
|
+
}
|
|
2174
|
+
case "tool.completed":
|
|
2175
|
+
closeTool(event.payload.toolCallId, event.timestamp, "completed");
|
|
2176
|
+
break;
|
|
2177
|
+
case "tool.failed":
|
|
2178
|
+
closeTool(event.payload.toolCallId, event.timestamp, "failed", event.payload.error);
|
|
2179
|
+
break;
|
|
2180
|
+
case "tool.execution_unknown":
|
|
2181
|
+
closeTool(event.payload.toolCallId, event.timestamp, "failed");
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
for (const step of [...openModels.keys()]) closeModel(step, terminal.timestamp, status);
|
|
2186
|
+
for (const open of [...openTools.values()]) if (open.event.type === "tool.started") closeTool(open.event.payload.toolCallId, terminal.timestamp, status);
|
|
2187
|
+
const finalUsage = run.usage ?? previousUsage;
|
|
2188
|
+
const totalCost = costFor(options, modelRegistryName, model, finalUsage);
|
|
2189
|
+
const traceContent = [];
|
|
2190
|
+
if (terminal.type === "run.completed") {
|
|
2191
|
+
const output = content(options, "run.output", terminal.payload.output);
|
|
2192
|
+
if (output) traceContent.push(output);
|
|
2193
|
+
} else if (terminal.type === "run.failed") {
|
|
2194
|
+
const error = content(options, "error.message", terminal.payload.error);
|
|
2195
|
+
if (error) traceContent.push(error);
|
|
2196
|
+
}
|
|
2197
|
+
return {
|
|
2198
|
+
schemaVersion: "1",
|
|
2199
|
+
traceId: run.id,
|
|
2200
|
+
runId: run.id,
|
|
2201
|
+
sessionId: run.sessionId,
|
|
2202
|
+
agentName: run.agentName,
|
|
2203
|
+
modelRegistryName,
|
|
2204
|
+
...model ? { model: { ...model } } : {},
|
|
2205
|
+
startedAt: started.timestamp,
|
|
2206
|
+
endedAt: terminal.timestamp,
|
|
2207
|
+
durationMs: durationMs(started.timestamp, terminal.timestamp),
|
|
2208
|
+
status,
|
|
2209
|
+
attributes: {
|
|
2210
|
+
eventCount: events.length,
|
|
2211
|
+
modelCallCount: spans.filter(({ kind }) => kind === "model").length,
|
|
2212
|
+
toolCallCount: spans.filter(({ kind }) => kind === "tool").length
|
|
2213
|
+
},
|
|
2214
|
+
annotations,
|
|
2215
|
+
spans,
|
|
2216
|
+
...Object.keys(finalUsage).length ? { usage: { ...finalUsage } } : {},
|
|
2217
|
+
...totalCost ? { cost: totalCost } : {},
|
|
2218
|
+
...traceContent.length ? { content: traceContent } : {}
|
|
2219
|
+
};
|
|
2220
|
+
}
|
|
2221
|
+
//#endregion
|
|
2222
|
+
//#region src/internal/run-core.ts
|
|
2223
|
+
/**
|
|
2224
|
+
* Shared foundation both engines build on: the registries that enforce one
|
|
2225
|
+
* active run per session, the single event sequencer, the durable commit path,
|
|
2226
|
+
* leases and policy authorization.
|
|
2227
|
+
*
|
|
2228
|
+
* It must be instantiated exactly once per runtime. `createEvent` advances
|
|
2229
|
+
* `state.eventSequence` in place and `runtimeOwner` identifies this process to
|
|
2230
|
+
* the lease table, so a second instance would corrupt event ordering and lease
|
|
2231
|
+
* ownership without raising an error.
|
|
2232
|
+
*/
|
|
2233
|
+
function createRunCore({ models, agents, tools, runStore, policies, onEvent, observability }) {
|
|
2234
|
+
const activeRuns = /* @__PURE__ */ new Map();
|
|
2235
|
+
const activeWorkflowRuns = /* @__PURE__ */ new Map();
|
|
2236
|
+
const activeSessions = /* @__PURE__ */ new Set();
|
|
2237
|
+
const pendingExports = /* @__PURE__ */ new Set();
|
|
2238
|
+
const exportFailures = [];
|
|
2239
|
+
const exportedRuns = /* @__PURE__ */ new Set();
|
|
2240
|
+
const runtimeOwner = crypto.randomUUID();
|
|
2241
|
+
const resolveModel = (agent, requestedModel) => {
|
|
2242
|
+
const modelName = requestedModel ?? (typeof agent.model === "string" ? agent.model : "default");
|
|
2243
|
+
if (!modelName.trim()) throw new FevexRunError("MODEL_NOT_FOUND", "Model name cannot be empty");
|
|
2244
|
+
const model = requestedModel !== void 0 || typeof agent.model === "string" ? models.get(modelName) : agent.model ?? models.get(modelName);
|
|
2245
|
+
if (!model) throw new FevexRunError("MODEL_NOT_FOUND", `Model "${modelName}" is not registered`);
|
|
2246
|
+
return {
|
|
2247
|
+
modelName,
|
|
2248
|
+
model
|
|
2249
|
+
};
|
|
2250
|
+
};
|
|
2251
|
+
const resolveCheckpointModel = async (agentName, agent, checkpoint) => {
|
|
2252
|
+
const preferred = resolveModel(agent, checkpoint.modelName);
|
|
2253
|
+
if (checkpoint.definitionHash === await definitionHash(agentName, preferred.modelName, agent, tools)) return preferred;
|
|
2254
|
+
if (checkpoint.modelName !== void 0) return void 0;
|
|
2255
|
+
for (const candidateName of models.keys()) {
|
|
2256
|
+
if (candidateName === preferred.modelName) continue;
|
|
2257
|
+
const candidate = resolveModel(agent, candidateName);
|
|
2258
|
+
if (checkpoint.definitionHash === await definitionHash(agentName, candidate.modelName, agent, tools)) return candidate;
|
|
2259
|
+
}
|
|
2260
|
+
};
|
|
2261
|
+
const scheduleTraceExport = (event) => {
|
|
2262
|
+
if (!observability || ![
|
|
2263
|
+
"run.completed",
|
|
2264
|
+
"run.failed",
|
|
2265
|
+
"run.cancelled"
|
|
2266
|
+
].includes(event.type) || exportedRuns.has(event.runId)) return;
|
|
2267
|
+
exportedRuns.add(event.runId);
|
|
2268
|
+
let task;
|
|
2269
|
+
task = (async () => {
|
|
2270
|
+
const run = await runStore.getRun(event.runId);
|
|
2271
|
+
if (!run) throw new Error(`Run "${event.runId}" does not exist`);
|
|
2272
|
+
if (isCoordinatorRun(run)) return;
|
|
2273
|
+
const agentRun = run;
|
|
2274
|
+
const agent = agents.get(agentRun.agentName);
|
|
2275
|
+
if (!agent) throw new Error(`Agent "${agentRun.agentName}" is not registered`);
|
|
2276
|
+
const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(agentRun.id) : void 0;
|
|
2277
|
+
const { modelName: modelRegistryName, model: gateway } = resolveModel(agent, checkpoint?.modelName);
|
|
2278
|
+
const trace = buildRunTrace(agentRun, await runStore.listEvents(agentRun.id), modelRegistryName, gateway?.metadata, observability);
|
|
2279
|
+
const failures = (await Promise.allSettled(observability.exporters.map((exporter) => Promise.resolve().then(() => exporter.export(trace))))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
|
|
2280
|
+
if (failures.length) throw new AggregateError(failures, "Trace export failed");
|
|
2281
|
+
})().catch((error) => {
|
|
2282
|
+
exportFailures.push(error);
|
|
2283
|
+
}).finally(() => pendingExports.delete(task));
|
|
2284
|
+
pendingExports.add(task);
|
|
2285
|
+
};
|
|
2286
|
+
const notifyObserver = (event) => {
|
|
2287
|
+
scheduleTraceExport(event);
|
|
2288
|
+
if (!onEvent) return;
|
|
2289
|
+
try {
|
|
2290
|
+
Promise.resolve(onEvent(event)).catch(() => {});
|
|
2291
|
+
} catch {}
|
|
2292
|
+
};
|
|
2293
|
+
const createEvent = (state, type, payload) => ({
|
|
2294
|
+
id: crypto.randomUUID(),
|
|
2295
|
+
sequence: state.eventSequence += 1,
|
|
2296
|
+
type,
|
|
2297
|
+
runId: state.run.id,
|
|
2298
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2299
|
+
...payload === void 0 ? {} : { payload }
|
|
2300
|
+
});
|
|
2301
|
+
const createCoordinatorEvent = (state, workflowType, workflowPayload, teamType, teamPayload = workflowPayload) => createEvent(state, isTeamRun(state.run) ? teamType : workflowType, isTeamRun(state.run) ? teamPayload : workflowPayload);
|
|
2302
|
+
const emitEvent = async (state, type, payload) => {
|
|
2303
|
+
const event = createEvent(state, type, payload);
|
|
2304
|
+
if (isDurableRunStore(runStore)) await commit(state, { events: [event] });
|
|
2305
|
+
else {
|
|
2306
|
+
await runStore.appendEvent(event);
|
|
2307
|
+
notifyObserver(event);
|
|
2308
|
+
}
|
|
2309
|
+
return event;
|
|
2310
|
+
};
|
|
2311
|
+
const releaseExecution = async (state) => {
|
|
2312
|
+
activeRuns.delete(state.run.id);
|
|
2313
|
+
activeSessions.delete(state.session.id);
|
|
2314
|
+
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
2315
|
+
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2316
|
+
state.leaseTimer = void 0;
|
|
2317
|
+
state.leaseExpiryTimer = void 0;
|
|
2318
|
+
if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
|
|
2319
|
+
};
|
|
2320
|
+
const commit = async (state, options = {}) => {
|
|
2321
|
+
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "This run requires a DurableRunStore", state.run.id);
|
|
2322
|
+
if (state.leaseLost && state.controller.signal.reason instanceof FevexRunError) throw state.controller.signal.reason;
|
|
2323
|
+
if (!state.lease || state.leaseLost) throw new FevexRunError("RUN_CONFLICT", "Run lease is unavailable", state.run.id);
|
|
2324
|
+
state.run.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
2325
|
+
if (!await runStore.commitExecution({
|
|
2326
|
+
expectedRevision: state.run.revision,
|
|
2327
|
+
lease: state.lease,
|
|
2328
|
+
run: state.run,
|
|
2329
|
+
...options
|
|
2330
|
+
})) {
|
|
2331
|
+
state.leaseLost = true;
|
|
2332
|
+
const error = new FevexRunError("RUN_CONFLICT", `Run "${state.run.id}" ownership or revision was lost`, state.run.id);
|
|
2333
|
+
state.controller.abort(error);
|
|
2334
|
+
throw error;
|
|
2335
|
+
}
|
|
2336
|
+
for (const event of options.events ?? []) notifyObserver(event);
|
|
2337
|
+
};
|
|
2338
|
+
const startLease = (state, store) => {
|
|
2339
|
+
let renewing = false;
|
|
2340
|
+
const loseLease = (cause) => {
|
|
2341
|
+
state.leaseLost = true;
|
|
2342
|
+
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
2343
|
+
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2344
|
+
state.leaseTimer = void 0;
|
|
2345
|
+
state.leaseExpiryTimer = void 0;
|
|
2346
|
+
state.controller.abort(new FevexRunError("RUN_CONFLICT", `Run lease lost: ${cause instanceof Error ? cause.message : String(cause)}`, state.run.id, { cause }));
|
|
2347
|
+
};
|
|
2348
|
+
const watchExpiry = () => {
|
|
2349
|
+
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2350
|
+
state.leaseExpiryTimer = setTimeout(() => loseLease(/* @__PURE__ */ new Error("Lease expired before renewal completed")), Math.max(0, Date.parse(state.lease.expiresAt) - Date.now()));
|
|
2351
|
+
};
|
|
2352
|
+
watchExpiry();
|
|
2353
|
+
state.leaseTimer = setInterval(() => {
|
|
2354
|
+
if (!state.lease || state.leaseLost || state.request.signal.aborted || renewing) return;
|
|
2355
|
+
renewing = true;
|
|
2356
|
+
const lease = {
|
|
2357
|
+
...state.lease,
|
|
2358
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
2359
|
+
};
|
|
2360
|
+
Promise.resolve().then(() => store.renewLease(lease)).then((renewed) => {
|
|
2361
|
+
if (state.leaseLost || state.request.signal.aborted || !state.leaseTimer) return;
|
|
2362
|
+
if (!renewed) loseLease(/* @__PURE__ */ new Error("Lease ownership changed"));
|
|
2363
|
+
else {
|
|
2364
|
+
state.lease.expiresAt = lease.expiresAt;
|
|
2365
|
+
watchExpiry();
|
|
2366
|
+
}
|
|
2367
|
+
}).catch((error) => {
|
|
2368
|
+
if (state.leaseTimer) loseLease(error);
|
|
2369
|
+
}).finally(() => {
|
|
2370
|
+
renewing = false;
|
|
2371
|
+
});
|
|
2372
|
+
}, LEASE_RENEW_MS);
|
|
2373
|
+
};
|
|
2374
|
+
const cancelExecution = async (state, reason = cancellationReason(state.request.signal)) => {
|
|
2375
|
+
if (state.leaseLost) {
|
|
2376
|
+
await releaseExecution(state);
|
|
2377
|
+
throw state.controller.signal.reason;
|
|
2378
|
+
}
|
|
2379
|
+
if (state.run.status === "completed" || state.run.status === "failed" || state.run.status === "cancelled") return;
|
|
2380
|
+
Object.assign(state.run, {
|
|
2381
|
+
status: "cancelled",
|
|
2382
|
+
pause: void 0,
|
|
2383
|
+
error: reason,
|
|
2384
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
2385
|
+
});
|
|
2386
|
+
const event = createEvent(state, "run.cancelled", { reason });
|
|
2387
|
+
if (isDurableRunStore(runStore)) await commit(state, {
|
|
2388
|
+
checkpoint: null,
|
|
2389
|
+
events: [event]
|
|
2390
|
+
});
|
|
2391
|
+
else {
|
|
2392
|
+
await runStore.saveRun(state.run);
|
|
2393
|
+
await runStore.appendEvent(event);
|
|
2394
|
+
notifyObserver(event);
|
|
2395
|
+
}
|
|
2396
|
+
await releaseExecution(state);
|
|
2397
|
+
return event;
|
|
2398
|
+
};
|
|
2399
|
+
const releaseWorkflowExecution = async (state) => {
|
|
2400
|
+
activeWorkflowRuns.delete(state.run.id);
|
|
2401
|
+
activeSessions.delete(state.session.id);
|
|
2402
|
+
if (state.leaseTimer) clearInterval(state.leaseTimer);
|
|
2403
|
+
if (state.leaseExpiryTimer) clearTimeout(state.leaseExpiryTimer);
|
|
2404
|
+
state.leaseTimer = void 0;
|
|
2405
|
+
state.leaseExpiryTimer = void 0;
|
|
2406
|
+
if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
|
|
2407
|
+
};
|
|
2408
|
+
const commitWorkflow = async (state, update, events = []) => {
|
|
2409
|
+
state.commitQueue = state.commitQueue.then(async () => {
|
|
2410
|
+
if (state.run.status !== "running") throw new FevexRunError("RUN_CONFLICT", `Workflow run "${state.run.id}" is not running`, state.run.id);
|
|
2411
|
+
update();
|
|
2412
|
+
await commit(state, {
|
|
2413
|
+
checkpoint: state.checkpoint,
|
|
2414
|
+
events
|
|
2415
|
+
});
|
|
2416
|
+
});
|
|
2417
|
+
await state.commitQueue;
|
|
2418
|
+
};
|
|
2419
|
+
const authorize = async (state, tool, input, action, context = state.request.context) => {
|
|
2420
|
+
let approval = false;
|
|
2421
|
+
for (const policy of policies) {
|
|
2422
|
+
const decision = await policy.authorize({
|
|
2423
|
+
runId: state.run.id,
|
|
2424
|
+
agentName: state.run.agentName,
|
|
2425
|
+
toolName: tool.name,
|
|
2426
|
+
risk: tool.risk ?? "read",
|
|
2427
|
+
input,
|
|
2428
|
+
action,
|
|
2429
|
+
context
|
|
2430
|
+
});
|
|
2431
|
+
if (decision === "deny") throw new FevexRunError("POLICY_DENIED", `Policy "${policy.name}" denied tool "${tool.name}"`, state.run.id);
|
|
2432
|
+
if (decision === "require_approval") approval = true;
|
|
2433
|
+
}
|
|
2434
|
+
return approval ? "require_approval" : "allow";
|
|
2435
|
+
};
|
|
2436
|
+
return {
|
|
2437
|
+
activeRuns,
|
|
2438
|
+
activeWorkflowRuns,
|
|
2439
|
+
activeSessions,
|
|
2440
|
+
pendingExports,
|
|
2441
|
+
exportFailures,
|
|
2442
|
+
runtimeOwner,
|
|
2443
|
+
resolveModel,
|
|
2444
|
+
resolveCheckpointModel,
|
|
2445
|
+
notifyObserver,
|
|
2446
|
+
createEvent,
|
|
2447
|
+
createCoordinatorEvent,
|
|
2448
|
+
emitEvent,
|
|
2449
|
+
releaseExecution,
|
|
2450
|
+
commit,
|
|
2451
|
+
startLease,
|
|
2452
|
+
cancelExecution,
|
|
2453
|
+
releaseWorkflowExecution,
|
|
2454
|
+
commitWorkflow,
|
|
2455
|
+
authorize
|
|
2456
|
+
};
|
|
2457
|
+
}
|
|
2458
|
+
//#endregion
|
|
2459
|
+
//#region src/internal/workflow-engine.ts
|
|
2460
|
+
/**
|
|
2461
|
+
* Coordination on top of the agent engine: durable workflow and team runs,
|
|
2462
|
+
* the step API, compensation, waits, and resume.
|
|
2463
|
+
*
|
|
2464
|
+
* It receives {@link AgentEngine} explicitly. That is what makes the old
|
|
2465
|
+
* forward reference from workflow cancellation into agent cancellation a
|
|
2466
|
+
* declared dependency instead of a call resolved 2,900 lines later.
|
|
2467
|
+
*/
|
|
2468
|
+
function createWorkflowEngine(composition, core, agentEngine) {
|
|
2469
|
+
const { models, agents, workflows, teams, runStore } = composition;
|
|
2470
|
+
const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
|
|
2471
|
+
const { prepareExecution, executeAgent, nextEvent, resumeAgent, cancelStoredAgent } = agentEngine;
|
|
2472
|
+
const cancelWorkflowExecution = async (state, reason = cancellationReason(state.request.signal)) => {
|
|
2473
|
+
if (state.leaseLost) {
|
|
2474
|
+
await releaseWorkflowExecution(state);
|
|
2475
|
+
throw state.controller.signal.reason;
|
|
2476
|
+
}
|
|
2477
|
+
if (state.run.status === "completed" || state.run.status === "failed" || state.run.status === "cancelled") return;
|
|
2478
|
+
const childRunIds = Object.values(state.checkpoint.steps).filter((record) => record.type === "agent" && record.status === "running").map(({ childRunId }) => childRunId);
|
|
2479
|
+
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId)) throw new FevexRunError("RUN_CONFLICT", `Workflow child "${childRunId}" could not be cancelled`, state.run.id);
|
|
2480
|
+
state.run.status = "cancelled";
|
|
2481
|
+
state.run.pause = void 0;
|
|
2482
|
+
state.run.error = reason;
|
|
2483
|
+
state.run.usage = state.checkpoint.budget?.usage;
|
|
2484
|
+
const event = createCoordinatorEvent(state, "workflow.run.cancelled", { reason }, "team.run.cancelled");
|
|
2485
|
+
await commit(state, {
|
|
2486
|
+
checkpoint: null,
|
|
2487
|
+
events: [event]
|
|
2488
|
+
});
|
|
2489
|
+
await releaseWorkflowExecution(state);
|
|
2490
|
+
return event;
|
|
2491
|
+
};
|
|
2492
|
+
const prepareWorkflowExecution = async (name, request, kind = "workflow") => {
|
|
2493
|
+
const workflow = workflows.get(name);
|
|
2494
|
+
const team = kind === "team" ? teams.get(name) : void 0;
|
|
2495
|
+
if (!workflow || kind === "team" && !team || kind === "workflow" && teams.has(name)) throw new FevexRunError(kind === "team" ? "TEAM_NOT_FOUND" : "WORKFLOW_NOT_FOUND", `${kind === "team" ? "Team" : "Workflow"} "${name}" is not registered`);
|
|
2496
|
+
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", `${kind === "team" ? "Teams" : "Workflows"} require a DurableRunStore`);
|
|
2497
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2498
|
+
let session;
|
|
2499
|
+
if (request.sessionId === void 0) session = {
|
|
2500
|
+
id: crypto.randomUUID(),
|
|
2501
|
+
history: [],
|
|
2502
|
+
createdAt: now,
|
|
2503
|
+
updatedAt: now
|
|
2504
|
+
};
|
|
2505
|
+
else {
|
|
2506
|
+
const stored = await runStore.getSession(request.sessionId);
|
|
2507
|
+
if (!stored) throw new FevexRunError("SESSION_NOT_FOUND", `Session "${request.sessionId}" does not exist`);
|
|
2508
|
+
session = stored;
|
|
2509
|
+
}
|
|
2510
|
+
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" already has an active run`);
|
|
2511
|
+
activeSessions.add(session.id);
|
|
2512
|
+
const controller = new AbortController();
|
|
2513
|
+
const signal = request.signal ? AbortSignal.any([controller.signal, request.signal]) : controller.signal;
|
|
2514
|
+
const input = toJsonValue(workflow.inputSchema === void 0 ? request.input : await abortable(() => validateSchema(workflow.inputSchema, request.input, `Input for ${kind} "${name}" does not match inputSchema`), signal), `${kind === "team" ? "Team" : "Workflow"} input must be a string or JSON-serializable value`);
|
|
2515
|
+
const id = crypto.randomUUID();
|
|
2516
|
+
const run = kind === "team" ? {
|
|
2517
|
+
kind: "team",
|
|
2518
|
+
id,
|
|
2519
|
+
sessionId: session.id,
|
|
2520
|
+
teamName: name,
|
|
2521
|
+
status: "running",
|
|
2522
|
+
revision: 0,
|
|
2523
|
+
createdAt: now,
|
|
2524
|
+
updatedAt: now
|
|
2525
|
+
} : {
|
|
2526
|
+
kind: "workflow",
|
|
2527
|
+
id,
|
|
2528
|
+
sessionId: session.id,
|
|
2529
|
+
workflowName: name,
|
|
2530
|
+
status: "running",
|
|
2531
|
+
revision: 0,
|
|
2532
|
+
createdAt: now,
|
|
2533
|
+
updatedAt: now
|
|
2534
|
+
};
|
|
2535
|
+
const checkpoint = kind === "team" ? {
|
|
2536
|
+
version: 2,
|
|
2537
|
+
kind: "team",
|
|
2538
|
+
runId: run.id,
|
|
2539
|
+
teamName: name,
|
|
2540
|
+
...request.model === void 0 ? {} : { model: request.model },
|
|
2541
|
+
...request.reasoning === void 0 ? {} : { reasoning: request.reasoning },
|
|
2542
|
+
definitionHash: await teamDefinitionHash(name, team),
|
|
2543
|
+
input,
|
|
2544
|
+
context: request.context,
|
|
2545
|
+
steps: {},
|
|
2546
|
+
limits: combineLimits(workflow.limits, request.limits)
|
|
2547
|
+
} : {
|
|
2548
|
+
version: 2,
|
|
2549
|
+
kind: "workflow",
|
|
2550
|
+
runId: run.id,
|
|
2551
|
+
workflowName: name,
|
|
2552
|
+
...request.model === void 0 ? {} : { model: request.model },
|
|
2553
|
+
...request.reasoning === void 0 ? {} : { reasoning: request.reasoning },
|
|
2554
|
+
definitionHash: await workflowDefinitionHash(name, workflow),
|
|
2555
|
+
input,
|
|
2556
|
+
context: request.context,
|
|
2557
|
+
steps: {},
|
|
2558
|
+
limits: combineLimits(workflow.limits, request.limits)
|
|
2559
|
+
};
|
|
2560
|
+
const state = {
|
|
2561
|
+
run,
|
|
2562
|
+
session,
|
|
2563
|
+
request: {
|
|
2564
|
+
...request,
|
|
2565
|
+
sessionId: session.id,
|
|
2566
|
+
signal
|
|
2567
|
+
},
|
|
2568
|
+
controller,
|
|
2569
|
+
eventSequence: 0,
|
|
2570
|
+
checkpoint,
|
|
2571
|
+
advancing: false,
|
|
2572
|
+
commitQueue: Promise.resolve()
|
|
2573
|
+
};
|
|
2574
|
+
try {
|
|
2575
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
2576
|
+
const lease = {
|
|
2577
|
+
generation: 0,
|
|
2578
|
+
runId: run.id,
|
|
2579
|
+
ownerId,
|
|
2580
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
2581
|
+
};
|
|
2582
|
+
const started = createCoordinatorEvent(state, "workflow.run.started", void 0, "team.run.started");
|
|
2583
|
+
if (!await runStore.createExecution({
|
|
2584
|
+
run,
|
|
2585
|
+
checkpoint,
|
|
2586
|
+
session,
|
|
2587
|
+
lease,
|
|
2588
|
+
events: [started]
|
|
2589
|
+
})) throw new FevexRunError("RUN_CONFLICT", `Run or session "${session.id}" is active or was modified`, run.id);
|
|
2590
|
+
state.lease = lease;
|
|
2591
|
+
state.initialEvents = [started];
|
|
2592
|
+
notifyObserver(started);
|
|
2593
|
+
startLease(state, runStore);
|
|
2594
|
+
} catch (error) {
|
|
2595
|
+
activeSessions.delete(session.id);
|
|
2596
|
+
throw error;
|
|
2597
|
+
}
|
|
2598
|
+
activeWorkflowRuns.set(run.id, state);
|
|
2599
|
+
return state;
|
|
2600
|
+
};
|
|
2601
|
+
async function* executeWorkflow(name, state) {
|
|
2602
|
+
const workflow = workflows.get(name);
|
|
2603
|
+
const events = [];
|
|
2604
|
+
let parallelDepth = 0;
|
|
2605
|
+
const compensationHandlers = /* @__PURE__ */ new Map();
|
|
2606
|
+
const pauseForChild = async (stepId, error) => {
|
|
2607
|
+
if (parallelDepth > 0) throw new WorkflowChildPausedError(stepId, error);
|
|
2608
|
+
const childPause = error.pause;
|
|
2609
|
+
if (childPause.type === "workflow_child" || childPause.type === "workflow_children") throw error;
|
|
2610
|
+
if (childPause.type === "workflow_timer" || childPause.type === "workflow_event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Workflow child "${error.runId}" paused for a workflow wait`, state.run.id);
|
|
2611
|
+
const event = createCoordinatorEvent(state, "workflow.run.paused", {
|
|
2612
|
+
stepId,
|
|
2613
|
+
reason: "child",
|
|
2614
|
+
childRunId: error.runId
|
|
2615
|
+
}, "team.run.paused", {
|
|
2616
|
+
delegationId: stepId,
|
|
2617
|
+
reason: "child",
|
|
2618
|
+
childRunId: error.runId
|
|
2619
|
+
});
|
|
2620
|
+
await commitWorkflow(state, () => {
|
|
2621
|
+
state.run.status = "paused";
|
|
2622
|
+
state.run.pause = {
|
|
2623
|
+
type: "workflow_child",
|
|
2624
|
+
childRunId: error.runId,
|
|
2625
|
+
childPause
|
|
2626
|
+
};
|
|
2627
|
+
}, [event]);
|
|
2628
|
+
events.push(event);
|
|
2629
|
+
await releaseWorkflowExecution(state);
|
|
2630
|
+
throw new RunPausedError(state.run.id, state.run.pause);
|
|
2631
|
+
};
|
|
2632
|
+
const compensationContext = (stepId) => ({
|
|
2633
|
+
runId: state.run.id,
|
|
2634
|
+
sessionId: state.session.id,
|
|
2635
|
+
stepId,
|
|
2636
|
+
context: state.request.context,
|
|
2637
|
+
signal: state.request.signal
|
|
2638
|
+
});
|
|
2639
|
+
const registerCompensation = (stepId, kind, result, options) => {
|
|
2640
|
+
if (!options?.compensate) return;
|
|
2641
|
+
compensationHandlers.set(stepId, {
|
|
2642
|
+
kind,
|
|
2643
|
+
async run() {
|
|
2644
|
+
await options.compensate(structuredClone(result), compensationContext(stepId));
|
|
2645
|
+
}
|
|
2646
|
+
});
|
|
2647
|
+
};
|
|
2648
|
+
const childBudget = async (result) => {
|
|
2649
|
+
const childEvents = await runStore.listEvents(result.runId);
|
|
2650
|
+
return {
|
|
2651
|
+
usage: result.usage,
|
|
2652
|
+
steps: childEvents.filter(({ type }) => type === "model.completed").length,
|
|
2653
|
+
toolCalls: childEvents.filter(({ type }) => type === "tool.completed").length
|
|
2654
|
+
};
|
|
2655
|
+
};
|
|
2656
|
+
const addChildBudget = (budget) => {
|
|
2657
|
+
state.checkpoint.budget = addWorkflowBudget(state.checkpoint.budget, budget);
|
|
2658
|
+
state.run.usage = state.checkpoint.budget?.usage;
|
|
2659
|
+
};
|
|
2660
|
+
const relayChildEvent = async (event, stepId, agentName) => {
|
|
2661
|
+
if (event.type !== "model.started" && event.type !== "model.output.delta" && event.type !== "model.completed" && event.type !== "tool.started" && event.type !== "tool.completed" && event.type !== "tool.failed" && event.type !== "tool.retrying" && event.type !== "tool.execution_unknown" && event.type !== "elicitation.requested" && event.type !== "elicitation.resolved" && event.type !== "approval.requested" && event.type !== "approval.resolved") return;
|
|
2662
|
+
const relayed = createEvent(state, event.type, {
|
|
2663
|
+
...event.payload,
|
|
2664
|
+
...isTeamRun(state.run) ? {
|
|
2665
|
+
teamDelegationId: stepId,
|
|
2666
|
+
teamAgentName: agentName
|
|
2667
|
+
} : {
|
|
2668
|
+
workflowStepId: stepId,
|
|
2669
|
+
workflowAgentName: agentName
|
|
2670
|
+
}
|
|
2671
|
+
});
|
|
2672
|
+
await commitWorkflow(state, () => {}, [relayed]);
|
|
2673
|
+
events.push(relayed);
|
|
2674
|
+
};
|
|
2675
|
+
const runCompensations = async () => {
|
|
2676
|
+
const entries = Object.entries(state.checkpoint.steps).reverse();
|
|
2677
|
+
for (const [stepId, record] of entries) {
|
|
2678
|
+
if (record.type !== "agent" && record.type !== "parallel" || record.status !== "completed" || record.compensation?.status !== "pending") continue;
|
|
2679
|
+
const handler = compensationHandlers.get(stepId);
|
|
2680
|
+
if (!handler) throw new Error(`Workflow step "${stepId}" has no compensation handler`);
|
|
2681
|
+
const started = createEvent(state, "workflow.compensation.started", {
|
|
2682
|
+
stepId,
|
|
2683
|
+
kind: handler.kind
|
|
2684
|
+
});
|
|
2685
|
+
await commitWorkflow(state, () => {}, [started]);
|
|
2686
|
+
events.push(started);
|
|
2687
|
+
try {
|
|
2688
|
+
await abortable(() => handler.run(record.result), state.request.signal);
|
|
2689
|
+
const completed = createEvent(state, "workflow.compensation.completed", {
|
|
2690
|
+
stepId,
|
|
2691
|
+
kind: handler.kind
|
|
2692
|
+
});
|
|
2693
|
+
await commitWorkflow(state, () => {
|
|
2694
|
+
record.compensation = { status: "completed" };
|
|
2695
|
+
}, [completed]);
|
|
2696
|
+
events.push(completed);
|
|
2697
|
+
} catch (error) {
|
|
2698
|
+
const failed = createEvent(state, "workflow.compensation.failed", {
|
|
2699
|
+
stepId,
|
|
2700
|
+
kind: handler.kind,
|
|
2701
|
+
error: toErrorMessage(error)
|
|
2702
|
+
});
|
|
2703
|
+
await commitWorkflow(state, () => {
|
|
2704
|
+
record.compensation = {
|
|
2705
|
+
status: "failed",
|
|
2706
|
+
error: toErrorMessage(error)
|
|
2707
|
+
};
|
|
2708
|
+
}, [failed]);
|
|
2709
|
+
events.push(failed);
|
|
2710
|
+
throw error;
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
};
|
|
2714
|
+
const completeAgentStep = async (stepId, agentName, childRunId, output, options) => {
|
|
2715
|
+
const child = await runStore.getRun(childRunId);
|
|
2716
|
+
const result = {
|
|
2717
|
+
runId: childRunId,
|
|
2718
|
+
sessionId: child.sessionId,
|
|
2719
|
+
output,
|
|
2720
|
+
...child.usage === void 0 ? {} : { usage: child.usage }
|
|
2721
|
+
};
|
|
2722
|
+
const budget = await childBudget(result);
|
|
2723
|
+
const event = createCoordinatorEvent(state, "workflow.step.completed", {
|
|
2724
|
+
stepId,
|
|
2725
|
+
kind: "agent"
|
|
2726
|
+
}, "team.task.completed", {
|
|
2727
|
+
delegationId: stepId,
|
|
2728
|
+
agentName,
|
|
2729
|
+
...eventUsage(child.usage) === void 0 ? {} : { usage: eventUsage(child.usage) }
|
|
2730
|
+
});
|
|
2731
|
+
await commitWorkflow(state, () => {
|
|
2732
|
+
addChildBudget(budget);
|
|
2733
|
+
state.checkpoint.steps[stepId] = {
|
|
2734
|
+
type: "agent",
|
|
2735
|
+
status: "completed",
|
|
2736
|
+
agentName,
|
|
2737
|
+
childRunId,
|
|
2738
|
+
result,
|
|
2739
|
+
...options?.metadata === void 0 ? {} : { metadata: structuredClone(options.metadata) },
|
|
2740
|
+
...options?.compensate === void 0 ? {} : { compensation: { status: "pending" } }
|
|
2741
|
+
};
|
|
2742
|
+
}, [event]);
|
|
2743
|
+
if (options?.compensate) registerCompensation(stepId, "agent", result, options);
|
|
2744
|
+
events.push(event);
|
|
2745
|
+
assertWorkflowBudget(name, state.checkpoint.limits, state.checkpoint.budget);
|
|
2746
|
+
return structuredClone(result);
|
|
2747
|
+
};
|
|
2748
|
+
const step = {
|
|
2749
|
+
async agent(stepId, agentName, request, options) {
|
|
2750
|
+
if (!stepId.trim()) throw new Error("Workflow step id cannot be empty");
|
|
2751
|
+
const existing = state.checkpoint.steps[stepId];
|
|
2752
|
+
if (existing?.type === "agent" && existing.status === "completed") {
|
|
2753
|
+
const result = structuredClone(existing.result);
|
|
2754
|
+
registerCompensation(stepId, "agent", result, options);
|
|
2755
|
+
return result;
|
|
2756
|
+
}
|
|
2757
|
+
if (existing?.type === "agent" && existing.status === "running") {
|
|
2758
|
+
let child = await runStore.getRun(existing.childRunId);
|
|
2759
|
+
if (!child) throw new Error(`Child run "${existing.childRunId}" does not exist`);
|
|
2760
|
+
if (child.status === "running") {
|
|
2761
|
+
if (!state.recoveryActor) throw new FevexRunError("RUN_NOT_RECOVERABLE", `Child run "${child.id}" is still running`, state.run.id);
|
|
2762
|
+
child = await resumeAgent(child.id, void 0, true, state.recoveryActor);
|
|
2763
|
+
}
|
|
2764
|
+
if (child.status === "completed") return completeAgentStep(stepId, existing.agentName, existing.childRunId, toJsonValue(child.output, "Child agent output must be JSON-serializable"), options);
|
|
2765
|
+
if (child.status === "paused" && child.pause) return pauseForChild(stepId, new RunPausedError(child.id, child.pause));
|
|
2766
|
+
if (child.status === "failed" || child.status === "cancelled") throw new Error(`Child run "${child.id}" ${child.status}`);
|
|
2767
|
+
}
|
|
2768
|
+
if (existing && existing.type !== "agent") throw new Error(`Workflow step "${stepId}" was already used as ${existing.type}`);
|
|
2769
|
+
if (!agents.has(agentName)) throw new FevexRunError("AGENT_NOT_FOUND", `Agent "${agentName}" is not registered`);
|
|
2770
|
+
const childSignal = request.signal ? AbortSignal.any([state.request.signal, request.signal]) : state.request.signal;
|
|
2771
|
+
const childState = await prepareExecution(agentName, {
|
|
2772
|
+
...request,
|
|
2773
|
+
model: request.model ?? state.request.model,
|
|
2774
|
+
reasoning: request.reasoning ?? state.request.reasoning,
|
|
2775
|
+
context: mergeExecutionContext(state.request.context, request.context),
|
|
2776
|
+
limits: combineLimits(request.limits, remainingWorkflowLimits(state.checkpoint.limits, state.checkpoint.budget)),
|
|
2777
|
+
signal: childSignal
|
|
2778
|
+
});
|
|
2779
|
+
const action = options?.metadata?.teamAction === "handoff" ? "handoff" : "delegate";
|
|
2780
|
+
const lifecycleEvents = [];
|
|
2781
|
+
if (isTeamRun(state.run) && action === "handoff") lifecycleEvents.push(createEvent(state, "team.handoff.created", {
|
|
2782
|
+
delegationId: stepId,
|
|
2783
|
+
from: String(options?.metadata?.from ?? ""),
|
|
2784
|
+
to: agentName,
|
|
2785
|
+
reason: String(options?.metadata?.reason ?? "")
|
|
2786
|
+
}));
|
|
2787
|
+
const started = createCoordinatorEvent(state, "workflow.step.started", {
|
|
2788
|
+
stepId,
|
|
2789
|
+
kind: "agent",
|
|
2790
|
+
agentName
|
|
2791
|
+
}, "team.agent.assigned", {
|
|
2792
|
+
delegationId: stepId,
|
|
2793
|
+
agentName,
|
|
2794
|
+
action,
|
|
2795
|
+
...typeof options?.metadata?.expectedOutput === "string" ? { expectedOutput: options.metadata.expectedOutput } : {},
|
|
2796
|
+
...Array.isArray(options?.metadata?.constraints) ? { constraints: options.metadata.constraints } : {}
|
|
2797
|
+
});
|
|
2798
|
+
lifecycleEvents.push(started);
|
|
2799
|
+
await commitWorkflow(state, () => {
|
|
2800
|
+
state.checkpoint.steps[stepId] = {
|
|
2801
|
+
type: "agent",
|
|
2802
|
+
status: "running",
|
|
2803
|
+
agentName,
|
|
2804
|
+
childRunId: childState.run.id,
|
|
2805
|
+
...options?.metadata === void 0 ? {} : { metadata: structuredClone(options.metadata) }
|
|
2806
|
+
};
|
|
2807
|
+
}, lifecycleEvents);
|
|
2808
|
+
events.push(...lifecycleEvents);
|
|
2809
|
+
try {
|
|
2810
|
+
const execution = executeAgent(agentName, childState);
|
|
2811
|
+
let result;
|
|
2812
|
+
while (true) {
|
|
2813
|
+
const next = await nextEvent(childState, execution);
|
|
2814
|
+
if (next.done) {
|
|
2815
|
+
if (next.value === void 0) {
|
|
2816
|
+
const run = await runStore.getRun(childState.run.id);
|
|
2817
|
+
if (run?.pause) throw new RunPausedError(run.id, run.pause);
|
|
2818
|
+
}
|
|
2819
|
+
result = next.value;
|
|
2820
|
+
break;
|
|
2821
|
+
}
|
|
2822
|
+
await relayChildEvent(next.value, stepId, agentName);
|
|
2823
|
+
}
|
|
2824
|
+
if (!result) throw new Error(`Agent "${agentName}" did not complete`);
|
|
2825
|
+
const normalized = {
|
|
2826
|
+
runId: result.runId,
|
|
2827
|
+
sessionId: result.sessionId,
|
|
2828
|
+
output: result.output,
|
|
2829
|
+
...result.usage === void 0 ? {} : { usage: result.usage }
|
|
2830
|
+
};
|
|
2831
|
+
const budget = await childBudget(normalized);
|
|
2832
|
+
const completed = createCoordinatorEvent(state, "workflow.step.completed", {
|
|
2833
|
+
stepId,
|
|
2834
|
+
kind: "agent"
|
|
2835
|
+
}, "team.task.completed", {
|
|
2836
|
+
delegationId: stepId,
|
|
2837
|
+
agentName,
|
|
2838
|
+
...eventUsage(normalized.usage) === void 0 ? {} : { usage: eventUsage(normalized.usage) }
|
|
2839
|
+
});
|
|
2840
|
+
await commitWorkflow(state, () => {
|
|
2841
|
+
addChildBudget(budget);
|
|
2842
|
+
state.checkpoint.steps[stepId] = {
|
|
2843
|
+
type: "agent",
|
|
2844
|
+
status: "completed",
|
|
2845
|
+
agentName,
|
|
2846
|
+
childRunId: normalized.runId,
|
|
2847
|
+
result: structuredClone(normalized),
|
|
2848
|
+
...options?.metadata === void 0 ? {} : { metadata: structuredClone(options.metadata) },
|
|
2849
|
+
...options?.compensate === void 0 ? {} : { compensation: { status: "pending" } }
|
|
2850
|
+
};
|
|
2851
|
+
}, [completed]);
|
|
2852
|
+
registerCompensation(stepId, "agent", normalized, options);
|
|
2853
|
+
events.push(completed);
|
|
2854
|
+
assertWorkflowBudget(name, state.checkpoint.limits, state.checkpoint.budget);
|
|
2855
|
+
return normalized;
|
|
2856
|
+
} catch (error) {
|
|
2857
|
+
if (error instanceof RunPausedError) return pauseForChild(stepId, error);
|
|
2858
|
+
const failed = createCoordinatorEvent(state, "workflow.step.failed", {
|
|
2859
|
+
stepId,
|
|
2860
|
+
kind: "agent",
|
|
2861
|
+
error: toErrorMessage(error)
|
|
2862
|
+
}, "team.task.failed", {
|
|
2863
|
+
delegationId: stepId,
|
|
2864
|
+
agentName,
|
|
2865
|
+
error: toErrorMessage(error)
|
|
2866
|
+
});
|
|
2867
|
+
await commitWorkflow(state, () => {}, [failed]);
|
|
2868
|
+
events.push(failed);
|
|
2869
|
+
throw error;
|
|
2870
|
+
}
|
|
2871
|
+
},
|
|
2872
|
+
async parallel(stepId, tasks, options) {
|
|
2873
|
+
if (!stepId.trim()) throw new Error("Workflow step id cannot be empty");
|
|
2874
|
+
const existing = state.checkpoint.steps[stepId];
|
|
2875
|
+
if (existing?.type === "parallel" && existing.status === "completed") {
|
|
2876
|
+
const result = structuredClone(existing.result);
|
|
2877
|
+
registerCompensation(stepId, "parallel", result, options);
|
|
2878
|
+
return result;
|
|
2879
|
+
}
|
|
2880
|
+
if (existing && existing.type !== "parallel") throw new Error(`Workflow step "${stepId}" was already used as ${existing.type}`);
|
|
2881
|
+
const started = createCoordinatorEvent(state, "workflow.step.started", {
|
|
2882
|
+
stepId,
|
|
2883
|
+
kind: "parallel"
|
|
2884
|
+
}, "team.merge.started", {
|
|
2885
|
+
stepId,
|
|
2886
|
+
width: typeof options?.metadata?.width === "number" ? options.metadata.width : Object.keys(tasks).length
|
|
2887
|
+
});
|
|
2888
|
+
await commitWorkflow(state, () => {
|
|
2889
|
+
state.checkpoint.steps[stepId] = {
|
|
2890
|
+
type: "parallel",
|
|
2891
|
+
status: "running",
|
|
2892
|
+
...options?.metadata === void 0 ? {} : { metadata: structuredClone(options.metadata) }
|
|
2893
|
+
};
|
|
2894
|
+
}, [started]);
|
|
2895
|
+
events.push(started);
|
|
2896
|
+
try {
|
|
2897
|
+
parallelDepth += 1;
|
|
2898
|
+
const taskEntries = Object.entries(tasks);
|
|
2899
|
+
let settled;
|
|
2900
|
+
try {
|
|
2901
|
+
settled = await Promise.allSettled(taskEntries.map(async ([key, task]) => [key, await task()]));
|
|
2902
|
+
} finally {
|
|
2903
|
+
parallelDepth -= 1;
|
|
2904
|
+
}
|
|
2905
|
+
const failures = settled.filter((item) => item.status === "rejected").map(({ reason }) => reason);
|
|
2906
|
+
const pauses = failures.filter((error) => error instanceof WorkflowChildPausedError);
|
|
2907
|
+
const errors = failures.filter((error) => !(error instanceof WorkflowChildPausedError));
|
|
2908
|
+
if (errors.length) {
|
|
2909
|
+
for (const pause of pauses) if (!await cancelStoredAgent(pause.paused.runId)) throw new FevexRunError("RUN_CONFLICT", `Paused workflow child "${pause.paused.runId}" could not be cancelled`, state.run.id);
|
|
2910
|
+
throw errors.length === 1 ? errors[0] : new AggregateError(errors, `Workflow parallel step "${stepId}" failed`);
|
|
2911
|
+
}
|
|
2912
|
+
if (pauses.length) {
|
|
2913
|
+
if (pauses.length > 1) {
|
|
2914
|
+
const event = createCoordinatorEvent(state, "workflow.run.paused", {
|
|
2915
|
+
stepId,
|
|
2916
|
+
reason: "child",
|
|
2917
|
+
childRunId: pauses[0].paused.runId
|
|
2918
|
+
}, "team.run.paused", {
|
|
2919
|
+
delegationId: stepId,
|
|
2920
|
+
reason: "child",
|
|
2921
|
+
childRunId: pauses[0].paused.runId
|
|
1202
2922
|
});
|
|
1203
|
-
|
|
2923
|
+
await commitWorkflow(state, () => {
|
|
2924
|
+
state.run.status = "paused";
|
|
2925
|
+
state.run.pause = {
|
|
2926
|
+
type: "workflow_children",
|
|
2927
|
+
children: pauses.map((pause) => ({
|
|
2928
|
+
stepId: pause.stepId,
|
|
2929
|
+
childRunId: pause.paused.runId,
|
|
2930
|
+
childPause: pause.paused.pause
|
|
2931
|
+
}))
|
|
2932
|
+
};
|
|
2933
|
+
}, [event]);
|
|
2934
|
+
events.push(event);
|
|
2935
|
+
await releaseWorkflowExecution(state);
|
|
2936
|
+
throw new RunPausedError(state.run.id, state.run.pause);
|
|
1204
2937
|
}
|
|
2938
|
+
return pauseForChild(pauses[0].stepId, pauses[0].paused);
|
|
1205
2939
|
}
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
2940
|
+
const entries = settled.map((item) => item.value);
|
|
2941
|
+
const result = toJsonValue(Object.fromEntries(entries), `Workflow parallel step "${stepId}" result must be JSON-serializable`);
|
|
2942
|
+
if (typeof result !== "object" || result === null || Array.isArray(result)) throw new Error(`Workflow parallel step "${stepId}" result must be an object`);
|
|
2943
|
+
const completed = createCoordinatorEvent(state, "workflow.step.completed", {
|
|
2944
|
+
stepId,
|
|
2945
|
+
kind: "parallel"
|
|
2946
|
+
}, "team.merge.completed", { stepId });
|
|
2947
|
+
await commitWorkflow(state, () => {
|
|
2948
|
+
state.checkpoint.steps[stepId] = {
|
|
2949
|
+
type: "parallel",
|
|
2950
|
+
status: "completed",
|
|
2951
|
+
result,
|
|
2952
|
+
...options?.metadata === void 0 ? {} : { metadata: structuredClone(options.metadata) },
|
|
2953
|
+
...options?.compensate === void 0 ? {} : { compensation: { status: "pending" } }
|
|
2954
|
+
};
|
|
2955
|
+
}, [completed]);
|
|
2956
|
+
registerCompensation(stepId, "parallel", structuredClone(result), options);
|
|
2957
|
+
events.push(completed);
|
|
2958
|
+
return structuredClone(result);
|
|
2959
|
+
} catch (error) {
|
|
2960
|
+
if (error instanceof RunPausedError) throw error;
|
|
2961
|
+
const failed = createCoordinatorEvent(state, "workflow.step.failed", {
|
|
2962
|
+
stepId,
|
|
2963
|
+
kind: "parallel",
|
|
2964
|
+
error: toErrorMessage(error)
|
|
2965
|
+
}, "team.merge.failed", {
|
|
2966
|
+
stepId,
|
|
2967
|
+
error: toErrorMessage(error)
|
|
1212
2968
|
});
|
|
1213
|
-
|
|
1214
|
-
|
|
2969
|
+
await commitWorkflow(state, () => {}, [failed]);
|
|
2970
|
+
events.push(failed);
|
|
2971
|
+
throw error;
|
|
1215
2972
|
}
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
step
|
|
2973
|
+
},
|
|
2974
|
+
async waitUntil(stepId, resumeAt) {
|
|
2975
|
+
if (!stepId.trim()) throw new Error("Workflow step id cannot be empty");
|
|
2976
|
+
const existing = state.checkpoint.steps[stepId];
|
|
2977
|
+
if (existing?.type === "wait" && existing.status === "completed") return;
|
|
2978
|
+
if (existing && existing.type !== "wait") throw new Error(`Workflow step "${stepId}" was already used as ${existing.type}`);
|
|
2979
|
+
const at = resumeAt instanceof Date ? resumeAt : new Date(resumeAt);
|
|
2980
|
+
if (!Number.isFinite(at.getTime())) throw new Error(`Workflow wait "${stepId}" resumeAt must be a valid date`);
|
|
2981
|
+
const wait = {
|
|
2982
|
+
type: "timer",
|
|
2983
|
+
resumeAt: at.toISOString()
|
|
2984
|
+
};
|
|
2985
|
+
if (Date.now() >= at.getTime()) {
|
|
2986
|
+
const started = createEvent(state, "workflow.wait.started", {
|
|
2987
|
+
stepId,
|
|
2988
|
+
kind: "timer",
|
|
2989
|
+
resumeAt: wait.resumeAt
|
|
2990
|
+
});
|
|
2991
|
+
const completed = createEvent(state, "workflow.wait.completed", {
|
|
2992
|
+
stepId,
|
|
2993
|
+
kind: "timer"
|
|
2994
|
+
});
|
|
2995
|
+
await commitWorkflow(state, () => {
|
|
2996
|
+
state.checkpoint.steps[stepId] = {
|
|
2997
|
+
type: "wait",
|
|
2998
|
+
status: "completed",
|
|
2999
|
+
wait
|
|
3000
|
+
};
|
|
3001
|
+
}, [started, completed]);
|
|
3002
|
+
events.push(started, completed);
|
|
3003
|
+
return;
|
|
3004
|
+
}
|
|
3005
|
+
const started = createEvent(state, "workflow.wait.started", {
|
|
3006
|
+
stepId,
|
|
3007
|
+
kind: "timer",
|
|
3008
|
+
resumeAt: wait.resumeAt
|
|
3009
|
+
});
|
|
3010
|
+
const paused = createEvent(state, "workflow.run.paused", {
|
|
3011
|
+
stepId,
|
|
3012
|
+
reason: "timer"
|
|
3013
|
+
});
|
|
3014
|
+
await commitWorkflow(state, () => {
|
|
3015
|
+
state.checkpoint.steps[stepId] = {
|
|
3016
|
+
type: "wait",
|
|
3017
|
+
status: "running",
|
|
3018
|
+
wait
|
|
3019
|
+
};
|
|
3020
|
+
state.run.status = "paused";
|
|
3021
|
+
state.run.pause = {
|
|
3022
|
+
type: "workflow_timer",
|
|
3023
|
+
stepId,
|
|
3024
|
+
resumeAt: wait.resumeAt
|
|
3025
|
+
};
|
|
3026
|
+
}, [started, paused]);
|
|
3027
|
+
events.push(started, paused);
|
|
3028
|
+
await releaseWorkflowExecution(state);
|
|
3029
|
+
throw new RunPausedError(state.run.id, state.run.pause);
|
|
3030
|
+
},
|
|
3031
|
+
async waitForEvent(stepId, eventName) {
|
|
3032
|
+
if (!stepId.trim()) throw new Error("Workflow step id cannot be empty");
|
|
3033
|
+
if (!eventName.trim()) throw new Error("Workflow event name cannot be empty");
|
|
3034
|
+
const definition = workflow.events?.[eventName];
|
|
3035
|
+
if (!definition) throw new Error(`Workflow event "${eventName}" is not declared`);
|
|
3036
|
+
const existing = state.checkpoint.steps[stepId];
|
|
3037
|
+
if (existing?.type === "wait" && existing.status === "completed") return {
|
|
3038
|
+
...existing.payload === void 0 ? {} : { payload: existing.payload },
|
|
3039
|
+
...existing.actor === void 0 ? {} : { actor: existing.actor },
|
|
3040
|
+
receivedAt: existing.receivedAt
|
|
3041
|
+
};
|
|
3042
|
+
if (existing && existing.type !== "wait") throw new Error(`Workflow step "${stepId}" was already used as ${existing.type}`);
|
|
3043
|
+
const wait = {
|
|
3044
|
+
type: "event",
|
|
3045
|
+
eventName,
|
|
3046
|
+
requireActor: definition.requireActor ?? false
|
|
3047
|
+
};
|
|
3048
|
+
const started = createEvent(state, "workflow.wait.started", {
|
|
3049
|
+
stepId,
|
|
3050
|
+
kind: "event",
|
|
3051
|
+
eventName
|
|
3052
|
+
});
|
|
3053
|
+
const paused = createEvent(state, "workflow.run.paused", {
|
|
3054
|
+
stepId,
|
|
3055
|
+
reason: "event"
|
|
3056
|
+
});
|
|
3057
|
+
await commitWorkflow(state, () => {
|
|
3058
|
+
state.checkpoint.steps[stepId] = {
|
|
3059
|
+
type: "wait",
|
|
3060
|
+
status: "running",
|
|
3061
|
+
wait
|
|
3062
|
+
};
|
|
3063
|
+
state.run.status = "paused";
|
|
3064
|
+
state.run.pause = {
|
|
3065
|
+
type: "workflow_event",
|
|
3066
|
+
stepId,
|
|
3067
|
+
eventName
|
|
3068
|
+
};
|
|
3069
|
+
}, [started, paused]);
|
|
3070
|
+
events.push(started, paused);
|
|
3071
|
+
await releaseWorkflowExecution(state);
|
|
3072
|
+
throw new RunPausedError(state.run.id, state.run.pause);
|
|
1219
3073
|
}
|
|
1220
|
-
|
|
3074
|
+
};
|
|
3075
|
+
const firstEvents = state.initialEvents ?? [];
|
|
3076
|
+
state.initialEvents = void 0;
|
|
3077
|
+
events.push(...firstEvents);
|
|
3078
|
+
try {
|
|
3079
|
+
for (const event of firstEvents) yield event;
|
|
3080
|
+
if (state.checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${state.run.id}" is unsupported`, state.run.id);
|
|
3081
|
+
const hash = isTeamRun(state.run) ? await teamDefinitionHash(name, teams.get(name)) : await workflowDefinitionHash(name, workflow);
|
|
3082
|
+
if (state.checkpoint.definitionHash !== hash) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Definition for ${(isTeamRun(state.run) ? "Team" : "Workflow").toLowerCase()} "${name}" changed`, state.run.id);
|
|
3083
|
+
state.request.signal.throwIfAborted();
|
|
3084
|
+
const rawOutput = await workflow.run(step, state.checkpoint.input);
|
|
3085
|
+
const output = toJsonValue(workflow.outputSchema === void 0 ? rawOutput : await validateSchema(workflow.outputSchema, rawOutput, `Output from ${isTeamRun(state.run) ? "team" : "workflow"} "${name}" does not match outputSchema`), `Output from ${isTeamRun(state.run) ? "team" : "workflow"} "${name}" must be JSON-serializable`);
|
|
3086
|
+
await state.commitQueue;
|
|
3087
|
+
state.run.status = "completed";
|
|
3088
|
+
state.run.pause = void 0;
|
|
3089
|
+
state.run.output = output;
|
|
3090
|
+
state.run.usage = state.checkpoint.budget?.usage;
|
|
3091
|
+
const completedPayload = {
|
|
3092
|
+
output,
|
|
3093
|
+
...eventUsage(state.run.usage) === void 0 ? {} : { usage: eventUsage(state.run.usage) }
|
|
3094
|
+
};
|
|
3095
|
+
const completed = createCoordinatorEvent(state, "workflow.run.completed", completedPayload, "team.run.completed");
|
|
3096
|
+
await commit(state, {
|
|
3097
|
+
checkpoint: null,
|
|
3098
|
+
events: [completed]
|
|
3099
|
+
});
|
|
3100
|
+
events.push(completed);
|
|
3101
|
+
await releaseWorkflowExecution(state);
|
|
3102
|
+
yield completed;
|
|
3103
|
+
return {
|
|
3104
|
+
runId: state.run.id,
|
|
3105
|
+
sessionId: state.session.id,
|
|
3106
|
+
output,
|
|
3107
|
+
events,
|
|
3108
|
+
...state.run.usage === void 0 ? {} : { usage: state.run.usage }
|
|
3109
|
+
};
|
|
1221
3110
|
} catch (error) {
|
|
1222
|
-
|
|
1223
|
-
|
|
3111
|
+
let thrown = error;
|
|
3112
|
+
if (state.request.signal.aborted) try {
|
|
3113
|
+
const event = await cancelWorkflowExecution(state);
|
|
1224
3114
|
if (event) {
|
|
1225
3115
|
events.push(event);
|
|
1226
3116
|
yield event;
|
|
1227
3117
|
}
|
|
3118
|
+
} catch (cancellationError) {
|
|
3119
|
+
await releaseWorkflowExecution(state);
|
|
3120
|
+
throw cancellationError;
|
|
3121
|
+
}
|
|
3122
|
+
else if (state.run.status === "paused") throw error;
|
|
3123
|
+
else if (error instanceof FevexRunError && error.code === "RUN_CONFLICT") {
|
|
3124
|
+
await releaseWorkflowExecution(state);
|
|
3125
|
+
throw error;
|
|
1228
3126
|
} else if (state.run.status === "running") {
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
3127
|
+
await state.commitQueue;
|
|
3128
|
+
let failure = error;
|
|
3129
|
+
try {
|
|
3130
|
+
await runCompensations();
|
|
3131
|
+
} catch (compensationError) {
|
|
3132
|
+
failure = new AggregateError([error, compensationError], `Workflow failed: ${toErrorMessage(error)}; compensation failed: ${toErrorMessage(compensationError)}`);
|
|
3133
|
+
thrown = failure;
|
|
3134
|
+
}
|
|
3135
|
+
state.run.status = "failed";
|
|
3136
|
+
state.run.error = toErrorMessage(failure);
|
|
3137
|
+
state.run.usage = state.checkpoint.budget?.usage;
|
|
3138
|
+
const failedPayload = {
|
|
3139
|
+
error: toErrorMessage(failure),
|
|
3140
|
+
...runErrorPayload(failure)
|
|
3141
|
+
};
|
|
3142
|
+
const failed = createCoordinatorEvent(state, "workflow.run.failed", failedPayload, "team.run.failed");
|
|
3143
|
+
await commit(state, {
|
|
1237
3144
|
checkpoint: null,
|
|
1238
3145
|
events: [failed]
|
|
1239
3146
|
});
|
|
1240
|
-
else {
|
|
1241
|
-
await runStore.saveRun(state.run);
|
|
1242
|
-
await runStore.appendEvent(failed);
|
|
1243
|
-
notifyObserver(failed);
|
|
1244
|
-
}
|
|
1245
3147
|
events.push(failed);
|
|
1246
|
-
await
|
|
3148
|
+
await releaseWorkflowExecution(state);
|
|
1247
3149
|
yield failed;
|
|
1248
|
-
}
|
|
1249
|
-
throw
|
|
3150
|
+
} else await releaseWorkflowExecution(state);
|
|
3151
|
+
throw thrown;
|
|
1250
3152
|
}
|
|
1251
3153
|
}
|
|
1252
|
-
const
|
|
3154
|
+
const nextWorkflowEvent = async (state, execution) => {
|
|
1253
3155
|
state.advancing = true;
|
|
1254
3156
|
try {
|
|
1255
3157
|
return await execution.next();
|
|
@@ -1257,10 +3159,10 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1257
3159
|
state.advancing = false;
|
|
1258
3160
|
}
|
|
1259
3161
|
};
|
|
1260
|
-
const
|
|
1261
|
-
const execution =
|
|
3162
|
+
const drainWorkflowExecution = async (name, state, throwOnPause) => {
|
|
3163
|
+
const execution = executeWorkflow(name, state);
|
|
1262
3164
|
while (true) {
|
|
1263
|
-
const next = await
|
|
3165
|
+
const next = await nextWorkflowEvent(state, execution);
|
|
1264
3166
|
if (next.done) {
|
|
1265
3167
|
if (next.value === void 0 && throwOnPause) {
|
|
1266
3168
|
const run = await runStore.getRun(state.run.id);
|
|
@@ -1270,141 +3172,216 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1270
3172
|
}
|
|
1271
3173
|
}
|
|
1272
3174
|
};
|
|
1273
|
-
const
|
|
3175
|
+
const resumeWorkflow = async (runId, resolution, recoveryActor) => {
|
|
1274
3176
|
if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "resumeRun requires DurableRunStore", runId);
|
|
1275
3177
|
const run = await runStore.getRun(runId);
|
|
1276
3178
|
const checkpoint = await runStore.getCheckpoint(runId);
|
|
1277
|
-
if (!run || !checkpoint) throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
1278
|
-
if (
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
const
|
|
1282
|
-
const
|
|
1283
|
-
if (!
|
|
1284
|
-
|
|
3179
|
+
if (!run || !checkpoint) throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be ${resolution ? "resumed" : "recovered"}`, runId);
|
|
3180
|
+
if (checkpoint.version !== 2) throw new FevexRunError("CHECKPOINT_UNSUPPORTED", `Checkpoint for run "${runId}" is unsupported`, runId);
|
|
3181
|
+
if (resolution && run.status !== "paused" && run.status !== "running" || !resolution && run.status !== "running") throw new FevexRunError(resolution ? "RUN_NOT_RESUMABLE" : "RUN_NOT_RECOVERABLE", `Run "${runId}" is not ${resolution ? "resumable" : "recoverable"}`, runId);
|
|
3182
|
+
const name = coordinatorName(run);
|
|
3183
|
+
const workflow = workflows.get(name);
|
|
3184
|
+
const team = isTeamRun(run) ? teams.get(name) : void 0;
|
|
3185
|
+
if (!workflow) throw new FevexRunError("RUN_DEFINITION_CHANGED", `${isTeamRun(run) ? "Team" : "Workflow"} "${name}" is unavailable`, runId);
|
|
3186
|
+
const currentHash = isTeamRun(run) ? team && await teamDefinitionHash(name, team) : await workflowDefinitionHash(name, workflow);
|
|
3187
|
+
if (!currentHash || checkpoint.definitionHash !== currentHash) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Definition for ${isTeamRun(run) ? "team" : "workflow"} "${name}" changed`, runId);
|
|
3188
|
+
if (resolution && run.pause?.type !== "workflow_child" && run.pause?.type !== "workflow_children" && run.pause?.type !== "workflow_timer" && run.pause?.type !== "workflow_event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
1285
3189
|
const session = await runStore.getSession(run.sessionId);
|
|
1286
3190
|
if (!session) throw new Error(`Session "${run.sessionId}" does not exist`);
|
|
1287
3191
|
if (activeSessions.has(session.id)) throw new FevexRunError("RUN_CONFLICT", `Session "${session.id}" is active`, runId);
|
|
1288
3192
|
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
1289
|
-
|
|
3193
|
+
const lease = {
|
|
3194
|
+
generation: 0,
|
|
1290
3195
|
runId,
|
|
1291
3196
|
ownerId,
|
|
1292
3197
|
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
1293
|
-
}
|
|
3198
|
+
};
|
|
3199
|
+
if (!await runStore.acquireLease(lease)) throw new FevexRunError("RUN_CONFLICT", `Run "${runId}" is leased`, runId);
|
|
1294
3200
|
try {
|
|
3201
|
+
if (!resolution) {
|
|
3202
|
+
if (!recoveryActor?.id.trim()) throw new FevexRunError("RUN_NOT_RECOVERABLE", "Recovery actor is required", runId);
|
|
3203
|
+
const controller = new AbortController();
|
|
3204
|
+
const state = {
|
|
3205
|
+
run,
|
|
3206
|
+
session,
|
|
3207
|
+
request: {
|
|
3208
|
+
input: checkpoint.input,
|
|
3209
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
3210
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3211
|
+
sessionId: session.id,
|
|
3212
|
+
context: checkpoint.context,
|
|
3213
|
+
limits: checkpoint.limits,
|
|
3214
|
+
signal: controller.signal
|
|
3215
|
+
},
|
|
3216
|
+
controller,
|
|
3217
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3218
|
+
checkpoint,
|
|
3219
|
+
advancing: false,
|
|
3220
|
+
lease,
|
|
3221
|
+
commitQueue: Promise.resolve(),
|
|
3222
|
+
recoveryActor
|
|
3223
|
+
};
|
|
3224
|
+
const recovered = createCoordinatorEvent(state, "workflow.run.recovered", { actorId: recoveryActor.id }, "team.run.recovered");
|
|
3225
|
+
await commit(state, {
|
|
3226
|
+
checkpoint,
|
|
3227
|
+
events: [recovered]
|
|
3228
|
+
});
|
|
3229
|
+
state.initialEvents = [recovered];
|
|
3230
|
+
activeWorkflowRuns.set(runId, state);
|
|
3231
|
+
activeSessions.add(session.id);
|
|
3232
|
+
startLease(state, runStore);
|
|
3233
|
+
drainWorkflowExecution(name, state, false).catch(() => {});
|
|
3234
|
+
return structuredClone(run);
|
|
3235
|
+
}
|
|
3236
|
+
const workflowPause = run.pause;
|
|
3237
|
+
if (!workflowPause || workflowPause.type !== "workflow_child" && workflowPause.type !== "workflow_children" && workflowPause.type !== "workflow_timer" && workflowPause.type !== "workflow_event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
3238
|
+
if (workflowPause.type === "workflow_timer" || workflowPause.type === "workflow_event") {
|
|
3239
|
+
const pause = workflowPause;
|
|
3240
|
+
if (pause.type === "workflow_timer") {
|
|
3241
|
+
if (resolution.type !== "timer") throw new FevexRunError("RUN_NOT_RESUMABLE", "Timer resolution is required", runId);
|
|
3242
|
+
if (Date.now() < Date.parse(pause.resumeAt)) throw new FevexRunError("RUN_NOT_RESUMABLE", "Workflow timer has not elapsed", runId);
|
|
3243
|
+
} else if (resolution.type !== "event" || resolution.eventName !== pause.eventName) throw new FevexRunError("RUN_NOT_RESUMABLE", "Workflow event does not match", runId);
|
|
3244
|
+
const waiting = checkpoint.steps[pause.stepId];
|
|
3245
|
+
if (waiting?.type !== "wait" || waiting.status !== "running") throw new FevexRunError("RUN_NOT_RESUMABLE", "Workflow wait is unavailable", runId);
|
|
3246
|
+
let eventPayload;
|
|
3247
|
+
const receivedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3248
|
+
if (resolution.type === "event") {
|
|
3249
|
+
const eventDefinition = workflow.events?.[resolution.eventName];
|
|
3250
|
+
if (!eventDefinition) throw new FevexRunError("RUN_DEFINITION_CHANGED", `Workflow event "${resolution.eventName}" is unavailable`, runId);
|
|
3251
|
+
if (waiting.wait.type !== "event") throw new FevexRunError("RUN_NOT_RESUMABLE", "Workflow wait is unavailable", runId);
|
|
3252
|
+
if (waiting.wait.requireActor && !resolution.actor?.id?.trim()) throw new FevexRunError("APPROVAL_INVALID", "Workflow event actor is required", runId);
|
|
3253
|
+
const validatedPayload = eventDefinition.payloadSchema === void 0 ? resolution.payload : await validateSchema(eventDefinition.payloadSchema, resolution.payload, `Payload for workflow event "${resolution.eventName}" does not match payloadSchema`);
|
|
3254
|
+
if (validatedPayload !== void 0) eventPayload = toJsonValue(validatedPayload, "Workflow event payload must be JSON-serializable");
|
|
3255
|
+
}
|
|
3256
|
+
const controller = new AbortController();
|
|
3257
|
+
const state = {
|
|
3258
|
+
run,
|
|
3259
|
+
session,
|
|
3260
|
+
request: {
|
|
3261
|
+
input: checkpoint.input,
|
|
3262
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
3263
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3264
|
+
sessionId: session.id,
|
|
3265
|
+
context: checkpoint.context,
|
|
3266
|
+
limits: checkpoint.limits,
|
|
3267
|
+
signal: controller.signal
|
|
3268
|
+
},
|
|
3269
|
+
controller,
|
|
3270
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3271
|
+
checkpoint,
|
|
3272
|
+
advancing: false,
|
|
3273
|
+
lease,
|
|
3274
|
+
commitQueue: Promise.resolve()
|
|
3275
|
+
};
|
|
3276
|
+
run.status = "running";
|
|
3277
|
+
run.pause = void 0;
|
|
3278
|
+
const resumed = createCoordinatorEvent(state, "workflow.run.resumed", void 0, "team.run.resumed");
|
|
3279
|
+
const completed = createEvent(state, "workflow.wait.completed", {
|
|
3280
|
+
stepId: pause.stepId,
|
|
3281
|
+
kind: waiting.wait.type,
|
|
3282
|
+
...eventPayload === void 0 ? {} : { payload: eventPayload },
|
|
3283
|
+
...resolution.type === "event" && resolution.actor ? { actorId: resolution.actor.id } : {},
|
|
3284
|
+
...resolution.type === "event" ? { receivedAt } : {}
|
|
3285
|
+
});
|
|
3286
|
+
await commitWorkflow(state, () => {
|
|
3287
|
+
state.checkpoint.steps[pause.stepId] = {
|
|
3288
|
+
type: "wait",
|
|
3289
|
+
status: "completed",
|
|
3290
|
+
wait: waiting.wait,
|
|
3291
|
+
...eventPayload === void 0 ? {} : { payload: eventPayload },
|
|
3292
|
+
...resolution.type === "event" && resolution.actor ? { actor: resolution.actor } : {},
|
|
3293
|
+
...resolution.type === "event" ? { receivedAt } : {}
|
|
3294
|
+
};
|
|
3295
|
+
}, [resumed, completed]);
|
|
3296
|
+
activeWorkflowRuns.set(runId, state);
|
|
3297
|
+
activeSessions.add(session.id);
|
|
3298
|
+
startLease(state, runStore);
|
|
3299
|
+
drainWorkflowExecution(name, state, false).catch(() => {});
|
|
3300
|
+
return structuredClone(run);
|
|
3301
|
+
}
|
|
3302
|
+
if (resolution.type === "timer" || resolution.type === "event") throw new FevexRunError("RUN_NOT_RESUMABLE", `Run "${runId}" cannot be resumed`, runId);
|
|
3303
|
+
const childRunId = workflowPause.type === "workflow_children" ? workflowPause.children.find(({ childPause }) => pauseMatchesResolution(childPause, resolution))?.childRunId : workflowPause.childRunId;
|
|
3304
|
+
if (!childRunId) throw new FevexRunError("RUN_NOT_RESUMABLE", "Resolution does not match a paused child run", runId);
|
|
3305
|
+
const childRun = await resumeAgent(childRunId, resolution, true);
|
|
1295
3306
|
const controller = new AbortController();
|
|
1296
3307
|
const state = {
|
|
1297
3308
|
run,
|
|
1298
3309
|
session,
|
|
1299
3310
|
request: {
|
|
1300
|
-
input:
|
|
3311
|
+
input: checkpoint.input,
|
|
3312
|
+
...checkpoint.model === void 0 ? {} : { model: checkpoint.model },
|
|
3313
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
1301
3314
|
sessionId: session.id,
|
|
1302
3315
|
context: checkpoint.context,
|
|
3316
|
+
limits: checkpoint.limits,
|
|
1303
3317
|
signal: controller.signal
|
|
1304
3318
|
},
|
|
1305
3319
|
controller,
|
|
1306
3320
|
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
1307
|
-
advancing: false,
|
|
1308
3321
|
checkpoint,
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
...resolution.decision === "retry" ? { forcedRetryToolCallId: resolution.toolCallId } : {}
|
|
1313
|
-
}
|
|
3322
|
+
advancing: false,
|
|
3323
|
+
lease,
|
|
3324
|
+
commitQueue: Promise.resolve()
|
|
1314
3325
|
};
|
|
1315
|
-
if (
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1325
|
-
const tool = pending && tools.get(pending.call.name);
|
|
1326
|
-
if (!pending || !tool) {
|
|
1327
|
-
await runStore.releaseLease(runId, ownerId);
|
|
1328
|
-
throw new FevexRunError("APPROVAL_INVALID", "Approval tool is unavailable", runId);
|
|
1329
|
-
}
|
|
1330
|
-
if (resolution.decision === "approve") await authorize(state, tool, pending.input, "approval.resolve", {
|
|
1331
|
-
...state.request.context,
|
|
1332
|
-
actor: resolution.actor
|
|
3326
|
+
if (childRun.status === "cancelled") {
|
|
3327
|
+
run.status = "cancelled";
|
|
3328
|
+
run.pause = void 0;
|
|
3329
|
+
run.error = childRun.error ?? "aborted";
|
|
3330
|
+
const reason = childRun.error === "approval_rejected" ? "approval_rejected" : "aborted";
|
|
3331
|
+
const cancelled = createCoordinatorEvent(state, "workflow.run.cancelled", { reason }, "team.run.cancelled");
|
|
3332
|
+
await commit(state, {
|
|
3333
|
+
checkpoint: null,
|
|
3334
|
+
events: [cancelled]
|
|
1333
3335
|
});
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
run.status = "cancelled";
|
|
1337
|
-
run.pause = void 0;
|
|
1338
|
-
run.error = "approval_rejected";
|
|
1339
|
-
const resolved = createEvent(state, "approval.resolved", {
|
|
1340
|
-
approvalId: resolution.approvalId,
|
|
1341
|
-
toolCallId,
|
|
1342
|
-
decision: "reject",
|
|
1343
|
-
actorId: resolution.actor.id
|
|
1344
|
-
});
|
|
1345
|
-
const cancelled = createEvent(state, "run.cancelled", { reason: "approval_rejected" });
|
|
1346
|
-
await commit(state, {
|
|
1347
|
-
checkpoint: null,
|
|
1348
|
-
events: [resolved, cancelled]
|
|
1349
|
-
});
|
|
1350
|
-
await releaseExecution(state);
|
|
1351
|
-
return structuredClone(run);
|
|
1352
|
-
}
|
|
1353
|
-
} else {
|
|
1354
|
-
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1355
|
-
const record = await runStore.getToolExecution(runId, resolution.toolCallId);
|
|
1356
|
-
const matchesUnknownPause = run.pause?.type === "tool_execution_unknown" && run.pause.toolCallId === resolution.toolCallId;
|
|
1357
|
-
const matchesInterruptedAttempt = run.status === "running" && pending?.call.id === resolution.toolCallId && (record?.status === "started" || record?.status === "completed");
|
|
1358
|
-
if (!matchesUnknownPause && !matchesInterruptedAttempt) {
|
|
1359
|
-
await runStore.releaseLease(runId, ownerId);
|
|
1360
|
-
throw new FevexRunError("RUN_NOT_RESUMABLE", "Tool execution resolution does not match a pending attempt", runId);
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
if (resolution.type === "tool_execution" && resolution.decision === "use_output") {
|
|
1364
|
-
const pending = checkpoint.pendingTools[checkpoint.pendingIndex];
|
|
1365
|
-
const tool = tools.get(pending.call.name);
|
|
1366
|
-
if (!tool) {
|
|
1367
|
-
await runStore.releaseLease(runId, ownerId);
|
|
1368
|
-
throw new FevexRunError("RUN_DEFINITION_CHANGED", "Pending tool is unavailable", runId);
|
|
1369
|
-
}
|
|
1370
|
-
const output = toJsonValue(tool.outputSchema === void 0 ? resolution.output : await validateSchema(tool.outputSchema, resolution.output, `Manual output for tool "${tool.name}" does not match outputSchema`), "Manual tool output must be JSON-serializable");
|
|
1371
|
-
const current = await runStore.getToolExecution(runId, resolution.toolCallId);
|
|
1372
|
-
await commit(state, { toolExecution: {
|
|
1373
|
-
runId,
|
|
1374
|
-
toolCallId: resolution.toolCallId,
|
|
1375
|
-
toolName: pending.call.name,
|
|
1376
|
-
input: pending.input,
|
|
1377
|
-
status: "completed",
|
|
1378
|
-
attempt: current?.attempt ?? 1,
|
|
1379
|
-
idempotencyKey: current?.idempotencyKey ?? pending.idempotencyKey,
|
|
1380
|
-
output,
|
|
1381
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1382
|
-
} });
|
|
3336
|
+
await releaseWorkflowExecution(state);
|
|
3337
|
+
return structuredClone(run);
|
|
1383
3338
|
}
|
|
1384
|
-
const approval = run.pause?.type === "approval" ? run.pause.approval : void 0;
|
|
1385
3339
|
run.status = "running";
|
|
1386
3340
|
run.pause = void 0;
|
|
1387
|
-
const
|
|
1388
|
-
if (approval && resolution.type === "approval") resumeEvents.push(createEvent(state, "approval.resolved", {
|
|
1389
|
-
approvalId: approval.id,
|
|
1390
|
-
toolCallId: approval.toolCallId,
|
|
1391
|
-
decision: "approve",
|
|
1392
|
-
actorId: resolution.actor.id
|
|
1393
|
-
}));
|
|
3341
|
+
const resumed = createCoordinatorEvent(state, "workflow.run.resumed", void 0, "team.run.resumed");
|
|
1394
3342
|
await commit(state, {
|
|
1395
3343
|
checkpoint,
|
|
1396
|
-
events:
|
|
3344
|
+
events: [resumed]
|
|
1397
3345
|
});
|
|
1398
|
-
|
|
3346
|
+
activeWorkflowRuns.set(runId, state);
|
|
1399
3347
|
activeSessions.add(session.id);
|
|
1400
3348
|
startLease(state, runStore);
|
|
1401
|
-
|
|
3349
|
+
drainWorkflowExecution(name, state, false).catch(() => {});
|
|
1402
3350
|
return structuredClone(run);
|
|
1403
3351
|
} catch (error) {
|
|
1404
|
-
await runStore.releaseLease(runId, ownerId).catch(() => {});
|
|
3352
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
1405
3353
|
throw error;
|
|
1406
3354
|
}
|
|
1407
3355
|
};
|
|
3356
|
+
return {
|
|
3357
|
+
prepareWorkflowExecution,
|
|
3358
|
+
executeWorkflow,
|
|
3359
|
+
nextWorkflowEvent,
|
|
3360
|
+
drainWorkflowExecution,
|
|
3361
|
+
resumeWorkflow,
|
|
3362
|
+
cancelWorkflowExecution
|
|
3363
|
+
};
|
|
3364
|
+
}
|
|
3365
|
+
//#endregion
|
|
3366
|
+
//#region src/internal/runtime.ts
|
|
3367
|
+
function createRuntime(composition) {
|
|
3368
|
+
const { models, agents, workflows, teams, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, policies } = composition;
|
|
3369
|
+
const core = createRunCore(composition);
|
|
3370
|
+
const { activeRuns, activeWorkflowRuns, activeSessions, pendingExports, exportFailures, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
|
|
3371
|
+
const agentEngine = createAgentEngine(composition, core);
|
|
3372
|
+
const { prepareExecution, executeAgent, nextEvent, drainExecution, resumeAgent, cancelStoredAgent } = agentEngine;
|
|
3373
|
+
const { prepareWorkflowExecution, executeWorkflow, nextWorkflowEvent, drainWorkflowExecution, resumeWorkflow, cancelWorkflowExecution } = createWorkflowEngine(composition, core, agentEngine);
|
|
3374
|
+
const resumeRun = async (runId, resolution) => {
|
|
3375
|
+
const run = await runStore.getRun(runId);
|
|
3376
|
+
if (run && isCoordinatorRun(run)) return resumeWorkflow(runId, resolution);
|
|
3377
|
+
return resumeAgent(runId, resolution);
|
|
3378
|
+
};
|
|
3379
|
+
const recoverRun = async (runId, actor) => {
|
|
3380
|
+
const run = await runStore.getRun(runId);
|
|
3381
|
+
if (!run) throw new FevexRunError("RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be recovered`, runId);
|
|
3382
|
+
if (isCoordinatorRun(run)) return resumeWorkflow(runId, void 0, actor);
|
|
3383
|
+
return resumeAgent(runId, void 0, false, actor);
|
|
3384
|
+
};
|
|
1408
3385
|
return {
|
|
1409
3386
|
async startAgent(name, request) {
|
|
1410
3387
|
const state = await prepareExecution(name, request);
|
|
@@ -1441,6 +3418,54 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1441
3418
|
}
|
|
1442
3419
|
} };
|
|
1443
3420
|
},
|
|
3421
|
+
async startWorkflow(name, request) {
|
|
3422
|
+
const state = await prepareWorkflowExecution(name, request);
|
|
3423
|
+
drainWorkflowExecution(name, state, false).catch(() => {});
|
|
3424
|
+
return structuredClone(state.run);
|
|
3425
|
+
},
|
|
3426
|
+
async runWorkflow(name, request) {
|
|
3427
|
+
const state = await prepareWorkflowExecution(name, request);
|
|
3428
|
+
return await drainWorkflowExecution(name, state, true);
|
|
3429
|
+
},
|
|
3430
|
+
async startTeam(name, request) {
|
|
3431
|
+
const state = await prepareWorkflowExecution(name, request, "team");
|
|
3432
|
+
drainWorkflowExecution(name, state, false).catch(() => {});
|
|
3433
|
+
return structuredClone(state.run);
|
|
3434
|
+
},
|
|
3435
|
+
async runTeam(name, request) {
|
|
3436
|
+
const state = await prepareWorkflowExecution(name, request, "team");
|
|
3437
|
+
return await drainWorkflowExecution(name, state, true);
|
|
3438
|
+
},
|
|
3439
|
+
streamTeam(name, request) {
|
|
3440
|
+
return { async *[Symbol.asyncIterator]() {
|
|
3441
|
+
const state = await prepareWorkflowExecution(name, request, "team");
|
|
3442
|
+
const execution = executeWorkflow(name, state);
|
|
3443
|
+
let finished = false;
|
|
3444
|
+
try {
|
|
3445
|
+
while (true) {
|
|
3446
|
+
const next = await nextWorkflowEvent(state, execution);
|
|
3447
|
+
if (next.done) {
|
|
3448
|
+
finished = true;
|
|
3449
|
+
return;
|
|
3450
|
+
}
|
|
3451
|
+
yield next.value;
|
|
3452
|
+
}
|
|
3453
|
+
} finally {
|
|
3454
|
+
if (!finished && state.run.status === "running") {
|
|
3455
|
+
state.controller.abort(new DOMException("Stream abandoned", "AbortError"));
|
|
3456
|
+
try {
|
|
3457
|
+
while (state.run.status === "running") if ((await nextWorkflowEvent(state, execution)).done) break;
|
|
3458
|
+
} catch {}
|
|
3459
|
+
if (state.run.status === "running") await cancelWorkflowExecution(state);
|
|
3460
|
+
}
|
|
3461
|
+
await execution.return(void 0);
|
|
3462
|
+
}
|
|
3463
|
+
} };
|
|
3464
|
+
},
|
|
3465
|
+
async getTeamRun(runId) {
|
|
3466
|
+
const run = await runStore.getRun(runId);
|
|
3467
|
+
return run?.kind === "team" ? run : void 0;
|
|
3468
|
+
},
|
|
1444
3469
|
getRun(runId) {
|
|
1445
3470
|
return runStore.getRun(runId);
|
|
1446
3471
|
},
|
|
@@ -1448,6 +3473,12 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1448
3473
|
return runStore.listEvents(runId, options);
|
|
1449
3474
|
},
|
|
1450
3475
|
async cancelRun(runId) {
|
|
3476
|
+
const activeWorkflow = activeWorkflowRuns.get(runId);
|
|
3477
|
+
if (activeWorkflow && !activeWorkflow.request.signal.aborted) {
|
|
3478
|
+
activeWorkflow.controller.abort(new DOMException("Run cancelled", "AbortError"));
|
|
3479
|
+
if (!activeWorkflow.advancing) await cancelWorkflowExecution(activeWorkflow);
|
|
3480
|
+
return true;
|
|
3481
|
+
}
|
|
1451
3482
|
const active = activeRuns.get(runId);
|
|
1452
3483
|
if (active && !active.request.signal.aborted) {
|
|
1453
3484
|
active.controller.abort(new DOMException("Run cancelled", "AbortError"));
|
|
@@ -1455,39 +3486,66 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1455
3486
|
return true;
|
|
1456
3487
|
}
|
|
1457
3488
|
const run = await runStore.getRun(runId);
|
|
1458
|
-
if (run
|
|
3489
|
+
if (!run || run.status !== "paused" && run.status !== "running") return false;
|
|
1459
3490
|
if (!isDurableRunStore(runStore)) return false;
|
|
1460
3491
|
const session = await runStore.getSession(run.sessionId);
|
|
1461
3492
|
if (!session) return false;
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
3493
|
+
if (isCoordinatorRun(run)) {
|
|
3494
|
+
const checkpoint = await runStore.getCheckpoint(runId);
|
|
3495
|
+
if (!checkpoint) return false;
|
|
3496
|
+
const ownerId = `${runtimeOwner}:${crypto.randomUUID()}`;
|
|
3497
|
+
const lease = {
|
|
3498
|
+
generation: 0,
|
|
3499
|
+
runId,
|
|
3500
|
+
ownerId,
|
|
3501
|
+
expiresAt: new Date(Date.now() + LEASE_MS).toISOString()
|
|
3502
|
+
};
|
|
3503
|
+
if (!await runStore.acquireLease(lease)) return false;
|
|
3504
|
+
try {
|
|
3505
|
+
const childRunIds = Object.values(checkpoint.steps).filter((record) => record.type === "agent" && record.status === "running").map(({ childRunId }) => childRunId);
|
|
3506
|
+
for (const childRunId of childRunIds) if (!await cancelStoredAgent(childRunId)) return false;
|
|
3507
|
+
const state = {
|
|
3508
|
+
run,
|
|
3509
|
+
session,
|
|
3510
|
+
request: {
|
|
3511
|
+
input: checkpoint.input,
|
|
3512
|
+
...checkpoint.reasoning === void 0 ? {} : { reasoning: checkpoint.reasoning },
|
|
3513
|
+
sessionId: session.id,
|
|
3514
|
+
context: checkpoint.context,
|
|
3515
|
+
limits: checkpoint.limits,
|
|
3516
|
+
signal: new AbortController().signal
|
|
3517
|
+
},
|
|
3518
|
+
controller: new AbortController(),
|
|
3519
|
+
eventSequence: (await runStore.listEvents(runId)).at(-1)?.sequence ?? 0,
|
|
3520
|
+
checkpoint,
|
|
3521
|
+
advancing: false,
|
|
3522
|
+
lease,
|
|
3523
|
+
commitQueue: Promise.resolve()
|
|
3524
|
+
};
|
|
3525
|
+
run.status = "cancelled";
|
|
3526
|
+
run.pause = void 0;
|
|
3527
|
+
run.error = "aborted";
|
|
3528
|
+
run.usage = checkpoint.budget?.usage;
|
|
3529
|
+
const cancelled = createCoordinatorEvent(state, "workflow.run.cancelled", { reason: "aborted" }, "team.run.cancelled");
|
|
3530
|
+
await commit(state, {
|
|
3531
|
+
checkpoint: null,
|
|
3532
|
+
events: [cancelled]
|
|
3533
|
+
});
|
|
3534
|
+
return true;
|
|
3535
|
+
} catch (error) {
|
|
3536
|
+
if (error instanceof FevexRunError && error.code === "RUN_CONFLICT" && (await runStore.getRun(runId))?.status !== "paused") return false;
|
|
3537
|
+
throw error;
|
|
3538
|
+
} finally {
|
|
3539
|
+
await runStore.releaseLease(runId, ownerId, lease.generation).catch(() => {});
|
|
3540
|
+
}
|
|
1487
3541
|
}
|
|
3542
|
+
return cancelStoredAgent(runId);
|
|
1488
3543
|
},
|
|
1489
3544
|
resumeRun(runId, resolution) {
|
|
1490
|
-
return
|
|
3545
|
+
return resumeRun(runId, resolution);
|
|
3546
|
+
},
|
|
3547
|
+
recoverRun(runId, options) {
|
|
3548
|
+
return recoverRun(runId, options.actor);
|
|
1491
3549
|
},
|
|
1492
3550
|
async compactSession(sessionId, summary) {
|
|
1493
3551
|
if (typeof summary !== "string" || !summary.trim()) throw new Error("Session summary must be a non-empty string");
|
|
@@ -1513,8 +3571,14 @@ function createRuntime({ models, agents, tools, runStore, credentialStore, polic
|
|
|
1513
3571
|
}
|
|
1514
3572
|
//#endregion
|
|
1515
3573
|
//#region src/fevex.ts
|
|
3574
|
+
/**
|
|
3575
|
+
* Validates a configuration snapshot and creates an isolated runtime.
|
|
3576
|
+
*
|
|
3577
|
+
* The runtime does not own external adapters; callers remain responsible for
|
|
3578
|
+
* closing model, connection, store, and exporter resources they provide.
|
|
3579
|
+
*/
|
|
1516
3580
|
function createFevex(config) {
|
|
1517
3581
|
return createRuntime(createComposition(config));
|
|
1518
3582
|
}
|
|
1519
3583
|
//#endregion
|
|
1520
|
-
export { FevexConfigurationError, FevexRunError, InMemoryRunStore, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, SCHEMA_NOT_TRANSPORTABLE, createFevex, defineAgent, definePolicy, defineTool, isDurableRunStore, toToolSpec };
|
|
3584
|
+
export { ChannelError, FEVEX_JSON_SCHEMA_PROFILE_VERSION, FevexConfigurationError, FevexRunError, InMemoryMemoryStore, InMemoryRunStore, IntegrationError, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, SCHEMA_NOT_TRANSPORTABLE, SandboxError, compileFevexJsonSchema, createFevex, createLocalSandbox, createMemoryContextProvider, defineAgent, defineConnection, defineContextProvider, definePolicy, defineSkill, defineTeam, defineTool, defineWorkflow, handleChannelInput, isDurableRunStore, sandboxResultToJson, toToolSpec, validateFevexJsonSchemaProfile, validateListEventsOptions };
|