@fevex/core 0.1.0-alpha.5 → 0.1.0-alpha.7

package/dist/index.mjs

@@ -2,7 +2,7 @@ import { defineAgent } from "./agents/index.mjs";
 import { a as CatalogedArtifactStore, c as assertArtifactId, i as FileSystemArtifactStore, n as InMemoryArtifactStore, o as ArtifactError, r as InMemoryArtifactCatalog, s as artifactTenantId, t as S3ArtifactStore } from "./artifacts-gUzemET7.mjs";
 import { n as RunPausedError, t as FevexRunError } from "./run-error-Cr17wPGH.mjs";
 import { ChannelError, deliverChannelRun, handleChannelInput } from "./channels/index.mjs";
-import { InMemoryMemoryStore, createMemoryContextProvider, defineContextProvider, defineSkill } from "./knowledge/index.mjs";
+import { InMemoryMemoryStore, createAgentsMdContextProvider, createMemoryContextProvider, defineContextProvider, defineSkill, isProgressiveSkillProvider, loadSkillFile, loadSkillFiles } from "./knowledge/index.mjs";
 import { n as teamAsWorkflow, t as defineTeam } from "./teams-Ctz676G4.mjs";
 import { InMemoryRunStore, isDurableRunStore, isRetainableRunStore, validateListEventsOptions, validatePurgeRunsOptions } from "./runtime/index.mjs";
 import { FEVEX_JSON_SCHEMA_PROFILE_VERSION, IntegrationError, compileFevexJsonSchema, defineConnection, defineTool, toToolSpec, validateFevexJsonSchemaProfile } from "./tools/index.mjs";
@@ -332,6 +332,18 @@ function createComposition(config) {
 				"error.message"
 			].includes(kind)) && (content.redact === void 0 || typeof content.redact === "function"), "INVALID_CONFIG", "Fevex config \"observability.content\" is invalid");
 		}
+		if (config.observability.exportQueue !== void 0) {
+			const queue = config.observability.exportQueue;
+			assertConfiguration(isRecord(queue) && [
+				"concurrency",
+				"maxQueued",
+				"timeoutMs",
+				"maxFailures"
+			].every((key) => {
+				const value = queue[key];
+				return value === void 0 || typeof value === "number" && Number.isSafeInteger(value) && value > 0;
+			}), "INVALID_CONFIG", "Fevex config \"observability.exportQueue\" must contain positive safe integers");
+		}
 	}
 	const models = /* @__PURE__ */ new Map();
 	const agents = /* @__PURE__ */ new Map();
@@ -624,7 +636,8 @@ function createComposition(config) {
 			...config.observability.content ? { content: {
 				include: [...config.observability.content.include],
 				...config.observability.content.redact ? { redact: config.observability.content.redact } : {}
-			} } : {}
+			} } : {},
+			...config.observability.exportQueue ? { exportQueue: { ...config.observability.exportQueue } } : {}
 		}
 	};
 }
@@ -636,6 +649,8 @@ const LEASE_MS = 3e4;
 const LEASE_RENEW_MS = 1e4;
 /** Reserved name of the internal elicitation tool exposed to the model. */
 const ELICIT_TOOL_NAME = "fevex__elicit";
+/** Reserved name of the internal progressive skill loader exposed to the model. */
+const LOAD_SKILL_TOOL_NAME = "fevex__load_skill";
 //#endregion
 //#region src/internal/run-helpers.ts
 function redact(message, secrets) {
@@ -890,8 +905,9 @@ function addWorkflowBudget(first, second) {
 */
 function createAgentEngine(composition, core) {
 	const { models, agents, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, artifacts, policies } = composition;
-	const { activeRuns, activeSessions, runtimeOwner, admitStart, markInternalRun, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
-	const prepareExecutionBody = async (name, request, internal = false) => {
+	const { activeRuns, activeSessions, runtimeOwner, isClosed, admitStart, markInternalRun, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
+	const prepareExecutionBody = async (name, request, internal = false, correlation = {}) => {
+		if (!internal && isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		const agent = agents.get(name);
 		if (!agent) throw new FevexRunError("AGENT_NOT_FOUND", `Agent "${name}" is not registered`);
 		const { modelName, model } = resolveModel(agent, request.model);
@@ -920,6 +936,8 @@ function createAgentEngine(composition, core) {
 			agentName: name,
 			modelName,
 			...model.metadata === void 0 ? {} : { modelMetadata: structuredClone(model.metadata) },
+			...correlation.parentRunId === void 0 ? {} : { parentRunId: correlation.parentRunId },
+			...correlation.rootRunId === void 0 ? {} : { rootRunId: correlation.rootRunId },
 			status: "running",
 			revision: 0,
 			createdAt: now,
@@ -1010,7 +1028,10 @@ function createAgentEngine(composition, core) {
 	const prepareExecution = async (name, request, options = {}) => {
 		const release = options.admit === false ? void 0 : await admitStart("agent", name, request);
 		try {
-			return await prepareExecutionBody(name, request, options.admit === false);
+			return await prepareExecutionBody(name, request, options.admit === false, {
+				...options.parentRunId === void 0 ? {} : { parentRunId: options.parentRunId },
+				...options.rootRunId === void 0 ? {} : { rootRunId: options.rootRunId }
+			});
 		} finally {
 			release?.();
 		}
@@ -1160,26 +1181,50 @@ function createAgentEngine(composition, core) {
 					description: remote.description
 				} : tool, inputSchema);
 			}));
-			const allTools = currentElicitationMode === "pause" ? [...agentTools, {
-				name: ELICIT_TOOL_NAME,
-				description: "Request external information required to continue this run. Use prompt for explanatory text shown outside the form. Put short field labels in responseSchema property titles and optional field help in property descriptions.",
+			const progressiveSkills = (agent.skills ?? []).map((skillName) => contextProviders.get(skillName)).filter((provider) => provider !== void 0 && isProgressiveSkillProvider(provider));
+			const skillTool = progressiveSkills.length === 0 ? void 0 : {
+				name: LOAD_SKILL_TOOL_NAME,
+				description: `Load the full instructions for one of this agent's skills. Available skills: ${progressiveSkills.map((skill) => skill.name).join(", ")}. Pass a resource path from the loaded skill to read one resource.`,
 				inputSchema: {
 					type: "object",
 					properties: {
-						prompt: {
+						name: {
 							type: "string",
-							description: "Explanatory text for the user; do not duplicate field labels here."
-						},
-						responseSchema: {
-							type: "object",
-							description: "JSON Schema for the expected response. Each property should include a short title for the form label and may include a description for field help."
+							enum: progressiveSkills.map((skill) => skill.name)
 						},
-						expiresAt: { type: "string" }
+						resource: {
+							type: "string",
+							description: "Optional resource path returned by a previous load."
+						}
 					},
-					required: ["prompt", "responseSchema"],
+					required: ["name"],
 					additionalProperties: false
 				}
-			}] : agentTools;
+			};
+			const allTools = [
+				...agentTools,
+				...skillTool === void 0 ? [] : [skillTool],
+				...currentElicitationMode === "pause" ? [{
+					name: ELICIT_TOOL_NAME,
+					description: "Request external information required to continue this run. Use prompt for explanatory text shown outside the form. Put short field labels in responseSchema property titles and optional field help in property descriptions.",
+					inputSchema: {
+						type: "object",
+						properties: {
+							prompt: {
+								type: "string",
+								description: "Explanatory text for the user; do not duplicate field labels here."
+							},
+							responseSchema: {
+								type: "object",
+								description: "JSON Schema for the expected response. Each property should include a short title for the form label and may include a description for field help."
+							},
+							expiresAt: { type: "string" }
+						},
+						required: ["prompt", "responseSchema"],
+						additionalProperties: false
+					}
+				}] : []
+			];
 			const effectiveLimits = state.checkpoint?.limits ?? combineLimits(agent.limits, request.limits);
 			const maxSteps = effectiveLimits?.maxSteps ?? 8;
 			const maxToolCalls = effectiveLimits?.maxToolCalls ?? 16;
@@ -1268,6 +1313,57 @@ function createAgentEngine(composition, core) {
 					assertContinuationBudget(name, "maxInputTokens", "inputTokens", effectiveLimits?.maxInputTokens, usage);
 					assertContinuationBudget(name, "maxOutputTokens", "outputTokens", effectiveLimits?.maxOutputTokens, usage);
 					if (result.toolCalls.length > maxToolCalls - toolCallCount) throw new Error(`Agent "${name}" exceeded maxToolCalls limit of ${maxToolCalls}`);
+					const loadSkillCall = progressiveSkills.length === 0 ? void 0 : result.toolCalls.find((call) => call.name === LOAD_SKILL_TOOL_NAME);
+					if (loadSkillCall) {
+						if (result.toolCalls.length !== 1) throw new Error(`${LOAD_SKILL_TOOL_NAME} must be the only tool call in a model turn`);
+						if (!loadSkillCall.id?.trim()) throw new Error("Tool call id cannot be empty");
+						if (seenToolCallIds.has(loadSkillCall.id)) throw new Error(`Tool call id "${loadSkillCall.id}" is duplicated in run "${state.run.id}"`);
+						if (toolCallCount + 1 > maxToolCalls) throw new Error(`Agent "${name}" exceeded maxToolCalls limit of ${maxToolCalls}`);
+						const rawInput = toJsonValue(loadSkillCall.input, `Input for tool "${LOAD_SKILL_TOOL_NAME}" must be JSON-serializable`);
+						const skillInput = readSkillInput(rawInput);
+						const skill = progressiveSkills.find((item) => item.name === skillInput.name);
+						if (!skill) throw new Error(`Skill "${skillInput.name}" is not available to agent "${name}"`);
+						seenToolCallIds.add(loadSkillCall.id);
+						messages.push({
+							role: "assistant",
+							content: result.output === void 0 ? "" : serializeValue(result.output, `Model output for agent "${name}" must be serializable`),
+							toolCalls: [{
+								...loadSkillCall,
+								input: rawInput
+							}]
+						});
+						yield await emit("tool.started", {
+							step,
+							toolCallId: loadSkillCall.id,
+							toolName: LOAD_SKILL_TOOL_NAME
+						});
+						let blocks;
+						try {
+							blocks = await abortable(() => skill.load(knowledgeContext(state, inputContent), skillInput.resource), request.signal);
+						} catch (error) {
+							yield await emit("tool.failed", {
+								step,
+								toolCallId: loadSkillCall.id,
+								toolName: LOAD_SKILL_TOOL_NAME,
+								error: toErrorMessage(error)
+							});
+							throw new Error(`Skill provider "${skill.name}" failed: ${toErrorMessage(error)}`, { cause: error });
+						}
+						messages.push({
+							role: "tool",
+							name: LOAD_SKILL_TOOL_NAME,
+							toolCallId: loadSkillCall.id,
+							content: blocks.filter((block) => block.content.trim()).map((block) => `[Skill: ${skill.name}/${block.id}]\n${block.content}`).join("\n\n")
+						});
+						yield await emit("tool.completed", {
+							step,
+							toolCallId: loadSkillCall.id,
+							toolName: LOAD_SKILL_TOOL_NAME
+						});
+						toolCallCount += 1;
+						step += 1;
+						continue;
+					}
 					const elicitationCall = result.toolCalls.find(isElicitationToolCall);
 					if (elicitationCall) {
 						if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "Elicitation requires a DurableRunStore", state.run.id);
@@ -2017,6 +2113,18 @@ function createAgentEngine(composition, core) {
 		cancelStoredAgent
 	};
 }
+function readSkillInput(input) {
+	if (typeof input !== "object" || input === null || Array.isArray(input)) throw new TypeError(`${LOAD_SKILL_TOOL_NAME} input must be an object`);
+	const record = input;
+	const name = record.name;
+	const resource = record.resource;
+	if (typeof name !== "string" || !name.trim()) throw new TypeError(`${LOAD_SKILL_TOOL_NAME} name must be a non-empty string`);
+	if (resource !== void 0 && (typeof resource !== "string" || !resource.trim())) throw new TypeError(`${LOAD_SKILL_TOOL_NAME} resource must be a non-empty string`);
+	return {
+		name: name.trim(),
+		...resource === void 0 ? {} : { resource: resource.trim() }
+	};
+}
 function scopedArtifacts(store, options) {
 	const scopedContext = (value) => {
 		const contextTenant = options.request.context?.attributes?.tenantId;
@@ -2053,16 +2161,31 @@ function scopedArtifacts(store, options) {
 const TERMINAL_TYPES = /* @__PURE__ */ new Set([
 	"run.completed",
 	"run.failed",
-	"run.cancelled"
+	"run.cancelled",
+	"workflow.run.completed",
+	"workflow.run.failed",
+	"workflow.run.cancelled",
+	"team.run.completed",
+	"team.run.failed",
+	"team.run.cancelled"
 ]);
 const SENSITIVE_KEY = /api[-_]?key|authorization|cookie|credential|password|secret|token/i;
+function isTraceTerminalEvent(event) {
+	return TERMINAL_TYPES.has(event.type);
+}
 function durationMs(startedAt, endedAt) {
 	return Math.max(0, Date.parse(endedAt) - Date.parse(startedAt));
 }
 function traceStatus(event) {
-	if (event.type === "run.completed") return "completed";
-	if (event.type === "run.cancelled") return "cancelled";
-	return "failed";
+	switch (event.type) {
+		case "run.completed":
+		case "workflow.run.completed":
+		case "team.run.completed": return "completed";
+		case "run.cancelled":
+		case "workflow.run.cancelled":
+		case "team.run.cancelled": return "cancelled";
+		default: return "failed";
+	}
 }
 function redactKeys(value) {
 	if (Array.isArray(value)) return value.map(redactKeys);
@@ -2169,20 +2292,74 @@ function annotation(event) {
 				toolName: event.payload.toolName
 			}
 		};
+		case "workflow.run.paused":
+		case "team.run.paused": {
+			const stepId = event.type === "workflow.run.paused" ? event.payload.stepId : event.payload.delegationId;
+			return {
+				name: event.type,
+				timestamp: event.timestamp,
+				attributes: {
+					stepId,
+					reason: event.payload.reason,
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				}
+			};
+		}
+		case "workflow.run.resumed":
+		case "team.run.resumed": return {
+			name: event.type,
+			timestamp: event.timestamp
+		};
+		case "team.handoff.created": return {
+			name: event.type,
+			timestamp: event.timestamp,
+			attributes: {
+				delegationId: event.payload.delegationId,
+				from: event.payload.from,
+				to: event.payload.to,
+				reason: event.payload.reason
+			}
+		};
 		default: return;
 	}
 }
-function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
-	const events = [...rawEvents].sort((left, right) => left.sequence - right.sequence);
-	const terminal = [...events].reverse().find((event) => TERMINAL_TYPES.has(event.type));
-	if (!terminal) throw new TypeError(`Run "${run.id}" has no terminal event`);
-	const started = events.find((event) => event.type === "run.started") ?? events[0];
-	if (!started) throw new TypeError(`Run "${run.id}" has no events`);
+function terminalContent(options, terminal) {
+	if (terminal.type === "run.completed" || terminal.type === "workflow.run.completed" || terminal.type === "team.run.completed") {
+		const output = content(options, "run.output", terminal.payload.output);
+		return output ? [output] : [];
+	}
+	if (terminal.type === "run.failed" || terminal.type === "workflow.run.failed" || terminal.type === "team.run.failed") {
+		const error = content(options, "error.message", terminal.payload.error);
+		return error ? [error] : [];
+	}
+	return [];
+}
+function sortedEvents(rawEvents) {
+	return [...rawEvents].sort((left, right) => left.sequence - right.sequence);
+}
+function terminalEvent(runId, events) {
+	const terminal = [...events].reverse().find((event) => isTraceTerminalEvent(event));
+	if (!terminal) throw new TypeError(`Run "${runId}" has no terminal event`);
+	return terminal;
+}
+function startEvent(events) {
+	const started = events.find((event) => event.type === "run.started" || event.type === "workflow.run.started" || event.type === "team.run.started") ?? events[0];
+	if (!started) throw new TypeError("Run has no events");
+	return started;
+}
+function traceKind(run) {
+	if (isTeamRun(run)) return "team";
+	return run.kind === "workflow" ? "workflow" : "agent";
+}
+function buildAgentTrace(run, events, modelRegistryName, model, options) {
+	const terminal = terminalEvent(run.id, events);
+	const started = startEvent(events);
 	const status = traceStatus(terminal);
 	const annotations = [];
 	const spans = [];
 	const openModels = /* @__PURE__ */ new Map();
 	const openTools = /* @__PURE__ */ new Map();
+	const modelSpanByStep = /* @__PURE__ */ new Map();
 	const previousUsage = {};
 	const closeModel = (step, endedAt, spanStatus) => {
 		const open = openModels.get(step);
@@ -2213,6 +2390,7 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		const spanCost = costFor(options, modelRegistryName, model, usage);
 		if (spanCost) span.cost = spanCost;
 		spans.push(span);
+		modelSpanByStep.set(step, span.id);
 	};
 	const findOpenTool = (toolCallId) => {
 		const entries = [...openTools.entries()];
@@ -2232,7 +2410,7 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		const errorContent = error ? content(options, "error.message", error) : void 0;
 		spans.push({
 			id: event.id,
-			parentId: run.id,
+			parentId: modelSpanByStep.get(event.payload.step) ?? run.id,
 			kind: "tool",
 			name: event.payload.toolName,
 			startedAt: event.timestamp,
@@ -2297,19 +2475,15 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 	for (const open of [...openTools.values()]) if (open.event.type === "tool.started") closeTool(open.event.payload.toolCallId, terminal.timestamp, status);
 	const finalUsage = run.usage ?? previousUsage;
 	const totalCost = costFor(options, modelRegistryName, model, finalUsage);
-	const traceContent = [];
-	if (terminal.type === "run.completed") {
-		const output = content(options, "run.output", terminal.payload.output);
-		if (output) traceContent.push(output);
-	} else if (terminal.type === "run.failed") {
-		const error = content(options, "error.message", terminal.payload.error);
-		if (error) traceContent.push(error);
-	}
+	const traceContent = terminalContent(options, terminal);
 	return {
 		schemaVersion: "1",
+		kind: "agent",
 		traceId: run.id,
 		runId: run.id,
 		sessionId: run.sessionId,
+		...run.parentRunId === void 0 ? {} : { parentRunId: run.parentRunId },
+		rootRunId: run.rootRunId ?? run.parentRunId ?? run.id,
 		agentName: run.agentName,
 		modelRegistryName,
 		...model ? { model: { ...model } } : {},
@@ -2329,6 +2503,249 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		...traceContent.length ? { content: traceContent } : {}
 	};
 }
+function buildCoordinatorTrace(run, events, options) {
+	const terminal = terminalEvent(run.id, events);
+	const started = startEvent(events);
+	const status = traceStatus(terminal);
+	const annotations = [];
+	const spans = [];
+	const openSteps = /* @__PURE__ */ new Map();
+	const stepKey = (scope, stepId) => `${scope}:${stepId}`;
+	const modelCallCount = events.filter(({ type }) => type === "model.completed").length;
+	const toolCallCount = events.filter(({ type }) => type === "tool.completed" || type === "tool.failed").length;
+	const openStep = (key, event, name, attributes) => {
+		openSteps.set(key, {
+			event,
+			name,
+			attributes
+		});
+	};
+	const closeStep = (key, endedAt, spanStatus, error) => {
+		const open = openSteps.get(key);
+		if (!open) return;
+		openSteps.delete(key);
+		const errorContent = error ? content(options, "error.message", error) : void 0;
+		spans.push({
+			id: open.event.id,
+			parentId: run.id,
+			kind: "step",
+			name: open.name,
+			startedAt: open.event.timestamp,
+			endedAt,
+			durationMs: durationMs(open.event.timestamp, endedAt),
+			status: spanStatus,
+			attributes: open.attributes,
+			annotations: [],
+			...errorContent ? { content: [errorContent] } : {}
+		});
+	};
+	for (const event of events) {
+		const item = annotation(event);
+		if (item) annotations.push(item);
+		switch (event.type) {
+			case "workflow.step.started":
+				openStep(stepKey("workflow", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: event.payload.kind,
+					...event.payload.agentName === void 0 ? {} : { agentName: event.payload.agentName },
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				});
+				break;
+			case "workflow.step.completed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "workflow.step.failed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "failed", event.payload.error);
+				break;
+			case "workflow.wait.started":
+				openStep(stepKey("workflow", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: "wait",
+					waitKind: event.payload.kind,
+					...event.payload.resumeAt === void 0 ? {} : { resumeAt: event.payload.resumeAt },
+					...event.payload.eventName === void 0 ? {} : { eventName: event.payload.eventName }
+				});
+				break;
+			case "workflow.wait.completed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "team.agent.assigned":
+				openStep(stepKey("team", event.payload.delegationId), event, event.payload.delegationId, {
+					stepId: event.payload.delegationId,
+					stepKind: "agent",
+					agentName: event.payload.agentName,
+					action: event.payload.action,
+					...event.payload.expectedOutput === void 0 ? {} : { expectedOutput: event.payload.expectedOutput },
+					...event.payload.constraints === void 0 ? {} : { constraints: event.payload.constraints },
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				});
+				break;
+			case "team.task.completed":
+				closeStep(stepKey("team", event.payload.delegationId), event.timestamp, "completed");
+				break;
+			case "team.task.failed":
+				closeStep(stepKey("team", event.payload.delegationId), event.timestamp, "failed", event.payload.error);
+				break;
+			case "team.merge.started":
+				openStep(stepKey("team", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: "merge",
+					width: event.payload.width
+				});
+				break;
+			case "team.merge.completed":
+				closeStep(stepKey("team", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "team.merge.failed":
+				closeStep(stepKey("team", event.payload.stepId), event.timestamp, "failed", event.payload.error);
+				break;
+		}
+	}
+	for (const key of [...openSteps.keys()]) closeStep(key, terminal.timestamp, status);
+	const traceContent = terminalContent(options, terminal);
+	return {
+		schemaVersion: "1",
+		kind: traceKind(run),
+		traceId: run.id,
+		runId: run.id,
+		sessionId: run.sessionId,
+		...run.parentRunId === void 0 ? {} : { parentRunId: run.parentRunId },
+		rootRunId: run.rootRunId ?? run.parentRunId ?? run.id,
+		agentName: coordinatorName(run),
+		startedAt: started.timestamp,
+		endedAt: terminal.timestamp,
+		durationMs: durationMs(started.timestamp, terminal.timestamp),
+		status,
+		attributes: {
+			eventCount: events.length,
+			stepCount: spans.filter(({ kind }) => kind === "step").length,
+			modelCallCount,
+			toolCallCount
+		},
+		annotations,
+		spans,
+		...run.usage && Object.keys(run.usage).length ? { usage: { ...run.usage } } : {},
+		...traceContent.length ? { content: traceContent } : {}
+	};
+}
+function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
+	const events = sortedEvents(rawEvents);
+	if (isCoordinatorRun(run)) return buildCoordinatorTrace(run, events, options);
+	if (!modelRegistryName) throw new TypeError(`Run "${run.id}" has no model registry name`);
+	return buildAgentTrace(run, events, modelRegistryName, model, options);
+}
+//#endregion
+//#region src/internal/trace-export.ts
+const DEFAULT_CONCURRENCY = 4;
+const DEFAULT_MAX_QUEUED = 100;
+const DEFAULT_TIMEOUT_MS = 3e4;
+const DEFAULT_MAX_FAILURES = 100;
+const DEDUPE_WINDOW = 1e4;
+function withTimeout(promise, timeoutMs, message) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+		promise.then((value) => {
+			clearTimeout(timer);
+			resolve(value);
+		}, (error) => {
+			clearTimeout(timer);
+			reject(error);
+		});
+	});
+}
+function withDefault(value, fallback) {
+	return value ?? fallback;
+}
+/**
+* Bounded, deduplicated trace export queue. Jobs include building the trace
+* from the store, so a slow store or exporter cannot grow memory without limit.
+*/
+function createTraceExportQueue(options = {}) {
+	const concurrency = withDefault(options.concurrency, DEFAULT_CONCURRENCY);
+	const maxQueued = withDefault(options.maxQueued, DEFAULT_MAX_QUEUED);
+	const timeoutMs = withDefault(options.timeoutMs, DEFAULT_TIMEOUT_MS);
+	const maxFailures = withDefault(options.maxFailures, DEFAULT_MAX_FAILURES);
+	const queue = [];
+	const pending = /* @__PURE__ */ new Set();
+	const exportedRuns = /* @__PURE__ */ new Set();
+	const failures = [];
+	let active = 0;
+	let exported = 0;
+	let failed = 0;
+	let dropped = 0;
+	let unreported = 0;
+	const rememberRun = (runId) => {
+		if (exportedRuns.has(runId)) return false;
+		if (exportedRuns.size >= DEDUPE_WINDOW) {
+			const oldest = exportedRuns.values().next().value;
+			if (oldest !== void 0) exportedRuns.delete(oldest);
+		}
+		exportedRuns.add(runId);
+		return true;
+	};
+	const recordFailure = (error) => {
+		failed += 1;
+		unreported += 1;
+		failures.push(error);
+		if (failures.length > maxFailures) failures.shift();
+	};
+	const startNext = () => {
+		while (active < concurrency && queue.length) {
+			const task = queue.shift();
+			active += 1;
+			let job;
+			job = Promise.resolve().then(task).then(() => {
+				exported += 1;
+			}, (error) => {
+				recordFailure(error);
+			}).finally(() => {
+				active -= 1;
+				pending.delete(job);
+				startNext();
+			});
+			pending.add(job);
+		}
+	};
+	return {
+		timeoutMs,
+		enqueue(runId, task) {
+			if (!rememberRun(runId)) return;
+			if (queue.length >= maxQueued) {
+				dropped += 1;
+				recordFailure(/* @__PURE__ */ new Error(`Observability export queue is full; trace for run "${runId}" was dropped`));
+				return;
+			}
+			queue.push(task);
+			startNext();
+		},
+		async flush(flushOptions) {
+			const timeout = flushOptions?.timeoutMs;
+			if (timeout !== void 0 && (typeof timeout !== "number" || !Number.isSafeInteger(timeout) || timeout <= 0)) throw new Error("Observability flush timeoutMs must be a positive safe integer");
+			const drain = async () => {
+				while (pending.size) await Promise.all([...pending]);
+			};
+			if (timeout === void 0) await drain();
+			else await withTimeout(drain(), timeout, `Observability flush timed out after ${timeout}ms`);
+			if (unreported === 0) return;
+			const errors = [...failures];
+			const omitted = unreported - failures.length;
+			if (omitted > 0) errors.push(/* @__PURE__ */ new Error(`${omitted} additional export failures were not retained`));
+			failures.length = 0;
+			unreported = 0;
+			throw new AggregateError(errors, `Observability export failed (${failed} failure(s), ${dropped} dropped trace(s))`);
+		},
+		stats() {
+			return {
+				exported,
+				failed,
+				dropped,
+				queued: queue.length,
+				active,
+				retainedFailures: failures.length
+			};
+		}
+	};
+}
 //#endregion
 //#region src/internal/run-core.ts
 /**
@@ -2347,11 +2764,25 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 	const activeSessions = /* @__PURE__ */ new Set();
 	const internalRuns = /* @__PURE__ */ new Set();
 	const pendingTenantStarts = /* @__PURE__ */ new Map();
+	const idleWaiters = /* @__PURE__ */ new Set();
 	let pendingStarts = 0;
-	const pendingExports = /* @__PURE__ */ new Set();
-	const exportFailures = [];
-	const exportedRuns = /* @__PURE__ */ new Set();
+	let closed = false;
+	const traceExports = createTraceExportQueue(observability?.exportQueue);
 	const runtimeOwner = crypto.randomUUID();
+	const isIdle = () => activeRuns.size === 0 && activeWorkflowRuns.size === 0;
+	const notifyIdle = () => {
+		if (!isIdle()) return;
+		for (const resolve of idleWaiters) resolve();
+		idleWaiters.clear();
+	};
+	const waitForIdle = () => {
+		if (isIdle()) return Promise.resolve();
+		return new Promise((resolve) => idleWaiters.add(resolve));
+	};
+	const closeAdmission = () => {
+		closed = true;
+	};
+	const isClosed = () => closed;
 	const unboundedEvents = /* @__PURE__ */ new Set([
 		"run.started",
 		"run.recovered",
@@ -2403,6 +2834,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		return count;
 	};
 	const admitStart = async (kind, name, request) => {
+		if (closed) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		if (!admission) return () => {};
 		const running = activeTopLevelRuns() + pendingStarts;
 		const tenant = tenantOf(request.context);
@@ -2459,35 +2891,30 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		}
 	};
 	const scheduleTraceExport = (event) => {
-		if (!observability || ![
-			"run.completed",
-			"run.failed",
-			"run.cancelled"
-		].includes(event.type) || exportedRuns.has(event.runId)) return;
-		exportedRuns.add(event.runId);
-		let task;
-		task = (async () => {
+		if (!observability || !isTraceTerminalEvent(event)) return;
+		traceExports.enqueue(event.runId, async () => {
 			const run = await runStore.getRun(event.runId);
 			if (!run) throw new Error(`Run "${event.runId}" does not exist`);
-			if (isCoordinatorRun(run)) return;
-			const agentRun = run;
-			let modelRegistryName = agentRun.modelName;
-			let modelMetadata = agentRun.modelMetadata;
-			if (modelRegistryName === void 0) {
-				const agent = agents.get(agentRun.agentName);
-				if (!agent) throw new Error(`Agent "${agentRun.agentName}" is not registered`);
-				const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(agentRun.id) : void 0;
-				const resolved = resolveModel(agent, checkpoint?.modelName);
-				modelRegistryName = resolved.modelName;
-				modelMetadata = resolved.model.metadata;
+			const coordinator = isCoordinatorRun(run);
+			let modelRegistryName;
+			let modelMetadata;
+			if (!coordinator) {
+				const agentRun = run;
+				modelRegistryName = agentRun.modelName;
+				modelMetadata = agentRun.modelMetadata;
+				if (modelRegistryName === void 0) {
+					const agent = agents.get(agentRun.agentName);
+					if (!agent) throw new Error(`Agent "${agentRun.agentName}" is not registered`);
+					const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(agentRun.id) : void 0;
+					const resolved = resolveModel(agent, checkpoint?.modelName);
+					modelRegistryName = resolved.modelName;
+					modelMetadata = resolved.model.metadata;
+				}
 			}
-			const trace = buildRunTrace(agentRun, await runStore.listEvents(agentRun.id), modelRegistryName, modelMetadata, observability);
-			const failures = (await Promise.allSettled(observability.exporters.map((exporter) => Promise.resolve().then(() => exporter.export(trace))))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
+			const trace = buildRunTrace(run, await runStore.listEvents(run.id), modelRegistryName, modelMetadata, observability);
+			const failures = (await Promise.allSettled(observability.exporters.map((exporter) => withTimeout(Promise.resolve().then(() => exporter.export(trace)), traceExports.timeoutMs, `Trace exporter timed out after ${traceExports.timeoutMs}ms for run "${run.id}"`)))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
 			if (failures.length) throw new AggregateError(failures, "Trace export failed");
-		})().catch((error) => {
-			exportFailures.push(error);
-		}).finally(() => pendingExports.delete(task));
-		pendingExports.add(task);
+		});
 	};
 	const notifyObserver = (event) => {
 		scheduleTraceExport(event);
@@ -2526,6 +2953,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		state.leaseTimer = void 0;
 		state.leaseExpiryTimer = void 0;
 		if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
+		notifyIdle();
 	};
 	const commit = async (state, options = {}) => {
 		if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "This run requires a DurableRunStore", state.run.id);
@@ -2615,6 +3043,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		state.leaseTimer = void 0;
 		state.leaseExpiryTimer = void 0;
 		if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
+		notifyIdle();
 	};
 	const commitWorkflow = async (state, update, events = []) => {
 		state.commitQueue = state.commitQueue.then(async () => {
@@ -2648,8 +3077,11 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		activeRuns,
 		activeWorkflowRuns,
 		activeSessions,
-		pendingExports,
-		exportFailures,
+		isIdle,
+		waitForIdle,
+		closeAdmission,
+		isClosed,
+		traceExports,
 		runtimeOwner,
 		admitStart,
 		markInternalRun,
@@ -2680,7 +3112,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 */
 function createWorkflowEngine(composition, core, agentEngine) {
 	const { models, agents, workflows, teams, runStore } = composition;
-	const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, admitStart, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
+	const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, isClosed, admitStart, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
 	const { prepareExecution, executeAgent, nextEvent, resumeAgent, cancelStoredAgent } = agentEngine;
 	const cancelWorkflowExecution = async (state, reason = cancellationReason(state.request.signal)) => {
 		if (state.leaseLost) {
@@ -2772,6 +3204,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
 		}
 	};
 	const prepareWorkflowExecutionBody = async (name, request, kind = "workflow") => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		const workflow = workflows.get(name);
 		const team = kind === "team" ? teams.get(name) : void 0;
 		if (!workflow || kind === "team" && !team || kind === "workflow" && teams.has(name)) throw new FevexRunError(kind === "team" ? "TEAM_NOT_FOUND" : "WORKFLOW_NOT_FOUND", `${kind === "team" ? "Team" : "Workflow"} "${name}" is not registered`);
@@ -3072,7 +3505,11 @@ function createWorkflowEngine(composition, core, agentEngine) {
 					context: mergeExecutionContext(state.request.context, request.context),
 					limits: combineLimits(request.limits, remainingWorkflowLimits(state.checkpoint.limits, state.checkpoint.budget)),
 					signal: childSignal
-				}, { admit: false });
+				}, {
+					admit: false,
+					parentRunId: state.run.id,
+					rootRunId: state.run.rootRunId ?? state.run.id
+				});
 				const action = options?.metadata?.teamAction === "handoff" ? "handoff" : "delegate";
 				const lifecycleEvents = [];
 				if (isTeamRun(state.run) && action === "handoff") lifecycleEvents.push(createEvent(state, "team.handoff.created", {
@@ -3084,11 +3521,13 @@ function createWorkflowEngine(composition, core, agentEngine) {
 				const started = createCoordinatorEvent(state, "workflow.step.started", {
 					stepId,
 					kind: "agent",
-					agentName
+					agentName,
+					childRunId: childState.run.id
 				}, "team.agent.assigned", {
 					delegationId: stepId,
 					agentName,
 					action,
+					childRunId: childState.run.id,
 					...typeof options?.metadata?.expectedOutput === "string" ? { expectedOutput: options.metadata.expectedOutput } : {},
 					...Array.isArray(options?.metadata?.constraints) ? { constraints: options.metadata.constraints } : {}
 				});
@@ -3772,24 +4211,76 @@ function createWorkflowEngine(composition, core, agentEngine) {
 }
 //#endregion
 //#region src/internal/runtime.ts
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 3e4;
+const SHUTDOWN_CLEANUP_MS = 5e3;
 function createRuntime(composition) {
 	const { models, agents, workflows, teams, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, policies } = composition;
 	const core = createRunCore(composition);
-	const { activeRuns, activeWorkflowRuns, activeSessions, pendingExports, exportFailures, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
+	const { activeRuns, activeWorkflowRuns, activeSessions, isIdle, waitForIdle, closeAdmission, isClosed, traceExports, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
 	const agentEngine = createAgentEngine(composition, core);
 	const { prepareExecution, executeAgent, nextEvent, drainExecution, resumeAgent, cancelStoredAgent } = agentEngine;
 	const { prepareWorkflowExecution, executeWorkflow, nextWorkflowEvent, drainWorkflowExecution, resumeWorkflow, cancelWorkflowExecution, cancelStoredWorkflow } = createWorkflowEngine(composition, core, agentEngine);
 	const resumeRun = async (runId, resolution) => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs", runId);
 		const run = await runStore.getRun(runId);
 		if (run && isCoordinatorRun(run)) return resumeWorkflow(runId, resolution);
 		return resumeAgent(runId, resolution);
 	};
 	const recoverRun = async (runId, actor) => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs", runId);
 		const run = await runStore.getRun(runId);
 		if (!run) throw new FevexRunError("RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be recovered`, runId);
 		if (isCoordinatorRun(run)) return resumeWorkflow(runId, void 0, actor);
 		return resumeAgent(runId, void 0, false, actor);
 	};
+	const waitForIdleUntil = async (deadline) => {
+		if (isIdle()) return true;
+		const remaining = deadline - Date.now();
+		if (remaining <= 0) return false;
+		return new Promise((resolve) => {
+			let settled = false;
+			let timer;
+			const finish = (idle) => {
+				if (settled) return;
+				settled = true;
+				if (timer) clearTimeout(timer);
+				resolve(idle);
+			};
+			timer = setTimeout(() => finish(false), remaining);
+			waitForIdle().then(() => finish(true));
+		});
+	};
+	const shutdown = async (options = {}) => {
+		const timeoutMs = options.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS;
+		if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0) throw new TypeError("Shutdown timeoutMs must be a non-negative safe integer");
+		closeAdmission();
+		const deadline = Date.now() + timeoutMs;
+		let cleanupDeadline = deadline;
+		let timedOut = false;
+		const cancelled = /* @__PURE__ */ new Set();
+		if (!await waitForIdleUntil(deadline)) {
+			timedOut = true;
+			cleanupDeadline = Date.now() + SHUTDOWN_CLEANUP_MS;
+			for (const { run } of activeWorkflowRuns.values()) cancelled.add(run.id);
+			for (const { run } of activeRuns.values()) cancelled.add(run.id);
+			for (const state of [...activeWorkflowRuns.values(), ...activeRuns.values()]) if (!state.request.signal.aborted) state.controller.abort(new DOMException("Run cancelled", "AbortError"));
+			for (const state of [...activeWorkflowRuns.values()]) if (!state.advancing) await cancelWorkflowExecution(state).catch(() => {});
+			for (const state of [...activeRuns.values()]) if (!state.advancing) await cancelExecution(state).catch(() => {});
+			await waitForIdleUntil(cleanupDeadline);
+		}
+		let exportError;
+		try {
+			await traceExports.flush({ timeoutMs: Math.max(1, cleanupDeadline - Date.now()) });
+		} catch (error) {
+			exportError = error;
+		}
+		return {
+			cancelledRunIds: [...cancelled],
+			timedOut,
+			...exportError === void 0 ? {} : { exportError },
+			observability: traceExports.stats()
+		};
+	};
 	return {
 		async startAgent(name, request) {
 			const state = await prepareExecution(name, request);
@@ -3921,13 +4412,13 @@ function createRuntime(composition) {
 			await runStore.saveSession(session);
 			return structuredClone(session);
 		},
-		async flushObservability() {
-			while (pendingExports.size) await Promise.all([...pendingExports]);
-			if (exportFailures.length) {
-				const failures = exportFailures.splice(0);
-				throw new AggregateError(failures, "Observability export failed");
-			}
-		}
+		flushObservability(options) {
+			return traceExports.flush(options);
+		},
+		getObservabilityStats() {
+			return traceExports.stats();
+		},
+		shutdown
 	};
 }
 //#endregion
@@ -3942,4 +4433,4 @@ function createFevex(config) {
 	return createRuntime(createComposition(config));
 }
 //#endregion
-export { ArtifactError, CatalogedArtifactStore, ChannelError, FEVEX_JSON_SCHEMA_PROFILE_VERSION, FevexConfigurationError, FevexRunError, FileSystemArtifactStore, InMemoryArtifactCatalog, InMemoryArtifactStore, InMemoryMemoryStore, InMemoryRunStore, IntegrationError, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, S3ArtifactStore, SCHEMA_NOT_TRANSPORTABLE, SandboxError, artifactTenantId, assertArtifactId, compileFevexJsonSchema, createFevex, createLocalSandbox, createMemoryContextProvider, defineAgent, defineConnection, defineContextProvider, definePolicy, defineSkill, defineTeam, defineTool, defineWorkflow, deliverChannelRun, handleChannelInput, isDurableRunStore, isRetainableRunStore, sandboxResultToJson, toToolSpec, validateFevexJsonSchemaProfile, validateListEventsOptions, validatePurgeRunsOptions };
+export { ArtifactError, CatalogedArtifactStore, ChannelError, FEVEX_JSON_SCHEMA_PROFILE_VERSION, FevexConfigurationError, FevexRunError, FileSystemArtifactStore, InMemoryArtifactCatalog, InMemoryArtifactStore, InMemoryMemoryStore, InMemoryRunStore, IntegrationError, PROVIDER_REASONING_UNSUPPORTED, PROVIDER_SCHEMA_UNSUPPORTED, RunPausedError, S3ArtifactStore, SCHEMA_NOT_TRANSPORTABLE, SandboxError, artifactTenantId, assertArtifactId, compileFevexJsonSchema, createAgentsMdContextProvider, createFevex, createLocalSandbox, createMemoryContextProvider, defineAgent, defineConnection, defineContextProvider, definePolicy, defineSkill, defineTeam, defineTool, defineWorkflow, deliverChannelRun, handleChannelInput, isDurableRunStore, isProgressiveSkillProvider, isRetainableRunStore, loadSkillFile, loadSkillFiles, sandboxResultToJson, toToolSpec, validateFevexJsonSchemaProfile, validateListEventsOptions, validatePurgeRunsOptions };