@fevex/core 0.1.0-alpha.5 → 0.1.0-alpha.6

package/dist/index.mjs

@@ -332,6 +332,18 @@ function createComposition(config) {
 				"error.message"
 			].includes(kind)) && (content.redact === void 0 || typeof content.redact === "function"), "INVALID_CONFIG", "Fevex config \"observability.content\" is invalid");
 		}
+		if (config.observability.exportQueue !== void 0) {
+			const queue = config.observability.exportQueue;
+			assertConfiguration(isRecord(queue) && [
+				"concurrency",
+				"maxQueued",
+				"timeoutMs",
+				"maxFailures"
+			].every((key) => {
+				const value = queue[key];
+				return value === void 0 || typeof value === "number" && Number.isSafeInteger(value) && value > 0;
+			}), "INVALID_CONFIG", "Fevex config \"observability.exportQueue\" must contain positive safe integers");
+		}
 	}
 	const models = /* @__PURE__ */ new Map();
 	const agents = /* @__PURE__ */ new Map();
@@ -624,7 +636,8 @@ function createComposition(config) {
 			...config.observability.content ? { content: {
 				include: [...config.observability.content.include],
 				...config.observability.content.redact ? { redact: config.observability.content.redact } : {}
-			} } : {}
+			} } : {},
+			...config.observability.exportQueue ? { exportQueue: { ...config.observability.exportQueue } } : {}
 		}
 	};
 }
@@ -890,8 +903,9 @@ function addWorkflowBudget(first, second) {
 */
 function createAgentEngine(composition, core) {
 	const { models, agents, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, artifacts, policies } = composition;
-	const { activeRuns, activeSessions, runtimeOwner, admitStart, markInternalRun, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
-	const prepareExecutionBody = async (name, request, internal = false) => {
+	const { activeRuns, activeSessions, runtimeOwner, isClosed, admitStart, markInternalRun, resolveModel, resolveCheckpointModel, notifyObserver, createEvent, emitEvent, releaseExecution, commit, startLease, cancelExecution, authorize } = core;
+	const prepareExecutionBody = async (name, request, internal = false, correlation = {}) => {
+		if (!internal && isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		const agent = agents.get(name);
 		if (!agent) throw new FevexRunError("AGENT_NOT_FOUND", `Agent "${name}" is not registered`);
 		const { modelName, model } = resolveModel(agent, request.model);
@@ -920,6 +934,8 @@ function createAgentEngine(composition, core) {
 			agentName: name,
 			modelName,
 			...model.metadata === void 0 ? {} : { modelMetadata: structuredClone(model.metadata) },
+			...correlation.parentRunId === void 0 ? {} : { parentRunId: correlation.parentRunId },
+			...correlation.rootRunId === void 0 ? {} : { rootRunId: correlation.rootRunId },
 			status: "running",
 			revision: 0,
 			createdAt: now,
@@ -1010,7 +1026,10 @@ function createAgentEngine(composition, core) {
 	const prepareExecution = async (name, request, options = {}) => {
 		const release = options.admit === false ? void 0 : await admitStart("agent", name, request);
 		try {
-			return await prepareExecutionBody(name, request, options.admit === false);
+			return await prepareExecutionBody(name, request, options.admit === false, {
+				...options.parentRunId === void 0 ? {} : { parentRunId: options.parentRunId },
+				...options.rootRunId === void 0 ? {} : { rootRunId: options.rootRunId }
+			});
 		} finally {
 			release?.();
 		}
@@ -2053,16 +2072,31 @@ function scopedArtifacts(store, options) {
 const TERMINAL_TYPES = /* @__PURE__ */ new Set([
 	"run.completed",
 	"run.failed",
-	"run.cancelled"
+	"run.cancelled",
+	"workflow.run.completed",
+	"workflow.run.failed",
+	"workflow.run.cancelled",
+	"team.run.completed",
+	"team.run.failed",
+	"team.run.cancelled"
 ]);
 const SENSITIVE_KEY = /api[-_]?key|authorization|cookie|credential|password|secret|token/i;
+function isTraceTerminalEvent(event) {
+	return TERMINAL_TYPES.has(event.type);
+}
 function durationMs(startedAt, endedAt) {
 	return Math.max(0, Date.parse(endedAt) - Date.parse(startedAt));
 }
 function traceStatus(event) {
-	if (event.type === "run.completed") return "completed";
-	if (event.type === "run.cancelled") return "cancelled";
-	return "failed";
+	switch (event.type) {
+		case "run.completed":
+		case "workflow.run.completed":
+		case "team.run.completed": return "completed";
+		case "run.cancelled":
+		case "workflow.run.cancelled":
+		case "team.run.cancelled": return "cancelled";
+		default: return "failed";
+	}
 }
 function redactKeys(value) {
 	if (Array.isArray(value)) return value.map(redactKeys);
@@ -2169,20 +2203,74 @@ function annotation(event) {
 				toolName: event.payload.toolName
 			}
 		};
+		case "workflow.run.paused":
+		case "team.run.paused": {
+			const stepId = event.type === "workflow.run.paused" ? event.payload.stepId : event.payload.delegationId;
+			return {
+				name: event.type,
+				timestamp: event.timestamp,
+				attributes: {
+					stepId,
+					reason: event.payload.reason,
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				}
+			};
+		}
+		case "workflow.run.resumed":
+		case "team.run.resumed": return {
+			name: event.type,
+			timestamp: event.timestamp
+		};
+		case "team.handoff.created": return {
+			name: event.type,
+			timestamp: event.timestamp,
+			attributes: {
+				delegationId: event.payload.delegationId,
+				from: event.payload.from,
+				to: event.payload.to,
+				reason: event.payload.reason
+			}
+		};
 		default: return;
 	}
 }
-function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
-	const events = [...rawEvents].sort((left, right) => left.sequence - right.sequence);
-	const terminal = [...events].reverse().find((event) => TERMINAL_TYPES.has(event.type));
-	if (!terminal) throw new TypeError(`Run "${run.id}" has no terminal event`);
-	const started = events.find((event) => event.type === "run.started") ?? events[0];
-	if (!started) throw new TypeError(`Run "${run.id}" has no events`);
+function terminalContent(options, terminal) {
+	if (terminal.type === "run.completed" || terminal.type === "workflow.run.completed" || terminal.type === "team.run.completed") {
+		const output = content(options, "run.output", terminal.payload.output);
+		return output ? [output] : [];
+	}
+	if (terminal.type === "run.failed" || terminal.type === "workflow.run.failed" || terminal.type === "team.run.failed") {
+		const error = content(options, "error.message", terminal.payload.error);
+		return error ? [error] : [];
+	}
+	return [];
+}
+function sortedEvents(rawEvents) {
+	return [...rawEvents].sort((left, right) => left.sequence - right.sequence);
+}
+function terminalEvent(runId, events) {
+	const terminal = [...events].reverse().find((event) => isTraceTerminalEvent(event));
+	if (!terminal) throw new TypeError(`Run "${runId}" has no terminal event`);
+	return terminal;
+}
+function startEvent(events) {
+	const started = events.find((event) => event.type === "run.started" || event.type === "workflow.run.started" || event.type === "team.run.started") ?? events[0];
+	if (!started) throw new TypeError("Run has no events");
+	return started;
+}
+function traceKind(run) {
+	if (isTeamRun(run)) return "team";
+	return run.kind === "workflow" ? "workflow" : "agent";
+}
+function buildAgentTrace(run, events, modelRegistryName, model, options) {
+	const terminal = terminalEvent(run.id, events);
+	const started = startEvent(events);
 	const status = traceStatus(terminal);
 	const annotations = [];
 	const spans = [];
 	const openModels = /* @__PURE__ */ new Map();
 	const openTools = /* @__PURE__ */ new Map();
+	const modelSpanByStep = /* @__PURE__ */ new Map();
 	const previousUsage = {};
 	const closeModel = (step, endedAt, spanStatus) => {
 		const open = openModels.get(step);
@@ -2213,6 +2301,7 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		const spanCost = costFor(options, modelRegistryName, model, usage);
 		if (spanCost) span.cost = spanCost;
 		spans.push(span);
+		modelSpanByStep.set(step, span.id);
 	};
 	const findOpenTool = (toolCallId) => {
 		const entries = [...openTools.entries()];
@@ -2232,7 +2321,7 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		const errorContent = error ? content(options, "error.message", error) : void 0;
 		spans.push({
 			id: event.id,
-			parentId: run.id,
+			parentId: modelSpanByStep.get(event.payload.step) ?? run.id,
 			kind: "tool",
 			name: event.payload.toolName,
 			startedAt: event.timestamp,
@@ -2297,19 +2386,15 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 	for (const open of [...openTools.values()]) if (open.event.type === "tool.started") closeTool(open.event.payload.toolCallId, terminal.timestamp, status);
 	const finalUsage = run.usage ?? previousUsage;
 	const totalCost = costFor(options, modelRegistryName, model, finalUsage);
-	const traceContent = [];
-	if (terminal.type === "run.completed") {
-		const output = content(options, "run.output", terminal.payload.output);
-		if (output) traceContent.push(output);
-	} else if (terminal.type === "run.failed") {
-		const error = content(options, "error.message", terminal.payload.error);
-		if (error) traceContent.push(error);
-	}
+	const traceContent = terminalContent(options, terminal);
 	return {
 		schemaVersion: "1",
+		kind: "agent",
 		traceId: run.id,
 		runId: run.id,
 		sessionId: run.sessionId,
+		...run.parentRunId === void 0 ? {} : { parentRunId: run.parentRunId },
+		rootRunId: run.rootRunId ?? run.parentRunId ?? run.id,
 		agentName: run.agentName,
 		modelRegistryName,
 		...model ? { model: { ...model } } : {},
@@ -2329,6 +2414,249 @@ function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
 		...traceContent.length ? { content: traceContent } : {}
 	};
 }
+function buildCoordinatorTrace(run, events, options) {
+	const terminal = terminalEvent(run.id, events);
+	const started = startEvent(events);
+	const status = traceStatus(terminal);
+	const annotations = [];
+	const spans = [];
+	const openSteps = /* @__PURE__ */ new Map();
+	const stepKey = (scope, stepId) => `${scope}:${stepId}`;
+	const modelCallCount = events.filter(({ type }) => type === "model.completed").length;
+	const toolCallCount = events.filter(({ type }) => type === "tool.completed" || type === "tool.failed").length;
+	const openStep = (key, event, name, attributes) => {
+		openSteps.set(key, {
+			event,
+			name,
+			attributes
+		});
+	};
+	const closeStep = (key, endedAt, spanStatus, error) => {
+		const open = openSteps.get(key);
+		if (!open) return;
+		openSteps.delete(key);
+		const errorContent = error ? content(options, "error.message", error) : void 0;
+		spans.push({
+			id: open.event.id,
+			parentId: run.id,
+			kind: "step",
+			name: open.name,
+			startedAt: open.event.timestamp,
+			endedAt,
+			durationMs: durationMs(open.event.timestamp, endedAt),
+			status: spanStatus,
+			attributes: open.attributes,
+			annotations: [],
+			...errorContent ? { content: [errorContent] } : {}
+		});
+	};
+	for (const event of events) {
+		const item = annotation(event);
+		if (item) annotations.push(item);
+		switch (event.type) {
+			case "workflow.step.started":
+				openStep(stepKey("workflow", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: event.payload.kind,
+					...event.payload.agentName === void 0 ? {} : { agentName: event.payload.agentName },
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				});
+				break;
+			case "workflow.step.completed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "workflow.step.failed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "failed", event.payload.error);
+				break;
+			case "workflow.wait.started":
+				openStep(stepKey("workflow", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: "wait",
+					waitKind: event.payload.kind,
+					...event.payload.resumeAt === void 0 ? {} : { resumeAt: event.payload.resumeAt },
+					...event.payload.eventName === void 0 ? {} : { eventName: event.payload.eventName }
+				});
+				break;
+			case "workflow.wait.completed":
+				closeStep(stepKey("workflow", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "team.agent.assigned":
+				openStep(stepKey("team", event.payload.delegationId), event, event.payload.delegationId, {
+					stepId: event.payload.delegationId,
+					stepKind: "agent",
+					agentName: event.payload.agentName,
+					action: event.payload.action,
+					...event.payload.expectedOutput === void 0 ? {} : { expectedOutput: event.payload.expectedOutput },
+					...event.payload.constraints === void 0 ? {} : { constraints: event.payload.constraints },
+					...event.payload.childRunId === void 0 ? {} : { childRunId: event.payload.childRunId }
+				});
+				break;
+			case "team.task.completed":
+				closeStep(stepKey("team", event.payload.delegationId), event.timestamp, "completed");
+				break;
+			case "team.task.failed":
+				closeStep(stepKey("team", event.payload.delegationId), event.timestamp, "failed", event.payload.error);
+				break;
+			case "team.merge.started":
+				openStep(stepKey("team", event.payload.stepId), event, event.payload.stepId, {
+					stepId: event.payload.stepId,
+					stepKind: "merge",
+					width: event.payload.width
+				});
+				break;
+			case "team.merge.completed":
+				closeStep(stepKey("team", event.payload.stepId), event.timestamp, "completed");
+				break;
+			case "team.merge.failed":
+				closeStep(stepKey("team", event.payload.stepId), event.timestamp, "failed", event.payload.error);
+				break;
+		}
+	}
+	for (const key of [...openSteps.keys()]) closeStep(key, terminal.timestamp, status);
+	const traceContent = terminalContent(options, terminal);
+	return {
+		schemaVersion: "1",
+		kind: traceKind(run),
+		traceId: run.id,
+		runId: run.id,
+		sessionId: run.sessionId,
+		...run.parentRunId === void 0 ? {} : { parentRunId: run.parentRunId },
+		rootRunId: run.rootRunId ?? run.parentRunId ?? run.id,
+		agentName: coordinatorName(run),
+		startedAt: started.timestamp,
+		endedAt: terminal.timestamp,
+		durationMs: durationMs(started.timestamp, terminal.timestamp),
+		status,
+		attributes: {
+			eventCount: events.length,
+			stepCount: spans.filter(({ kind }) => kind === "step").length,
+			modelCallCount,
+			toolCallCount
+		},
+		annotations,
+		spans,
+		...run.usage && Object.keys(run.usage).length ? { usage: { ...run.usage } } : {},
+		...traceContent.length ? { content: traceContent } : {}
+	};
+}
+function buildRunTrace(run, rawEvents, modelRegistryName, model, options) {
+	const events = sortedEvents(rawEvents);
+	if (isCoordinatorRun(run)) return buildCoordinatorTrace(run, events, options);
+	if (!modelRegistryName) throw new TypeError(`Run "${run.id}" has no model registry name`);
+	return buildAgentTrace(run, events, modelRegistryName, model, options);
+}
+//#endregion
+//#region src/internal/trace-export.ts
+const DEFAULT_CONCURRENCY = 4;
+const DEFAULT_MAX_QUEUED = 100;
+const DEFAULT_TIMEOUT_MS = 3e4;
+const DEFAULT_MAX_FAILURES = 100;
+const DEDUPE_WINDOW = 1e4;
+function withTimeout(promise, timeoutMs, message) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+		promise.then((value) => {
+			clearTimeout(timer);
+			resolve(value);
+		}, (error) => {
+			clearTimeout(timer);
+			reject(error);
+		});
+	});
+}
+function withDefault(value, fallback) {
+	return value ?? fallback;
+}
+/**
+* Bounded, deduplicated trace export queue. Jobs include building the trace
+* from the store, so a slow store or exporter cannot grow memory without limit.
+*/
+function createTraceExportQueue(options = {}) {
+	const concurrency = withDefault(options.concurrency, DEFAULT_CONCURRENCY);
+	const maxQueued = withDefault(options.maxQueued, DEFAULT_MAX_QUEUED);
+	const timeoutMs = withDefault(options.timeoutMs, DEFAULT_TIMEOUT_MS);
+	const maxFailures = withDefault(options.maxFailures, DEFAULT_MAX_FAILURES);
+	const queue = [];
+	const pending = /* @__PURE__ */ new Set();
+	const exportedRuns = /* @__PURE__ */ new Set();
+	const failures = [];
+	let active = 0;
+	let exported = 0;
+	let failed = 0;
+	let dropped = 0;
+	let unreported = 0;
+	const rememberRun = (runId) => {
+		if (exportedRuns.has(runId)) return false;
+		if (exportedRuns.size >= DEDUPE_WINDOW) {
+			const oldest = exportedRuns.values().next().value;
+			if (oldest !== void 0) exportedRuns.delete(oldest);
+		}
+		exportedRuns.add(runId);
+		return true;
+	};
+	const recordFailure = (error) => {
+		failed += 1;
+		unreported += 1;
+		failures.push(error);
+		if (failures.length > maxFailures) failures.shift();
+	};
+	const startNext = () => {
+		while (active < concurrency && queue.length) {
+			const task = queue.shift();
+			active += 1;
+			let job;
+			job = Promise.resolve().then(task).then(() => {
+				exported += 1;
+			}, (error) => {
+				recordFailure(error);
+			}).finally(() => {
+				active -= 1;
+				pending.delete(job);
+				startNext();
+			});
+			pending.add(job);
+		}
+	};
+	return {
+		timeoutMs,
+		enqueue(runId, task) {
+			if (!rememberRun(runId)) return;
+			if (queue.length >= maxQueued) {
+				dropped += 1;
+				recordFailure(/* @__PURE__ */ new Error(`Observability export queue is full; trace for run "${runId}" was dropped`));
+				return;
+			}
+			queue.push(task);
+			startNext();
+		},
+		async flush(flushOptions) {
+			const timeout = flushOptions?.timeoutMs;
+			if (timeout !== void 0 && (typeof timeout !== "number" || !Number.isSafeInteger(timeout) || timeout <= 0)) throw new Error("Observability flush timeoutMs must be a positive safe integer");
+			const drain = async () => {
+				while (pending.size) await Promise.all([...pending]);
+			};
+			if (timeout === void 0) await drain();
+			else await withTimeout(drain(), timeout, `Observability flush timed out after ${timeout}ms`);
+			if (unreported === 0) return;
+			const errors = [...failures];
+			const omitted = unreported - failures.length;
+			if (omitted > 0) errors.push(/* @__PURE__ */ new Error(`${omitted} additional export failures were not retained`));
+			failures.length = 0;
+			unreported = 0;
+			throw new AggregateError(errors, `Observability export failed (${failed} failure(s), ${dropped} dropped trace(s))`);
+		},
+		stats() {
+			return {
+				exported,
+				failed,
+				dropped,
+				queued: queue.length,
+				active,
+				retainedFailures: failures.length
+			};
+		}
+	};
+}
 //#endregion
 //#region src/internal/run-core.ts
 /**
@@ -2347,11 +2675,25 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 	const activeSessions = /* @__PURE__ */ new Set();
 	const internalRuns = /* @__PURE__ */ new Set();
 	const pendingTenantStarts = /* @__PURE__ */ new Map();
+	const idleWaiters = /* @__PURE__ */ new Set();
 	let pendingStarts = 0;
-	const pendingExports = /* @__PURE__ */ new Set();
-	const exportFailures = [];
-	const exportedRuns = /* @__PURE__ */ new Set();
+	let closed = false;
+	const traceExports = createTraceExportQueue(observability?.exportQueue);
 	const runtimeOwner = crypto.randomUUID();
+	const isIdle = () => activeRuns.size === 0 && activeWorkflowRuns.size === 0;
+	const notifyIdle = () => {
+		if (!isIdle()) return;
+		for (const resolve of idleWaiters) resolve();
+		idleWaiters.clear();
+	};
+	const waitForIdle = () => {
+		if (isIdle()) return Promise.resolve();
+		return new Promise((resolve) => idleWaiters.add(resolve));
+	};
+	const closeAdmission = () => {
+		closed = true;
+	};
+	const isClosed = () => closed;
 	const unboundedEvents = /* @__PURE__ */ new Set([
 		"run.started",
 		"run.recovered",
@@ -2403,6 +2745,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		return count;
 	};
 	const admitStart = async (kind, name, request) => {
+		if (closed) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		if (!admission) return () => {};
 		const running = activeTopLevelRuns() + pendingStarts;
 		const tenant = tenantOf(request.context);
@@ -2459,35 +2802,30 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		}
 	};
 	const scheduleTraceExport = (event) => {
-		if (!observability || ![
-			"run.completed",
-			"run.failed",
-			"run.cancelled"
-		].includes(event.type) || exportedRuns.has(event.runId)) return;
-		exportedRuns.add(event.runId);
-		let task;
-		task = (async () => {
+		if (!observability || !isTraceTerminalEvent(event)) return;
+		traceExports.enqueue(event.runId, async () => {
 			const run = await runStore.getRun(event.runId);
 			if (!run) throw new Error(`Run "${event.runId}" does not exist`);
-			if (isCoordinatorRun(run)) return;
-			const agentRun = run;
-			let modelRegistryName = agentRun.modelName;
-			let modelMetadata = agentRun.modelMetadata;
-			if (modelRegistryName === void 0) {
-				const agent = agents.get(agentRun.agentName);
-				if (!agent) throw new Error(`Agent "${agentRun.agentName}" is not registered`);
-				const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(agentRun.id) : void 0;
-				const resolved = resolveModel(agent, checkpoint?.modelName);
-				modelRegistryName = resolved.modelName;
-				modelMetadata = resolved.model.metadata;
+			const coordinator = isCoordinatorRun(run);
+			let modelRegistryName;
+			let modelMetadata;
+			if (!coordinator) {
+				const agentRun = run;
+				modelRegistryName = agentRun.modelName;
+				modelMetadata = agentRun.modelMetadata;
+				if (modelRegistryName === void 0) {
+					const agent = agents.get(agentRun.agentName);
+					if (!agent) throw new Error(`Agent "${agentRun.agentName}" is not registered`);
+					const checkpoint = isDurableRunStore(runStore) ? await runStore.getCheckpoint(agentRun.id) : void 0;
+					const resolved = resolveModel(agent, checkpoint?.modelName);
+					modelRegistryName = resolved.modelName;
+					modelMetadata = resolved.model.metadata;
+				}
 			}
-			const trace = buildRunTrace(agentRun, await runStore.listEvents(agentRun.id), modelRegistryName, modelMetadata, observability);
-			const failures = (await Promise.allSettled(observability.exporters.map((exporter) => Promise.resolve().then(() => exporter.export(trace))))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
+			const trace = buildRunTrace(run, await runStore.listEvents(run.id), modelRegistryName, modelMetadata, observability);
+			const failures = (await Promise.allSettled(observability.exporters.map((exporter) => withTimeout(Promise.resolve().then(() => exporter.export(trace)), traceExports.timeoutMs, `Trace exporter timed out after ${traceExports.timeoutMs}ms for run "${run.id}"`)))).filter((result) => result.status === "rejected").map(({ reason }) => reason);
 			if (failures.length) throw new AggregateError(failures, "Trace export failed");
-		})().catch((error) => {
-			exportFailures.push(error);
-		}).finally(() => pendingExports.delete(task));
-		pendingExports.add(task);
+		});
 	};
 	const notifyObserver = (event) => {
 		scheduleTraceExport(event);
@@ -2526,6 +2864,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		state.leaseTimer = void 0;
 		state.leaseExpiryTimer = void 0;
 		if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
+		notifyIdle();
 	};
 	const commit = async (state, options = {}) => {
 		if (!isDurableRunStore(runStore)) throw new FevexRunError("DURABLE_STORE_REQUIRED", "This run requires a DurableRunStore", state.run.id);
@@ -2615,6 +2954,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		state.leaseTimer = void 0;
 		state.leaseExpiryTimer = void 0;
 		if (state.lease && isDurableRunStore(runStore)) await runStore.releaseLease(state.run.id, state.lease.ownerId, state.lease.generation).catch(() => {});
+		notifyIdle();
 	};
 	const commitWorkflow = async (state, update, events = []) => {
 		state.commitQueue = state.commitQueue.then(async () => {
@@ -2648,8 +2988,11 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 		activeRuns,
 		activeWorkflowRuns,
 		activeSessions,
-		pendingExports,
-		exportFailures,
+		isIdle,
+		waitForIdle,
+		closeAdmission,
+		isClosed,
+		traceExports,
 		runtimeOwner,
 		admitStart,
 		markInternalRun,
@@ -2680,7 +3023,7 @@ function createRunCore({ models, agents, tools, runStore, policies, onEvent, obs
 */
 function createWorkflowEngine(composition, core, agentEngine) {
 	const { models, agents, workflows, teams, runStore } = composition;
-	const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, admitStart, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
+	const { activeRuns, activeWorkflowRuns, activeSessions, runtimeOwner, isClosed, admitStart, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, releaseWorkflowExecution, commitWorkflow } = core;
 	const { prepareExecution, executeAgent, nextEvent, resumeAgent, cancelStoredAgent } = agentEngine;
 	const cancelWorkflowExecution = async (state, reason = cancellationReason(state.request.signal)) => {
 		if (state.leaseLost) {
@@ -2772,6 +3115,7 @@ function createWorkflowEngine(composition, core, agentEngine) {
 		}
 	};
 	const prepareWorkflowExecutionBody = async (name, request, kind = "workflow") => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs");
 		const workflow = workflows.get(name);
 		const team = kind === "team" ? teams.get(name) : void 0;
 		if (!workflow || kind === "team" && !team || kind === "workflow" && teams.has(name)) throw new FevexRunError(kind === "team" ? "TEAM_NOT_FOUND" : "WORKFLOW_NOT_FOUND", `${kind === "team" ? "Team" : "Workflow"} "${name}" is not registered`);
@@ -3072,7 +3416,11 @@ function createWorkflowEngine(composition, core, agentEngine) {
 					context: mergeExecutionContext(state.request.context, request.context),
 					limits: combineLimits(request.limits, remainingWorkflowLimits(state.checkpoint.limits, state.checkpoint.budget)),
 					signal: childSignal
-				}, { admit: false });
+				}, {
+					admit: false,
+					parentRunId: state.run.id,
+					rootRunId: state.run.rootRunId ?? state.run.id
+				});
 				const action = options?.metadata?.teamAction === "handoff" ? "handoff" : "delegate";
 				const lifecycleEvents = [];
 				if (isTeamRun(state.run) && action === "handoff") lifecycleEvents.push(createEvent(state, "team.handoff.created", {
@@ -3084,11 +3432,13 @@ function createWorkflowEngine(composition, core, agentEngine) {
 				const started = createCoordinatorEvent(state, "workflow.step.started", {
 					stepId,
 					kind: "agent",
-					agentName
+					agentName,
+					childRunId: childState.run.id
 				}, "team.agent.assigned", {
 					delegationId: stepId,
 					agentName,
 					action,
+					childRunId: childState.run.id,
 					...typeof options?.metadata?.expectedOutput === "string" ? { expectedOutput: options.metadata.expectedOutput } : {},
 					...Array.isArray(options?.metadata?.constraints) ? { constraints: options.metadata.constraints } : {}
 				});
@@ -3772,24 +4122,76 @@ function createWorkflowEngine(composition, core, agentEngine) {
 }
 //#endregion
 //#region src/internal/runtime.ts
+const DEFAULT_SHUTDOWN_TIMEOUT_MS = 3e4;
+const SHUTDOWN_CLEANUP_MS = 5e3;
 function createRuntime(composition) {
 	const { models, agents, workflows, teams, tools, contextProviders, memoryStore, runStore, credentialStore, sandbox, policies } = composition;
 	const core = createRunCore(composition);
-	const { activeRuns, activeWorkflowRuns, activeSessions, pendingExports, exportFailures, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
+	const { activeRuns, activeWorkflowRuns, activeSessions, isIdle, waitForIdle, closeAdmission, isClosed, traceExports, runtimeOwner, createEvent, createCoordinatorEvent, notifyObserver, commit, startLease, cancelExecution, releaseWorkflowExecution, commitWorkflow } = core;
 	const agentEngine = createAgentEngine(composition, core);
 	const { prepareExecution, executeAgent, nextEvent, drainExecution, resumeAgent, cancelStoredAgent } = agentEngine;
 	const { prepareWorkflowExecution, executeWorkflow, nextWorkflowEvent, drainWorkflowExecution, resumeWorkflow, cancelWorkflowExecution, cancelStoredWorkflow } = createWorkflowEngine(composition, core, agentEngine);
 	const resumeRun = async (runId, resolution) => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs", runId);
 		const run = await runStore.getRun(runId);
 		if (run && isCoordinatorRun(run)) return resumeWorkflow(runId, resolution);
 		return resumeAgent(runId, resolution);
 	};
 	const recoverRun = async (runId, actor) => {
+		if (isClosed()) throw new FevexRunError("RUNTIME_CLOSED", "Runtime is not accepting new runs", runId);
 		const run = await runStore.getRun(runId);
 		if (!run) throw new FevexRunError("RUN_NOT_RECOVERABLE", `Run "${runId}" cannot be recovered`, runId);
 		if (isCoordinatorRun(run)) return resumeWorkflow(runId, void 0, actor);
 		return resumeAgent(runId, void 0, false, actor);
 	};
+	const waitForIdleUntil = async (deadline) => {
+		if (isIdle()) return true;
+		const remaining = deadline - Date.now();
+		if (remaining <= 0) return false;
+		return new Promise((resolve) => {
+			let settled = false;
+			let timer;
+			const finish = (idle) => {
+				if (settled) return;
+				settled = true;
+				if (timer) clearTimeout(timer);
+				resolve(idle);
+			};
+			timer = setTimeout(() => finish(false), remaining);
+			waitForIdle().then(() => finish(true));
+		});
+	};
+	const shutdown = async (options = {}) => {
+		const timeoutMs = options.timeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS;
+		if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0) throw new TypeError("Shutdown timeoutMs must be a non-negative safe integer");
+		closeAdmission();
+		const deadline = Date.now() + timeoutMs;
+		let cleanupDeadline = deadline;
+		let timedOut = false;
+		const cancelled = /* @__PURE__ */ new Set();
+		if (!await waitForIdleUntil(deadline)) {
+			timedOut = true;
+			cleanupDeadline = Date.now() + SHUTDOWN_CLEANUP_MS;
+			for (const { run } of activeWorkflowRuns.values()) cancelled.add(run.id);
+			for (const { run } of activeRuns.values()) cancelled.add(run.id);
+			for (const state of [...activeWorkflowRuns.values(), ...activeRuns.values()]) if (!state.request.signal.aborted) state.controller.abort(new DOMException("Run cancelled", "AbortError"));
+			for (const state of [...activeWorkflowRuns.values()]) if (!state.advancing) await cancelWorkflowExecution(state).catch(() => {});
+			for (const state of [...activeRuns.values()]) if (!state.advancing) await cancelExecution(state).catch(() => {});
+			await waitForIdleUntil(cleanupDeadline);
+		}
+		let exportError;
+		try {
+			await traceExports.flush({ timeoutMs: Math.max(1, cleanupDeadline - Date.now()) });
+		} catch (error) {
+			exportError = error;
+		}
+		return {
+			cancelledRunIds: [...cancelled],
+			timedOut,
+			...exportError === void 0 ? {} : { exportError },
+			observability: traceExports.stats()
+		};
+	};
 	return {
 		async startAgent(name, request) {
 			const state = await prepareExecution(name, request);
@@ -3921,13 +4323,13 @@ function createRuntime(composition) {
 			await runStore.saveSession(session);
 			return structuredClone(session);
 		},
-		async flushObservability() {
-			while (pendingExports.size) await Promise.all([...pendingExports]);
-			if (exportFailures.length) {
-				const failures = exportFailures.splice(0);
-				throw new AggregateError(failures, "Observability export failed");
-			}
-		}
+		flushObservability(options) {
+			return traceExports.flush(options);
+		},
+		getObservabilityStats() {
+			return traceExports.stats();
+		},
+		shutdown
 	};
 }
 //#endregion