@agentuity/runtime 0.1.23 → 0.1.25

package/src/agent.ts

@@ -10,7 +10,6 @@ import {
 	type InferInput,
 	type InferOutput,
 	toCamelCase,
-	type EvalRunStartEvent,
 } from '@agentuity/core';
 import { context, SpanStatusCode, type Tracer, trace } from '@opentelemetry/api';
 import { TraceState } from '@opentelemetry/core';
@@ -27,7 +26,7 @@ import {
 	type RequestAgentContextArgs,
 } from './_context';
 import type { Logger } from './logger';
-import type { Eval, EvalContext, EvalHandlerResult, EvalRunResult, EvalFunction } from './eval';
+import type { Eval, EvalHandlerResult, EvalRunResult, EvalFunction } from './eval';
 import { internal } from './logger/internal';
 import { fireEvent } from './_events';
 import type { Thread, Session } from './session';
@@ -1659,20 +1658,10 @@ export function createAgent<
 		await fireAgentEvent(runtime, agent as Agent, 'started', agentCtx);
 
 		try {
-			// Wrap agent execution with span tracking if tracer is available
+			// Execute the handler directly - span creation is handled by the caller (AgentRunner.run)
+			// This avoids duplicate spans when agents call other agents
 			const result = await (async () => {
-				if (agentCtx.tracer && inHTTPContext()) {
-					const honoCtx = getHTTPContext();
-					return runWithSpan<any, TInput, TOutput, TStream>(
-						agentCtx.tracer,
-						agent as Agent<TInput, TOutput, TStream>,
-						honoCtx,
-						async () =>
-							inputSchema
-								? await (config.handler as any)(agentCtx, validatedInput)
-								: await (config.handler as any)(agentCtx)
-					);
-				} else if (agent.metadata.id) {
+				if (agent.metadata.id && !inHTTPContext()) {
 					// For standalone contexts, wrap with agent context to set aid in trace state
 					return runWithAgentContext(agent.metadata.id, () =>
 						inputSchema
@@ -1680,7 +1669,8 @@ export function createAgent<
 							: (config.handler as any)(agentCtx)
 					);
 				} else {
-					// No agent ID, invoke handler directly
+					// HTTP context or no agent ID - invoke handler directly
+					// Span is created by AgentRunner.run or createAgentRunner
 					return inputSchema
 						? (config.handler as any)(agentCtx, validatedInput)
 						: (config.handler as any)(agentCtx);
@@ -1917,273 +1907,258 @@ export function createAgent<
 				// HTTP context may not be available, spanId will be undefined
 			}
 
+			// Capture the agent span context so eval spans are parented to the agent
+			const agentSpanContext = context.active();
+
 			// Execute each eval using waitUntil to avoid blocking the response
 			for (const evalItem of agentEvals) {
 				const evalName = evalItem.metadata.name || 'unnamed';
 				const agentName = _agent?.metadata?.name || name;
+				const evalRunId = generateId('evalrun');
+
+				// Look up eval metadata synchronously before async execution
+				const evalMeta = getEvalMetadata(agentName, evalName);
+				const evalId = evalMeta?.id || '';
+				const evalIdentifier = evalMeta?.identifier || '';
+
+				// Create eval span FIRST, parented to agent, then call waitUntil inside it
+				// This makes waitUntil a child of the eval span
+				const tracer = ctx.tracer;
+				if (tracer) {
+					const evalSpan = tracer.startSpan(evalName, {}, agentSpanContext);
+					evalSpan.setAttributes({
+						'@agentuity/evalId': evalId,
+						'@agentuity/evalIdentifier': evalIdentifier,
+						'@agentuity/evalName': evalName,
+						'@agentuity/evalRunId': evalRunId,
+						'@agentuity/agentName': agentName,
+						'@agentuity/evalDescription':
+							evalMeta?.description || evalItem.metadata.description || '',
+						'@agentuity/evalFilename':
+							evalMeta?.filename || evalItem.metadata.filename || '',
+					});
 
-				ctx.waitUntil(
-					(async () => {
-						internal.info(`[EVALRUN] Starting eval run tracking for '${evalName}'`);
-						const evalRunId = generateId('evalrun');
-
-						// Look up eval metadata from agentuity.metadata.json by agent name and eval name
-						internal.info(
-							`[EVALRUN] Looking up eval metadata: agentName='${agentName}', evalName='${evalName}'`
-						);
-						const evalMeta = getEvalMetadata(agentName, evalName);
-						internal.info(`[EVALRUN] Eval metadata lookup result:`, {
-							found: !!evalMeta,
-							identifier: evalMeta?.identifier,
-							id: evalMeta?.id,
-							filename: evalMeta?.filename,
-						});
-
-						// evalId = deployment-specific ID (evalid_...), evalIdentifier = stable (eval_...)
-						const evalId = evalMeta?.id || '';
-						const evalIdentifier = evalMeta?.identifier || '';
-						internal.info(
-							`[EVALRUN] Resolved evalId='${evalId}', evalIdentifier='${evalIdentifier}'`
-						);
-
-						// Log eval metadata using structured logging and tracing
-						ctx.logger.debug('Starting eval run with metadata', {
-							evalName,
-							agentName,
-							evalRunId,
-							evalId,
-							evalMetaFromFile: !!evalMeta,
-							evalMetadata: evalItem.metadata,
-						});
-
-						// Add eval metadata to the active span for observability
-						const activeSpan = ctx.tracer ? trace.getActiveSpan() : undefined;
-						if (activeSpan) {
-							activeSpan.setAttributes({
-								'eval.name': evalName,
-								'eval.id': evalId,
-								'eval.runId': evalRunId,
-								'eval.description':
-									evalMeta?.description || evalItem.metadata.description || '',
-								'eval.filename': evalMeta?.filename || evalItem.metadata.filename || '',
-							});
-						}
+					const evalSpanContext = trace.setSpan(agentSpanContext, evalSpan);
 
-						const orgId = runtimeConfig.getOrganizationId();
-						const projectId = runtimeConfig.getProjectId();
-						const devMode = runtimeConfig.isDevMode() ?? false;
-						const evalRunEventProvider = getEvalRunEventProvider();
-
-						// Only send events if we have required context (devmode flag will be set based on devMode)
-						const shouldSendEvalRunEvents =
-							orgId && projectId && evalId !== '' && evalIdentifier !== '';
-
-						internal.info(`[EVALRUN] Checking conditions for eval '${evalName}':`, {
-							orgId: orgId,
-							projectId: projectId,
-							evalId: evalId,
-							evalIdentifier: evalIdentifier,
-							devMode,
-							hasEvalRunEventProvider: !!evalRunEventProvider,
-							shouldSendEvalRunEvents,
-						});
+					// Run waitUntil INSIDE the eval span context - this makes waitUntil a child of eval
+					// Pass a function (not an already-executing promise) so waitUntil executes it
+					// AFTER setting up its span context, making operations children of waitUntil
+					context.with(evalSpanContext, () => {
+						ctx.waitUntil(async () => {
+								try {
+									internal.info(`[EVALRUN] Starting eval run tracking for '${evalName}'`);
+
+									const orgId = runtimeConfig.getOrganizationId();
+									const projectId = runtimeConfig.getProjectId();
+									const devMode = runtimeConfig.isDevMode() ?? false;
+									const evalRunEventProvider = getEvalRunEventProvider();
+
+									const shouldSendEvalRunEvents =
+										orgId && projectId && evalId !== '' && evalIdentifier !== '';
+
+									// Send eval run start event
+									if (shouldSendEvalRunEvents && evalRunEventProvider) {
+										try {
+											const deploymentId = runtimeConfig.getDeploymentId();
+											await evalRunEventProvider.start({
+												id: evalRunId,
+												sessionId: ctx.sessionId,
+												evalId,
+												evalIdentifier,
+												orgId: orgId!,
+												projectId: projectId!,
+												devmode: Boolean(devMode),
+												deploymentId: deploymentId || undefined,
+												spanId: agentRunSpanId,
+											});
+										} catch (error) {
+											internal.error(
+												`[EVALRUN] Error sending start event for '${evalName}'`,
+												{ error }
+											);
+										}
+									}
 
-						if (!shouldSendEvalRunEvents) {
-							const reasons: string[] = [];
-							if (!orgId) reasons.push('missing orgId');
-							if (!projectId) reasons.push('missing projectId');
-							if (!evalId || evalId === '') reasons.push('empty evalId');
-							if (!evalIdentifier || evalIdentifier === '')
-								reasons.push('empty evalIdentifier');
-							internal.info(
-								`[EVALRUN] Skipping eval run events for '${evalName}': ${reasons.join(', ')}`
-							);
-						}
+									// Validate eval input/output if schemas exist
+									let evalValidatedInput: any = validatedInput;
+									let evalValidatedOutput: any = validatedOutput;
+
+									if (evalItem.inputSchema) {
+										const result =
+											await evalItem.inputSchema['~standard'].validate(validatedInput);
+										if (result.issues) {
+											throw new ValidationError({
+												issues: result.issues,
+												message: `Eval input validation failed`,
+											});
+										}
+										evalValidatedInput = result.value;
+									}
 
-						try {
-							internal.debug(`Executing eval: ${evalName}`);
+									if (evalItem.outputSchema) {
+										const result =
+											await evalItem.outputSchema['~standard'].validate(validatedOutput);
+										if (result.issues) {
+											throw new ValidationError({
+												issues: result.issues,
+												message: `Eval output validation failed`,
+											});
+										}
+										evalValidatedOutput = result.value;
+									}
 
-							// Send eval run start event
-							if (shouldSendEvalRunEvents && evalRunEventProvider) {
-								internal.info(
-									`[EVALRUN] Sending start event for eval '${evalName}' (id: ${evalRunId}, evalId: ${evalId})`
-								);
-								try {
-									const deploymentId = runtimeConfig.getDeploymentId();
-									// Use captured agentRunSpanId (may be undefined if HTTP context unavailable)
-									if (!agentRunSpanId) {
-										internal.warn(
-											`[EVALRUN] agentRunSpanId not available for eval '${evalName}' (id: ${evalRunId}). This may occur if waitUntil runs outside AsyncLocalStorage context.`
+									// Execute the eval handler
+									let handlerResult: EvalHandlerResult;
+									if (inputSchema && outputSchema) {
+										handlerResult = await (evalItem.handler as any)(
+											ctx,
+											evalValidatedInput,
+											evalValidatedOutput
 										);
+									} else if (inputSchema) {
+										handlerResult = await (evalItem.handler as any)(ctx, evalValidatedInput);
+									} else if (outputSchema) {
+										handlerResult = await (evalItem.handler as any)(ctx, evalValidatedOutput);
+									} else {
+										handlerResult = await (evalItem.handler as any)(ctx);
 									}
-									const startEvent: EvalRunStartEvent = {
-										id: evalRunId,
-										sessionId: ctx.sessionId,
-										evalId: evalId, // deployment-specific ID (evalid_...)
-										evalIdentifier: evalIdentifier, // stable identifier (eval_...)
-										orgId: orgId!,
-										projectId: projectId!,
-										devmode: Boolean(devMode),
-										deploymentId: deploymentId || undefined,
-										spanId: agentRunSpanId,
-									};
-									internal.debug(
-										'[EVALRUN] Start event payload: %s',
-										JSON.stringify(startEvent, null, 2)
-									);
-									await evalRunEventProvider.start(startEvent);
-									internal.info(
-										`[EVALRUN] Start event sent successfully for eval '${evalName}' (id: ${evalRunId})`
-									);
-								} catch (error) {
-									internal.error(
-										`[EVALRUN] Error sending eval run start event for '${evalName}' (id: ${evalRunId})`,
-										{
-											error,
+
+									const result: EvalRunResult = { success: true, ...handlerResult };
+
+									// Send eval run complete event
+									if (shouldSendEvalRunEvents && evalRunEventProvider) {
+										try {
+											await evalRunEventProvider.complete({ id: evalRunId, result });
+										} catch (error) {
+											internal.error(
+												`[EVALRUN] Error sending complete event for '${evalName}'`,
+												{ error }
+											);
 										}
-									);
-									// Don't throw - continue with eval execution even if start event fails
-								}
-							} else if (shouldSendEvalRunEvents && !evalRunEventProvider) {
-								internal.warn(
-									`[EVALRUN] Conditions met but no evalRunEventProvider available for '${evalName}'`
-								);
-							} else {
-								internal.debug(
-									`[EVALRUN] Not sending start event for '${evalName}': shouldSendEvalRunEvents=${shouldSendEvalRunEvents}, hasProvider=${!!evalRunEventProvider}`
-								);
-							}
+									}
 
-							// Validate eval input if schema exists
-							let evalValidatedInput: any = validatedInput;
-							if (evalItem.inputSchema) {
-								const evalInputResult =
-									await evalItem.inputSchema['~standard'].validate(validatedInput);
-								if (evalInputResult.issues) {
-									throw new ValidationError({
-										issues: evalInputResult.issues,
-										message: `Eval input validation failed: ${evalInputResult.issues.map((i: any) => i.message).join(', ')}`,
+									internal.debug(`Eval '${evalName}' completed successfully`);
+								} catch (error) {
+									const errorMessage =
+										error instanceof Error ? error.message : String(error);
+									evalSpan.recordException(error as Error);
+									evalSpan.setStatus({
+										code: SpanStatusCode.ERROR,
+										message: errorMessage,
 									});
+									internal.error(`Error executing eval '${evalName}'`, { error });
+
+									// Send error event
+									const orgId = runtimeConfig.getOrganizationId();
+									const projectId = runtimeConfig.getProjectId();
+									const evalRunEventProvider = getEvalRunEventProvider();
+									if (orgId && projectId && evalId && evalRunEventProvider) {
+										try {
+											await evalRunEventProvider.complete({
+												id: evalRunId,
+												error: errorMessage,
+												result: {
+													success: false,
+													passed: false,
+													error: errorMessage,
+													metadata: {},
+												},
+											});
+										} catch (e) {
+											internal.debug('Failed to send eval run complete event', {
+												evalRunId,
+												errorMessage,
+												error: e instanceof Error ? e.message : String(e),
+											});
+										}
+									}
+								} finally {
+									evalSpan.end();
 								}
-								evalValidatedInput = evalInputResult.value;
-							}
-
-							// Validate eval output if schema exists
-							let evalValidatedOutput: any = validatedOutput;
-							if (evalItem.outputSchema) {
-								const evalOutputResult =
-									await evalItem.outputSchema['~standard'].validate(validatedOutput);
-								if (evalOutputResult.issues) {
-									throw new ValidationError({
-										issues: evalOutputResult.issues,
-										message: `Eval output validation failed: ${evalOutputResult.issues.map((i: any) => i.message).join(', ')}`,
-									});
+						});
+					});
+				} else {
+					// No tracer - execute without span
+					ctx.waitUntil(async () => {
+							try {
+								const orgId = runtimeConfig.getOrganizationId();
+								const projectId = runtimeConfig.getProjectId();
+								const devMode = runtimeConfig.isDevMode() ?? false;
+								const evalRunEventProvider = getEvalRunEventProvider();
+								const shouldSendEvalRunEvents =
+									orgId && projectId && evalId !== '' && evalIdentifier !== '';
+
+								if (shouldSendEvalRunEvents && evalRunEventProvider) {
+									try {
+										await evalRunEventProvider.start({
+											id: evalRunId,
+											sessionId: ctx.sessionId,
+											evalId,
+											evalIdentifier,
+											orgId: orgId!,
+											projectId: projectId!,
+											devmode: Boolean(devMode),
+											deploymentId: runtimeConfig.getDeploymentId() || undefined,
+											spanId: agentRunSpanId,
+										});
+									} catch (e) {
+										internal.debug('Failed to send eval run start event', {
+											evalRunId,
+											evalId,
+											evalIdentifier,
+											sessionId: ctx.sessionId,
+											error: e instanceof Error ? e.message : String(e),
+										});
+									}
 								}
-								evalValidatedOutput = evalOutputResult.value;
-							}
 
-							// Create EvalContext (just an alias for AgentContext)
-							const evalContext: EvalContext = ctx;
-
-							// Execute the eval handler conditionally based on agent schema
-							let handlerResult: EvalHandlerResult;
-							if (inputSchema && outputSchema) {
-								// Both input and output defined
-								handlerResult = await (evalItem.handler as any)(
-									evalContext,
-									evalValidatedInput,
-									evalValidatedOutput
-								);
-							} else if (inputSchema) {
-								// Only input defined
-								handlerResult = await (evalItem.handler as any)(
-									evalContext,
-									evalValidatedInput
-								);
-							} else if (outputSchema) {
-								// Only output defined
-								handlerResult = await (evalItem.handler as any)(
-									evalContext,
-									evalValidatedOutput
-								);
-							} else {
-								// Neither defined
-								handlerResult = await (evalItem.handler as any)(evalContext);
-							}
+								let evalValidatedInput: any = validatedInput;
+								let evalValidatedOutput: any = validatedOutput;
 
-							// Wrap handler result with success for catalyst
-							const result: EvalRunResult = {
-								success: true,
-								...handlerResult,
-							};
-
-							// Log the result
-							if (result.score !== undefined) {
-								internal.info(
-									`Eval '${evalName}' pass: ${result.passed}, score: ${result.score}`,
-									result.metadata
-								);
-							} else {
-								internal.info(`Eval '${evalName}' pass: ${result.passed}`, result.metadata);
-							}
+								if (evalItem.inputSchema) {
+									const result =
+										await evalItem.inputSchema['~standard'].validate(validatedInput);
+									if (!result.issues) evalValidatedInput = result.value;
+								}
+								if (evalItem.outputSchema) {
+									const result =
+										await evalItem.outputSchema['~standard'].validate(validatedOutput);
+									if (!result.issues) evalValidatedOutput = result.value;
+								}
 
-							// Send eval run complete event
-							if (shouldSendEvalRunEvents && evalRunEventProvider) {
-								internal.info(
-									`[EVALRUN] Sending complete event for eval '${evalName}' (id: ${evalRunId})`
-								);
-								try {
-									await evalRunEventProvider.complete({
-										id: evalRunId,
-										result,
-									});
-									internal.info(
-										`[EVALRUN] Complete event sent successfully for eval '${evalName}' (id: ${evalRunId})`
-									);
-								} catch (error) {
-									internal.error(
-										`[EVALRUN] Error sending eval run complete event for '${evalName}' (id: ${evalRunId})`,
-										{
-											error,
-										}
+								let handlerResult: EvalHandlerResult;
+								if (inputSchema && outputSchema) {
+									handlerResult = await (evalItem.handler as any)(
+										ctx,
+										evalValidatedInput,
+										evalValidatedOutput
 									);
+								} else if (inputSchema) {
+									handlerResult = await (evalItem.handler as any)(ctx, evalValidatedInput);
+								} else if (outputSchema) {
+									handlerResult = await (evalItem.handler as any)(ctx, evalValidatedOutput);
+								} else {
+									handlerResult = await (evalItem.handler as any)(ctx);
 								}
-							}
-
-							internal.debug(`Eval '${evalName}' completed successfully`);
-						} catch (error) {
-							const errorMessage = error instanceof Error ? error.message : String(error);
-							internal.error(`Error executing eval '${evalName}'`, { error });
 
-							// Send eval run complete event with error
-							if (shouldSendEvalRunEvents && evalRunEventProvider) {
-								internal.info(
-									`[EVALRUN] Sending complete event (error) for eval '${evalName}' (id: ${evalRunId})`
-								);
-								try {
-									await evalRunEventProvider.complete({
-										id: evalRunId,
-										error: errorMessage,
-										result: {
-											success: false,
-											passed: false,
-											error: errorMessage,
-											metadata: {},
-										},
-									});
-									internal.info(
-										`[EVALRUN] Complete event (error) sent successfully for eval '${evalName}' (id: ${evalRunId})`
-									);
-								} catch (eventError) {
-									internal.error(
-										`[EVALRUN] Error sending eval run complete event (error) for '${evalName}' (id: ${evalRunId})`,
-										{ error: eventError }
-									);
+								if (shouldSendEvalRunEvents && evalRunEventProvider) {
+									try {
+										await evalRunEventProvider.complete({
+											id: evalRunId,
+											result: { success: true, ...handlerResult },
+										});
+									} catch (e) {
+										internal.debug('Failed to send eval run complete event', {
+											evalRunId,
+											error: e instanceof Error ? e.message : String(e),
+										});
+									}
 								}
+							} catch (error) {
+								internal.error(`Error executing eval '${evalName}'`, { error });
 							}
-						}
-					})()
-				);
+					});
+				}
 			}
 		}
 	});
@@ -2331,9 +2306,35 @@ export function createAgent<
 		removeEventListener: agent.removeEventListener,
 		run: inputSchema
 			? async (input: InferSchemaInput<Exclude<TInput, undefined>>) => {
+					// Wrap with span if in HTTP context with tracer
+					if (inHTTPContext()) {
+						const honoCtx = getHTTPContext();
+						const tracer = honoCtx.var.tracer;
+						if (tracer) {
+							return runWithSpan(
+								tracer,
+								agent as Agent<TInput, TOutput, TStream>,
+								honoCtx,
+								async () => await agent.handler(input)
+							);
+						}
+					}
 					return await agent.handler(input);
 				}
 			: async () => {
+					// Wrap with span if in HTTP context with tracer
+					if (inHTTPContext()) {
+						const honoCtx = getHTTPContext();
+						const tracer = honoCtx.var.tracer;
+						if (tracer) {
+							return runWithSpan(
+								tracer,
+								agent as Agent<TInput, TOutput, TStream>,
+								honoCtx,
+								async () => await agent.handler()
+							);
+						}
+					}
 					return await agent.handler();
 				},
 		[INTERNAL_AGENT]: agent, // Store reference to internal agent for testing