@agentuity/runtime 0.0.99 → 0.0.101

package/src/_metadata.ts

@@ -0,0 +1,307 @@
+/**
+ * Build metadata utilities
+ * Provides cached access to agentuity.metadata.json
+ */
+
+import { join } from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+import { internal } from './logger/internal';
+
+export interface BuildMetadataAgent {
+	filename: string;
+	id: string;
+	agentId: string;
+	version: string;
+	name: string;
+	description?: string;
+	projectId?: string;
+	schema?: {
+		input?: string;
+		output?: string;
+	};
+	evals?: Array<{
+		filename: string;
+		id: string;
+		evalId: string;
+		name: string;
+		version: string;
+		description?: string;
+		agentIdentifier?: string;
+		projectId?: string;
+	}>;
+}
+
+export interface BuildMetadataRoute {
+	id: string;
+	filename: string;
+	path: string;
+	method: 'get' | 'post' | 'put' | 'delete' | 'patch';
+	version: string;
+	type?: string;
+	agentIds?: string[];
+	config?: Record<string, unknown>;
+	schema?: {
+		input?: string;
+		output?: string;
+	};
+}
+
+export interface BuildMetadata {
+	routes: BuildMetadataRoute[];
+	agents: BuildMetadataAgent[];
+	assets?: string[];
+	project: {
+		id: string;
+		name: string;
+		version?: string;
+		description?: string;
+		keywords?: string[];
+		orgId?: string;
+	};
+	deployment: {
+		id: string;
+		date: string;
+		build: {
+			bun: string;
+			agentuity: string;
+			arch: string;
+			platform: string;
+		};
+		git?: {
+			branch?: string;
+			repo?: string;
+			provider?: string;
+			tags?: string[];
+			commit?: string;
+			message?: string;
+		};
+	};
+}
+
+// Cached metadata - null means not yet loaded, undefined means file not found
+let _metadataCache: BuildMetadata | null | undefined = null;
+
+/**
+ * Get the path to agentuity.metadata.json
+ */
+export function getMetadataPath(): string {
+	return join(process.cwd(), '.agentuity', 'agentuity.metadata.json');
+}
+
+/**
+ * Load and cache the build metadata from agentuity.metadata.json
+ * Returns undefined if the file doesn't exist or can't be parsed
+ */
+export function loadBuildMetadata(): BuildMetadata | undefined {
+	// Return cached value if already loaded
+	if (_metadataCache !== null) {
+		internal.info(
+			'[metadata] loadBuildMetadata: returning cached value (exists: %s)',
+			_metadataCache !== undefined
+		);
+		return _metadataCache;
+	}
+
+	const metadataPath = getMetadataPath();
+	internal.info('[metadata] loadBuildMetadata: checking path %s', metadataPath);
+	internal.info('[metadata] loadBuildMetadata: cwd=%s', process.cwd());
+
+	if (!existsSync(metadataPath)) {
+		internal.info('[metadata] agentuity.metadata.json not found at %s', metadataPath);
+		_metadataCache = undefined;
+		return undefined;
+	}
+
+	try {
+		internal.info('[metadata] loadBuildMetadata: file exists, reading...');
+		const content = readFileSync(metadataPath, 'utf-8');
+		const metadata = JSON.parse(content) as BuildMetadata;
+		_metadataCache = metadata;
+
+		// Log agent and eval counts
+		let totalEvals = 0;
+		for (const agent of metadata.agents ?? []) {
+			totalEvals += agent.evals?.length ?? 0;
+		}
+
+		internal.info(
+			'[metadata] loaded agentuity.metadata.json: %d agents, %d routes, %d total evals',
+			metadata.agents?.length ?? 0,
+			metadata.routes?.length ?? 0,
+			totalEvals
+		);
+
+		// Log agent names and their eval counts
+		for (const agent of metadata.agents ?? []) {
+			internal.info('[metadata]   agent: %s (evals: %d)', agent.name, agent.evals?.length ?? 0);
+		}
+
+		return metadata;
+	} catch (err) {
+		internal.info('[metadata] failed to load agentuity.metadata.json: %s', err);
+		_metadataCache = undefined;
+		return undefined;
+	}
+}
+
+// Eval metadata type (extracted from agent's evals array)
+export type BuildMetadataEval = NonNullable<BuildMetadataAgent['evals']>[number];
+
+// Agent lookup cache - built lazily from metadata
+let _agentsByName: Map<string, BuildMetadataAgent> | null = null;
+let _agentsByAgentId: Map<string, BuildMetadataAgent> | null = null;
+
+// Eval lookup cache - nested map: agentName -> evalName -> evalMetadata
+let _evalsByAgentName: Map<string, Map<string, BuildMetadataEval>> | null = null;
+let _evalsByAgentId: Map<string, Map<string, BuildMetadataEval>> | null = null;
+
+// Track if we've already attempted a reload for empty eval map
+let _evalReloadAttempted = false;
+
+/**
+ * Build agent lookup maps from metadata
+ */
+function ensureAgentMaps(): void {
+	if (_agentsByName !== null) {
+		internal.info(`[metadata] ensureAgentMaps: already initialized, skipping`);
+		return;
+	}
+
+	internal.info(`[metadata] ensureAgentMaps: initializing agent and eval maps`);
+
+	_agentsByName = new Map();
+	_agentsByAgentId = new Map();
+	_evalsByAgentName = new Map();
+	_evalsByAgentId = new Map();
+
+	const metadata = loadBuildMetadata();
+	if (!metadata?.agents) {
+		internal.info(`[metadata] ensureAgentMaps: no metadata or no agents found`);
+		return;
+	}
+
+	internal.info(`[metadata] ensureAgentMaps: processing ${metadata.agents.length} agents`);
+
+	for (const agent of metadata.agents) {
+		if (agent.name) {
+			_agentsByName.set(agent.name, agent);
+		}
+		if (agent.agentId) {
+			_agentsByAgentId.set(agent.agentId, agent);
+		}
+
+		// Build eval lookup maps
+		if (agent.evals && agent.evals.length > 0) {
+			const evalsByName = new Map<string, BuildMetadataEval>();
+			for (const evalMeta of agent.evals) {
+				if (evalMeta.name) {
+					evalsByName.set(evalMeta.name, evalMeta);
+					internal.info(
+						`[metadata] Indexed eval: agent='${agent.name}' eval='${evalMeta.name}' evalId='${evalMeta.evalId}'`
+					);
+				}
+			}
+			if (agent.name) {
+				_evalsByAgentName.set(agent.name, evalsByName);
+			}
+			if (agent.agentId) {
+				_evalsByAgentId.set(agent.agentId, evalsByName);
+			}
+		} else {
+			internal.info(`[metadata] Agent '${agent.name}' has no evals`);
+		}
+	}
+	internal.info(`[metadata] Eval maps built: ${_evalsByAgentName?.size ?? 0} agents with evals`);
+}
+
+/**
+ * Look up agent metadata by name
+ */
+export function getAgentMetadataByName(agentName: string): BuildMetadataAgent | undefined {
+	ensureAgentMaps();
+	return _agentsByName?.get(agentName);
+}
+
+/**
+ * Look up agent metadata by agentId
+ */
+export function getAgentMetadataByAgentId(agentId: string): BuildMetadataAgent | undefined {
+	ensureAgentMaps();
+	return _agentsByAgentId?.get(agentId);
+}
+
+/**
+ * Look up eval metadata by agent name and eval name
+ */
+export function getEvalMetadata(
+	agentName: string,
+	evalName: string
+): BuildMetadataEval | undefined {
+	ensureAgentMaps();
+
+	// If eval map is empty, the cache may have been built before metadata was ready
+	// Try clearing and reloading once (only attempt once to avoid repeated reloads)
+	if (_evalsByAgentName?.size === 0 && !_evalReloadAttempted) {
+		_evalReloadAttempted = true;
+		internal.info(
+			`[metadata] getEvalMetadata: eval map is empty, attempting cache clear and reload`
+		);
+		clearMetadataCache();
+		ensureAgentMaps();
+		internal.info(
+			`[metadata] getEvalMetadata: after reload, eval map size: ${_evalsByAgentName?.size ?? 0}`
+		);
+	}
+
+	const agentEvals = _evalsByAgentName?.get(agentName);
+	internal.info(
+		`[metadata] getEvalMetadata('${agentName}', '${evalName}'): agentEvals=${agentEvals ? `Map(${agentEvals.size})` : 'undefined'}`
+	);
+	if (agentEvals) {
+		internal.info(
+			`[metadata] Available evals for agent '${agentName}': [${[...agentEvals.keys()].join(', ')}]`
+		);
+	}
+	if (!agentEvals) {
+		internal.info(
+			`[metadata] Available agents in eval map: [${[...(_evalsByAgentName?.keys() ?? [])].join(', ')}]`
+		);
+	}
+	const result = agentEvals?.get(evalName);
+	internal.info(
+		`[metadata] getEvalMetadata result: ${result ? `found evalId=${result.evalId}` : 'not found'}`
+	);
+	return result;
+}
+
+/**
+ * Look up eval metadata by agentId and eval name
+ */
+export function getEvalMetadataByAgentId(
+	agentId: string,
+	evalName: string
+): BuildMetadataEval | undefined {
+	ensureAgentMaps();
+	return _evalsByAgentId?.get(agentId)?.get(evalName);
+}
+
+/**
+ * Check if metadata file exists (uses cache)
+ */
+export function hasMetadata(): boolean {
+	return loadBuildMetadata() !== undefined;
+}
+
+/**
+ * Clear the metadata cache (useful for testing or hot reload)
+ */
+export function clearMetadataCache(): void {
+	internal.info('[metadata] clearMetadataCache: clearing all caches');
+	_metadataCache = null;
+	_agentsByName = null;
+	_agentsByAgentId = null;
+	_evalsByAgentName = null;
+	_evalsByAgentId = null;
+	// Note: _evalReloadAttempted is intentionally NOT reset here
+	// to prevent infinite reload loops in getEvalMetadata
+}

package/src/_process-protection.ts

@@ -31,6 +31,9 @@ export function enableProcessExitProtection(): void {
 
 	protectionEnabled = true;
 
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	(globalThis as any).AGENTUITY_PROCESS_EXIT = originalExit;
+
 	// Replace process.exit with a function that throws
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
 	(process as any).exit = function (code?: number | string | null | undefined): never {
@@ -46,6 +49,9 @@ export function disableProcessExitProtection(): void {
 		return;
 	}
 
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	(globalThis as any).AGENTUITY_PROCESS_EXIT = undefined;
+
 	protectionEnabled = false;
 	process.exit = originalExit;
 }

package/src/_services.ts

@@ -50,15 +50,16 @@ import {
 } from './services/local';
 
 const userAgent = `Agentuity SDK/${getSDKVersion()}`;
-const sdkKey = process.env.AGENTUITY_SDK_KEY;
-const bearerKey = `Bearer ${sdkKey}`;
 
-const region = process.env.AGENTUITY_REGION ?? 'usc';
-const serviceUrls = getServiceUrls(region);
-const kvBaseUrl = serviceUrls.keyvalue;
-const streamBaseUrl = serviceUrls.stream;
-const vectorBaseUrl = serviceUrls.vector;
-const catalystBaseUrl = serviceUrls.catalyst;
+// Lazy getters - these must be functions to read env vars AFTER bootstrapRuntimeEnv() runs
+const getSdkKey = () => process.env.AGENTUITY_SDK_KEY;
+const getBearerKey = () => `Bearer ${getSdkKey()}`;
+const getRegion = () => process.env.AGENTUITY_REGION ?? 'usc';
+const getLazyServiceUrls = () => getServiceUrls(getRegion());
+const getKvBaseUrl = () => getLazyServiceUrls().keyvalue;
+const getStreamBaseUrl = () => getLazyServiceUrls().stream;
+const getVectorBaseUrl = () => getLazyServiceUrls().vector;
+const getCatalystBaseUrl = () => getLazyServiceUrls().catalyst;
 
 let adapter: FetchAdapter;
 
@@ -66,7 +67,7 @@ const createFetchAdapter = (logger: Logger) =>
 	createServerFetchAdapter(
 		{
 			headers: {
-				Authorization: bearerKey,
+				Authorization: getBearerKey(),
 				'User-Agent': userAgent,
 			},
 			onBefore: async (url, options, callback) => {
@@ -115,7 +116,7 @@ const createFetchAdapter = (logger: Logger) =>
 							const res = result.data as { id: string };
 							span?.setAttributes({
 								'stream.id': res.id,
-								'stream.url': `${streamBaseUrl}/${res.id}`,
+								'stream.url': `${getStreamBaseUrl()}/${res.id}`,
 							});
 						}
 						break;
@@ -225,30 +226,31 @@ export function createServices(logger: Logger, config?: AppConfig<any>, serverUr
 	localRouter = null;
 
 	// At this point we must be authenticated (since !authenticated would trigger local services above)
-	kv = config?.services?.keyvalue || new KeyValueStorageService(kvBaseUrl, adapter);
-	stream = config?.services?.stream || new StreamStorageService(streamBaseUrl, adapter);
-	vector = config?.services?.vector || new VectorStorageService(vectorBaseUrl, adapter);
+	const catalystUrl = getCatalystBaseUrl();
+	kv = config?.services?.keyvalue || new KeyValueStorageService(getKvBaseUrl(), adapter);
+	stream = config?.services?.stream || new StreamStorageService(getStreamBaseUrl(), adapter);
+	vector = config?.services?.vector || new VectorStorageService(getVectorBaseUrl(), adapter);
 	session = config?.services?.session || new DefaultSessionProvider();
 	thread = config?.services?.thread || new DefaultThreadProvider();
 	// FIXME: this is turned off for now for production until we have the new changes deployed
 	sessionEvent =
 		isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
 			? new JSONSessionEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
-			: new HTTPSessionEventProvider(new APIClient(catalystBaseUrl, logger), logger);
-	new LocalSessionEventProvider();
+			: new HTTPSessionEventProvider(new APIClient(catalystUrl, logger), logger);
 	if (config?.services?.sessionEvent) {
 		sessionEvent = new CompositeSessionEventProvider(sessionEvent, config.services.sessionEvent);
 	}
 	// FIXME: this is turned off for now for production until we have the new changes deployed
+	logger.debug(
+		'[SERVICES] Initializing eval run provider - region: %s, catalystBaseUrl: %s, isProduction: %s',
+		getRegion(),
+		catalystUrl,
+		isProduction()
+	);
 	evalRunEvent =
 		isProduction() && process.env.AGENTUITY_CLOUD_EXPORT_DIR
 			? new JSONEvalRunEventProvider(process.env.AGENTUITY_CLOUD_EXPORT_DIR)
-			: new HTTPEvalRunEventProvider(
-					new APIClient(catalystBaseUrl, logger),
-					logger,
-					catalystBaseUrl
-				);
-	new LocalEvalRunEventProvider();
+			: new HTTPEvalRunEventProvider(new APIClient(catalystUrl, logger), logger, catalystUrl);
 	if (config?.services?.evalRunEvent) {
 		evalRunEvent = new CompositeEvalRunEventProvider(evalRunEvent, config.services.evalRunEvent);
 	}

package/src/_standalone.ts

@@ -143,6 +143,7 @@ export class StandaloneAgentContext<
 			({
 				id: 'pending',
 				state: new Map(),
+				metadata: {},
 				addEventListener: () => {},
 				removeEventListener: () => {},
 				destroy: async () => {},
@@ -155,6 +156,7 @@ export class StandaloneAgentContext<
 				id: 'pending',
 				thread: this.thread,
 				state: new Map(),
+				metadata: {},
 				addEventListener: () => {},
 				removeEventListener: () => {},
 				serializeUserData: () => undefined,
@@ -317,6 +319,10 @@ export class StandaloneAgentContext<
 								method: 'STANDALONE',
 								url: '',
 								trigger: this.trigger,
+								metadata:
+									Object.keys(invocationSession.metadata).length > 0
+										? invocationSession.metadata
+										: undefined,
 							})
 							.catch((ex) => {
 								canSendSessionEvents = false;
@@ -352,6 +358,10 @@ export class StandaloneAgentContext<
 												statusCode: 200, // Success
 												agentIds: Array.from(agentIds),
 												userData,
+												metadata:
+													Object.keys(invocationSession.metadata).length > 0
+														? invocationSession.metadata
+														: undefined,
 											})
 											.then(() => {})
 											.catch((ex) => this.logger.error(ex));
@@ -382,6 +392,10 @@ export class StandaloneAgentContext<
 												error: message,
 												agentIds: Array.from(agentIds),
 												userData,
+												metadata:
+													Object.keys(invocationSession.metadata).length > 0
+														? invocationSession.metadata
+														: undefined,
 											})
 											.then(() => {})
 											.catch((ex) => this.logger.error(ex));
@@ -401,6 +415,10 @@ export class StandaloneAgentContext<
 										statusCode: 200,
 										agentIds: Array.from(agentIds),
 										userData,
+										metadata:
+											Object.keys(invocationSession.metadata).length > 0
+												? invocationSession.metadata
+												: undefined,
 									})
 									.then(() => {})
 									.catch((ex) => this.logger.error(ex));
@@ -428,6 +446,10 @@ export class StandaloneAgentContext<
 									error: message,
 									agentIds: Array.from(agentIds),
 									userData,
+									metadata:
+										Object.keys(invocationSession.metadata).length > 0
+											? invocationSession.metadata
+											: undefined,
 								})
 								.then(() => {})
 								.catch((ex) => this.logger.error(ex));

package/src/agent.ts

@@ -5,6 +5,7 @@ import {
 	type StandardSchemaV1,
 	type StreamStorage,
 	type VectorStorage,
+	type InferInput,
 	type InferOutput,
 	toCamelCase,
 	type EvalRunStartEvent,
@@ -33,6 +34,7 @@ import { getEvalRunEventProvider } from './_services';
 import * as runtimeConfig from './_config';
 import type { AppState } from './index';
 import { validateSchema, formatValidationIssues } from './_validation';
+import { getAgentMetadataByName, getEvalMetadata } from './_metadata';
 
 export type AgentEventName = 'started' | 'completed' | 'errored';
 
@@ -422,7 +424,7 @@ export interface AgentValidator<
 				{
 					// eslint-disable-next-line @typescript-eslint/no-empty-object-type
 					in: {};
-					out: { json: InferOutput<TInput> };
+					out: { json: InferInput<TInput> };
 				}
 			>
 		: Handler<any, any, any>;
@@ -502,7 +504,7 @@ export interface AgentValidator<
 			// eslint-disable-next-line @typescript-eslint/no-empty-object-type
 			in: {};
 			out: {
-				json: InferOutput<TOverrideInput>;
+				json: InferInput<TOverrideInput>;
 			};
 		}
 	>;
@@ -1526,6 +1528,7 @@ export function createAgent<
 			'@agentuity/agentInstanceId': agent.metadata.agentId,
 			'@agentuity/agentDescription': agent.metadata.description,
 			'@agentuity/agentName': agent.metadata.name,
+			'@agentuity/threadId': agentCtx.thread.id,
 		};
 
 		// Set agent attributes on the current active span
@@ -1537,15 +1540,15 @@ export function createAgent<
 		if (inHTTPContext()) {
 			const honoCtx = privateContext(getHTTPContext());
 			if (honoCtx.var.agentIds) {
-				honoCtx.var.agentIds.add(agent.metadata.id);
-				honoCtx.var.agentIds.add(agent.metadata.agentId);
+				if (agent.metadata.id) honoCtx.var.agentIds.add(agent.metadata.id);
+				if (agent.metadata.agentId) honoCtx.var.agentIds.add(agent.metadata.agentId);
 			}
 		} else {
 			// For standalone contexts, check for AGENT_IDS symbol
 			const agentIds = (agentCtx as any)[AGENT_IDS] as Set<string> | undefined;
 			if (agentIds) {
-				agentIds.add(agent.metadata.id);
-				agentIds.add(agent.metadata.agentId);
+				if (agent.metadata.id) agentIds.add(agent.metadata.id);
+				if (agent.metadata.agentId) agentIds.add(agent.metadata.agentId);
 			}
 		}
 
@@ -1666,7 +1669,7 @@ export function createAgent<
 
 	// Build metadata - merge user-provided metadata with defaults
 	// The build plugin injects metadata via config.metadata during AST transformation
-	const metadata: Partial<AgentMetadata> = {
+	let metadata: Partial<AgentMetadata> = {
 		// Defaults (used when running without build, e.g., dev mode)
 		name,
 		description: config.description,
@@ -1680,6 +1683,26 @@ export function createAgent<
 		...config.metadata,
 	};
 
+	// If id/agentId are empty, try to load from agentuity.metadata.json
+	if (!metadata.id || !metadata.agentId) {
+		const fileMetadata = getAgentMetadataByName(name);
+		if (fileMetadata) {
+			internal.info(
+				'[agent] loaded metadata for "%s" from file: id=%s, agentId=%s',
+				name,
+				fileMetadata.id,
+				fileMetadata.agentId
+			);
+			metadata = {
+				...metadata,
+				id: fileMetadata.id || metadata.id,
+				agentId: fileMetadata.agentId || metadata.agentId,
+				filename: fileMetadata.filename || metadata.filename,
+				version: fileMetadata.version || metadata.version,
+			};
+		}
+	}
+
 	const agent: any = {
 		handler,
 		metadata,
@@ -1736,19 +1759,39 @@ export function createAgent<
 			// Execute each eval using waitUntil to avoid blocking the response
 			for (const evalItem of agentEvals) {
 				const evalName = evalItem.metadata.name || 'unnamed';
+				const agentName = _agent?.metadata?.name || name;
 
 				ctx.waitUntil(
 					(async () => {
 						internal.info(`[EVALRUN] Starting eval run tracking for '${evalName}'`);
 						const evalRunId = generateId('evalrun');
-						// Use build-time injected eval ID (now properly injected via AST transformation)
-						const evalId = evalItem.metadata.id || '';
+
+						// Look up eval metadata from agentuity.metadata.json by agent name and eval name
+						internal.info(
+							`[EVALRUN] Looking up eval metadata: agentName='${agentName}', evalName='${evalName}'`
+						);
+						const evalMeta = getEvalMetadata(agentName, evalName);
+						internal.info(`[EVALRUN] Eval metadata lookup result:`, {
+							found: !!evalMeta,
+							evalId: evalMeta?.evalId,
+							id: evalMeta?.id,
+							filename: evalMeta?.filename,
+						});
+
+						// evalId = deployment-specific ID (evalid_...), evalIdentifier = stable (eval_...)
+						const evalId = evalMeta?.id || '';
+						const evalIdentifier = evalMeta?.evalId || '';
+						internal.info(
+							`[EVALRUN] Resolved evalId='${evalId}', evalIdentifier='${evalIdentifier}'`
+						);
 
 						// Log eval metadata using structured logging and tracing
 						ctx.logger.debug('Starting eval run with metadata', {
 							evalName,
+							agentName,
 							evalRunId,
 							evalId,
+							evalMetaFromFile: !!evalMeta,
 							evalMetadata: evalItem.metadata,
 						});
 
@@ -1759,8 +1802,9 @@ export function createAgent<
 								'eval.name': evalName,
 								'eval.id': evalId,
 								'eval.runId': evalRunId,
-								'eval.description': evalItem.metadata.description || '',
-								'eval.filename': evalItem.metadata.filename || '',
+								'eval.description':
+									evalMeta?.description || evalItem.metadata.description || '',
+								'eval.filename': evalMeta?.filename || evalItem.metadata.filename || '',
 							});
 						}
 
@@ -1770,12 +1814,14 @@ export function createAgent<
 						const evalRunEventProvider = getEvalRunEventProvider();
 
 						// Only send events if we have required context (devmode flag will be set based on devMode)
-						const shouldSendEvalRunEvents = orgId && projectId && evalId !== '';
+						const shouldSendEvalRunEvents =
+							orgId && projectId && evalId !== '' && evalIdentifier !== '';
 
 						internal.info(`[EVALRUN] Checking conditions for eval '${evalName}':`, {
 							orgId: orgId,
 							projectId: projectId,
 							evalId: evalId,
+							evalIdentifier: evalIdentifier,
 							devMode,
 							hasEvalRunEventProvider: !!evalRunEventProvider,
 							shouldSendEvalRunEvents,
@@ -1786,6 +1832,8 @@ export function createAgent<
 							if (!orgId) reasons.push('missing orgId');
 							if (!projectId) reasons.push('missing projectId');
 							if (!evalId || evalId === '') reasons.push('empty evalId');
+							if (!evalIdentifier || evalIdentifier === '')
+								reasons.push('empty evalIdentifier');
 							internal.info(
 								`[EVALRUN] Skipping eval run events for '${evalName}': ${reasons.join(', ')}`
 							);
@@ -1810,7 +1858,8 @@ export function createAgent<
 									const startEvent: EvalRunStartEvent = {
 										id: evalRunId,
 										sessionId: ctx.sessionId,
-										evalId: evalId,
+										evalId: evalId, // deployment-specific ID (evalid_...)
+										evalIdentifier: evalIdentifier, // stable identifier (eval_...)
 										orgId: orgId!,
 										projectId: projectId!,
 										devmode: Boolean(devMode),
@@ -2139,6 +2188,7 @@ const runWithSpan = async <
 		'@agentuity/agentInstanceId': agent.metadata.agentId,
 		'@agentuity/agentDescription': agent.metadata.description,
 		'@agentuity/agentName': agent.metadata.name,
+		'@agentuity/threadId': ctx.var.thread.id,
 	});
 
 	const spanId = span.spanContext().spanId;
@@ -2243,9 +2293,11 @@ export const createAgentMiddleware = (agentName: AgentName | ''): MiddlewareHand
 			const agentKey = toCamelCase(agentName);
 			const agent = agentsObj[agentKey];
 			const _ctx = privateContext(ctx);
+			// we add both so that you can query by either
 			if (agent?.metadata?.id) {
-				// we add both so that you can query by either
 				_ctx.var.agentIds.add(agent.metadata.id);
+			}
+			if (agent?.metadata?.agentId) {
 				_ctx.var.agentIds.add(agent.metadata.agentId);
 			}
 		}