@inkeep/agents-api 0.0.1

package/dist/domains/evals/services/EvaluationService.js

@@ -0,0 +1,863 @@
+import { getLogger as getLogger$1 } from "../../../logger.js";
+import { env } from "../../../env.js";
+import manageDbClient_default from "../../../data/db/manageDbClient.js";
+import manageDbPool_default from "../../../data/db/manageDbPool.js";
+import runDbClient_default from "../../../data/db/runDbClient.js";
+import { ModelFactory, createEvaluationResult, createEvaluationRun, filterConversationsForJob, generateId, getConversationHistory, getEvaluationJobConfigById, getEvaluationJobConfigEvaluatorRelations, getEvaluatorById, getFullAgent, getProjectScopedRef, resolveRef, updateEvaluationResult, withRef } from "@inkeep/agents-core";
+import { generateObject, generateText } from "ai";
+import { z } from "zod";
+
+//#region src/domains/evals/services/EvaluationService.ts
+const logger = getLogger$1("EvaluationService");
+/**
+* Converts JSON Schema objects to Zod schema types
+*/
+function jsonSchemaToZod(jsonSchema) {
+	if (!jsonSchema || typeof jsonSchema !== "object") {
+		logger.warn({ jsonSchema }, "Invalid JSON schema provided, using string fallback");
+		return z.string();
+	}
+	switch (jsonSchema.type) {
+		case "object":
+			if (jsonSchema.properties) {
+				const shape = {};
+				const required = jsonSchema.required || [];
+				for (const [key, prop] of Object.entries(jsonSchema.properties)) {
+					const propSchema = prop;
+					let zodType = jsonSchemaToZod(propSchema);
+					if (propSchema.description) zodType = zodType.describe(String(propSchema.description));
+					if (!required.includes(key)) zodType = zodType.optional();
+					shape[key] = zodType;
+				}
+				return z.object(shape);
+			}
+			return z.record(z.string(), z.unknown());
+		case "array": {
+			const itemSchema = jsonSchema.items ? jsonSchemaToZod(jsonSchema.items) : z.unknown();
+			return z.array(itemSchema);
+		}
+		case "string": return z.string();
+		case "number": return z.number();
+		case "integer": return z.number().int();
+		case "boolean": return z.boolean();
+		case "null": return z.null();
+		default:
+			logger.warn({
+				unsupportedType: jsonSchema.type,
+				schema: jsonSchema
+			}, "Unsupported JSON schema type, using unknown validation");
+			return z.unknown();
+	}
+}
+/**
+* Service for running dataset items through the chat API endpoint
+*/
+var EvaluationService = class {
+	agentsApiUrl;
+	runApiBypassSecret;
+	constructor() {
+		this.runApiBypassSecret = env.INKEEP_AGENTS_RUN_API_BYPASS_SECRET ?? void 0;
+		this.agentsApiUrl = env.INKEEP_AGENTS_API_URL ?? "";
+	}
+	/**
+	* Run a dataset item through the chat API endpoint
+	* Supports multi-turn conversations with simulation agents
+	*/
+	async runDatasetItem(options) {
+		const { tenantId, projectId, agentId, datasetItem, datasetRunId, apiKey } = options;
+		try {
+			const initialMessages = this.extractMessagesFromDatasetItem(datasetItem);
+			if (!initialMessages || initialMessages.length === 0) return { error: "Dataset item has no valid input messages" };
+			const finalConversationId = generateId();
+			if (datasetItem.simulationAgent && typeof datasetItem.simulationAgent === "object" && datasetItem.simulationAgent !== null && !Array.isArray(datasetItem.simulationAgent) && datasetItem.simulationAgent.prompt && datasetItem.simulationAgent.model?.model) return await this.runDatasetItemWithSimulation({
+				tenantId,
+				projectId,
+				agentId,
+				datasetItem,
+				datasetRunId,
+				conversationId: finalConversationId,
+				apiKey,
+				initialMessages,
+				simulationAgent: datasetItem.simulationAgent
+			});
+			return await this.runSingleTurn({
+				tenantId,
+				projectId,
+				agentId,
+				datasetItem,
+				datasetRunId,
+				conversationId: finalConversationId,
+				apiKey,
+				messages: initialMessages
+			});
+		} catch (error) {
+			logger.error({
+				error: error instanceof Error ? error.message : String(error),
+				datasetItemId: datasetItem.id
+			}, "Error running dataset item through chat API");
+			return { error: error instanceof Error ? error.message : "Unknown error occurred" };
+		}
+	}
+	/**
+	* Run a single-turn conversation (original behavior)
+	*/
+	async runSingleTurn(params) {
+		const { tenantId, projectId, agentId, datasetItem, datasetRunId, conversationId, apiKey, messages } = params;
+		const chatUrl = `${this.agentsApiUrl}/run/api/chat`;
+		const chatPayload = {
+			messages,
+			conversationId,
+			stream: true
+		};
+		const authToken = apiKey || this.runApiBypassSecret;
+		const headers$1 = {
+			"Content-Type": "application/json",
+			...authToken && { Authorization: `Bearer ${authToken}` },
+			"x-inkeep-tenant-id": tenantId,
+			"x-inkeep-project-id": projectId,
+			"x-inkeep-agent-id": agentId,
+			...datasetRunId && { "x-inkeep-dataset-run-id": datasetRunId }
+		};
+		logger.info({
+			tenantId,
+			projectId,
+			agentId,
+			datasetItemId: datasetItem.id,
+			datasetRunId,
+			conversationId
+		}, "Running dataset item through chat API");
+		const response = await fetch(chatUrl, {
+			method: "POST",
+			headers: headers$1,
+			body: JSON.stringify(chatPayload)
+		});
+		if (!response.ok) {
+			const errorText = await response.text();
+			logger.error({
+				status: response.status,
+				statusText: response.statusText,
+				errorText,
+				datasetItemId: datasetItem.id,
+				conversationId
+			}, "Chat API request failed");
+			return {
+				conversationId,
+				error: `Chat API error: ${response.status} ${response.statusText}`
+			};
+		}
+		const responseText = await response.text();
+		const parseResult = this.parseSSEResponse(responseText);
+		if (parseResult.error) {
+			logger.error({
+				datasetItemId: datasetItem.id,
+				conversationId,
+				errorMessage: parseResult.error
+			}, "Chat API returned error operation");
+			return {
+				conversationId,
+				error: parseResult.error
+			};
+		}
+		logger.info({
+			datasetItemId: datasetItem.id,
+			conversationId,
+			responseLength: parseResult.text?.length || 0
+		}, "Successfully processed dataset item");
+		return {
+			conversationId,
+			response: parseResult.text
+		};
+	}
+	/**
+	* Run a multi-turn conversation with simulation agent
+	*/
+	async runDatasetItemWithSimulation(params) {
+		const { tenantId, projectId, agentId, datasetItem, datasetRunId, conversationId, apiKey, initialMessages, simulationAgent } = params;
+		logger.info({
+			tenantId,
+			projectId,
+			agentId,
+			datasetItemId: datasetItem.id,
+			conversationId,
+			hasSimulationAgent: true,
+			stopWhen: simulationAgent.stopWhen
+		}, "Running dataset item with simulation agent");
+		const simulationModelConfig = ModelFactory.prepareGenerationConfig(simulationAgent.model);
+		const maxSteps = (simulationAgent.stopWhen || {}).stepCountIs ?? 10;
+		const conversationHistory = [];
+		let stepCount = 0;
+		const initialResult = await this.runSingleTurn({
+			tenantId,
+			projectId,
+			agentId,
+			datasetItem,
+			datasetRunId,
+			conversationId,
+			apiKey,
+			messages: initialMessages
+		});
+		if (initialResult.error || !initialResult.response) return initialResult;
+		const lastUserMessage = initialMessages.filter((m) => m.role === "user").map((m) => typeof m.content === "string" ? m.content : JSON.stringify(m.content)).join("\n");
+		conversationHistory.push({
+			role: "user",
+			content: lastUserMessage
+		});
+		conversationHistory.push({
+			role: "assistant",
+			content: initialResult.response
+		});
+		stepCount++;
+		while (stepCount < maxSteps) try {
+			const simulationPrompt = this.buildSimulationPrompt(simulationAgent.prompt, conversationHistory);
+			logger.debug({
+				stepCount,
+				maxSteps,
+				conversationHistoryLength: conversationHistory.length
+			}, "Generating next user message with simulation agent");
+			const nextUserMessage = (await generateText({
+				...simulationModelConfig,
+				prompt: simulationPrompt
+			})).text.trim();
+			if (!nextUserMessage) {
+				logger.warn({
+					stepCount,
+					datasetItemId: datasetItem.id
+				}, "Simulation agent returned empty message, stopping conversation");
+				break;
+			}
+			conversationHistory.push({
+				role: "user",
+				content: nextUserMessage
+			});
+			const agentResponse = await this.runSingleTurn({
+				tenantId,
+				projectId,
+				agentId,
+				datasetItem,
+				datasetRunId,
+				conversationId,
+				apiKey,
+				messages: [{
+					role: "user",
+					content: nextUserMessage
+				}]
+			});
+			if (agentResponse.error || !agentResponse.response) {
+				logger.warn({
+					stepCount,
+					error: agentResponse.error,
+					datasetItemId: datasetItem.id
+				}, "Agent response failed, stopping conversation");
+				break;
+			}
+			conversationHistory.push({
+				role: "assistant",
+				content: agentResponse.response
+			});
+			stepCount++;
+		} catch (error) {
+			logger.error({
+				error: error instanceof Error ? error.message : String(error),
+				stepCount,
+				datasetItemId: datasetItem.id
+			}, "Error in simulation agent loop");
+			break;
+		}
+		logger.info({
+			datasetItemId: datasetItem.id,
+			conversationId,
+			finalStepCount: stepCount,
+			maxSteps,
+			conversationHistoryLength: conversationHistory.length
+		}, "Completed multi-turn conversation with simulation agent");
+		return {
+			conversationId,
+			response: conversationHistory.filter((m) => m.role === "assistant").pop()?.content || ""
+		};
+	}
+	/**
+	* Build prompt for simulation agent based on persona and conversation history
+	*/
+	buildSimulationPrompt(personaPrompt, conversationHistory) {
+		return `${personaPrompt}
+
+You are simulating a user in a conversation. Based on the conversation history below, generate the next user message that would naturally follow. Keep your response concise and realistic.
+
+Conversation History:
+${conversationHistory.map((msg) => `${msg.role === "user" ? "User" : "Assistant"}: ${msg.content}`).join("\n\n")}
+
+Generate the next user message:`;
+	}
+	/**
+	* Extract messages from dataset item input
+	*/
+	extractMessagesFromDatasetItem(datasetItem) {
+		if (!datasetItem.input) return null;
+		const roleMap = {
+			agent: "assistant",
+			user: "user",
+			system: "system",
+			assistant: "assistant",
+			function: "function",
+			tool: "tool"
+		};
+		if (typeof datasetItem.input === "object" && "messages" in datasetItem.input) {
+			const input = datasetItem.input;
+			const validMessages = input.messages.map((msg) => {
+				const mappedRole = roleMap[msg.role.toLowerCase()];
+				if (!mappedRole) {
+					logger.warn({
+						datasetItemId: datasetItem.id,
+						invalidRole: msg.role
+					}, "Invalid message role found, skipping message");
+					return null;
+				}
+				return {
+					role: mappedRole,
+					content: msg.content
+				};
+			}).filter((msg) => msg !== null);
+			if (validMessages.length === 0) {
+				logger.warn({
+					datasetItemId: datasetItem.id,
+					totalMessages: input.messages.length
+				}, "No valid messages found after filtering roles");
+				return null;
+			}
+			return validMessages;
+		}
+		if (typeof datasetItem.input === "string") try {
+			const parsed = JSON.parse(datasetItem.input);
+			if (parsed.messages && Array.isArray(parsed.messages)) {
+				const validMessages = parsed.messages.map((msg) => {
+					const mappedRole = roleMap[msg.role?.toLowerCase()];
+					if (!mappedRole) {
+						logger.warn({
+							datasetItemId: datasetItem.id,
+							invalidRole: msg.role
+						}, "Invalid message role found in parsed input, skipping message");
+						return null;
+					}
+					return {
+						role: mappedRole,
+						content: msg.content
+					};
+				}).filter((msg) => msg !== null);
+				return validMessages.length > 0 ? validMessages : null;
+			}
+		} catch {
+			return [{
+				role: "user",
+				content: datasetItem.input
+			}];
+		}
+		return null;
+	}
+	/**
+	* Parse SSE (Server-Sent Events) response from chat API
+	* Handles text deltas, error operations, and other data operations
+	*/
+	parseSSEResponse(sseText) {
+		let textContent = "";
+		let hasError = false;
+		let errorMessage = "";
+		const lines = sseText.split("\n").filter((line) => line.startsWith("data: "));
+		for (const line of lines) try {
+			const data = JSON.parse(line.slice(6));
+			if (data.object === "chat.completion.chunk" && data.choices?.[0]?.delta) {
+				const delta = data.choices[0].delta;
+				if (delta.content) textContent += delta.content;
+				if (delta.content && typeof delta.content === "string") try {
+					const parsedContent = JSON.parse(delta.content);
+					if (parsedContent.type === "data-operation" && parsedContent.data?.type === "error") {
+						hasError = true;
+						errorMessage = parsedContent.data.message || "Unknown error occurred";
+						logger.warn({
+							errorMessage,
+							errorData: parsedContent.data
+						}, "Received error operation from chat API");
+					}
+				} catch {}
+			} else if (data.type === "text-delta" && data.delta) textContent += data.delta;
+			else if (data.type === "data-operation" && data.data?.type === "error") {
+				hasError = true;
+				errorMessage = data.data.message || "Unknown error occurred";
+				logger.warn({
+					errorMessage,
+					errorData: data.data
+				}, "Received error operation from chat API");
+			} else if (data.type === "error") {
+				hasError = true;
+				errorMessage = data.message || "Unknown error occurred";
+				logger.warn({
+					errorMessage,
+					errorData: data
+				}, "Received error event from chat API");
+			} else if (data.content) textContent += typeof data.content === "string" ? data.content : JSON.stringify(data.content);
+		} catch {}
+		if (hasError) return {
+			text: textContent.trim(),
+			error: errorMessage
+		};
+		return { text: textContent.trim() };
+	}
+	/**
+	* Run an evaluation job based on an evaluation job config
+	* Filters conversations based on jobFilters and runs evaluations with configured evaluators
+	*/
+	async runEvaluationJob(params) {
+		const { tenantId, projectId, evaluationJobConfigId, sampleRate } = params;
+		const ref = getProjectScopedRef(tenantId, projectId, "main");
+		const resolvedRef = await resolveRef(manageDbClient_default)(ref);
+		if (!resolvedRef) throw new Error("Failed to resolve ref");
+		logger.info({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			sampleRate
+		}, "Starting evaluation job");
+		const config = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationJobConfigById(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationJobConfigId
+		} }));
+		if (!config) throw new Error(`Evaluation job config not found: ${evaluationJobConfigId}`);
+		const evaluatorRelations = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationJobConfigEvaluatorRelations(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationJobConfigId
+		} }));
+		if (evaluatorRelations.length === 0) throw new Error(`No evaluators found for job config: ${evaluationJobConfigId}`);
+		const validEvaluators = (await Promise.all(evaluatorRelations.map((relation) => withRef(manageDbPool_default, resolvedRef, (db) => getEvaluatorById(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluatorId: relation.evaluatorId
+		} }))))).filter((e) => e !== null);
+		if (validEvaluators.length === 0) throw new Error(`No valid evaluators found for job config: ${evaluationJobConfigId}`);
+		logger.info({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			evaluatorCount: validEvaluators.length
+		}, "Found evaluators for job config");
+		let conversationsToEvaluate = await filterConversationsForJob(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			jobFilters: config.jobFilters
+		});
+		if (sampleRate !== void 0 && sampleRate !== null) {
+			const originalCount = conversationsToEvaluate.length;
+			conversationsToEvaluate = this.applySampleRate(conversationsToEvaluate, sampleRate);
+			logger.info({
+				tenantId,
+				projectId,
+				evaluationJobConfigId,
+				originalCount,
+				sampledCount: conversationsToEvaluate.length,
+				sampleRate
+			}, "Applied sample rate to conversations");
+		}
+		logger.info({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			conversationCount: conversationsToEvaluate.length
+		}, "Found conversations for evaluation");
+		if (conversationsToEvaluate.length === 0) {
+			logger.warn({
+				tenantId,
+				projectId,
+				evaluationJobConfigId
+			}, "No conversations found matching job filters");
+			return [];
+		}
+		const evaluationRun = await createEvaluationRun(runDbClient_default)({
+			id: generateId(),
+			tenantId,
+			projectId,
+			evaluationJobConfigId
+		});
+		const results = [];
+		for (const conversation of conversationsToEvaluate) for (const evaluator of validEvaluators) try {
+			logger.info({
+				tenantId,
+				conversationId: conversation.id,
+				evaluatorId: evaluator.id
+			}, "Running evaluation");
+			const evalResult = await createEvaluationResult(runDbClient_default)({
+				id: generateId(),
+				tenantId,
+				projectId,
+				conversationId: conversation.id,
+				evaluatorId: evaluator.id,
+				evaluationRunId: evaluationRun.id
+			});
+			try {
+				const evaluationResult = await this.executeEvaluation({
+					conversation,
+					evaluator,
+					tenantId,
+					projectId
+				});
+				const updatedResult = await updateEvaluationResult(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId,
+						evaluationResultId: evalResult.id
+					},
+					data: { output: evaluationResult.output }
+				});
+				if (updatedResult) results.push(updatedResult);
+				logger.info({
+					tenantId,
+					conversationId: conversation.id,
+					evaluatorId: evaluator.id,
+					resultId: evalResult.id
+				}, "Evaluation completed successfully");
+			} catch (error) {
+				logger.error({
+					error,
+					tenantId,
+					conversationId: conversation.id,
+					evaluatorId: evaluator.id,
+					resultId: evalResult.id
+				}, "Evaluation execution failed");
+				const errorMessage = error instanceof Error ? error.message : "Unknown error";
+				const failedResult = await updateEvaluationResult(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId,
+						evaluationResultId: evalResult.id
+					},
+					data: { output: { text: `Evaluation failed: ${errorMessage}` } }
+				});
+				if (failedResult) results.push(failedResult);
+			}
+		} catch (error) {
+			logger.error({
+				error: error instanceof Error ? error.message : String(error),
+				tenantId,
+				conversationId: conversation.id,
+				evaluatorId: evaluator.id
+			}, "Failed to create or update eval result");
+		}
+		logger.info({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			resultCount: results.length
+		}, "Evaluation job completed");
+		return results;
+	}
+	/**
+	* Apply sample rate to conversations
+	*/
+	applySampleRate(items, sampleRate) {
+		if (!sampleRate || sampleRate >= 1) return items;
+		if (sampleRate <= 0) return [];
+		const targetCount = Math.ceil(items.length * sampleRate);
+		const sampled = [];
+		const indices = /* @__PURE__ */ new Set();
+		while (sampled.length < targetCount && sampled.length < items.length) {
+			const randomIndex = Math.floor(Math.random() * items.length);
+			if (!indices.has(randomIndex)) {
+				indices.add(randomIndex);
+				sampled.push(items[randomIndex]);
+			}
+		}
+		return sampled;
+	}
+	/**
+	* Execute an evaluation by calling the LLM with the evaluator prompt and conversation data
+	*/
+	async executeEvaluation(params) {
+		const { conversation, evaluator, tenantId, projectId, expectedOutput } = params;
+		let resolvedRef = null;
+		if (conversation.ref) resolvedRef = conversation.ref;
+		else {
+			const ref = getProjectScopedRef(tenantId, projectId, "main");
+			resolvedRef = await resolveRef(manageDbClient_default)(ref);
+		}
+		if (!resolvedRef) throw new Error("Failed to resolve ref");
+		const conversationHistory = await getConversationHistory(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			conversationId: conversation.id,
+			options: {
+				includeInternal: false,
+				limit: 100
+			}
+		});
+		let agentDefinition = null;
+		let agentId = null;
+		try {
+			agentId = conversation.agentId ?? null;
+			if (agentId) {
+				const agentIdForLookup = agentId;
+				agentDefinition = await withRef(manageDbPool_default, resolvedRef, (db) => getFullAgent(db)({ scopes: {
+					tenantId,
+					projectId,
+					agentId: agentIdForLookup
+				} }));
+			} else logger.warn({
+				conversationId: conversation.id,
+				agentId: conversation.agentId
+			}, "AgentId not found, cannot get agent definition");
+		} catch (error) {
+			logger.warn({
+				error,
+				conversationId: conversation.id,
+				agentId: conversation.agentId
+			}, "Failed to fetch agent definition for evaluation");
+		}
+		const prettifiedTrace = await this.fetchTraceFromSigNoz(conversation.id);
+		logger.info({
+			conversationId: conversation.id,
+			hasTrace: !!prettifiedTrace,
+			traceActivityCount: prettifiedTrace?.timeline?.length || 0
+		}, "Trace fetch completed");
+		const conversationText = JSON.stringify(conversationHistory, null, 2);
+		const agentDefinitionText = agentDefinition ? JSON.stringify(agentDefinition, null, 2) : "Agent definition not available";
+		const traceText = prettifiedTrace ? JSON.stringify(prettifiedTrace, null, 2) : "Trace data not available";
+		const modelConfig = evaluator.model ?? {};
+		let schemaObj;
+		if (typeof evaluator.schema === "string") try {
+			schemaObj = JSON.parse(evaluator.schema);
+		} catch (error) {
+			logger.error({
+				error,
+				schemaString: evaluator.schema
+			}, "Failed to parse evaluator schema string");
+			throw new Error("Invalid evaluator schema format");
+		}
+		else schemaObj = evaluator.schema;
+		logger.info({
+			evaluatorId: evaluator.id,
+			schemaType: typeof schemaObj,
+			schemaKeys: schemaObj && typeof schemaObj === "object" ? Object.keys(schemaObj) : []
+		}, "Using evaluator schema");
+		const expectedOutputText = expectedOutput ? JSON.stringify(expectedOutput, null, 2) : void 0;
+		const evaluationPrompt = this.buildEvalInputEvaluationPrompt(evaluator.prompt, agentDefinitionText, conversationText, traceText, schemaObj, expectedOutputText);
+		const llmResponse = await this.callLLM({
+			prompt: evaluationPrompt,
+			modelConfig,
+			schema: schemaObj
+		});
+		return {
+			output: llmResponse.result,
+			metadata: {
+				...llmResponse.metadata,
+				model: modelConfig.model || "unknown",
+				agentId,
+				hasAgentDefinition: !!agentDefinition,
+				hasTrace: !!prettifiedTrace,
+				traceActivityCount: prettifiedTrace?.timeline?.length || 0
+			}
+		};
+	}
+	/**
+	* Build evaluation prompt with agent definition, conversation history, trace, and expected output
+	*/
+	buildEvalInputEvaluationPrompt(evaluatorPrompt, agentDefinitionText, conversationText, traceText, schema, expectedOutputText) {
+		const schemaDescription = JSON.stringify(schema, null, 2);
+		return `${evaluatorPrompt}
+
+Agent Definition:
+
+${agentDefinitionText}
+
+Conversation History:
+
+${conversationText}
+
+Execution Trace:
+
+${traceText}
+${expectedOutputText ? `
+
+Expected Output:
+
+${expectedOutputText}
+` : ""}
+Please evaluate this conversation according to the following schema and return your evaluation as JSON:
+
+${schemaDescription}
+
+Return your evaluation as a JSON object matching the schema above.`;
+	}
+	/**
+	* Call LLM API using AI SDK's generateObject for structured output
+	*/
+	async callLLM(params) {
+		const { prompt, modelConfig, schema } = params;
+		const languageModel = ModelFactory.prepareGenerationConfig(modelConfig);
+		const providerOptions = modelConfig?.providerOptions || {};
+		let resultSchema;
+		try {
+			resultSchema = jsonSchemaToZod(schema);
+			logger.info({
+				schemaType: typeof schema,
+				schemaKeys: schema && typeof schema === "object" ? Object.keys(schema) : [],
+				convertedSchema: "success"
+			}, "Converted JSON schema to Zod");
+		} catch (error) {
+			logger.error({
+				error,
+				schema
+			}, "Failed to convert JSON schema to Zod, using fallback");
+			resultSchema = z.record(z.string(), z.unknown());
+		}
+		const evaluationSchema = resultSchema;
+		try {
+			logger.info({
+				promptLength: prompt.length,
+				model: modelConfig.model
+			}, "Calling generateObject");
+			const result = await generateObject({
+				...languageModel,
+				schema: evaluationSchema,
+				prompt,
+				temperature: providerOptions.temperature ?? .3
+			});
+			return {
+				result: result.object,
+				metadata: { usage: result.usage }
+			};
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			logger.error({
+				error: errorMessage,
+				schema: JSON.stringify(schema, null, 2),
+				promptPreview: prompt.substring(0, 500)
+			}, "Evaluation failed with generateObject");
+			throw new Error(`Evaluation failed: ${errorMessage}`);
+		}
+	}
+	/**
+	* Fetch trace from SigNoz (similar to the example)
+	*/
+	async fetchTraceFromSigNoz(conversationId) {
+		const manageUIUrl = env.INKEEP_AGENTS_MANAGE_UI_URL;
+		const maxRetries = 2;
+		const retryDelayMs = 2e4;
+		const initialDelayMs = 3e4;
+		try {
+			logger.info({
+				conversationId,
+				manageUIUrl,
+				initialDelayMs
+			}, "Waiting 30s before fetching trace from SigNoz");
+			await new Promise((resolve) => setTimeout(resolve, initialDelayMs));
+			for (let attempt = 0; attempt <= maxRetries; attempt++) try {
+				logger.info({
+					conversationId,
+					attempt: attempt + 1,
+					maxRetries: maxRetries + 1
+				}, "Fetching trace from SigNoz");
+				const traceResponse = await fetch(`${manageUIUrl}/api/signoz/conversations/${conversationId}`);
+				if (!traceResponse.ok) {
+					logger.warn({
+						conversationId,
+						status: traceResponse.status,
+						statusText: traceResponse.statusText,
+						attempt: attempt + 1
+					}, "Failed to fetch trace from SigNoz");
+					if (attempt < maxRetries) {
+						logger.info({
+							conversationId,
+							retryDelayMs
+						}, "Retrying trace fetch after delay");
+						await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+						continue;
+					}
+					return null;
+				}
+				const conversationDetail = await traceResponse.json();
+				logger.debug({
+					conversationId,
+					activityTypes: conversationDetail.activities?.map((a) => a.type) || [],
+					activityCount: conversationDetail.activities?.length || 0
+				}, "Checking activities for ai_assistant_message type");
+				if (!conversationDetail.activities?.some((activity) => activity.type === "ai_assistant_message")) {
+					logger.warn({
+						conversationId,
+						attempt: attempt + 1,
+						activityCount: conversationDetail.activities?.length || 0,
+						activityTypes: conversationDetail.activities?.slice(0, 5).map((a) => a.type) || []
+					}, "Trace fetched but ai_assistant_message not found in activities");
+					if (attempt < maxRetries) {
+						logger.info({
+							conversationId,
+							retryDelayMs
+						}, "Retrying trace fetch after delay to wait for assistant message");
+						await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+						continue;
+					}
+					logger.warn({
+						conversationId,
+						maxRetries,
+						activityCount: conversationDetail.activities?.length || 0
+					}, "Max retries reached, ai_assistant_message not found - proceeding with available trace data");
+				} else logger.info({
+					conversationId,
+					activityCount: conversationDetail.activities?.length || 0,
+					attempt: attempt + 1
+				}, "Trace fetched successfully with ai_assistant_message");
+				return this.formatConversationAsPrettifiedTrace(conversationDetail);
+			} catch (fetchError) {
+				logger.warn({
+					error: fetchError,
+					conversationId,
+					attempt: attempt + 1
+				}, "Error fetching trace from SigNoz");
+				if (attempt < maxRetries) {
+					logger.info({
+						conversationId,
+						retryDelayMs
+					}, "Retrying trace fetch after delay");
+					await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+					continue;
+				}
+				return null;
+			}
+			return null;
+		} catch (error) {
+			logger.warn({
+				error,
+				conversationId,
+				manageUIUrl
+			}, "Failed to fetch trace from SigNoz, will continue without trace");
+			return null;
+		}
+	}
+	/**
+	* Format conversation detail as prettified trace
+	*/
+	formatConversationAsPrettifiedTrace(conversation) {
+		return {
+			metadata: {
+				conversationId: conversation.conversationId,
+				traceId: conversation.traceId,
+				agentName: conversation.agentName,
+				agentId: conversation.agentId,
+				exportedAt: (/* @__PURE__ */ new Date()).toISOString()
+			},
+			timing: {
+				startTime: conversation.conversationStartTime || "",
+				endTime: conversation.conversationEndTime || "",
+				durationMs: conversation.duration || 0
+			},
+			timeline: (conversation.activities || []).map((activity) => {
+				const { id: _id, ...rest } = activity;
+				return { ...rest };
+			})
+		};
+	}
+};
+
+//#endregion
+export { EvaluationService };