@inkeep/agents-api 0.0.1

package/dist/domains/evals/services/conversationEvaluation.d.ts

@@ -0,0 +1,15 @@
+import { ResolvedRef } from "@inkeep/agents-core";
+
+//#region src/domains/evals/services/conversationEvaluation.d.ts
+declare const triggerConversationEvaluation: (params: {
+  tenantId: string;
+  projectId: string;
+  conversationId: string;
+  resolvedRef: ResolvedRef;
+}) => Promise<{
+  success: boolean;
+  message: string;
+  evaluationsTriggered: number;
+}>;
+//#endregion
+export { triggerConversationEvaluation };

package/dist/domains/evals/services/conversationEvaluation.js

@@ -0,0 +1,102 @@
+import { getLogger as getLogger$1 } from "../../../logger.js";
+import manageDbPool_default from "../../../data/db/manageDbPool.js";
+import runDbClient_default from "../../../data/db/runDbClient.js";
+import { evaluateConversationWorkflow } from "../workflow/functions/evaluateConversation.js";
+import "../workflow/index.js";
+import { createEvaluationRun, generateId, getConversation, getEvaluationSuiteConfigById, getEvaluationSuiteConfigEvaluatorRelations, listEvaluationRunConfigsWithSuiteConfigs, withRef } from "@inkeep/agents-core";
+import { start } from "workflow/api";
+
+//#region src/domains/evals/services/conversationEvaluation.ts
+const logger = getLogger$1("ConversationEvaluation");
+const triggerConversationEvaluation = async (params) => {
+	const { tenantId, projectId, conversationId, resolvedRef } = params;
+	try {
+		logger.info({
+			tenantId,
+			projectId,
+			conversationId
+		}, "Triggering conversation evaluation (eval-api handling all logic)");
+		if (!await getConversation(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			conversationId
+		})) throw new Error(`Conversation not found: ${conversationId}`);
+		const runConfigs = (await withRef(manageDbPool_default, resolvedRef, (db) => listEvaluationRunConfigsWithSuiteConfigs(db)({ scopes: {
+			tenantId,
+			projectId
+		} }))).filter((config) => config.isActive);
+		if (runConfigs.length === 0) throw new Error("No active evaluation run configs found");
+		let evaluationsTriggered = 0;
+		for (const runConfig of runConfigs) for (const suiteConfigId of runConfig.suiteConfigIds) {
+			const suiteConfig = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationSuiteConfigById(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationSuiteConfigId: suiteConfigId
+			} }));
+			if (!suiteConfig) {
+				logger.warn({ suiteConfigId }, "Suite config not found, skipping");
+				continue;
+			}
+			if (suiteConfig.sampleRate !== null && suiteConfig.sampleRate !== void 0) {
+				const random = Math.random();
+				if (random > suiteConfig.sampleRate) {
+					logger.info({
+						suiteConfigId: suiteConfig.id,
+						sampleRate: suiteConfig.sampleRate,
+						random,
+						conversationId
+					}, "Conversation filtered out by sample rate");
+					continue;
+				}
+			}
+			const evaluatorIds = (await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationSuiteConfigEvaluatorRelations(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationSuiteConfigId: suiteConfigId
+			} }))).map((r) => r.evaluatorId);
+			if (evaluatorIds.length === 0) continue;
+			const evaluationRunId = generateId();
+			await createEvaluationRun(runDbClient_default)({
+				id: evaluationRunId,
+				tenantId,
+				projectId,
+				evaluationRunConfigId: runConfig.id
+			});
+			logger.info({
+				conversationId,
+				runConfigId: runConfig.id,
+				evaluationRunId,
+				evaluatorCount: evaluatorIds.length,
+				sampleRate: suiteConfig.sampleRate
+			}, "Created evaluation run, starting workflow");
+			await start(evaluateConversationWorkflow, [{
+				tenantId,
+				projectId,
+				conversationId,
+				evaluatorIds,
+				evaluationRunId
+			}]);
+			evaluationsTriggered++;
+		}
+		return {
+			success: true,
+			message: evaluationsTriggered > 0 ? `Triggered ${evaluationsTriggered} evaluation(s)` : "No evaluations matched (filtered by sample rate or no evaluators)",
+			evaluationsTriggered
+		};
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			conversationId
+		}, "Failed to trigger conversation evaluation");
+		logger.error({ error: error?.stack }, "Failed to trigger conversation evaluation");
+		logger.error({ error: error?.message }, "Failed to trigger conversation evaluation");
+		throw error;
+	}
+};
+
+//#endregion
+export { triggerConversationEvaluation };

package/dist/domains/evals/services/datasetRun.d.ts

@@ -0,0 +1,16 @@
+import { DatasetRunItem } from "@inkeep/agents-core";
+
+//#region src/domains/evals/services/datasetRun.d.ts
+declare function queueDatasetRunItems(params: {
+  tenantId: string;
+  projectId: string;
+  datasetRunId: string;
+  items: DatasetRunItem[];
+  evaluatorIds?: string[];
+  evaluationRunId?: string;
+}): Promise<{
+  queued: number;
+  failed: number;
+}>;
+//#endregion
+export { queueDatasetRunItems };

package/dist/domains/evals/services/datasetRun.js

@@ -0,0 +1,43 @@
+import { getLogger } from "../../../logger.js";
+import { runDatasetItemWorkflow } from "../workflow/functions/runDatasetItem.js";
+import { start } from "workflow/api";
+
+//#region src/domains/evals/services/datasetRun.ts
+async function queueDatasetRunItems(params) {
+	const { tenantId, projectId, datasetRunId, items, evaluatorIds, evaluationRunId } = params;
+	const logger = getLogger("workflow-triggers");
+	let queued = 0;
+	let failed = 0;
+	for (const item of items) {
+		const payload = {
+			tenantId,
+			projectId,
+			agentId: item.agentId,
+			datasetItemId: item.id ?? "",
+			datasetItemInput: item.input,
+			datasetItemExpectedOutput: item.expectedOutput,
+			datasetItemSimulationAgent: item.simulationAgent,
+			datasetRunId,
+			evaluatorIds,
+			evaluationRunId
+		};
+		try {
+			await start(runDatasetItemWorkflow, [payload]);
+			queued++;
+		} catch (err) {
+			logger.error({
+				err,
+				datasetItemId: item.id,
+				agentId: item.agentId
+			}, "Failed to queue dataset item workflow");
+			failed++;
+		}
+	}
+	return {
+		queued,
+		failed
+	};
+}
+
+//#endregion
+export { queueDatasetRunItems };

package/dist/domains/evals/services/evaluationJob.d.ts

@@ -0,0 +1,17 @@
+import { EvaluationJobFilterCriteria } from "@inkeep/agents-core";
+
+//#region src/domains/evals/services/evaluationJob.d.ts
+declare function queueEvaluationJobConversations(params: {
+  tenantId: string;
+  projectId: string;
+  evaluationJobConfigId: string;
+  evaluatorIds: string[];
+  jobFilters: EvaluationJobFilterCriteria | null | undefined;
+}): Promise<{
+  conversationCount: number;
+  queued: number;
+  failed: number;
+  evaluationRunId: string;
+}>;
+//#endregion
+export { queueEvaluationJobConversations };

package/dist/domains/evals/services/evaluationJob.js

@@ -0,0 +1,65 @@
+import { getLogger as getLogger$1 } from "../../../logger.js";
+import runDbClient_default from "../../../data/db/runDbClient.js";
+import { evaluateConversationWorkflow } from "../workflow/functions/evaluateConversation.js";
+import "../workflow/index.js";
+import { createEvaluationRun, filterConversationsForJob, generateId } from "@inkeep/agents-core";
+import { start } from "workflow/api";
+
+//#region src/domains/evals/services/evaluationJob.ts
+const logger = getLogger$1("evaluation-job");
+async function queueEvaluationJobConversations(params) {
+	const { tenantId, projectId, evaluationJobConfigId, evaluatorIds, jobFilters } = params;
+	const conversations = await filterConversationsForJob(runDbClient_default)({
+		scopes: {
+			tenantId,
+			projectId
+		},
+		jobFilters
+	});
+	if (conversations.length === 0) {
+		logger.warn({
+			tenantId,
+			projectId,
+			evaluationJobConfigId
+		}, "No conversations found for job");
+		return {
+			conversationCount: 0,
+			queued: 0,
+			failed: 0,
+			evaluationRunId: ""
+		};
+	}
+	const evaluationRun = await createEvaluationRun(runDbClient_default)({
+		id: generateId(),
+		tenantId,
+		projectId,
+		evaluationJobConfigId
+	});
+	let queued = 0;
+	let failed = 0;
+	for (const conv of conversations) try {
+		await start(evaluateConversationWorkflow, [{
+			tenantId,
+			projectId,
+			conversationId: conv.id,
+			evaluatorIds,
+			evaluationRunId: evaluationRun.id
+		}]);
+		queued++;
+	} catch (err) {
+		logger.error({
+			err,
+			conversationId: conv.id
+		}, "Failed to queue conversation evaluation");
+		failed++;
+	}
+	return {
+		conversationCount: conversations.length,
+		queued,
+		failed,
+		evaluationRunId: evaluationRun.id
+	};
+}
+
+//#endregion
+export { queueEvaluationJobConversations };

package/dist/domains/evals/services/startEvaluation.d.ts

@@ -0,0 +1,19 @@
+//#region src/domains/evals/services/startEvaluation.d.ts
+/**
+ * Service to start evaluation workflows.
+ * This encapsulates the workflow logic so consumers don't need to import workflow packages.
+ */
+interface StartEvaluationParams {
+  tenantId: string;
+  projectId: string;
+  conversationId: string;
+  evaluatorIds: string[];
+  evaluationRunId: string;
+}
+/**
+ * Start an evaluation workflow for a conversation.
+ * This is a convenience wrapper that handles workflow initialization internally.
+ */
+declare function startConversationEvaluation(params: StartEvaluationParams): Promise<void>;
+//#endregion
+export { StartEvaluationParams, startConversationEvaluation };

package/dist/domains/evals/services/startEvaluation.js

@@ -0,0 +1,18 @@
+import { evaluateConversationWorkflow } from "../workflow/functions/evaluateConversation.js";
+import { start } from "workflow/api";
+
+//#region src/domains/evals/services/startEvaluation.ts
+/**
+* Service to start evaluation workflows.
+* This encapsulates the workflow logic so consumers don't need to import workflow packages.
+*/
+/**
+* Start an evaluation workflow for a conversation.
+* This is a convenience wrapper that handles workflow initialization internally.
+*/
+async function startConversationEvaluation(params) {
+	await start(evaluateConversationWorkflow, [params]);
+}
+
+//#endregion
+export { startConversationEvaluation };

package/dist/domains/evals/workflow/functions/evaluateConversation.d.ts

@@ -0,0 +1,28 @@
+//#region src/domains/evals/workflow/functions/evaluateConversation.d.ts
+type EvaluationPayload = {
+  tenantId: string;
+  projectId: string;
+  conversationId: string;
+  evaluatorIds: string[];
+  evaluationRunId: string;
+};
+/**
+ * Main workflow function - orchestrates the evaluation steps.
+ *
+ * IMPORTANT: This runs in a deterministic sandbox.
+ * - Do NOT call Node.js APIs directly here (no DB, no fs, etc.)
+ * - All side effects must happen in step functions
+ */
+declare function evaluateConversationWorkflow(payload: EvaluationPayload): Promise<{
+  success: boolean;
+  reason: string;
+  conversationId?: undefined;
+  resultCount?: undefined;
+} | {
+  success: boolean;
+  conversationId: string;
+  resultCount: number;
+  reason?: undefined;
+}>;
+//#endregion
+export { evaluateConversationWorkflow };

package/dist/domains/evals/workflow/functions/evaluateConversation.js

@@ -0,0 +1,134 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import manageDbClient_default from "../../../../data/db/manageDbClient.js";
+import manageDbPool_default from "../../../../data/db/manageDbPool.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import "../../../../data/db/index.js";
+import { EvaluationService } from "../../services/EvaluationService.js";
+import { createEvaluationResult, generateId, getConversation, getEvaluatorById, getEvaluatorsByIds, getProjectMainResolvedRef, updateEvaluationResult, withRef } from "@inkeep/agents-core";
+
+//#region src/domains/evals/workflow/functions/evaluateConversation.ts
+const logger = getLogger$1("workflow-evaluate-conversation");
+async function getConversationStep(payload) {
+	"use step";
+	const { tenantId, projectId, conversationId } = payload;
+	const conv = await getConversation(runDbClient_default)({
+		scopes: {
+			tenantId,
+			projectId
+		},
+		conversationId
+	});
+	if (!conv) throw new Error(`Conversation not found: ${conversationId}`);
+	return conv;
+}
+async function getEvaluatorsStep(payload) {
+	"use step";
+	const { tenantId, projectId, evaluatorIds } = payload;
+	return await withRef(manageDbPool_default, await getProjectMainResolvedRef(manageDbClient_default)(tenantId, projectId), (db) => getEvaluatorsByIds(db)({
+		scopes: {
+			tenantId,
+			projectId
+		},
+		evaluatorIds
+	}));
+}
+async function executeEvaluatorStep(payload, evaluatorId, conversation) {
+	"use step";
+	const { tenantId, projectId, conversationId, evaluationRunId } = payload;
+	const evaluator = await withRef(manageDbPool_default, await getProjectMainResolvedRef(manageDbClient_default)(tenantId, projectId), (db) => getEvaluatorById(db)({ scopes: {
+		tenantId,
+		projectId,
+		evaluatorId
+	} }));
+	if (!evaluator) throw new Error(`Evaluator not found: ${evaluatorId}`);
+	const evalResult = await createEvaluationResult(runDbClient_default)({
+		id: generateId(),
+		tenantId,
+		projectId,
+		conversationId,
+		evaluatorId: evaluator.id,
+		evaluationRunId
+	});
+	try {
+		const output = await new EvaluationService().executeEvaluation({
+			conversation,
+			evaluator,
+			tenantId,
+			projectId
+		});
+		const updated = await updateEvaluationResult(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationResultId: evalResult.id
+			},
+			data: { output }
+		});
+		logger.info({
+			conversationId,
+			evaluatorId: evaluator.id,
+			resultId: evalResult.id
+		}, "Evaluation completed successfully");
+		return updated;
+	} catch (error) {
+		const errorMessage = error instanceof Error ? error.message : "Unknown error";
+		logger.error({
+			error,
+			conversationId,
+			evaluatorId: evaluator.id,
+			resultId: evalResult.id
+		}, "Evaluation execution failed");
+		return await updateEvaluationResult(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationResultId: evalResult.id
+			},
+			data: { output: { text: `Evaluation failed: ${errorMessage}` } }
+		});
+	}
+}
+/**
+* Step: Log workflow progress
+*/
+async function logStep(message, data) {
+	"use step";
+	logger.info(data, message);
+}
+/**
+* Main workflow function - orchestrates the evaluation steps.
+*
+* IMPORTANT: This runs in a deterministic sandbox.
+* - Do NOT call Node.js APIs directly here (no DB, no fs, etc.)
+* - All side effects must happen in step functions
+*/
+async function evaluateConversationWorkflow(payload) {
+	"use workflow";
+	const { conversationId, evaluatorIds } = payload;
+	await logStep("Starting conversation evaluation", payload);
+	const conversation = await getConversationStep(payload);
+	const evaluators = await getEvaluatorsStep(payload);
+	if (evaluators.length === 0) {
+		await logStep("No valid evaluators found", {
+			conversationId,
+			evaluatorIds
+		});
+		return {
+			success: false,
+			reason: "No valid evaluators"
+		};
+	}
+	const results = [];
+	for (const evaluator of evaluators) {
+		const result = await executeEvaluatorStep(payload, evaluator.id, conversation);
+		results.push(result);
+	}
+	return {
+		success: true,
+		conversationId,
+		resultCount: results.length
+	};
+}
+
+//#endregion
+export { evaluateConversationWorkflow };

package/dist/domains/evals/workflow/functions/runDatasetItem.d.ts

@@ -0,0 +1,36 @@
+//#region src/domains/evals/workflow/functions/runDatasetItem.d.ts
+type RunDatasetItemPayload = {
+  tenantId: string;
+  projectId: string;
+  agentId: string;
+  datasetItemId: string;
+  datasetItemInput: unknown;
+  datasetItemExpectedOutput?: unknown;
+  datasetItemSimulationAgent?: {
+    prompt: string;
+    model: {
+      model: string;
+      providerOptions?: Record<string, unknown>;
+    };
+    stopWhen?: {
+      transferCountIs?: number;
+      stepCountIs?: number;
+    };
+  };
+  datasetRunId: string;
+  evaluatorIds?: string[];
+  evaluationRunId?: string;
+};
+/**
+ * Main workflow function - processes a single dataset item through the chat API.
+ * Optionally runs evaluators on the resulting conversation.
+ */
+declare function runDatasetItemWorkflow(payload: RunDatasetItemPayload): Promise<{
+  success: boolean;
+  datasetItemId: string;
+  datasetRunId: string;
+  conversationId: string | null;
+  error: string | null;
+}>;
+//#endregion
+export { runDatasetItemWorkflow };