@inkeep/agents-api 0.0.1

package/dist/domains/evals/routes/evaluationTriggers.js

@@ -0,0 +1,311 @@
+import { getLogger as getLogger$1 } from "../../../logger.js";
+import manageDbPool_default from "../../../data/db/manageDbPool.js";
+import runDbClient_default from "../../../data/db/runDbClient.js";
+import { evaluateConversationWorkflow } from "../workflow/functions/evaluateConversation.js";
+import "../workflow/index.js";
+import { queueEvaluationJobConversations } from "../services/evaluationJob.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { TenantProjectParamsSchema, TriggerEvaluationJobSchema, commonGetErrorResponses, createApiError, createEvaluationRun, generateId, getConversation, getEvaluationSuiteConfigById, getEvaluationSuiteConfigEvaluatorRelations, getEvaluatorsByIds, listEvaluationRunConfigsWithSuiteConfigs, withRef } from "@inkeep/agents-core";
+import { start } from "workflow/api";
+
+//#region src/domains/evals/routes/evaluationTriggers.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("ConversationEvaluations");
+const TriggerConversationSchema = z.object({ conversationId: z.string() });
+app.openapi(createRoute({
+	method: "post",
+	path: "/evaluate-conversation",
+	summary: "Trigger evaluation on a single conversation",
+	operationId: "evaluate-conversation",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: TriggerConversationSchema } } }
+	},
+	responses: {
+		200: {
+			description: "Evaluation triggered successfully",
+			content: { "application/json": { schema: z.object({
+				success: z.boolean(),
+				message: z.string(),
+				evaluationsTriggered: z.number()
+			}) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const body = c.req.valid("json");
+	const resolvedRef = c.get("resolvedRef");
+	const { conversationId } = body;
+	try {
+		logger.info({
+			tenantId,
+			projectId,
+			conversationId
+		}, "Triggering conversation evaluation (eval-api handling all logic)");
+		if (!await getConversation(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			conversationId
+		})) {
+			logger.warn({ conversationId }, "Conversation not found");
+			return c.json(createApiError({
+				code: "not_found",
+				message: "Conversation not found"
+			}), 404);
+		}
+		const runConfigs = (await withRef(manageDbPool_default, resolvedRef, (db) => listEvaluationRunConfigsWithSuiteConfigs(db)({ scopes: {
+			tenantId,
+			projectId
+		} }))).filter((config) => config.isActive);
+		if (runConfigs.length === 0) {
+			logger.debug({
+				tenantId,
+				projectId
+			}, "No active evaluation run configs found");
+			return c.json({
+				success: true,
+				message: "No active evaluation run configs",
+				evaluationsTriggered: 0
+			});
+		}
+		let evaluationsTriggered = 0;
+		for (const runConfig of runConfigs) for (const suiteConfigId of runConfig.suiteConfigIds) {
+			const suiteConfig = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationSuiteConfigById(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationSuiteConfigId: suiteConfigId
+			} }));
+			if (!suiteConfig) {
+				logger.warn({ suiteConfigId }, "Suite config not found, skipping");
+				continue;
+			}
+			if (suiteConfig.sampleRate !== null && suiteConfig.sampleRate !== void 0) {
+				const random = Math.random();
+				if (random > suiteConfig.sampleRate) {
+					logger.info({
+						suiteConfigId: suiteConfig.id,
+						sampleRate: suiteConfig.sampleRate,
+						random,
+						conversationId
+					}, "Conversation filtered out by sample rate");
+					continue;
+				}
+			}
+			const evaluatorIds = (await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluationSuiteConfigEvaluatorRelations(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationSuiteConfigId: suiteConfigId
+			} }))).map((r) => r.evaluatorId);
+			if (evaluatorIds.length === 0) continue;
+			const evaluationRunId = generateId();
+			await createEvaluationRun(runDbClient_default)({
+				id: evaluationRunId,
+				tenantId,
+				projectId,
+				evaluationRunConfigId: runConfig.id
+			});
+			logger.info({
+				conversationId,
+				runConfigId: runConfig.id,
+				evaluationRunId,
+				evaluatorCount: evaluatorIds.length,
+				sampleRate: suiteConfig.sampleRate
+			}, "Created evaluation run, starting workflow");
+			await start(evaluateConversationWorkflow, [{
+				tenantId,
+				projectId,
+				conversationId,
+				evaluatorIds,
+				evaluationRunId
+			}]);
+			evaluationsTriggered++;
+		}
+		return c.json({
+			success: true,
+			message: evaluationsTriggered > 0 ? `Triggered ${evaluationsTriggered} evaluation(s)` : "No evaluations matched (filtered by sample rate or no evaluators)",
+			evaluationsTriggered
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			errorStack: error?.stack,
+			tenantId,
+			projectId,
+			conversationId
+		}, "Failed to trigger conversation evaluation");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: error?.message || "Failed to trigger evaluation"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/evaluate-conversations",
+	summary: "Trigger evaluations on conversations with specified evaluators",
+	operationId: "start-conversations-evaluations",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: z.object({
+			conversationIds: z.array(z.string()).min(1, "At least one conversation is required"),
+			evaluatorIds: z.array(z.string()).min(1, "At least one evaluator is required")
+		}) } } }
+	},
+	responses: {
+		202: {
+			description: "Evaluations triggered",
+			content: { "application/json": { schema: z.object({
+				message: z.string(),
+				evaluationRunId: z.string(),
+				conversationIds: z.array(z.string()),
+				evaluatorIds: z.array(z.string())
+			}) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const { conversationIds, evaluatorIds } = c.req.valid("json");
+	const resolvedRef = c.get("resolvedRef");
+	try {
+		const conversations = await Promise.all(conversationIds.map((conversationId) => getConversation(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			conversationId
+		})));
+		const missingConversations = conversationIds.filter((_id, index) => !conversations[index]);
+		if (missingConversations.length > 0) return c.json(createApiError({
+			code: "not_found",
+			message: `Conversations not found: ${missingConversations.join(", ")}`
+		}), 404);
+		const evaluators = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluatorsByIds(db)({
+			scopes: {
+				tenantId,
+				projectId
+			},
+			evaluatorIds
+		}));
+		const missingEvaluators = evaluatorIds.filter((_id, index) => !evaluators[index]);
+		if (missingEvaluators.length > 0) return c.json(createApiError({
+			code: "not_found",
+			message: `Evaluators not found: ${missingEvaluators.join(", ")}`
+		}), 404);
+		const evaluationRunId = generateId();
+		await createEvaluationRun(runDbClient_default)({
+			id: evaluationRunId,
+			tenantId,
+			projectId
+		});
+		await Promise.all(conversationIds.map((conversationId) => start(evaluateConversationWorkflow, [{
+			tenantId,
+			projectId,
+			conversationId,
+			evaluatorIds,
+			evaluationRunId
+		}])));
+		logger.info({
+			tenantId,
+			projectId,
+			conversationIds,
+			evaluatorIds,
+			evaluationRunId
+		}, "Conversation evaluations triggered");
+		return c.json({
+			message: "Evaluations triggered successfully",
+			evaluationRunId,
+			conversationIds,
+			evaluatorIds
+		}, 202);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			conversationIds
+		}, "Failed to trigger conversation evaluations");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to trigger evaluations"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/evaluate-conversations-by-job",
+	summary: "Trigger evaluations on conversations by evaluation job config",
+	description: "Filters conversations based on job filters, creates an evaluation run, and enqueues workflows",
+	operationId: "evaluate-conversations-by-job",
+	tags: ["Workflows"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: TriggerEvaluationJobSchema } } }
+	},
+	responses: {
+		202: {
+			description: "Evaluation job triggered successfully",
+			content: { "application/json": { schema: z.object({
+				queued: z.number(),
+				failed: z.number(),
+				evaluationRunId: z.string(),
+				conversationCount: z.number()
+			}) } }
+		},
+		400: { description: "Invalid request" },
+		500: { description: "Internal server error" }
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const { evaluationJobConfigId, evaluatorIds, jobFilters } = c.req.valid("json");
+	logger.info({
+		tenantId,
+		projectId,
+		evaluationJobConfigId,
+		evaluatorCount: evaluatorIds.length
+	}, "Triggering evaluation job");
+	try {
+		const { conversationCount, queued, failed, evaluationRunId } = await queueEvaluationJobConversations({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			evaluatorIds,
+			jobFilters
+		});
+		logger.info({
+			tenantId,
+			projectId,
+			evaluationJobConfigId,
+			evaluationRunId,
+			conversationCount,
+			queued,
+			failed
+		}, "Evaluation job triggered successfully");
+		return c.json({
+			queued,
+			failed,
+			evaluationRunId,
+			conversationCount
+		}, 202);
+	} catch (err) {
+		logger.error({
+			err,
+			tenantId,
+			projectId,
+			evaluationJobConfigId
+		}, "Failed to trigger evaluation job");
+		return c.json({
+			error: "Failed to trigger evaluation job",
+			message: err.message
+		}, 500);
+	}
+});
+var evaluationTriggers_default = app;
+
+//#endregion
+export { evaluationTriggers_default as default };

package/dist/domains/evals/routes/index.d.ts

@@ -0,0 +1,7 @@
+import { OpenAPIHono } from "@hono/zod-openapi";
+import * as hono1 from "hono";
+
+//#region src/domains/evals/routes/index.d.ts
+declare const app: OpenAPIHono<hono1.Env, {}, "/">;
+//#endregion
+export { app as default };

package/dist/domains/evals/routes/index.js

@@ -0,0 +1,12 @@
+import datasetTriggers_default from "./datasetTriggers.js";
+import evaluationTriggers_default from "./evaluationTriggers.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/evals/routes/index.ts
+const app = new OpenAPIHono();
+app.route("/", datasetTriggers_default);
+app.route("/", evaluationTriggers_default);
+var routes_default = app;
+
+//#endregion
+export { routes_default as default };

package/dist/domains/evals/scripts/build-workflow.d.ts

@@ -0,0 +1 @@
+export { };

package/dist/domains/evals/scripts/build-workflow.js

@@ -0,0 +1,31 @@
+import { StandaloneBuilder } from "@workflow/builders";
+
+//#region src/domains/evals/scripts/build-workflow.ts
+/**
+* Custom workflow build script that supports externalizing native modules.
+*
+* The default `workflow build` CLI doesn't expose the externalPackages option,
+* so we use the builder directly to exclude native modules like keytar.
+*/
+const config = {
+	dirs: ["./src/domains/evals/workflow"],
+	workingDir: process.cwd(),
+	buildTarget: "standalone",
+	stepsBundlePath: "./.well-known/workflow/v1/step.cjs",
+	workflowsBundlePath: "./.well-known/workflow/v1/flow.cjs",
+	webhookBundlePath: "./.well-known/workflow/v1/webhook.mjs",
+	externalPackages: ["keytar"]
+};
+async function build() {
+	console.log("Building workflow bundles...");
+	console.log("External packages:", config.externalPackages);
+	await new StandaloneBuilder(config).build();
+	console.log("Workflow build completed!");
+}
+build().catch((err) => {
+	console.error("Build failed:", err);
+	process.exit(1);
+});
+
+//#endregion
+export {  };

package/dist/domains/evals/services/EvaluationService.d.ts

@@ -0,0 +1,96 @@
+import { ConversationSelect, DatasetItemSelect, EvaluationResultSelect, EvaluatorSelect } from "@inkeep/agents-core";
+
+//#region src/domains/evals/services/EvaluationService.d.ts
+interface RunDatasetItemOptions {
+  tenantId: string;
+  projectId: string;
+  agentId: string;
+  datasetItem: DatasetItemSelect;
+  datasetRunId: string;
+  conversationId?: string;
+  apiKey?: string;
+}
+interface ChatApiResponse {
+  conversationId?: string;
+  response?: string;
+  error?: string;
+}
+/**
+ * Service for running dataset items through the chat API endpoint
+ */
+declare class EvaluationService {
+  private readonly agentsApiUrl;
+  private readonly runApiBypassSecret;
+  constructor();
+  /**
+   * Run a dataset item through the chat API endpoint
+   * Supports multi-turn conversations with simulation agents
+   */
+  runDatasetItem(options: RunDatasetItemOptions): Promise<ChatApiResponse>;
+  /**
+   * Run a single-turn conversation (original behavior)
+   */
+  private runSingleTurn;
+  /**
+   * Run a multi-turn conversation with simulation agent
+   */
+  private runDatasetItemWithSimulation;
+  /**
+   * Build prompt for simulation agent based on persona and conversation history
+   */
+  private buildSimulationPrompt;
+  /**
+   * Extract messages from dataset item input
+   */
+  private extractMessagesFromDatasetItem;
+  /**
+   * Parse SSE (Server-Sent Events) response from chat API
+   * Handles text deltas, error operations, and other data operations
+   */
+  private parseSSEResponse;
+  /**
+   * Run an evaluation job based on an evaluation job config
+   * Filters conversations based on jobFilters and runs evaluations with configured evaluators
+   */
+  runEvaluationJob(params: {
+    tenantId: string;
+    projectId: string;
+    evaluationJobConfigId: string;
+    sampleRate?: number | null;
+  }): Promise<Array<EvaluationResultSelect>>;
+  /**
+   * Apply sample rate to conversations
+   */
+  applySampleRate<T>(items: T[], sampleRate: number | null | undefined): T[];
+  /**
+   * Execute an evaluation by calling the LLM with the evaluator prompt and conversation data
+   */
+  executeEvaluation(params: {
+    conversation: ConversationSelect;
+    evaluator: EvaluatorSelect;
+    tenantId: string;
+    projectId: string;
+    expectedOutput?: unknown;
+  }): Promise<{
+    output: any;
+    metadata: Record<string, unknown>;
+  }>;
+  /**
+   * Build evaluation prompt with agent definition, conversation history, trace, and expected output
+   */
+  private buildEvalInputEvaluationPrompt;
+  /**
+   * Call LLM API using AI SDK's generateObject for structured output
+   */
+  private callLLM;
+  /**
+   * Fetch trace from SigNoz (similar to the example)
+   */
+  private fetchTraceFromSigNoz;
+  /**
+   * Format conversation detail as prettified trace
+   */
+  private formatConversationAsPrettifiedTrace;
+}
+//#endregion
+export { ChatApiResponse, EvaluationService, RunDatasetItemOptions };