@inkeep/agents-api 0.0.0-dev-20260121145510

package/dist/domains/manage/routes/evals/evaluationRunConfigs.js

@@ -0,0 +1,403 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { EvaluationResultApiSelectSchema, EvaluationRunConfigApiInsertSchema, EvaluationRunConfigApiUpdateSchema, EvaluationRunConfigWithSuiteConfigsApiSelectSchema, ListResponseSchema, SingleResponseSchema, TenantProjectParamsSchema, commonGetErrorResponses, createApiError, createEvaluationRunConfig, createEvaluationRunConfigEvaluationSuiteConfigRelation, deleteEvaluationRunConfig, deleteEvaluationRunConfigEvaluationSuiteConfigRelation, generateId, getConversation, getEvaluationRunConfigById, getEvaluationRunConfigEvaluationSuiteConfigRelations, getMessagesByConversation, listEvaluationResultsByRun, listEvaluationRunConfigsWithSuiteConfigs, listEvaluationRuns, updateEvaluationRunConfig } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/evaluationRunConfigs.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("evaluationRunConfigs");
+app.openapi(createRoute({
+	method: "get",
+	path: "/",
+	summary: "List Evaluation Run Configs",
+	operationId: "list-evaluation-run-configs",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema },
+	responses: {
+		200: {
+			description: "List of evaluation run configs",
+			content: { "application/json": { schema: ListResponseSchema(EvaluationRunConfigWithSuiteConfigsApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId } = c.req.valid("param");
+	try {
+		const configsWithSuiteConfigs = await listEvaluationRunConfigsWithSuiteConfigs(db)({ scopes: {
+			tenantId,
+			projectId
+		} });
+		return c.json({
+			data: configsWithSuiteConfigs,
+			pagination: {
+				page: 1,
+				limit: configsWithSuiteConfigs.length,
+				total: configsWithSuiteConfigs.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId
+		}, "Failed to list evaluation run configs");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to list evaluation run configs"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{configId}",
+	summary: "Get Evaluation Run Config by ID",
+	operationId: "get-evaluation-run-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		200: {
+			description: "Evaluation run config details",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationRunConfigWithSuiteConfigsApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	try {
+		const config = await getEvaluationRunConfigById(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunConfigId: configId
+		} });
+		if (!config) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation run config not found"
+		}), 404);
+		const suiteConfigRelations = await getEvaluationRunConfigEvaluationSuiteConfigRelations(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunConfigId: configId
+		} });
+		return c.json({ data: {
+			...config,
+			suiteConfigIds: suiteConfigRelations.map((rel) => rel.evaluationSuiteConfigId)
+		} });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to get evaluation run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get evaluation run config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{configId}/results",
+	summary: "Get Evaluation Results by Run Config ID",
+	operationId: "get-evaluation-run-config-results",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		200: {
+			description: "Evaluation results retrieved",
+			content: { "application/json": { schema: ListResponseSchema(EvaluationResultApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	try {
+		const runConfigRuns = (await listEvaluationRuns(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId
+		} })).filter((run) => run.evaluationRunConfigId === configId);
+		if (runConfigRuns.length === 0) return c.json({
+			data: [],
+			pagination: {
+				page: 1,
+				limit: 100,
+				total: 0,
+				pages: 0
+			}
+		});
+		const results = (await Promise.all(runConfigRuns.map(async (run) => {
+			return await listEvaluationResultsByRun(runDbClient_default)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationRunId: run.id
+			} });
+		}))).flat();
+		const uniqueConversationIds = [...new Set(results.map((r) => r.conversationId))];
+		const conversationInputs = /* @__PURE__ */ new Map();
+		const conversationAgents = /* @__PURE__ */ new Map();
+		const conversationCreatedAts = /* @__PURE__ */ new Map();
+		await Promise.all(uniqueConversationIds.map(async (conversationId) => {
+			try {
+				const conversation = await getConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId
+				});
+				if (conversation?.agentId) conversationAgents.set(conversationId, conversation.agentId);
+				if (conversation?.createdAt) conversationCreatedAts.set(conversationId, conversation.createdAt);
+				const firstUserMessage = [...await getMessagesByConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId,
+					pagination: {
+						page: 1,
+						limit: 100
+					}
+				})].reverse().find((msg) => msg.role === "user");
+				if (firstUserMessage?.content) {
+					const text = typeof firstUserMessage.content === "string" ? firstUserMessage.content : firstUserMessage.content.text || "";
+					conversationInputs.set(conversationId, text);
+				}
+			} catch (error) {
+				logger.warn({
+					error,
+					conversationId
+				}, "Failed to fetch conversation input");
+			}
+		}));
+		const enrichedResults = results.map((result) => ({
+			...result,
+			input: conversationInputs.get(result.conversationId) || null,
+			agentId: conversationAgents.get(result.conversationId) || null,
+			conversationCreatedAt: conversationCreatedAts.get(result.conversationId) || null
+		}));
+		return c.json({
+			data: enrichedResults,
+			pagination: {
+				page: 1,
+				limit: enrichedResults.length,
+				total: enrichedResults.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to get evaluation results for run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get evaluation results"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/",
+	summary: "Create Evaluation Run Config",
+	operationId: "create-evaluation-run-config",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: EvaluationRunConfigApiInsertSchema } } }
+	},
+	responses: {
+		201: {
+			description: "Evaluation run config created",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationRunConfigWithSuiteConfigsApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId } = c.req.valid("param");
+	const configData = c.req.valid("json");
+	const { suiteConfigIds, ...runConfigData } = configData;
+	try {
+		const id = runConfigData.id || generateId();
+		const created = await createEvaluationRunConfig(db)({
+			...runConfigData,
+			id,
+			tenantId,
+			projectId,
+			isActive: runConfigData.isActive !== void 0 ? runConfigData.isActive : true
+		});
+		if (suiteConfigIds && Array.isArray(suiteConfigIds) && suiteConfigIds.length > 0) await Promise.all(suiteConfigIds.map((suiteConfigId) => createEvaluationRunConfigEvaluationSuiteConfigRelation(db)({
+			tenantId,
+			projectId,
+			id: generateId(),
+			evaluationRunConfigId: id,
+			evaluationSuiteConfigId: suiteConfigId
+		})));
+		const suiteConfigRelations = await getEvaluationRunConfigEvaluationSuiteConfigRelations(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunConfigId: id
+		} });
+		logger.info({
+			tenantId,
+			projectId,
+			configId: id
+		}, "Evaluation run config created");
+		return c.json({ data: {
+			...created,
+			suiteConfigIds: suiteConfigRelations.map((rel) => rel.evaluationSuiteConfigId)
+		} }, 201);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configData
+		}, "Failed to create evaluation run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to create evaluation run config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "patch",
+	path: "/{configId}",
+	summary: "Update Evaluation Run Config",
+	operationId: "update-evaluation-run-config",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema.extend({ configId: z.string() }),
+		body: { content: { "application/json": { schema: EvaluationRunConfigApiUpdateSchema } } }
+	},
+	responses: {
+		200: {
+			description: "Evaluation run config updated",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationRunConfigWithSuiteConfigsApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	const configData = c.req.valid("json");
+	const { suiteConfigIds, ...runConfigUpdateData } = configData;
+	try {
+		const updated = await updateEvaluationRunConfig(db)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationRunConfigId: configId
+			},
+			data: runConfigUpdateData
+		});
+		if (!updated) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation run config not found"
+		}), 404);
+		if (suiteConfigIds !== void 0) {
+			const existingSuiteConfigIds = (await getEvaluationRunConfigEvaluationSuiteConfigRelations(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationRunConfigId: configId
+			} })).map((rel) => rel.evaluationSuiteConfigId);
+			const newSuiteConfigIds = Array.isArray(suiteConfigIds) ? suiteConfigIds : [];
+			const toDelete = existingSuiteConfigIds.filter((id) => !newSuiteConfigIds.includes(id));
+			await Promise.all(toDelete.map((suiteConfigId) => deleteEvaluationRunConfigEvaluationSuiteConfigRelation(db)({ scopes: {
+				tenantId,
+				projectId,
+				evaluationRunConfigId: configId,
+				evaluationSuiteConfigId: suiteConfigId
+			} })));
+			const toCreate = newSuiteConfigIds.filter((id) => !existingSuiteConfigIds.includes(id));
+			await Promise.all(toCreate.map((suiteConfigId) => createEvaluationRunConfigEvaluationSuiteConfigRelation(db)({
+				tenantId,
+				projectId,
+				id: generateId(),
+				evaluationRunConfigId: configId,
+				evaluationSuiteConfigId: suiteConfigId
+			})));
+		}
+		const suiteConfigRelations = await getEvaluationRunConfigEvaluationSuiteConfigRelations(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunConfigId: configId
+		} });
+		logger.info({
+			tenantId,
+			projectId,
+			configId
+		}, "Evaluation run config updated");
+		return c.json({ data: {
+			...updated,
+			suiteConfigIds: suiteConfigRelations.map((rel) => rel.evaluationSuiteConfigId)
+		} });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId,
+			configData
+		}, "Failed to update evaluation run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to update evaluation run config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "delete",
+	path: "/{configId}",
+	summary: "Delete Evaluation Run Config",
+	operationId: "delete-evaluation-run-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		204: { description: "Evaluation run config deleted" },
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	try {
+		if (!await deleteEvaluationRunConfig(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunConfigId: configId
+		} })) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation run config not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			configId
+		}, "Evaluation run config deleted");
+		return c.body(null, 204);
+	} catch (error) {
+		logger.error({
+			error: error?.message || error,
+			errorStack: error?.stack,
+			errorCode: error?.cause?.code,
+			errorDetail: error?.cause?.detail,
+			errorConstraint: error?.cause?.constraint,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to delete evaluation run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: error?.cause?.detail || error?.message || "Failed to delete evaluation run config"
+		}), 500);
+	}
+});
+var evaluationRunConfigs_default = app;
+
+//#endregion
+export { evaluationRunConfigs_default as default };

package/dist/domains/manage/routes/evals/evaluationSuiteConfigEvaluatorRelations.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/evals/evaluationSuiteConfigEvaluatorRelations.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };

package/dist/domains/manage/routes/evals/evaluationSuiteConfigEvaluatorRelations.js

@@ -0,0 +1,146 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { TenantProjectParamsSchema, commonGetErrorResponses, createApiError, createEvaluationSuiteConfigEvaluatorRelation, deleteEvaluationSuiteConfigEvaluatorRelation, generateId, getEvaluationSuiteConfigEvaluatorRelations } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/evaluationSuiteConfigEvaluatorRelations.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("evaluationSuiteConfigEvaluatorRelations");
+app.openapi(createRoute({
+	method: "get",
+	path: "/{configId}/evaluators",
+	summary: "List Evaluators for Evaluation Suite Config",
+	operationId: "list-evaluation-suite-config-evaluators",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		200: {
+			description: "List of evaluator relations",
+			content: { "application/json": { schema: z.array(z.any()) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	try {
+		const relations = await getEvaluationSuiteConfigEvaluatorRelations(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationSuiteConfigId: configId
+		} });
+		return c.json({ data: relations });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to list evaluator relations");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to list evaluator relations"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/{configId}/evaluators/{evaluatorId}",
+	summary: "Add Evaluator to Evaluation Suite Config",
+	operationId: "add-evaluator-to-suite-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({
+		configId: z.string(),
+		evaluatorId: z.string()
+	}) },
+	responses: {
+		201: {
+			description: "Evaluator relation created",
+			content: { "application/json": { schema: z.any() } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId, evaluatorId } = c.req.valid("param");
+	try {
+		const id = generateId();
+		const created = await createEvaluationSuiteConfigEvaluatorRelation(db)({
+			id,
+			tenantId,
+			projectId,
+			evaluationSuiteConfigId: configId,
+			evaluatorId
+		});
+		logger.info({
+			tenantId,
+			projectId,
+			configId,
+			evaluatorId
+		}, "Evaluator relation created");
+		return c.json({ data: created }, 201);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId,
+			evaluatorId
+		}, "Failed to create evaluator relation");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to create evaluator relation"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "delete",
+	path: "/{configId}/evaluators/{evaluatorId}",
+	summary: "Remove Evaluator from Evaluation Suite Config",
+	operationId: "remove-evaluator-from-suite-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({
+		configId: z.string(),
+		evaluatorId: z.string()
+	}) },
+	responses: {
+		204: { description: "Evaluator relation deleted" },
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, configId, evaluatorId } = c.req.valid("param");
+	try {
+		if (!await deleteEvaluationSuiteConfigEvaluatorRelation(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationSuiteConfigId: configId,
+			evaluatorId
+		} })) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluator relation not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			configId,
+			evaluatorId
+		}, "Evaluator relation deleted");
+		return c.body(null, 204);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId,
+			evaluatorId
+		}, "Failed to delete evaluator relation");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to delete evaluator relation"
+		}), 500);
+	}
+});
+var evaluationSuiteConfigEvaluatorRelations_default = app;
+
+//#endregion
+export { evaluationSuiteConfigEvaluatorRelations_default as default };

package/dist/domains/manage/routes/evals/evaluationSuiteConfigs.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/evals/evaluationSuiteConfigs.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };