@inkeep/agents-api 0.0.0-dev-20260121145510

package/dist/domains/manage/routes/evals/evaluationJobConfigs.js

@@ -0,0 +1,364 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import { queueEvaluationJobConversations } from "../../../evals/services/evaluationJob.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { EvaluationJobConfigApiInsertSchema, EvaluationJobConfigApiSelectSchema, EvaluationResultApiSelectSchema, ListResponseSchema, SingleResponseSchema, TenantProjectParamsSchema, commonGetErrorResponses, createApiError, createEvaluationJobConfig, createEvaluationJobConfigEvaluatorRelation, deleteEvaluationJobConfig, generateId, getConversation, getEvaluationJobConfigById, getMessagesByConversation, listEvaluationJobConfigs, listEvaluationResultsByRun, listEvaluationRuns } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/evaluationJobConfigs.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("evaluationJobConfigs");
+/**
+* Extract plain filter criteria from a potential Filter wrapper.
+* Returns null if the filter is a complex and/or combinator.
+*/
+function getPlainJobFilters(filter) {
+	if (!filter) return null;
+	if ("and" in filter || "or" in filter) return null;
+	return filter;
+}
+app.openapi(createRoute({
+	method: "get",
+	path: "/",
+	summary: "List Evaluation Job Configs",
+	operationId: "list-evaluation-job-configs",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema },
+	responses: {
+		200: {
+			description: "List of evaluation job configs",
+			content: { "application/json": { schema: ListResponseSchema(EvaluationJobConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		const configs = await listEvaluationJobConfigs(db)({ scopes: {
+			tenantId,
+			projectId
+		} });
+		return c.json({
+			data: configs,
+			pagination: {
+				page: 1,
+				limit: configs.length,
+				total: configs.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId
+		}, "Failed to list evaluation job configs");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to list evaluation job configs"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{configId}",
+	summary: "Get Evaluation Job Config by ID",
+	operationId: "get-evaluation-job-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		200: {
+			description: "Evaluation job config details",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationJobConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		const config = await getEvaluationJobConfigById(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationJobConfigId: configId
+		} });
+		if (!config) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation job config not found"
+		}), 404);
+		return c.json({ data: config });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to get evaluation job config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get evaluation job config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/",
+	summary: "Create Evaluation Job Config",
+	operationId: "create-evaluation-job-config",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: EvaluationJobConfigApiInsertSchema } } }
+	},
+	responses: {
+		201: {
+			description: "Evaluation job config created",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationJobConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const db = c.get("db");
+	const configData = c.req.valid("json");
+	const { evaluatorIds, ...jobConfigData } = configData;
+	try {
+		const id = jobConfigData.id || generateId();
+		const created = await createEvaluationJobConfig(db)({
+			...jobConfigData,
+			id,
+			tenantId,
+			projectId
+		});
+		if (evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0) await Promise.all(evaluatorIds.map((evaluatorId) => createEvaluationJobConfigEvaluatorRelation(db)({
+			tenantId,
+			projectId,
+			id: generateId(),
+			evaluationJobConfigId: id,
+			evaluatorId
+		})));
+		logger.info({
+			tenantId,
+			projectId,
+			configId: id
+		}, "Evaluation job config created");
+		if (evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0) queueEvaluationJobConversations({
+			tenantId,
+			projectId,
+			evaluationJobConfigId: id,
+			evaluatorIds,
+			jobFilters: getPlainJobFilters(created.jobFilters)
+		}).then((result) => {
+			logger.info({
+				tenantId,
+				projectId,
+				configId: id,
+				conversationCount: result.conversationCount,
+				queued: result.queued,
+				failed: result.failed,
+				evaluationRunId: result.evaluationRunId
+			}, "Manual bulk evaluation job triggered");
+		}).catch((error) => {
+			logger.error({
+				error,
+				tenantId,
+				projectId,
+				configId: id
+			}, "Failed to trigger manual bulk evaluation job");
+		});
+		else logger.warn({
+			tenantId,
+			projectId,
+			configId: id
+		}, "Evaluation job config created without evaluators, skipping job execution");
+		return c.json({ data: created }, 201);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configData
+		}, "Failed to create evaluation job config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to create evaluation job config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "delete",
+	path: "/{configId}",
+	summary: "Delete Evaluation Job Config",
+	operationId: "delete-evaluation-job-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		204: { description: "Evaluation job config deleted" },
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		if (!await deleteEvaluationJobConfig(db)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationJobConfigId: configId
+		} })) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation job config not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			configId
+		}, "Evaluation job config deleted");
+		return c.body(null, 204);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to delete evaluation job config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to delete evaluation job config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{configId}/results",
+	summary: "Get Evaluation Results by Job Config ID",
+	operationId: "get-evaluation-job-config-results",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ configId: z.string() }) },
+	responses: {
+		200: {
+			description: "Evaluation results retrieved",
+			content: { "application/json": { schema: ListResponseSchema(EvaluationResultApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, configId } = c.req.valid("param");
+	try {
+		const jobRuns = (await listEvaluationRuns(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId
+		} })).filter((run) => run.evaluationJobConfigId === configId);
+		if (jobRuns.length === 0) return c.json({
+			data: [],
+			pagination: {
+				page: 1,
+				limit: 100,
+				total: 0,
+				pages: 0
+			}
+		});
+		const results = (await Promise.all(jobRuns.map((run) => listEvaluationResultsByRun(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationRunId: run.id
+		} })))).flat();
+		const uniqueConversationIds = [...new Set(results.map((r) => r.conversationId))];
+		const conversationInputs = /* @__PURE__ */ new Map();
+		const conversationAgents = /* @__PURE__ */ new Map();
+		const conversationCreatedAts = /* @__PURE__ */ new Map();
+		logger.info({ uniqueConversationIds }, "=== FETCHING INPUTS FOR JOB CONFIG CONVERSATIONS ===");
+		await Promise.all(uniqueConversationIds.map(async (conversationId) => {
+			try {
+				const conversation = await getConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId
+				});
+				if (conversation?.agentId) conversationAgents.set(conversationId, conversation.agentId);
+				if (conversation?.createdAt) conversationCreatedAts.set(conversationId, conversation.createdAt);
+				logger.info({ conversationId }, "Fetching messages for conversation");
+				const messages = await getMessagesByConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId,
+					pagination: {
+						page: 1,
+						limit: 100
+					}
+				});
+				logger.info({
+					conversationId,
+					messageCount: messages.length,
+					messages: messages.map((m) => ({
+						role: m.role,
+						content: m.content
+					}))
+				}, "Found messages for conversation");
+				const firstUserMessage = [...messages].reverse().find((msg) => msg.role === "user");
+				logger.info({
+					conversationId,
+					firstUserMessage
+				}, "First user message found");
+				if (firstUserMessage?.content) {
+					const text = typeof firstUserMessage.content === "string" ? firstUserMessage.content : firstUserMessage.content.text || "";
+					logger.info({
+						conversationId,
+						text
+					}, "Extracted text from message");
+					conversationInputs.set(conversationId, text);
+				} else logger.info({ conversationId }, "No user message found for conversation");
+			} catch (error) {
+				logger.error({
+					error,
+					conversationId
+				}, "Error fetching conversation");
+			}
+		}));
+		logger.info({ conversationInputs: Array.from(conversationInputs.entries()) }, "=== CONVERSATION INPUTS MAP ===");
+		const enrichedResults = results.map((result) => ({
+			...result,
+			input: conversationInputs.get(result.conversationId) || null,
+			agentId: conversationAgents.get(result.conversationId) || null,
+			conversationCreatedAt: conversationCreatedAts.get(result.conversationId) || null
+		}));
+		logger.info({ enrichedResults: enrichedResults.map((r) => ({
+			id: r.id,
+			conversationId: r.conversationId,
+			input: r.input
+		})) }, "=== ENRICHED RESULTS ===");
+		logger.info({
+			tenantId,
+			projectId,
+			configId,
+			resultCount: enrichedResults.length
+		}, "Retrieved evaluation results for job config");
+		return c.json({
+			data: enrichedResults,
+			pagination: {
+				page: 1,
+				limit: enrichedResults.length,
+				total: enrichedResults.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configId
+		}, "Failed to get evaluation results for job config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get evaluation results"
+		}), 500);
+	}
+});
+var evaluationJobConfigs_default = app;
+
+//#endregion
+export { evaluationJobConfigs_default as default };

package/dist/domains/manage/routes/evals/evaluationResults.d.ts

@@ -0,0 +1,7 @@
+import { OpenAPIHono } from "@hono/zod-openapi";
+import * as hono14 from "hono";
+
+//#region src/domains/manage/routes/evals/evaluationResults.d.ts
+declare const app: OpenAPIHono<hono14.Env, {}, "/">;
+//#endregion
+export { app as default };

package/dist/domains/manage/routes/evals/evaluationResults.js

@@ -0,0 +1,192 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { EvaluationResultApiInsertSchema, EvaluationResultApiSelectSchema, EvaluationResultApiUpdateSchema, SingleResponseSchema, TenantProjectParamsSchema, commonGetErrorResponses, createApiError, createEvaluationResult, deleteEvaluationResult, generateId, getEvaluationResultById, updateEvaluationResult } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/evaluationResults.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("evaluationResults");
+app.openapi(createRoute({
+	method: "get",
+	path: "/{resultId}",
+	summary: "Get Evaluation Result by ID",
+	operationId: "get-evaluation-result",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ resultId: z.string() }) },
+	responses: {
+		200: {
+			description: "Evaluation result details",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationResultApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, resultId } = c.req.valid("param");
+	try {
+		const result = await getEvaluationResultById(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationResultId: resultId
+		} });
+		if (!result) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation result not found"
+		}), 404);
+		return c.json({ data: result });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			resultId
+		}, "Failed to get evaluation result");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get evaluation result"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/",
+	summary: "Create Evaluation Result",
+	operationId: "create-evaluation-result",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: EvaluationResultApiInsertSchema } } }
+	},
+	responses: {
+		201: {
+			description: "Evaluation result created",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationResultApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const resultData = c.req.valid("json");
+	try {
+		const id = resultData.id || generateId();
+		const created = await createEvaluationResult(runDbClient_default)({
+			...resultData,
+			id,
+			tenantId,
+			projectId
+		});
+		logger.info({
+			tenantId,
+			projectId,
+			resultId: id
+		}, "Evaluation result created");
+		return c.json({ data: created }, 201);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			resultData
+		}, "Failed to create evaluation result");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to create evaluation result"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "patch",
+	path: "/{resultId}",
+	summary: "Update Evaluation Result",
+	operationId: "update-evaluation-result",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema.extend({ resultId: z.string() }),
+		body: { content: { "application/json": { schema: EvaluationResultApiUpdateSchema } } }
+	},
+	responses: {
+		200: {
+			description: "Evaluation result updated",
+			content: { "application/json": { schema: SingleResponseSchema(EvaluationResultApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, resultId } = c.req.valid("param");
+	const updateData = c.req.valid("json");
+	try {
+		const updated = await updateEvaluationResult(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationResultId: resultId
+			},
+			data: updateData
+		});
+		if (!updated) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation result not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			resultId
+		}, "Evaluation result updated");
+		return c.json({ data: updated });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			resultId
+		}, "Failed to update evaluation result");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to update evaluation result"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "delete",
+	path: "/{resultId}",
+	summary: "Delete Evaluation Result",
+	operationId: "delete-evaluation-result",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ resultId: z.string() }) },
+	responses: {
+		204: { description: "Evaluation result deleted" },
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, resultId } = c.req.valid("param");
+	try {
+		if (!await deleteEvaluationResult(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId,
+			evaluationResultId: resultId
+		} })) return c.json(createApiError({
+			code: "not_found",
+			message: "Evaluation result not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			resultId
+		}, "Evaluation result deleted");
+		return c.body(null, 204);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			resultId
+		}, "Failed to delete evaluation result");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to delete evaluation result"
+		}), 500);
+	}
+});
+var evaluationResults_default = app;
+
+//#endregion
+export { evaluationResults_default as default };

package/dist/domains/manage/routes/evals/evaluationRunConfigs.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/evals/evaluationRunConfigs.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };