@inkeep/agents-api 0.0.0-dev-20260121145510

package/dist/domains/manage/routes/evals/datasetRunConfigs.js

@@ -0,0 +1,402 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import { queueDatasetRunItems } from "../../../evals/services/datasetRun.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { DatasetRunConfigApiInsertSchema, DatasetRunConfigApiSelectSchema, DatasetRunConfigApiUpdateSchema, ListResponseSchema, SingleResponseSchema, TenantProjectParamsSchema, commonGetErrorResponses, createApiError, createDatasetRun, createDatasetRunConfig, createDatasetRunConfigAgentRelation, createEvaluationJobConfig, createEvaluationJobConfigEvaluatorRelation, createEvaluationRun, deleteDatasetRunConfig, deleteDatasetRunConfigAgentRelation, generateId, getDatasetRunConfigAgentRelations, getDatasetRunConfigById, linkDatasetRunToEvaluationJobConfig, listDatasetItems, listDatasetRunConfigs, updateDatasetRunConfig } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/datasetRunConfigs.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("datasetRunConfigs");
+app.openapi(createRoute({
+	method: "get",
+	path: "/by-dataset/{datasetId}",
+	summary: "List Dataset Run Configs by Dataset ID",
+	operationId: "list-dataset-run-configs",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ datasetId: z.string() }) },
+	responses: {
+		200: {
+			description: "List of dataset run configs",
+			content: { "application/json": { schema: ListResponseSchema(DatasetRunConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		const filteredConfigs = (await listDatasetRunConfigs(db)({ scopes: {
+			tenantId,
+			projectId
+		} })).filter((config) => config.datasetId === c.req.valid("param").datasetId);
+		return c.json({
+			data: filteredConfigs,
+			pagination: {
+				page: 1,
+				limit: filteredConfigs.length,
+				total: filteredConfigs.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId
+		}, "Failed to list dataset run configs");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to list dataset run configs"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{runConfigId}",
+	summary: "Get Dataset Run Config by ID",
+	operationId: "get-dataset-run-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ runConfigId: z.string() }) },
+	responses: {
+		200: {
+			description: "Dataset run config details",
+			content: { "application/json": { schema: SingleResponseSchema(DatasetRunConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, runConfigId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		const config = await getDatasetRunConfigById(db)({ scopes: {
+			tenantId,
+			projectId,
+			datasetRunConfigId: runConfigId
+		} });
+		if (!config) return c.json(createApiError({
+			code: "not_found",
+			message: "Dataset run config not found"
+		}), 404);
+		return c.json({ data: config });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			runConfigId
+		}, "Failed to get dataset run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get dataset run config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "post",
+	path: "/",
+	summary: "Create Dataset Run Config",
+	operationId: "create-dataset-run-config",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema,
+		body: { content: { "application/json": { schema: DatasetRunConfigApiInsertSchema.extend({
+			agentIds: z.array(z.string()).optional(),
+			evaluatorIds: z.array(z.string()).optional()
+		}) } } }
+	},
+	responses: {
+		201: {
+			description: "Dataset run config created",
+			content: { "application/json": { schema: SingleResponseSchema(DatasetRunConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId } = c.req.valid("param");
+	const db = c.get("db");
+	const configData = c.req.valid("json");
+	const { agentIds, evaluatorIds, ...runConfigData } = configData;
+	logger.info({
+		tenantId,
+		projectId,
+		agentIds,
+		evaluatorIds,
+		evaluatorIdsType: typeof evaluatorIds,
+		evaluatorIdsIsArray: Array.isArray(evaluatorIds),
+		evaluatorIdsLength: Array.isArray(evaluatorIds) ? evaluatorIds.length : 0,
+		hasEvaluators: evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0,
+		configDataKeys: Object.keys(configData),
+		runConfigDataKeys: Object.keys(runConfigData)
+	}, "Creating dataset run config with evaluators");
+	try {
+		const id = runConfigData.id || generateId();
+		const created = await createDatasetRunConfig(db)({
+			...runConfigData,
+			id,
+			tenantId,
+			projectId
+		});
+		if (agentIds && Array.isArray(agentIds) && agentIds.length > 0) await Promise.all(agentIds.map((agentId) => createDatasetRunConfigAgentRelation(db)({
+			tenantId,
+			projectId,
+			id: generateId(),
+			datasetRunConfigId: id,
+			agentId
+		})));
+		logger.info({
+			tenantId,
+			projectId,
+			runConfigId: id
+		}, "Dataset run config created");
+		try {
+			const datasetRunId = generateId();
+			await createDatasetRun(runDbClient_default)({
+				id: datasetRunId,
+				tenantId,
+				projectId,
+				datasetId: runConfigData.datasetId,
+				datasetRunConfigId: id,
+				evaluationJobConfigId: void 0
+			});
+			logger.info({
+				tenantId,
+				projectId,
+				runConfigId: id,
+				datasetRunId,
+				hasEvaluators: !!(evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0)
+			}, "Dataset run created, processing items");
+			logger.info({
+				tenantId,
+				projectId,
+				datasetRunId,
+				runConfigId: id,
+				evaluatorIds,
+				evaluatorIdsType: typeof evaluatorIds,
+				evaluatorIdsIsArray: Array.isArray(evaluatorIds),
+				evaluatorIdsLength: Array.isArray(evaluatorIds) ? evaluatorIds.length : 0,
+				hasEvaluators: evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0
+			}, "Starting dataset run processing with evaluators");
+			const datasetItems = await listDatasetItems(db)({ scopes: {
+				tenantId,
+				projectId,
+				datasetId: runConfigData.datasetId
+			} });
+			const agentRelations = await getDatasetRunConfigAgentRelations(db)({ scopes: {
+				tenantId,
+				projectId,
+				datasetRunConfigId: id
+			} });
+			let evaluationRunId;
+			let evalJobConfigId;
+			if (evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0) {
+				const jobConfigId = generateId();
+				evalJobConfigId = jobConfigId;
+				await createEvaluationJobConfig(db)({
+					id: jobConfigId,
+					tenantId,
+					projectId,
+					jobFilters: { datasetRunIds: [datasetRunId] }
+				});
+				await Promise.all(evaluatorIds.map((evaluatorId) => createEvaluationJobConfigEvaluatorRelation(db)({
+					tenantId,
+					projectId,
+					id: generateId(),
+					evaluationJobConfigId: jobConfigId,
+					evaluatorId
+				})));
+				await linkDatasetRunToEvaluationJobConfig(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId,
+						datasetRunId
+					},
+					evaluationJobConfigId: evalJobConfigId
+				});
+				evaluationRunId = generateId();
+				await createEvaluationRun(runDbClient_default)({
+					id: evaluationRunId,
+					tenantId,
+					projectId,
+					evaluationJobConfigId: evalJobConfigId
+				});
+			}
+			const result = await queueDatasetRunItems({
+				tenantId,
+				projectId,
+				datasetRunId,
+				items: agentRelations.flatMap((agentRelation) => datasetItems.map((datasetItem) => ({
+					agentId: agentRelation.agentId,
+					id: datasetItem.id,
+					input: datasetItem.input,
+					expectedOutput: datasetItem.expectedOutput,
+					simulationAgent: datasetItem.simulationAgent
+				}))),
+				evaluatorIds,
+				evaluationRunId
+			});
+			logger.info({
+				tenantId,
+				projectId,
+				runConfigId: id,
+				datasetRunId,
+				itemsQueued: result.queued,
+				itemsFailed: result.failed,
+				agentsUsed: agentRelations.length,
+				datasetItemCount: datasetItems.length,
+				hasEvaluators: !!(evaluatorIds && Array.isArray(evaluatorIds) && evaluatorIds.length > 0)
+			}, "Dataset run items queued via eval API");
+			if (result.queued === 0 && result.failed > 0) throw new Error(`All ${result.failed} workflow items failed to queue`);
+		} catch (runError) {
+			logger.error({
+				error: runError,
+				tenantId,
+				projectId,
+				runConfigId: id
+			}, "Failed to create/execute dataset run, but config was created");
+		}
+		return c.json({ data: created }, 201);
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			configData
+		}, "Failed to create dataset run config");
+		const errorMessage = error instanceof Error ? error.message : typeof error === "object" && error !== null && "message" in error ? String(error.message) : "Failed to create dataset run config";
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: errorMessage
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "patch",
+	path: "/{runConfigId}",
+	summary: "Update Dataset Run Config",
+	operationId: "update-dataset-run-config",
+	tags: ["Evaluations"],
+	request: {
+		params: TenantProjectParamsSchema.extend({ runConfigId: z.string() }),
+		body: { content: { "application/json": { schema: DatasetRunConfigApiUpdateSchema.extend({
+			agentIds: z.array(z.string()).optional(),
+			evaluatorIds: z.array(z.string()).optional()
+		}) } } }
+	},
+	responses: {
+		200: {
+			description: "Dataset run config updated",
+			content: { "application/json": { schema: SingleResponseSchema(DatasetRunConfigApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, runConfigId } = c.req.valid("param");
+	const db = c.get("db");
+	const configData = c.req.valid("json");
+	const { agentIds, ...runConfigUpdateData } = configData;
+	try {
+		const updated = await updateDatasetRunConfig(db)({
+			scopes: {
+				tenantId,
+				projectId,
+				datasetRunConfigId: runConfigId
+			},
+			data: runConfigUpdateData
+		});
+		if (!updated) return c.json(createApiError({
+			code: "not_found",
+			message: "Dataset run config not found"
+		}), 404);
+		if (agentIds !== void 0) {
+			const existingAgentIds = (await getDatasetRunConfigAgentRelations(db)({ scopes: {
+				tenantId,
+				projectId,
+				datasetRunConfigId: runConfigId
+			} })).map((rel) => rel.agentId);
+			const newAgentIds = Array.isArray(agentIds) ? agentIds : [];
+			const toDelete = existingAgentIds.filter((id) => !newAgentIds.includes(id));
+			await Promise.all(toDelete.map((agentId) => deleteDatasetRunConfigAgentRelation(db)({ scopes: {
+				tenantId,
+				projectId,
+				datasetRunConfigId: runConfigId,
+				agentId
+			} })));
+			const toCreate = newAgentIds.filter((id) => !existingAgentIds.includes(id));
+			await Promise.all(toCreate.map((agentId) => createDatasetRunConfigAgentRelation(db)({
+				tenantId,
+				projectId,
+				id: generateId(),
+				datasetRunConfigId: runConfigId,
+				agentId
+			})));
+		}
+		logger.info({
+			tenantId,
+			projectId,
+			runConfigId
+		}, "Dataset run config updated");
+		return c.json({ data: updated });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			runConfigId,
+			configData
+		}, "Failed to update dataset run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to update dataset run config"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "delete",
+	path: "/{runConfigId}",
+	summary: "Delete Dataset Run Config",
+	operationId: "delete-dataset-run-config",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ runConfigId: z.string() }) },
+	responses: {
+		204: { description: "Dataset run config deleted" },
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const { tenantId, projectId, runConfigId } = c.req.valid("param");
+	const db = c.get("db");
+	try {
+		if (!await deleteDatasetRunConfig(db)({ scopes: {
+			tenantId,
+			projectId,
+			datasetRunConfigId: runConfigId
+		} })) return c.json(createApiError({
+			code: "not_found",
+			message: "Dataset run config not found"
+		}), 404);
+		logger.info({
+			tenantId,
+			projectId,
+			runConfigId
+		}, "Dataset run config deleted");
+		return c.body(null, 204);
+	} catch (error) {
+		logger.error({
+			error: error?.message || error,
+			errorStack: error?.stack,
+			errorCode: error?.cause?.code,
+			errorDetail: error?.cause?.detail,
+			errorConstraint: error?.cause?.constraint,
+			tenantId,
+			projectId,
+			runConfigId
+		}, "Failed to delete dataset run config");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: error?.cause?.detail || error?.message || "Failed to delete dataset run config"
+		}), 500);
+	}
+});
+var datasetRunConfigs_default = app;
+
+//#endregion
+export { datasetRunConfigs_default as default };

package/dist/domains/manage/routes/evals/datasetRuns.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/evals/datasetRuns.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };

package/dist/domains/manage/routes/evals/datasetRuns.js

@@ -0,0 +1,256 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import { OpenAPIHono, createRoute, z } from "@hono/zod-openapi";
+import { DatasetRunApiSelectSchema, ListResponseSchema, SingleResponseSchema, TenantProjectParamsSchema, commonGetErrorResponses, createApiError, getConversation, getDatasetRunById, getDatasetRunConfigById, getDatasetRunConversationRelations, getMessagesByConversation, listDatasetItems, listDatasetRuns } from "@inkeep/agents-core";
+
+//#region src/domains/manage/routes/evals/datasetRuns.ts
+const app = new OpenAPIHono();
+const logger = getLogger$1("datasetRuns");
+app.openapi(createRoute({
+	method: "get",
+	path: "/by-dataset/{datasetId}",
+	summary: "List Dataset Runs by Dataset ID",
+	operationId: "list-dataset-runs",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ datasetId: z.string() }) },
+	responses: {
+		200: {
+			description: "List of dataset runs",
+			content: { "application/json": { schema: ListResponseSchema(DatasetRunApiSelectSchema) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, datasetId } = c.req.valid("param");
+	try {
+		const filteredRuns = (await listDatasetRuns(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId
+		} })).filter((run) => run.datasetId === datasetId);
+		const runsWithNames = await Promise.all(filteredRuns.map(async (run) => {
+			if (run.datasetRunConfigId) {
+				const runConfig = await getDatasetRunConfigById(db)({ scopes: {
+					tenantId,
+					projectId,
+					datasetRunConfigId: run.datasetRunConfigId
+				} });
+				return {
+					...run,
+					runConfigName: runConfig?.name || null
+				};
+			}
+			return {
+				...run,
+				runConfigName: null
+			};
+		}));
+		return c.json({
+			data: runsWithNames,
+			pagination: {
+				page: 1,
+				limit: runsWithNames.length,
+				total: runsWithNames.length,
+				pages: 1
+			}
+		});
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			datasetId
+		}, "Failed to list dataset runs");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to list dataset runs"
+		}), 500);
+	}
+});
+app.openapi(createRoute({
+	method: "get",
+	path: "/{runId}",
+	summary: "Get Dataset Run by ID",
+	operationId: "get-dataset-run",
+	tags: ["Evaluations"],
+	request: { params: TenantProjectParamsSchema.extend({ runId: z.string() }) },
+	responses: {
+		200: {
+			description: "Dataset run with conversations",
+			content: { "application/json": { schema: SingleResponseSchema(DatasetRunApiSelectSchema.extend({
+				conversations: z.array(z.object({
+					id: z.string(),
+					conversationId: z.string(),
+					datasetRunId: z.string(),
+					agentId: z.string().nullable().optional(),
+					output: z.string().nullable().optional(),
+					createdAt: z.string(),
+					updatedAt: z.string()
+				})),
+				items: z.array(z.object({
+					id: z.string(),
+					tenantId: z.string(),
+					projectId: z.string(),
+					datasetId: z.string(),
+					input: z.any().nullable().optional(),
+					expectedOutput: z.any().nullable().optional(),
+					simulationAgent: z.any().nullable().optional(),
+					createdAt: z.string(),
+					updatedAt: z.string(),
+					conversations: z.array(z.object({
+						id: z.string(),
+						conversationId: z.string(),
+						datasetRunId: z.string(),
+						agentId: z.string().nullable().optional(),
+						output: z.string().nullable().optional(),
+						createdAt: z.string(),
+						updatedAt: z.string()
+					}))
+				}))
+			})) } }
+		},
+		...commonGetErrorResponses
+	}
+}), async (c) => {
+	const db = c.get("db");
+	const { tenantId, projectId, runId } = c.req.valid("param");
+	try {
+		const run = await getDatasetRunById(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId,
+			datasetRunId: runId
+		} });
+		if (!run) return c.json(createApiError({
+			code: "not_found",
+			message: "Dataset run not found"
+		}), 404);
+		let runConfigName = null;
+		if (run.datasetRunConfigId) runConfigName = (await getDatasetRunConfigById(db)({ scopes: {
+			tenantId,
+			projectId,
+			datasetRunConfigId: run.datasetRunConfigId
+		} }))?.name || null;
+		const conversationRelations = await getDatasetRunConversationRelations(runDbClient_default)({ scopes: {
+			tenantId,
+			projectId,
+			datasetRunId: runId
+		} });
+		const datasetItems = await listDatasetItems(db)({ scopes: {
+			tenantId,
+			projectId,
+			datasetId: run.datasetId
+		} });
+		const itemsWithConversations = await Promise.all(datasetItems.map(async (item) => {
+			const itemConversations = conversationRelations.filter((conv) => conv.datasetItemId === item.id);
+			const conversationsWithOutput$1 = await Promise.all(itemConversations.map(async (conv) => {
+				try {
+					const agentId = (await getConversation(runDbClient_default)({
+						scopes: {
+							tenantId,
+							projectId
+						},
+						conversationId: conv.conversationId
+					}))?.agentId || null;
+					const assistantMessage = (await getMessagesByConversation(runDbClient_default)({
+						scopes: {
+							tenantId,
+							projectId
+						},
+						conversationId: conv.conversationId,
+						pagination: {
+							page: 1,
+							limit: 100
+						}
+					})).filter((msg) => msg.role === "assistant" || msg.role === "agent").sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())[0];
+					let output = null;
+					if (assistantMessage?.content) {
+						if (typeof assistantMessage.content === "string") output = assistantMessage.content;
+						else if (typeof assistantMessage.content === "object" && assistantMessage.content !== null && "text" in assistantMessage.content) output = typeof assistantMessage.content.text === "string" ? assistantMessage.content.text : null;
+					}
+					return {
+						...conv,
+						output,
+						agentId
+					};
+				} catch (error) {
+					logger.warn({
+						error,
+						conversationId: conv.conversationId
+					}, "Failed to fetch conversation output");
+					return {
+						...conv,
+						output: null,
+						agentId: null
+					};
+				}
+			}));
+			return {
+				...item,
+				conversations: conversationsWithOutput$1
+			};
+		}));
+		const conversationsWithOutput = await Promise.all(conversationRelations.map(async (conv) => {
+			try {
+				const agentId = (await getConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId: conv.conversationId
+				}))?.agentId || null;
+				const assistantMessage = (await getMessagesByConversation(runDbClient_default)({
+					scopes: {
+						tenantId,
+						projectId
+					},
+					conversationId: conv.conversationId,
+					pagination: {
+						page: 1,
+						limit: 100
+					}
+				})).filter((msg) => msg.role === "assistant" || msg.role === "agent").sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime())[0];
+				let output = null;
+				if (assistantMessage?.content) {
+					if (typeof assistantMessage.content === "string") output = assistantMessage.content;
+					else if (typeof assistantMessage.content === "object" && assistantMessage.content !== null && "text" in assistantMessage.content) output = typeof assistantMessage.content.text === "string" ? assistantMessage.content.text : null;
+				}
+				return {
+					...conv,
+					output,
+					agentId
+				};
+			} catch (error) {
+				logger.warn({
+					error,
+					conversationId: conv.conversationId
+				}, "Failed to fetch conversation output");
+				return {
+					...conv,
+					output: null,
+					agentId: null
+				};
+			}
+		}));
+		return c.json({ data: {
+			...run,
+			runConfigName,
+			conversations: conversationsWithOutput,
+			items: itemsWithConversations
+		} });
+	} catch (error) {
+		logger.error({
+			error,
+			tenantId,
+			projectId,
+			runId
+		}, "Failed to get dataset run");
+		return c.json(createApiError({
+			code: "internal_server_error",
+			message: "Failed to get dataset run"
+		}), 500);
+	}
+});
+var datasetRuns_default = app;
+
+//#endregion
+export { datasetRuns_default as default };

package/dist/domains/manage/routes/evals/datasets.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/evals/datasets.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };