@inkeep/agents-api 0.0.0-dev-20260121145510

package/dist/domains/evals/workflow/functions/runDatasetItem.js

@@ -0,0 +1,205 @@
+import { getLogger as getLogger$1 } from "../../../../logger.js";
+import manageDbClient_default from "../../../../data/db/manageDbClient.js";
+import manageDbPool_default from "../../../../data/db/manageDbPool.js";
+import runDbClient_default from "../../../../data/db/runDbClient.js";
+import "../../../../data/db/index.js";
+import { EvaluationService } from "../../services/EvaluationService.js";
+import { createDatasetRunConversationRelation, createEvaluationResult, generateId, getConversation, getEvaluatorById, getProjectScopedRef, resolveRef, updateEvaluationResult, withRef } from "@inkeep/agents-core";
+
+//#region src/domains/evals/workflow/functions/runDatasetItem.ts
+/**
+* Workflow for running dataset items through the chat API.
+*
+* This makes dataset run processing fire-and-forget - each item is queued
+* independently and processed in parallel by the workflow system.
+*/
+const logger = getLogger$1("workflow-run-dataset-item");
+/**
+* Step: Call the chat API to process the dataset item
+*/
+async function callChatApiStep(payload) {
+	"use step";
+	const { tenantId, projectId, agentId, datasetItemId, datasetItemInput, datasetItemSimulationAgent, datasetRunId } = payload;
+	const evaluationService = new EvaluationService();
+	const datasetItem = {
+		id: datasetItemId,
+		input: datasetItemInput,
+		simulationAgent: datasetItemSimulationAgent
+	};
+	const result = await evaluationService.runDatasetItem({
+		tenantId,
+		projectId,
+		agentId,
+		datasetItem,
+		datasetRunId
+	});
+	logger.info({
+		tenantId,
+		projectId,
+		datasetItemId,
+		datasetRunId,
+		conversationId: result.conversationId,
+		hasError: !!result.error
+	}, "Chat API call completed");
+	return result;
+}
+/**
+* Step: Create conversation relation in database
+*/
+async function createRelationStep(payload, conversationId) {
+	"use step";
+	const { tenantId, projectId, datasetItemId, datasetRunId } = payload;
+	const relationId = generateId();
+	try {
+		await createDatasetRunConversationRelation(runDbClient_default)({
+			tenantId,
+			projectId,
+			id: relationId,
+			datasetRunId,
+			conversationId,
+			datasetItemId
+		});
+		logger.info({
+			tenantId,
+			projectId,
+			datasetItemId,
+			datasetRunId,
+			conversationId,
+			relationId
+		}, "Created conversation relation");
+		return {
+			relationId,
+			success: true
+		};
+	} catch (error) {
+		if (error?.cause?.code === "23503" || error?.code === "23503") {
+			logger.warn({
+				tenantId,
+				projectId,
+				datasetItemId,
+				datasetRunId,
+				conversationId
+			}, "Conversation does not exist, skipping relation creation");
+			return {
+				relationId: null,
+				success: false,
+				reason: "conversation_not_found"
+			};
+		}
+		throw error;
+	}
+}
+/**
+* Step: Execute a single evaluator on a conversation
+*/
+async function executeEvaluatorStep(tenantId, projectId, conversationId, evaluatorId, evaluationRunId, expectedOutput) {
+	"use step";
+	const ref = getProjectScopedRef(tenantId, projectId, "main");
+	const resolvedRef = await resolveRef(manageDbClient_default)(ref);
+	if (!resolvedRef) throw new Error("Failed to resolve ref");
+	const evaluator = await withRef(manageDbPool_default, resolvedRef, (db) => getEvaluatorById(db)({ scopes: {
+		tenantId,
+		projectId,
+		evaluatorId
+	} }));
+	if (!evaluator) {
+		logger.warn({ evaluatorId }, "Evaluator not found");
+		return null;
+	}
+	const conversation = await getConversation(runDbClient_default)({
+		scopes: {
+			tenantId,
+			projectId
+		},
+		conversationId
+	});
+	if (!conversation) throw new Error(`Conversation not found: ${conversationId}`);
+	const evalResult = await createEvaluationResult(runDbClient_default)({
+		id: generateId(),
+		tenantId,
+		projectId,
+		conversationId,
+		evaluatorId: evaluator.id,
+		evaluationRunId
+	});
+	try {
+		const output = await new EvaluationService().executeEvaluation({
+			conversation,
+			evaluator,
+			tenantId,
+			projectId,
+			expectedOutput
+		});
+		await updateEvaluationResult(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationResultId: evalResult.id
+			},
+			data: { output }
+		});
+		logger.info({
+			conversationId,
+			evaluatorId: evaluator.id,
+			resultId: evalResult.id
+		}, "Evaluation completed");
+		return evalResult.id;
+	} catch (error) {
+		const errorMessage = error instanceof Error ? error.message : "Unknown error";
+		logger.error({
+			error,
+			conversationId,
+			evaluatorId: evaluator.id
+		}, "Evaluation failed");
+		await updateEvaluationResult(runDbClient_default)({
+			scopes: {
+				tenantId,
+				projectId,
+				evaluationResultId: evalResult.id
+			},
+			data: { output: { text: `Evaluation failed: ${errorMessage}` } }
+		});
+		return evalResult.id;
+	}
+}
+/**
+* Step: Log workflow progress
+*/
+async function logStep(message, data) {
+	"use step";
+	logger.info(data, message);
+}
+/**
+* Main workflow function - processes a single dataset item through the chat API.
+* Optionally runs evaluators on the resulting conversation.
+*/
+async function _runDatasetItemWorkflow(payload) {
+	"use workflow";
+	const { datasetItemId, datasetRunId, agentId, evaluatorIds, evaluationRunId } = payload;
+	await logStep("Starting dataset item processing", {
+		datasetItemId,
+		datasetRunId,
+		agentId,
+		hasEvaluators: !!(evaluatorIds && evaluatorIds.length > 0)
+	});
+	const result = await callChatApiStep(payload);
+	if (result.conversationId) {
+		await createRelationStep(payload, result.conversationId);
+		if (evaluatorIds && evaluatorIds.length > 0 && evaluationRunId) for (const evaluatorId of evaluatorIds) await executeEvaluatorStep(payload.tenantId, payload.projectId, result.conversationId, evaluatorId, evaluationRunId, payload.datasetItemExpectedOutput);
+	} else await logStep("No conversation created", {
+		datasetItemId,
+		datasetRunId,
+		error: result.error
+	});
+	return {
+		success: !result.error,
+		datasetItemId,
+		datasetRunId,
+		conversationId: result.conversationId || null,
+		error: result.error || null
+	};
+}
+const runDatasetItemWorkflow = Object.assign(_runDatasetItemWorkflow, { workflowId: "workflow//src/domains/evals/workflow/functions/runDatasetItem.ts//_runDatasetItemWorkflow" });
+
+//#endregion
+export { runDatasetItemWorkflow };

package/dist/domains/evals/workflow/index.d.ts

@@ -0,0 +1,4 @@
+import { evaluateConversationWorkflow } from "./functions/evaluateConversation.js";
+import { runDatasetItemWorkflow } from "./functions/runDatasetItem.js";
+import { world } from "./world.js";
+export { evaluateConversationWorkflow, runDatasetItemWorkflow, world };

package/dist/domains/evals/workflow/index.js

@@ -0,0 +1,5 @@
+import { runDatasetItemWorkflow } from "./functions/runDatasetItem.js";
+import { evaluateConversationWorkflow } from "./functions/evaluateConversation.js";
+import { world } from "./world.js";
+
+export { evaluateConversationWorkflow, runDatasetItemWorkflow, world };

package/dist/domains/evals/workflow/routes.d.ts

@@ -0,0 +1,7 @@
+import { Hono } from "hono";
+import * as hono_types6 from "hono/types";
+
+//#region src/domains/evals/workflow/routes.d.ts
+declare const workflowRoutes: Hono<hono_types6.BlankEnv, hono_types6.BlankSchema, "/">;
+//#endregion
+export { workflowRoutes };

package/dist/domains/evals/workflow/routes.js

@@ -0,0 +1,106 @@
+import { createRequire } from "node:module";
+import { Hono } from "hono";
+import { existsSync } from "node:fs";
+import { dirname, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+
+//#region src/domains/evals/workflow/routes.ts
+/**
+* Workflow route handlers for Hono.
+*
+* These routes expose the generated workflow handlers from `workflow build`.
+* The postgres world queues jobs via pg-boss, then calls these endpoints.
+*
+* Generated files:
+* - .well-known/workflow/v1/flow.cjs (CJS bundle)
+* - .well-known/workflow/v1/step.cjs (CJS bundle)
+* - .well-known/workflow/v1/webhook.mjs (ESM)
+*/
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+function findPackageRoot(startDir) {
+	let current = startDir;
+	for (let i = 0; i < 30; i += 1) {
+		if (existsSync(resolve(current, "package.json"))) return current;
+		const parent = dirname(current);
+		if (parent === current) break;
+		current = parent;
+	}
+	return startDir;
+}
+const packageRoot = findPackageRoot(__dirname);
+const distRoot = resolve(packageRoot, "dist");
+const isRunningFromDist = __dirname.startsWith(`${distRoot}${sep}`);
+/**
+* Resolve paths to generated handlers.
+* In dev: bundles live at <packageRoot>/.well-known
+* In prod: bundles live at <packageRoot>/dist/.well-known
+*/
+function resolveWorkflowPath(filename) {
+	const devPath = resolve(packageRoot, ".well-known/workflow/v1", filename);
+	const prodPath = resolve(packageRoot, "dist/.well-known/workflow/v1", filename);
+	const preferred = isRunningFromDist ? prodPath : devPath;
+	if (existsSync(preferred)) return preferred;
+	const fallback = isRunningFromDist ? devPath : prodPath;
+	if (existsSync(fallback)) return fallback;
+	return preferred;
+}
+const flowPath = resolveWorkflowPath("flow.cjs");
+const stepPath = resolveWorkflowPath("step.cjs");
+const webhookPath = resolveWorkflowPath("webhook.mjs");
+let flowHandler;
+let stepHandler;
+let webhook;
+function loadCjsHandlers() {
+	if (!flowHandler) {
+		flowHandler = require(flowPath);
+		stepHandler = require(stepPath);
+	}
+}
+async function loadWebhookHandler() {
+	if (!webhook) webhook = await import(webhookPath);
+}
+const workflowRoutes = new Hono();
+/**
+* Smart dispatcher that routes to flow or step handler based on queueName.
+* Handles cases where Vercel Queue delivers step messages to /flow endpoint.
+*/
+async function dispatchFlowOrStep(c) {
+	loadCjsHandlers();
+	const bodyBuf = await c.req.arrayBuffer();
+	let queueName = c.req.header("x-vqs-queue-name");
+	if (!queueName) try {
+		queueName = JSON.parse(new TextDecoder().decode(bodyBuf))?.data?.queueName;
+	} catch {}
+	const isStep = typeof queueName === "string" && queueName.startsWith("__wkf_step_");
+	const rawUrl = c.req.raw.url;
+	const url = new URL(rawUrl);
+	url.pathname = isStep ? "/.well-known/workflow/v1/step" : "/.well-known/workflow/v1/flow";
+	const fixedRequest = new Request(url.toString(), {
+		method: "POST",
+		headers: c.req.raw.headers,
+		body: bodyBuf
+	});
+	const flow = flowHandler?.POST || flowHandler?.default?.POST || flowHandler?.default || flowHandler;
+	const step = stepHandler?.POST || stepHandler?.default?.POST || stepHandler?.default || stepHandler;
+	const handler = isStep ? step : flow;
+	if (typeof handler !== "function") return c.json({ error: "Handler not found" }, 500);
+	return handler(fixedRequest);
+}
+workflowRoutes.post("/workflow/v1/flow", dispatchFlowOrStep);
+workflowRoutes.post("/workflow/v1/step", dispatchFlowOrStep);
+workflowRoutes.all("/workflow/v1/webhook/:token", async (c) => {
+	try {
+		await loadWebhookHandler();
+		const req = c.req.raw;
+		const method = req.method;
+		const handler = webhook[method] ?? webhook.default?.[method] ?? webhook.default;
+		if (handler) return handler(req);
+		return c.json({ error: "Method not allowed" }, 405);
+	} catch (err) {
+		return c.json({ error: String(err) }, 500);
+	}
+});
+
+//#endregion
+export { workflowRoutes };

package/dist/domains/evals/workflow/world.d.ts

@@ -0,0 +1,4 @@
+//#region src/domains/evals/workflow/world.d.ts
+declare let world: any;
+//#endregion
+export { world };

package/dist/domains/evals/workflow/world.js

@@ -0,0 +1,36 @@
+import { env } from "../../../env.js";
+import { createLocalWorld } from "@workflow/world-local";
+import { createWorld } from "@workflow/world-postgres";
+import { createVercelWorld } from "@workflow/world-vercel";
+
+//#region src/domains/evals/workflow/world.ts
+/**
+* Workflow world configuration.
+*
+* Static imports are required instead of dynamic imports because
+* Vercel's NFT can't trace dynamic imports in bundled code.
+*
+* Set WORKFLOW_TARGET_WORLD to: 'local' | 'vercel' | '@workflow/world-postgres'
+*/
+const targetWorld = env.WORKFLOW_TARGET_WORLD;
+let world;
+if (targetWorld === "vercel") {
+	const token = process.env.WORKFLOW_VERCEL_AUTH_TOKEN;
+	world = createVercelWorld({
+		token: token?.trim() || void 0,
+		baseUrl: process.env.WORKFLOW_VERCEL_BASE_URL || void 0,
+		projectConfig: {
+			projectId: process.env.VERCEL_PROJECT_ID,
+			teamId: process.env.VERCEL_TEAM_ID,
+			environment: process.env.VERCEL_ENV
+		}
+	});
+} else if (targetWorld === "@workflow/world-postgres") world = createWorld({
+	connectionString: env.WORKFLOW_POSTGRES_URL || "postgres://world:world@localhost:5432/world",
+	jobPrefix: env.WORKFLOW_POSTGRES_JOB_PREFIX,
+	queueConcurrency: Number(env.WORKFLOW_POSTGRES_WORKER_CONCURRENCY) || 10
+});
+else if (targetWorld === "local") world = createLocalWorld();
+
+//#endregion
+export { world };

package/dist/domains/index.d.ts

@@ -0,0 +1,4 @@
+import { createEvalRoutes, evalRoutes } from "./evals/index.js";
+import { createManageRoutes, manageRoutes } from "./manage/index.js";
+import { createRunRoutes, runRoutes } from "./run/index.js";
+export { createEvalRoutes, createManageRoutes, createRunRoutes, evalRoutes, manageRoutes, runRoutes };

package/dist/domains/index.js

@@ -0,0 +1,5 @@
+import { createEvalRoutes, evalRoutes } from "./evals/index.js";
+import { createManageRoutes, manageRoutes } from "./manage/index.js";
+import { createRunRoutes, runRoutes } from "./run/index.js";
+
+export { createEvalRoutes, createManageRoutes, createRunRoutes, evalRoutes, manageRoutes, runRoutes };

package/dist/domains/manage/index.d.ts

@@ -0,0 +1,12 @@
+import { ManageAppVariables } from "../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/index.d.ts
+declare function createManageRoutes(): OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+declare const manageRoutes: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { createManageRoutes, manageRoutes };

package/dist/domains/manage/index.js

@@ -0,0 +1,31 @@
+import cliAuth_default from "./routes/cliAuth.js";
+import evals_default from "./routes/evals/index.js";
+import routes_default from "./routes/index.js";
+import invitations_default from "./routes/invitations.js";
+import mcp_default from "./routes/mcp.js";
+import oauth_default from "./routes/oauth.js";
+import playgroundToken_default from "./routes/playgroundToken.js";
+import projectFull_default from "./routes/projectFull.js";
+import signoz_default from "./routes/signoz.js";
+import userOrganizations_default from "./routes/userOrganizations.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/index.ts
+function createManageRoutes() {
+	const app = new OpenAPIHono();
+	app.route("/api/users/:userId/organizations", userOrganizations_default);
+	app.route("/api/cli", cliAuth_default);
+	app.route("/api/invitations", invitations_default);
+	app.route("/tenants/:tenantId", routes_default);
+	app.route("/tenants/:tenantId/playground/token", playgroundToken_default);
+	app.route("/tenants/:tenantId/signoz", signoz_default);
+	app.route("/tenants/:tenantId", projectFull_default);
+	app.route("/tenants/:tenantId/projects/:projectId/evals", evals_default);
+	app.route("/oauth", oauth_default);
+	app.route("/mcp", mcp_default);
+	return app;
+}
+const manageRoutes = createManageRoutes();
+
+//#endregion
+export { createManageRoutes, manageRoutes };

package/dist/domains/manage/routes/agent.d.ts

@@ -0,0 +1,9 @@
+import { ManageAppVariables } from "../../../types/app.js";
+import { OpenAPIHono } from "@hono/zod-openapi";
+
+//#region src/domains/manage/routes/agent.d.ts
+declare const app: OpenAPIHono<{
+  Variables: ManageAppVariables;
+}, {}, "/">;
+//#endregion
+export { app as default };