n8n 2.33.2 → 2.34.0

package/dist/modules/agent-evals/agent-eval-case-generation.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-case-generation.service.js","sourceRoot":"","sources":["../../../src/modules/agent-evals/agent-eval-case-generation.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,iEAAmE;AACnE,8CAMwB;AACxB,wDAA6C;AAE7C,gCAAqD;AACrD,gCAAkC;AAClC,+CAA2D;AAE3D,2EAAuE;AACvE,8EAA0E;AAC1E,6HAAoH;AAEpH,mEAA6D;AAC7D,yEAAoE;AACpE,qFAMkD;AAClD,6DAAqE;AACrE,6FAA0F;AAC1F,qEAAuF;AACvF,yFAA0F;AAC1F,yEAAoE;AACpE,wGAAiG;AAEjG,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,cAAc,GAAG,EAAE,CAAC;AAI1B,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAGlC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAKnC,MAAM,YAAY,GAAG,OAAO,CAAC;AAC7B,MAAM,eAAe,GAAG,UAAU,CAAC;AAInC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAuBtB,IAAM,8BAA8B,GAApC,MAAM,8BAA8B;IAC1C,YACkB,MAAc,EACd,kBAAsC,EACtC,kBAAsC,EACtC,gBAAkC,EAClC,iBAA6C,EAC7C,QAA4B,EAC5B,+BAAgE;sBANhE,MAAM;kCACN,kBAAkB;kCAClB,kBAAkB;gCAClB,gBAAgB;iCAChB,iBAAiB;wBACjB,QAAQ;+CACR,+BAA+B;IAC9C,CAAC;IAQJ,KAAK,CAAC,kBAAkB,CACvB,IAAU,EACV,SAAiB,EACjB,OAAe,EACf,OAAO,GAA8B,EAAE;QAEvC,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC3E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAE1E,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,YAAY,GAAG,IAAA,2CAAkB,EAAC,MAAM,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAA,kCAAqB,EAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAE1D,MAAM,OAAO,GAAG,IAAA,0CAAiB,EAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CACvC,WAAW,EACX,IAAA,sDAA6B,EAAC,OAAO,EAAE,MAAM,CAAC,EAC9C,MAAM,CAAC,MAAM,CACb,CAAC;QAIF,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAGnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QAChD,MAAM,QAAQ,GACb,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvF,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAC3D,SAAS,EACT,OAAO,EACP,IAAI,CAAC,EAAE,EACP,QAAQ,EACR,KAAK,CACL,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;YAC/C,OAAO;YACP,SAAS;YACT,SAAS,EAAE,KAAK,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IASO,yBAAyB;QAChC,IAAI,IAAI,CAAC,+BAA+B,CAAC,cAAc,EAAE,CAAC,cAAc,EAAE,CAAC;YAC1E,MAAM,IAAI,gCAAc,CACvB,yFAAyF,CACzF,CAAC;QACH,CAAC;IACF,CAAC;IAQO,KAAK,CAAC,iBAAiB,CAAC,MAAuB,EAAE,SAAiB,EAAE,IAAU;QACrF,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,IAAI,UAAU,KAAK,oCAAwB,EAAE,CAAC;YACtE,MAAM,IAAI,wBAAS,CAClB,iGAAiG,CACjG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAA,mDAAwB,EAAC,IAAA,gCAAiB,EAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,wBAAS,CAClB,qEAAqE,KAAK,KAAK,CAC/E,CAAC;QACH,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAA,yDAA6B,EACvD,IAAI,CAAC,kBAAkB,EACvB,SAAS,EACT,IAAI,CACJ,CAAC;QACF,OAAO,MAAM,IAAA,gDAAiC,EAAC,KAAK,EAAE,UAAU,EAAE,kBAAkB,CAAC,CAAC;IACvF,CAAC;IAWO,KAAK,CAAC,WAAW,CACxB,WAA0E,EAC1E,UAAkB,EAClB,aAAqB;QAIrB,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAU,IAAI,KAAK,CAAC,4BAA4B,CAAC;aAC1D,KAAK,CAAC,WAAW,CAAC;aAClB,YAAY,CAAC,sDAA6B,CAAC;aAC3C,gBAAgB,CAAC,6CAAoB,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,KAAK,EAAE,MAAc,EAAwC,EAAE;YAC9E,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE;gBAC3C,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,mBAAmB,CAAC;aACrD,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,6CAAoB,CAAC,SAAS,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACvE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAGjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK;iBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;iBAC1E,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEhE,OAAO,KAAK,CAAC,MAAM,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;QACxC,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;QAExB,MAAM,KAAK,GAAG,MAAM,OAAO,CAC1B,GAAG,UAAU,8CAA8C,aAAa,gEAAgE,CACxI,CAAC;QACF,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;QAExB,MAAM,IAAI,+BAAgB,CACzB,yEAAyE,CACzE,CAAC;IACH,CAAC;IAUO,KAAK,CAAC,cAAc,CAC3B,SAAiB,EACjB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,KAA2B;QAE3B,MAAM,OAAO,GAAG;YACf,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAiB,EAAE;YAC/C,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,QAAiB,EAAE;SAClD,CAAC;QAEF,IAAI,KAA2E,CAAC;QAChF,IAAI,IAAI,GAAG,QAAQ,CAAC;QACpB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,KAAK,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YACtE,IAAI,CAAC;gBACJ,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,KAAK,YAAY,2DAA0B,IAAI,OAAO,GAAG,iBAAiB,GAAG,CAAC,EAAE,CAAC;oBACpF,SAAS;gBACV,CAAC;gBACD,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9B,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,KAAK;gBACvB,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,WAAW;aAChC,CAAC,CAAC,CAAC;YACJ,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YAElE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC;gBAC1D,IAAI;gBACJ,OAAO;gBACP,aAAa,EAAE,YAAY;gBAC3B,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE;gBACrC,aAAa,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE;gBACjE,WAAW;aACX,CAAC,CAAC;YACH,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;QACzD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,iBAAiB,CAAC,WAAmB,EAAE,SAAiB;QACrE,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6DAA6D,EAAE;gBAChF,WAAW;gBACX,KAAK,EAAE,YAAY,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;aAClF,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;CACD,CAAA;;yCAjOY,8BAA8B;IAD1C,IAAA,YAAO,GAAE;qCAGiB,uBAAM,EACM,yCAAkB,EAClB,wCAAkB,EACpB,qCAAgB,EACf,+BAA0B,EACnC,0CAAkB,EACK,uEAA+B;GARtE,8BAA8B,CAiO1C;AAGD,SAAS,UAAU,CAAC,KAA2B,EAAE,KAAa;IAC7D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,EAAE,mBAAmB,CAAC;QACjD,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,WAAW,EAAE,mBAAmB,CAAC;KAC7D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,GAAW;IAC9C,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,UAAU,CAAC,KAAyB;IAC5C,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAiB;IAC5C,OAAO,YAAY,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;AACrD,CAAC;AAGD,SAAS,YAAY,CAAC,QAAgB,EAAE,CAAS;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC;IACzB,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,GAAG,GAAG,GAAG;IAC5C,OAAO,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC"}

package/dist/modules/agent-evals/agent-eval-rating.service.d.ts

@@ -0,0 +1,23 @@
+import { type CreateAgentEvalRatingPayload } from '@n8n/api-types';
+import { Logger } from '@n8n/backend-common';
+import type { AgentEvalRating, User } from '@n8n/db';
+import { AgentEvalDatasetRepository, AgentEvalRatingRepository, AgentEvalResultRepository, AgentEvalRunRepository } from '@n8n/db';
+import { AgentRepository } from '../../modules/agents/repositories/agent.repository';
+import { PostHogClient } from '../../posthog';
+export declare class AgentEvalRatingService {
+    private readonly logger;
+    private readonly ratingRepository;
+    private readonly resultRepository;
+    private readonly runRepository;
+    private readonly datasetRepository;
+    private readonly agentRepository;
+    private readonly postHogClient;
+    constructor(logger: Logger, ratingRepository: AgentEvalRatingRepository, resultRepository: AgentEvalResultRepository, runRepository: AgentEvalRunRepository, datasetRepository: AgentEvalDatasetRepository, agentRepository: AgentRepository, postHogClient: PostHogClient);
+    rateResult(user: User, projectId: string, resultId: string, payload: CreateAgentEvalRatingPayload): Promise<AgentEvalRating>;
+    listRatingsForResult(user: User, projectId: string, resultId: string): Promise<AgentEvalRating[]>;
+    listLatestRatingsForRun(user: User, projectId: string, runId: string): Promise<AgentEvalRating[]>;
+    private assertFeatureEnabled;
+    private assertProjectScopes;
+    private resolveResultInProject;
+    private resolveRunInProject;
+}

package/dist/modules/agent-evals/agent-eval-rating.service.js

@@ -0,0 +1,124 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentEvalRatingService = void 0;
+const api_types_1 = require("@n8n/api-types");
+const backend_common_1 = require("@n8n/backend-common");
+const db_1 = require("@n8n/db");
+const di_1 = require("@n8n/di");
+const bad_request_error_1 = require("../../errors/response-errors/bad-request.error");
+const forbidden_error_1 = require("../../errors/response-errors/forbidden.error");
+const not_found_error_1 = require("../../errors/response-errors/not-found.error");
+const agent_repository_1 = require("../../modules/agents/repositories/agent.repository");
+const check_access_1 = require("../../permissions.ee/check-access");
+const posthog_1 = require("../../posthog");
+const MAX_COMMENT_CHARS = 2_000;
+const MAX_CORRECTION_TEXT_CHARS = 20_000;
+const MAX_CORRECTION_CHARS = 32_000;
+let AgentEvalRatingService = class AgentEvalRatingService {
+    constructor(logger, ratingRepository, resultRepository, runRepository, datasetRepository, agentRepository, postHogClient) {
+        this.logger = logger;
+        this.ratingRepository = ratingRepository;
+        this.resultRepository = resultRepository;
+        this.runRepository = runRepository;
+        this.datasetRepository = datasetRepository;
+        this.agentRepository = agentRepository;
+        this.postHogClient = postHogClient;
+    }
+    async rateResult(user, projectId, resultId, payload) {
+        await this.assertFeatureEnabled(user);
+        await this.assertProjectScopes(user, projectId, ['agent:execute']);
+        assertPayloadWithinBounds(payload);
+        const result = await this.resolveResultInProject(projectId, resultId);
+        if (result.status === 'new' || result.status === 'running') {
+            throw new bad_request_error_1.BadRequestError('This case has not finished running yet.');
+        }
+        const rating = await this.ratingRepository.createRating({
+            resultId: result.id,
+            vote: payload.vote,
+            comment: payload.comment ?? null,
+            correction: payload.correction ?? null,
+            ratedById: user.id,
+        });
+        this.logger.debug('Recorded agent eval rating', {
+            resultId: result.id,
+            vote: rating.vote,
+            hasCorrection: rating.correction !== null,
+        });
+        return rating;
+    }
+    async listRatingsForResult(user, projectId, resultId) {
+        await this.assertFeatureEnabled(user);
+        await this.assertProjectScopes(user, projectId, ['agent:read']);
+        const result = await this.resolveResultInProject(projectId, resultId);
+        return await this.ratingRepository.findByResultId(result.id);
+    }
+    async listLatestRatingsForRun(user, projectId, runId) {
+        await this.assertFeatureEnabled(user);
+        await this.assertProjectScopes(user, projectId, ['agent:read']);
+        await this.resolveRunInProject(projectId, runId);
+        return await this.ratingRepository.findLatestByRunId(runId);
+    }
+    async assertFeatureEnabled(user) {
+        const flags = await this.postHogClient.getFeatureFlags(user);
+        if (flags?.[api_types_1.AGENT_EVALS_FLAG] !== true) {
+            throw new not_found_error_1.NotFoundError('Not found');
+        }
+    }
+    async assertProjectScopes(user, projectId, scopes) {
+        if (!(await (0, check_access_1.userHasScopes)(user, scopes, false, { projectId }))) {
+            throw new forbidden_error_1.ForbiddenError('You do not have permission to review agent evals in this project.');
+        }
+    }
+    async resolveResultInProject(projectId, resultId) {
+        const result = await this.resultRepository.findById(resultId);
+        if (!result)
+            throw new not_found_error_1.NotFoundError(`Agent eval result ${resultId} not found.`);
+        await this.resolveRunInProject(projectId, result.runId);
+        return result;
+    }
+    async resolveRunInProject(projectId, runId) {
+        const notFound = () => new not_found_error_1.NotFoundError(`Agent eval run ${runId} not found.`);
+        const run = await this.runRepository.findById(runId);
+        if (!run)
+            throw notFound();
+        const dataset = await this.datasetRepository.findById(run.datasetId);
+        if (!dataset)
+            throw notFound();
+        const inProject = await this.agentRepository.existsByIdAndProjectId(dataset.agentId, projectId);
+        if (!inProject)
+            throw notFound();
+    }
+};
+exports.AgentEvalRatingService = AgentEvalRatingService;
+exports.AgentEvalRatingService = AgentEvalRatingService = __decorate([
+    (0, di_1.Service)(),
+    __metadata("design:paramtypes", [backend_common_1.Logger, db_1.AgentEvalRatingRepository, db_1.AgentEvalResultRepository, db_1.AgentEvalRunRepository, db_1.AgentEvalDatasetRepository, agent_repository_1.AgentRepository, posthog_1.PostHogClient])
+], AgentEvalRatingService);
+function assertPayloadWithinBounds(payload) {
+    if (payload.comment !== undefined && payload.comment.length > MAX_COMMENT_CHARS) {
+        throw new bad_request_error_1.BadRequestError(`A rating comment cannot exceed ${MAX_COMMENT_CHARS} characters.`);
+    }
+    const { correction } = payload;
+    if (correction === undefined)
+        return;
+    const finalText = correction.finalText;
+    if (typeof finalText !== 'string' || finalText.trim().length === 0) {
+        throw new bad_request_error_1.BadRequestError("A correction must carry the edited answer in 'finalText'.");
+    }
+    if (finalText.length > MAX_CORRECTION_TEXT_CHARS) {
+        throw new bad_request_error_1.BadRequestError(`A corrected answer cannot exceed ${MAX_CORRECTION_TEXT_CHARS} characters.`);
+    }
+    if (JSON.stringify(correction).length > MAX_CORRECTION_CHARS) {
+        throw new bad_request_error_1.BadRequestError(`A correction cannot exceed ${MAX_CORRECTION_CHARS} characters in total.`);
+    }
+}
+//# sourceMappingURL=agent-eval-rating.service.js.map

package/dist/modules/agent-evals/agent-eval-rating.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-rating.service.js","sourceRoot":"","sources":["../../../src/modules/agent-evals/agent-eval-rating.service.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAqF;AACrF,wDAA6C;AAE7C,gCAKiB;AACjB,gCAAkC;AAGlC,kFAA6E;AAC7E,8EAA0E;AAC1E,8EAAyE;AACzE,qFAAiF;AACjF,gEAA8D;AAC9D,uCAA0C;AAG1C,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAChC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAO7B,IAAM,sBAAsB,GAA5B,MAAM,sBAAsB;IAClC,YACkB,MAAc,EACd,gBAA2C,EAC3C,gBAA2C,EAC3C,aAAqC,EACrC,iBAA6C,EAC7C,eAAgC,EAChC,aAA4B;sBAN5B,MAAM;gCACN,gBAAgB;gCAChB,gBAAgB;6BAChB,aAAa;iCACb,iBAAiB;+BACjB,eAAe;6BACf,aAAa;IAC5B,CAAC;IAGJ,KAAK,CAAC,UAAU,CACf,IAAU,EACV,SAAiB,EACjB,QAAgB,EAChB,OAAqC;QAErC,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAGtC,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;QACnE,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAEnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAGtE,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,mCAAe,CAAC,yCAAyC,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YACvD,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;YAChC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,SAAS,EAAE,IAAI,CAAC,EAAE;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;YAC/C,QAAQ,EAAE,MAAM,CAAC,EAAE;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,aAAa,EAAE,MAAM,CAAC,UAAU,KAAK,IAAI;SACzC,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAGD,KAAK,CAAC,oBAAoB,CACzB,IAAU,EACV,SAAiB,EACjB,QAAgB;QAEhB,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEtE,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;IAGD,KAAK,CAAC,uBAAuB,CAC5B,IAAU,EACV,SAAiB,EACjB,KAAa;QAEb,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAEhE,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAEjD,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IAQO,KAAK,CAAC,oBAAoB,CAAC,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,KAAK,EAAE,CAAC,4BAAgB,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,+BAAa,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,mBAAmB,CAAC,IAAU,EAAE,SAAiB,EAAE,MAAe;QAC/E,IAAI,CAAC,CAAC,MAAM,IAAA,4BAAa,EAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,gCAAc,CAAC,mEAAmE,CAAC,CAAC;QAC/F,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,sBAAsB,CACnC,SAAiB,EACjB,QAAgB;QAEhB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,+BAAa,CAAC,qBAAqB,QAAQ,aAAa,CAAC,CAAC;QAEjF,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAExD,OAAO,MAAM,CAAC;IACf,CAAC;IAMO,KAAK,CAAC,mBAAmB,CAAC,SAAiB,EAAE,KAAa;QACjE,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,IAAI,+BAAa,CAAC,kBAAkB,KAAK,aAAa,CAAC,CAAC;QAE/E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,GAAG;YAAE,MAAM,QAAQ,EAAE,CAAC;QAE3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,CAAC,OAAO;YAAE,MAAM,QAAQ,EAAE,CAAC;QAG/B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,sBAAsB,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAChG,IAAI,CAAC,SAAS;YAAE,MAAM,QAAQ,EAAE,CAAC;IAClC,CAAC;CACD,CAAA;;iCA7HY,sBAAsB;IADlC,IAAA,YAAO,GAAE;qCAGiB,uBAAM,EACI,8BAAyB,EACzB,8BAAyB,EAC5B,2BAAsB,EAClB,+BAA0B,EAC5B,kCAAe,EACjB,uBAAa;GARlC,sBAAsB,CA6HlC;AAGD,SAAS,yBAAyB,CAAC,OAAqC;IACvE,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACjF,MAAM,IAAI,mCAAe,CAAC,kCAAkC,iBAAiB,cAAc,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAC/B,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO;IAIrC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,mCAAe,CAAC,2DAA2D,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,yBAAyB,EAAE,CAAC;QAClD,MAAM,IAAI,mCAAe,CACxB,oCAAoC,yBAAyB,cAAc,CAC3E,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC9D,MAAM,IAAI,mCAAe,CACxB,8BAA8B,oBAAoB,uBAAuB,CACzE,CAAC;IACH,CAAC;AACF,CAAC"}

package/dist/modules/agent-evals/agent-eval-record-mappers.d.ts

@@ -0,0 +1,5 @@
+import type { AgentEvalDatasetRecord, AgentEvalResultRecord, AgentEvalRunRecord } from '@n8n/api-types';
+import type { AgentEvalDataset, AgentEvalResult, AgentEvalRun } from '@n8n/db';
+export declare function toDatasetRecord(dataset: AgentEvalDataset): AgentEvalDatasetRecord;
+export declare function toRunRecord(run: AgentEvalRun): AgentEvalRunRecord;
+export declare function toResultRecord(result: AgentEvalResult): AgentEvalResultRecord;

package/dist/modules/agent-evals/agent-eval-record-mappers.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toDatasetRecord = toDatasetRecord;
+exports.toRunRecord = toRunRecord;
+exports.toResultRecord = toResultRecord;
+const n8n_workflow_1 = require("n8n-workflow");
+const toIso = (date) => date?.toISOString() ?? null;
+const isDataTableRef = (ref) => 'dataTableId' in ref;
+const isGoogleSheetsRef = (ref) => 'spreadsheetId' in ref;
+function toDatasetRefPair(dataset) {
+    const { datasetSource, datasetRef } = dataset;
+    if (datasetSource === 'data_table' && isDataTableRef(datasetRef)) {
+        return { datasetSource, datasetRef };
+    }
+    if (datasetSource === 'google_sheets' && isGoogleSheetsRef(datasetRef)) {
+        return { datasetSource, datasetRef };
+    }
+    throw new n8n_workflow_1.UnexpectedError(`Agent eval dataset ${dataset.id} has a '${datasetSource}' source whose ref does not match that shape.`);
+}
+function toDatasetRecord(dataset) {
+    return {
+        id: dataset.id,
+        name: dataset.name,
+        description: dataset.description,
+        agentId: dataset.agentId,
+        columnMapping: dataset.columnMapping,
+        createdById: dataset.createdById,
+        createdAt: dataset.createdAt.toISOString(),
+        updatedAt: dataset.updatedAt.toISOString(),
+        ...toDatasetRefPair(dataset),
+    };
+}
+function toRunRecord(run) {
+    return {
+        id: run.id,
+        datasetId: run.datasetId,
+        agentVersionId: run.agentVersionId,
+        status: run.status,
+        runAt: toIso(run.runAt),
+        completedAt: toIso(run.completedAt),
+        metrics: run.metrics,
+        errorCode: run.errorCode,
+        errorDetails: run.errorDetails,
+        createdById: run.createdById,
+        createdAt: run.createdAt.toISOString(),
+        updatedAt: run.updatedAt.toISOString(),
+    };
+}
+function toResultRecord(result) {
+    return {
+        id: result.id,
+        runId: result.runId,
+        sourceRowId: result.sourceRowId,
+        runIndex: result.runIndex,
+        status: result.status,
+        input: result.input,
+        output: result.output,
+        toolCalls: result.toolCalls,
+        metrics: result.metrics,
+        runAt: toIso(result.runAt),
+        completedAt: toIso(result.completedAt),
+        errorCode: result.errorCode,
+        errorDetails: result.errorDetails,
+        createdAt: result.createdAt.toISOString(),
+        updatedAt: result.updatedAt.toISOString(),
+    };
+}
+//# sourceMappingURL=agent-eval-record-mappers.js.map

package/dist/modules/agent-evals/agent-eval-record-mappers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-record-mappers.js","sourceRoot":"","sources":["../../../src/modules/agent-evals/agent-eval-record-mappers.ts"],"names":[],"mappings":";;;;;AASA,+CAA+C;AAQ/C,MAAM,KAAK,GAAG,CAAC,IAAiB,EAAiB,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,IAAI,CAAC;AAGhF,MAAM,cAAc,GAAG,CAAC,GAA6B,EAA8B,EAAE,CACpF,aAAa,IAAI,GAAG,CAAC;AAEtB,MAAM,iBAAiB,GAAG,CAAC,GAA6B,EAAiC,EAAE,CAC1F,eAAe,IAAI,GAAG,CAAC;AAQxB,SAAS,gBAAgB,CAAC,OAAyB;IAClD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAE9C,IAAI,aAAa,KAAK,YAAY,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;QAClE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;IACtC,CAAC;IACD,IAAI,aAAa,KAAK,eAAe,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;QACxE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;IACtC,CAAC;IAED,MAAM,IAAI,8BAAe,CACxB,sBAAsB,OAAO,CAAC,EAAE,WAAW,aAAa,+CAA+C,CACvG,CAAC;AACH,CAAC;AAED,yBAAgC,OAAyB;IACxD,OAAO;QACN,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;QAC1C,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE;QAC1C,GAAG,gBAAgB,CAAC,OAAO,CAAC;KAC5B,CAAC;AACH,CAAC;AAED,qBAA4B,GAAiB;IAC5C,OAAO;QACN,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;QACvB,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC;QACnC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;QACtC,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE;KACtC,CAAC;AACH,CAAC;AAED,wBAA+B,MAAuB;IACrD,OAAO;QACN,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;QAC1B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;KACzC,CAAC;AACH,CAAC"}

package/dist/modules/agent-evals/agent-eval-runner.service.d.ts

@@ -0,0 +1,42 @@
+import type { AgentEvalRunSummary } from '@n8n/api-types';
+import { Logger, ModuleRegistry } from '@n8n/backend-common';
+import { GlobalConfig } from '@n8n/config';
+import type { User } from '@n8n/db';
+import { AgentEvalDatasetRepository, AgentEvalResultRepository, AgentEvalRunRepository } from '@n8n/db';
+import { InstanceSettings } from 'n8n-core';
+import { ConcurrencyControlService } from '../../concurrency/concurrency-control.service';
+import { License } from '../../license';
+import { AgentRepository } from '../../modules/agents/repositories/agent.repository';
+import { DataTableService } from '../../modules/data-table/data-table.service';
+import { EvalAgentExecutionService } from '../../modules/instance-ai/eval/agent-execution.service';
+import { AgentEvalsFlagGate } from './agent-evals-flag-gate';
+export declare class AgentEvalRunnerService {
+    private readonly logger;
+    private readonly globalConfig;
+    private readonly instanceSettings;
+    private readonly moduleRegistry;
+    private readonly datasetRepository;
+    private readonly runRepository;
+    private readonly resultRepository;
+    private readonly agentRepository;
+    private readonly dataTableService;
+    private readonly evalAgentExecutionService;
+    private readonly concurrencyControl;
+    private readonly license;
+    private readonly flagGate;
+    constructor(logger: Logger, globalConfig: GlobalConfig, instanceSettings: InstanceSettings, moduleRegistry: ModuleRegistry, datasetRepository: AgentEvalDatasetRepository, runRepository: AgentEvalRunRepository, resultRepository: AgentEvalResultRepository, agentRepository: AgentRepository, dataTableService: DataTableService, evalAgentExecutionService: EvalAgentExecutionService, concurrencyControl: ConcurrencyControlService, license: License, flagGate: AgentEvalsFlagGate);
+    startRun(datasetId: string, projectId: string, user: User, options?: {
+        timeoutMs?: number;
+    }): Promise<{
+        runId: string;
+        finished: Promise<void>;
+    }>;
+    getRunSummary(runId: string, agentId: string): Promise<AgentEvalRunSummary>;
+    private executeRun;
+    private startRunDeadline;
+    private acquireEvaluationSlot;
+    private failRun;
+    cleanupInterruptedRuns(): Promise<void>;
+    private runCase;
+    private resolveCases;
+}