n8n 1.93.0 → 1.95.0

package/dist/evaluation.ee/test-runner/evaluation-metrics.ee.js

@@ -3,35 +3,28 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EvaluationMetrics = void 0;
 const errors_ee_1 = require("../../evaluation.ee/test-runner/errors.ee");
 class EvaluationMetrics {
-    constructor(metricNames) {
-        this.metricNames = metricNames;
+    constructor() {
         this.rawMetricsByName = new Map();
-        for (const metricName of metricNames) {
-            this.rawMetricsByName.set(metricName, []);
-        }
     }
     addResults(result) {
         const addResultsInfo = {
             addedMetrics: {},
-            missingMetrics: new Set(),
-            unknownMetrics: new Set(),
             incorrectTypeMetrics: new Set(),
         };
         for (const [metricName, metricValue] of Object.entries(result)) {
-            if (this.metricNames.has(metricName)) {
-                if (typeof metricValue === 'number') {
-                    addResultsInfo.addedMetrics[metricName] = metricValue;
-                    this.rawMetricsByName.get(metricName).push(metricValue);
-                }
-                else {
-                    throw new errors_ee_1.TestCaseExecutionError('INVALID_METRICS', {
-                        metricName,
-                        metricValue,
-                    });
+            if (typeof metricValue === 'number') {
+                addResultsInfo.addedMetrics[metricName] = metricValue;
+                if (!this.rawMetricsByName.has(metricName)) {
+                    this.rawMetricsByName.set(metricName, []);
                 }
+                this.rawMetricsByName.get(metricName).push(metricValue);
             }
             else {
-                addResultsInfo.unknownMetrics.add(metricName);
+                addResultsInfo.incorrectTypeMetrics.add(metricName);
+                throw new errors_ee_1.TestCaseExecutionError('INVALID_METRICS', {
+                    metricName,
+                    metricValue,
+                });
             }
         }
         return addResultsInfo;

package/dist/evaluation.ee/test-runner/evaluation-metrics.ee.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluation-metrics.ee.js","sourceRoot":"","sources":["../../../src/evaluation.ee/test-runner/evaluation-metrics.ee.ts"],"names":[],"mappings":";;;AAEA,qEAA+E;AAS/E,MAAa,iBAAiB;IAG7B,YAA6B,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;QAFpC,qBAAgB,GAAG,IAAI,GAAG,EAAoB,CAAC;QAG/D,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACtC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;IACF,CAAC;IAED,UAAU,CAAC,MAAmB;QAC7B,MAAM,cAAc,GAAoC;YACvD,YAAY,EAAE,EAAE;YAChB,cAAc,EAAE,IAAI,GAAG,EAAU;YACjC,cAAc,EAAE,IAAI,GAAG,EAAU;YACjC,oBAAoB,EAAE,IAAI,GAAG,EAAU;SACvC,CAAC;QAEF,KAAK,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChE,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;oBACrC,cAAc,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;oBACtD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACP,MAAM,IAAI,kCAAsB,CAAC,iBAAiB,EAAE;wBACnD,UAAU;wBACV,WAAW;qBACX,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC/C,CAAC;QACF,CAAC;QAED,OAAO,cAAc,CAAC;IACvB,CAAC;IAED,oBAAoB;QACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;gBAClE,iBAAiB,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YACjE,CAAC;QACF,CAAC;QAED,OAAO,iBAAiB,CAAC;IAC1B,CAAC;CACD;AAhDD,8CAgDC"}
+{"version":3,"file":"evaluation-metrics.ee.js","sourceRoot":"","sources":["../../../src/evaluation.ee/test-runner/evaluation-metrics.ee.ts"],"names":[],"mappings":";;;AAEA,qEAA+E;AAO/E,MAAa,iBAAiB;IAA9B;QACkB,qBAAgB,GAAG,IAAI,GAAG,EAAoB,CAAC;IA0CjE,CAAC;IAxCA,UAAU,CAAC,MAAmB;QAC7B,MAAM,cAAc,GAAoC;YACvD,YAAY,EAAE,EAAE;YAChB,oBAAoB,EAAE,IAAI,GAAG,EAAU;SACvC,CAAC;QAEF,KAAK,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChE,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;gBACrC,cAAc,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,WAAW,CAAC;gBAGtD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC3C,CAAC;gBAED,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACP,cAAc,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACpD,MAAM,IAAI,kCAAsB,CAAC,iBAAiB,EAAE;oBACnD,UAAU;oBACV,WAAW;iBACX,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAED,OAAO,cAAc,CAAC;IACvB,CAAC;IAED,oBAAoB;QACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;gBAClE,iBAAiB,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YACjE,CAAC;QACF,CAAC;QAED,OAAO,iBAAiB,CAAC;IAC1B,CAAC;CACD;AA3CD,8CA2CC"}

package/dist/evaluation.ee/test-runner/test-run-cleanup.service.ee.d.ts

@@ -0,0 +1,8 @@
+import { TestRunRepository } from '@n8n/db';
+import { Logger } from 'n8n-core';
+export declare class TestRunCleanupService {
+    private readonly logger;
+    private readonly testRunRepository;
+    constructor(logger: Logger, testRunRepository: TestRunRepository);
+    cleanupIncompleteRuns(): Promise<void>;
+}

package/dist/{databases/repositories/webhook.repository.js → evaluation.ee/test-runner/test-run-cleanup.service.ee.js}

@@ -9,18 +9,26 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebhookRepository = void 0;
+exports.TestRunCleanupService = void 0;
 const db_1 = require("@n8n/db");
 const di_1 = require("@n8n/di");
-const typeorm_1 = require("@n8n/typeorm");
-let WebhookRepository = class WebhookRepository extends typeorm_1.Repository {
-    constructor(dataSource) {
-        super(db_1.WebhookEntity, dataSource.manager);
+const n8n_core_1 = require("n8n-core");
+let TestRunCleanupService = class TestRunCleanupService {
+    constructor(logger, testRunRepository) {
+        this.logger = logger;
+        this.testRunRepository = testRunRepository;
+    }
+    async cleanupIncompleteRuns() {
+        const result = await this.testRunRepository.markAllIncompleteAsFailed();
+        if (result.affected && result.affected > 0) {
+            this.logger.debug(`Marked ${result.affected} incomplete test runs as failed`);
+        }
     }
 };
-exports.WebhookRepository = WebhookRepository;
-exports.WebhookRepository = WebhookRepository = __decorate([
+exports.TestRunCleanupService = TestRunCleanupService;
+exports.TestRunCleanupService = TestRunCleanupService = __decorate([
     (0, di_1.Service)(),
-    __metadata("design:paramtypes", [typeorm_1.DataSource])
-], WebhookRepository);
-//# sourceMappingURL=webhook.repository.js.map
+    __metadata("design:paramtypes", [n8n_core_1.Logger,
+        db_1.TestRunRepository])
+], TestRunCleanupService);
+//# sourceMappingURL=test-run-cleanup.service.ee.js.map

package/dist/evaluation.ee/test-runner/test-run-cleanup.service.ee.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-run-cleanup.service.ee.js","sourceRoot":"","sources":["../../../src/evaluation.ee/test-runner/test-run-cleanup.service.ee.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,gCAA4C;AAC5C,gCAAkC;AAClC,uCAAkC;AAM3B,IAAM,qBAAqB,GAA3B,MAAM,qBAAqB;IACjC,YACkB,MAAc,EACd,iBAAoC;QADpC,WAAM,GAAN,MAAM,CAAQ;QACd,sBAAiB,GAAjB,iBAAiB,CAAmB;IACnD,CAAC;IAMJ,KAAK,CAAC,qBAAqB;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,EAAE,CAAC;QACxE,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,QAAQ,iCAAiC,CAAC,CAAC;QAC/E,CAAC;IACF,CAAC;CACD,CAAA;AAhBY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,YAAO,GAAE;qCAGiB,iBAAM;QACK,sBAAiB;GAH1C,qBAAqB,CAgBjC"}

package/dist/evaluation.ee/test-runner/test-runner.service.ee.d.ts

@@ -1,55 +1,41 @@
-import type { User, ExecutionEntity, TestDefinition, TestRun } from '@n8n/db';
-import { ExecutionRepository, TestCaseExecutionRepository, TestMetricRepository, TestRunRepository } from '@n8n/db';
+import type { User, TestRun } from '@n8n/db';
+import { TestCaseExecutionRepository, TestRunRepository, WorkflowRepository } from '@n8n/db';
 import { ErrorReporter, Logger } from 'n8n-core';
-import type { IWorkflowBase } from 'n8n-workflow';
+import type { IRun, IWorkflowBase } from 'n8n-workflow';
 import { ActiveExecutions } from '../../active-executions';
-import { WorkflowRepository } from '../../databases/repositories/workflow.repository';
-import { NodeTypes } from '../../node-types';
 import { Telemetry } from '../../telemetry';
 import { WorkflowRunner } from '../../workflow-runner';
 export interface TestRunMetadata {
     testRunId: string;
     userId: string;
 }
-export interface TestCaseRunMetadata extends TestRunMetadata {
-    pastExecutionId: string;
-    annotation: ExecutionEntity['annotation'];
-    highlightedData: ExecutionEntity['metadata'];
+export interface TestCaseExecutionResult {
+    executionData: IRun;
+    executionId: string;
 }
 export declare class TestRunnerService {
     private readonly logger;
     private readonly telemetry;
     private readonly workflowRepository;
     private readonly workflowRunner;
-    private readonly executionRepository;
     private readonly activeExecutions;
     private readonly testRunRepository;
     private readonly testCaseExecutionRepository;
-    private readonly testMetricRepository;
-    private readonly nodeTypes;
     private readonly errorReporter;
     private abortControllers;
-    constructor(logger: Logger, telemetry: Telemetry, workflowRepository: WorkflowRepository, workflowRunner: WorkflowRunner, executionRepository: ExecutionRepository, activeExecutions: ActiveExecutions, testRunRepository: TestRunRepository, testCaseExecutionRepository: TestCaseExecutionRepository, testMetricRepository: TestMetricRepository, nodeTypes: NodeTypes, errorReporter: ErrorReporter);
-    cleanupIncompleteRuns(): Promise<void>;
-    private getStartNodesData;
+    constructor(logger: Logger, telemetry: Telemetry, workflowRepository: WorkflowRepository, workflowRunner: WorkflowRunner, activeExecutions: ActiveExecutions, testRunRepository: TestRunRepository, testCaseExecutionRepository: TestCaseExecutionRepository, errorReporter: ErrorReporter);
+    private findEvaluationTriggerNode;
+    private validateEvaluationTriggerNode;
+    private validateSetMetricsNodes;
+    private validateSetOutputsNodes;
+    private validateWorkflowConfiguration;
     private runTestCase;
-    syncMetrics(testDefinitionId: string, evaluationWorkflow: IWorkflowBase): Promise<Set<string>>;
-    private runTestCaseEvaluation;
+    private runDatasetTrigger;
     static getEvaluationMetricsNodes(workflow: IWorkflowBase): import("n8n-workflow").INode[];
+    static getEvaluationSetOutputsNodes(workflow: IWorkflowBase): import("n8n-workflow").INode[];
+    private extractDatasetTriggerOutput;
     private extractEvaluationResult;
-    private getUsedTestMetricNames;
-    runTest(user: User, test: TestDefinition): Promise<void>;
+    runTest(user: User, workflowId: string): Promise<void>;
     canBeCancelled(testRun: TestRun): boolean;
     cancelTestRun(testRunId: string): Promise<void>;
-    getExampleEvaluationInputData(test: TestDefinition, annotationTagId: string): Promise<{
-        annotations: {
-            vote: import("n8n-workflow").AnnotationVote | null | undefined;
-            tags: Pick<import("@n8n/db").AnnotationTagEntity, "id" | "name">[] | undefined;
-            highlightedData: {
-                [k: string]: string;
-            };
-        };
-        originalExecution: Record<string, any>;
-        newExecution: Record<string, any>;
-    } | null>;
 }