@n8n/instance-ai 1.6.1 → 1.7.0

package/dist/tools/evals/detect-agent-named-refs.service.d.ts

@@ -0,0 +1,9 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+export interface NamedRef {
+    nodeName: string;
+    field: string;
+    originalExpression: string;
+    column: string;
+    targetNodeName: string;
+}
+export declare function detectAgentNamedRefs(workflow: WorkflowJSON, agentNodeName: string): NamedRef[];

package/dist/tools/evals/detect-agent-named-refs.service.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectAgentNamedRefs = detectAgentNamedRefs;
+const column_ref_utils_1 = require("./column-ref-utils");
+function slug(name) {
+    const result = name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '');
+    return result.length > 0 ? result : 'node';
+}
+function detectAgentNamedRefs(workflow, agentNodeName) {
+    const node = (workflow.nodes ?? []).find((n) => (0, column_ref_utils_1.nodeHasName)(n) && n.name === agentNodeName);
+    if (!node) {
+        throw new Error(`Agent node "${agentNodeName}" not found in workflow`);
+    }
+    const targets = [agentNodeName, ...(0, column_ref_utils_1.findAgentSubComponents)(workflow, agentNodeName)];
+    const nodesByName = new Map((workflow.nodes ?? []).filter(column_ref_utils_1.nodeHasName).map((n) => [n.name, n]));
+    const targetMatches = new Map();
+    for (const target of targets) {
+        const tNode = nodesByName.get(target);
+        if (!tNode)
+            continue;
+        const matches = (0, column_ref_utils_1.collectStrings)(tNode.parameters).flatMap((text) => (0, column_ref_utils_1.extractNamedRefMatches)(text));
+        const dedup = new Map();
+        for (const m of matches) {
+            const key = `${m.nodeName}\x00${m.field}`;
+            if (!dedup.has(key))
+                dedup.set(key, m);
+        }
+        if (dedup.size > 0)
+            targetMatches.set(target, [...dedup.values()]);
+    }
+    const allPairs = new Map();
+    for (const matches of targetMatches.values()) {
+        for (const m of matches) {
+            const key = `${m.nodeName}\x00${m.field}`;
+            if (!allPairs.has(key)) {
+                allPairs.set(key, { nodeName: m.nodeName, field: m.field });
+            }
+        }
+    }
+    const byField = new Map();
+    for (const p of allPairs.values()) {
+        const arr = byField.get(p.field) ?? [];
+        arr.push(p);
+        byField.set(p.field, arr);
+    }
+    function columnFor(nodeName, field) {
+        const group = byField.get(field) ?? [];
+        return group.length === 1 ? field : `${slug(nodeName)}_${field}`;
+    }
+    const result = [];
+    for (const [target, matches] of targetMatches) {
+        for (const m of matches) {
+            result.push({
+                nodeName: m.nodeName,
+                field: m.field,
+                originalExpression: m.originalExpression,
+                column: columnFor(m.nodeName, m.field),
+                targetNodeName: target,
+            });
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=detect-agent-named-refs.service.js.map

package/dist/tools/evals/detect-agent-named-refs.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-agent-named-refs.service.js","sourceRoot":"","sources":["../../../src/tools/evals/detect-agent-named-refs.service.ts"],"names":[],"mappings":";;AAiCA,oDA8DC;AA7FD,yDAM4B;AAe5B,SAAS,IAAI,CAAC,IAAY;IACzB,MAAM,MAAM,GAAG,IAAI;SACjB,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5C,CAAC;AAID,SAAgB,oBAAoB,CAAC,QAAsB,EAAE,aAAqB;IACjF,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,8BAAW,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC;IAC5F,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,eAAe,aAAa,yBAAyB,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,aAAa,EAAE,GAAG,IAAA,yCAAsB,EAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;IACpF,MAAM,WAAW,GAAG,IAAI,GAAG,CAC1B,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,8BAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAU,CAAC,CAC3E,CAAC;IAGF,MAAM,aAAa,GAAG,IAAI,GAAG,EAAsB,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,OAAO,GAAG,IAAA,iCAAc,EAAC,KAAK,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACjE,IAAA,yCAAsB,EAAC,IAAI,CAAC,CAC5B,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,QAAQ,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC;YAAE,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAGD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+C,CAAC;IACxE,KAAK,MAAM,OAAO,IAAI,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,QAAQ,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;QACF,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsD,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IACD,SAAS,SAAS,CAAC,QAAgB,EAAE,KAAa;QACjD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,CAAC;IAClE,CAAC;IAGD,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,aAAa,EAAE,CAAC;QAC/C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC;gBACX,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,kBAAkB,EAAE,CAAC,CAAC,kBAAkB;gBACxC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC;gBACtC,cAAc,EAAE,MAAM;aACtB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC"}

package/dist/tools/evals/detect-ai-nodes.d.ts

@@ -0,0 +1,7 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+export interface DetectAiNodesResult {
+    isAiWorkflow: boolean;
+    aiNodeNames: string[];
+    alreadyConfigured: boolean;
+}
+export declare function detectAiNodes(workflow: WorkflowJSON): DetectAiNodesResult;

package/dist/tools/evals/detect-ai-nodes.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectAiNodes = detectAiNodes;
+const LANGCHAIN_TYPE_PREFIX = '@n8n/n8n-nodes-langchain.';
+const EVALUATION_TYPES = new Set([
+    'n8n-nodes-base.evaluation',
+    'n8n-nodes-base.evaluationTrigger',
+]);
+const ROOT_AI_TYPES = new Set([
+    `${LANGCHAIN_TYPE_PREFIX}agent`,
+    `${LANGCHAIN_TYPE_PREFIX}openAiAssistant`,
+    `${LANGCHAIN_TYPE_PREFIX}chainLlm`,
+    `${LANGCHAIN_TYPE_PREFIX}chainRetrievalQa`,
+    `${LANGCHAIN_TYPE_PREFIX}chainSummarization`,
+    `${LANGCHAIN_TYPE_PREFIX}informationExtractor`,
+    `${LANGCHAIN_TYPE_PREFIX}sentimentAnalysis`,
+    `${LANGCHAIN_TYPE_PREFIX}textClassifier`,
+    `${LANGCHAIN_TYPE_PREFIX}openAi`,
+    `${LANGCHAIN_TYPE_PREFIX}anthropic`,
+    `${LANGCHAIN_TYPE_PREFIX}googleGemini`,
+    `${LANGCHAIN_TYPE_PREFIX}ollama`,
+    `${LANGCHAIN_TYPE_PREFIX}alibabaCloud`,
+    `${LANGCHAIN_TYPE_PREFIX}miniMax`,
+    `${LANGCHAIN_TYPE_PREFIX}moonshot`,
+]);
+const SUB_COMPONENT_PREFIXES = [
+    'lm',
+    'embedding',
+    'memory',
+    'tool',
+    'outputparser',
+    'document',
+    'textsplitter',
+    'vectorstore',
+    'retriever',
+    'chattrigger',
+    'manualchattrigger',
+];
+function isRootAgentType(type) {
+    return ROOT_AI_TYPES.has(type);
+}
+function isAnyLangchainNode(type) {
+    return type.startsWith(LANGCHAIN_TYPE_PREFIX);
+}
+function isLikelySubComponent(type) {
+    if (!type.startsWith(LANGCHAIN_TYPE_PREFIX))
+        return false;
+    const localPart = type.slice(LANGCHAIN_TYPE_PREFIX.length).toLowerCase();
+    return SUB_COMPONENT_PREFIXES.some((prefix) => localPart.startsWith(prefix));
+}
+function detectAiNodes(workflow) {
+    const aiNodeNames = [];
+    const fallbackCandidates = [];
+    let alreadyConfigured = false;
+    for (const node of workflow.nodes ?? []) {
+        if (!node.name)
+            continue;
+        if (isRootAgentType(node.type)) {
+            aiNodeNames.push(node.name);
+        }
+        else if (isAnyLangchainNode(node.type) && !isLikelySubComponent(node.type)) {
+            fallbackCandidates.push(node.name);
+        }
+        if (EVALUATION_TYPES.has(node.type)) {
+            alreadyConfigured = true;
+        }
+    }
+    const finalAiNodeNames = aiNodeNames.length > 0 ? aiNodeNames : fallbackCandidates;
+    return {
+        isAiWorkflow: finalAiNodeNames.length > 0,
+        aiNodeNames: finalAiNodeNames,
+        alreadyConfigured,
+    };
+}
+//# sourceMappingURL=detect-ai-nodes.js.map

package/dist/tools/evals/detect-ai-nodes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-ai-nodes.js","sourceRoot":"","sources":["../../../src/tools/evals/detect-ai-nodes.ts"],"names":[],"mappings":";;AA4EA,sCA6BC;AAvGD,MAAM,qBAAqB,GAAG,2BAA2B,CAAC;AAC1D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS;IACxC,2BAA2B;IAC3B,kCAAkC;CAClC,CAAC,CAAC;AASH,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS;IAErC,GAAG,qBAAqB,OAAO;IAC/B,GAAG,qBAAqB,iBAAiB;IACzC,GAAG,qBAAqB,UAAU;IAClC,GAAG,qBAAqB,kBAAkB;IAC1C,GAAG,qBAAqB,oBAAoB;IAC5C,GAAG,qBAAqB,sBAAsB;IAC9C,GAAG,qBAAqB,mBAAmB;IAC3C,GAAG,qBAAqB,gBAAgB;IAGxC,GAAG,qBAAqB,QAAQ;IAChC,GAAG,qBAAqB,WAAW;IACnC,GAAG,qBAAqB,cAAc;IACtC,GAAG,qBAAqB,QAAQ;IAChC,GAAG,qBAAqB,cAAc;IACtC,GAAG,qBAAqB,SAAS;IACjC,GAAG,qBAAqB,UAAU;CAClC,CAAC,CAAC;AAKH,MAAM,sBAAsB,GAAG;IAC9B,IAAI;IACJ,WAAW;IACX,QAAQ;IACR,MAAM;IACN,cAAc;IACd,UAAU;IACV,cAAc;IACd,aAAa;IACb,WAAW;IACX,aAAa;IACb,mBAAmB;CACnB,CAAC;AAQF,SAAS,eAAe,CAAC,IAAY;IACpC,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;AAC/C,CAAC;AAKD,SAAS,oBAAoB,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,qBAAqB,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IACzE,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,SAAgB,aAAa,CAAC,QAAsB;IACnD,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,SAAS;QACzB,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAChC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAI9E,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrC,iBAAiB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACF,CAAC;IAID,MAAM,gBAAgB,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAEnF,OAAO;QACN,YAAY,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACzC,WAAW,EAAE,gBAAgB;QAC7B,iBAAiB;KACjB,CAAC;AACH,CAAC"}

package/dist/tools/evals/detect-tool-refs.service.d.ts

@@ -0,0 +1,7 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+export interface ToolRef {
+    sourceNodeName: string;
+    field: string;
+    toolNodeName: string;
+}
+export declare function detectToolRefs(workflow: WorkflowJSON, agentNodeName: string): ToolRef[];

package/dist/tools/evals/detect-tool-refs.service.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectToolRefs = detectToolRefs;
+const column_ref_utils_1 = require("./column-ref-utils");
+function reachableDownstreamMain(workflow, startNodeName) {
+    const reached = new Set();
+    const queue = [startNodeName];
+    while (queue.length > 0) {
+        const current = queue.shift();
+        const slot = workflow.connections?.[current]?.main;
+        if (!Array.isArray(slot))
+            continue;
+        for (const inner of slot) {
+            if (!Array.isArray(inner))
+                continue;
+            for (const conn of inner) {
+                if (!(0, column_ref_utils_1.isRecord)(conn) || typeof conn.node !== 'string')
+                    continue;
+                if (reached.has(conn.node))
+                    continue;
+                reached.add(conn.node);
+                queue.push(conn.node);
+            }
+        }
+    }
+    return reached;
+}
+function detectToolRefs(workflow, agentNodeName) {
+    const agent = (workflow.nodes ?? []).find((n) => (0, column_ref_utils_1.nodeHasName)(n) && n.name === agentNodeName);
+    if (!agent) {
+        throw new Error(`Agent node "${agentNodeName}" not found in workflow`);
+    }
+    const subComponents = (0, column_ref_utils_1.findAgentSubComponents)(workflow, agentNodeName);
+    if (subComponents.length === 0)
+        return [];
+    const nodesByName = new Map((workflow.nodes ?? []).filter(column_ref_utils_1.nodeHasName).map((n) => [n.name, n]));
+    const downstream = reachableDownstreamMain(workflow, agentNodeName);
+    const seen = new Set();
+    const result = [];
+    for (const toolNodeName of subComponents) {
+        const toolNode = nodesByName.get(toolNodeName);
+        if (!toolNode)
+            continue;
+        const matches = (0, column_ref_utils_1.collectStrings)(toolNode.parameters).flatMap((text) => (0, column_ref_utils_1.extractNamedRefMatches)(text));
+        for (const m of matches) {
+            if (m.nodeName === agentNodeName)
+                continue;
+            if (downstream.has(m.nodeName))
+                continue;
+            if (!nodesByName.has(m.nodeName))
+                continue;
+            const key = `${toolNodeName}\x00${m.nodeName}\x00${m.field}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            result.push({
+                sourceNodeName: m.nodeName,
+                field: m.field,
+                toolNodeName,
+            });
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=detect-tool-refs.service.js.map

package/dist/tools/evals/detect-tool-refs.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-tool-refs.service.js","sourceRoot":"","sources":["../../../src/tools/evals/detect-tool-refs.service.ts"],"names":[],"mappings":";;AAwDA,wCAqCC;AA3FD,yDAM4B;AAiB5B,SAAS,uBAAuB,CAAC,QAAsB,EAAE,aAAqB;IAC7E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAa,CAAC,aAAa,CAAC,CAAC;IACxC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,SAAS;YACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,CAAC,IAAA,2BAAQ,EAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;oBAAE,SAAS;gBAC/D,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBACrC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAaD,SAAgB,cAAc,CAAC,QAAsB,EAAE,aAAqB;IAC3E,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,8BAAW,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC;IAC7F,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,eAAe,aAAa,yBAAyB,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,aAAa,GAAG,IAAA,yCAAsB,EAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACtE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE1C,MAAM,WAAW,GAAG,IAAI,GAAG,CAC1B,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,8BAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAU,CAAC,CAC3E,CAAC;IACF,MAAM,UAAU,GAAG,uBAAuB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,OAAO,GAAG,IAAA,iCAAc,EAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACpE,IAAA,yCAAsB,EAAC,IAAI,CAAC,CAC5B,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,CAAC,QAAQ,KAAK,aAAa;gBAAE,SAAS;YAC3C,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAAE,SAAS;YACzC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAC3C,MAAM,GAAG,GAAG,GAAG,YAAY,OAAO,CAAC,CAAC,QAAQ,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,IAAI,CAAC;gBACX,cAAc,EAAE,CAAC,CAAC,QAAQ;gBAC1B,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,YAAY;aACZ,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC"}

package/dist/tools/evals/eval-data-requirements.service.d.ts

@@ -0,0 +1,19 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+export interface EvalDataTarget {
+    dataTableId: string;
+    evaluationTriggerName: string;
+    targetNodeName?: string;
+    inputColumns: string[];
+    expectedOutputColumns: string[];
+    actualOutputColumns: string[];
+    metricNodeNames: string[];
+    expectedToActualPairs: Array<{
+        expectedColumn: string;
+        actualField: string;
+    }>;
+}
+export interface EvalDataRequirements {
+    targets: EvalDataTarget[];
+    reason?: string;
+}
+export declare function analyzeEvalDataRequirements(workflow: WorkflowJSON): EvalDataRequirements;

package/dist/tools/evals/eval-data-requirements.service.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyzeEvalDataRequirements = analyzeEvalDataRequirements;
+const analyze_agent_input_columns_service_1 = require("./analyze-agent-input-columns.service");
+const column_ref_utils_1 = require("./column-ref-utils");
+const detect_ai_nodes_1 = require("./detect-ai-nodes");
+function readOperation(node) {
+    const parameters = node.parameters;
+    if (!(0, column_ref_utils_1.isRecord)(parameters))
+        return undefined;
+    const operation = parameters.operation;
+    return typeof operation === 'string' ? operation : undefined;
+}
+function readDataTableId(node) {
+    const parameters = node.parameters;
+    if (!(0, column_ref_utils_1.isRecord)(parameters))
+        return undefined;
+    const dataTableId = parameters.dataTableId;
+    if (typeof dataTableId === 'string')
+        return dataTableId;
+    if (!(0, column_ref_utils_1.isRecord)(dataTableId))
+        return undefined;
+    const value = dataTableId.value;
+    return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+function childNames(workflow, sourceName, outputIndex) {
+    const connectionsByType = workflow.connections?.[sourceName];
+    if (!connectionsByType || typeof connectionsByType !== 'object')
+        return [];
+    const main = connectionsByType.main;
+    if (!Array.isArray(main))
+        return [];
+    const slots = outputIndex === undefined
+        ? main
+        : [main[outputIndex]].filter((slot) => Array.isArray(slot));
+    return (0, column_ref_utils_1.unique)(slots.flatMap((slot) => {
+        if (!Array.isArray(slot))
+            return [];
+        return slot.flatMap((connection) => {
+            if (!(0, column_ref_utils_1.isRecord)(connection))
+                return [];
+            const node = connection.node;
+            return typeof node === 'string' ? [node] : [];
+        });
+    }));
+}
+function collectReachableNodeNames(workflow, startName) {
+    const seen = new Set();
+    const queue = [startName];
+    while (queue.length > 0) {
+        const current = queue.shift();
+        if (!current || seen.has(current))
+            continue;
+        seen.add(current);
+        for (const child of childNames(workflow, current)) {
+            if (!seen.has(child))
+                queue.push(child);
+        }
+    }
+    seen.delete(startName);
+    return [...seen];
+}
+function expectedColumnsFromMetricNodes(nodes) {
+    return (0, column_ref_utils_1.unique)(nodes.flatMap((node) => {
+        if (!(0, column_ref_utils_1.nodeTypeEndsWith)(node, 'evaluation') || readOperation(node) !== 'setMetrics')
+            return [];
+        const parameters = node.parameters;
+        if (!(0, column_ref_utils_1.isRecord)(parameters))
+            return [];
+        const expectedRaw = parameters.expectedAnswer;
+        return typeof expectedRaw === 'string' ? (0, column_ref_utils_1.extractJsonColumnRefs)(expectedRaw) : [];
+    }));
+}
+function actualColumnsFromSetOutputs(nodes) {
+    return (0, column_ref_utils_1.unique)(nodes.flatMap((node) => {
+        if (!(0, column_ref_utils_1.nodeTypeEndsWith)(node, 'evaluation') || readOperation(node) !== 'setOutputs')
+            return [];
+        const parameters = node.parameters;
+        if (!(0, column_ref_utils_1.isRecord)(parameters))
+            return [];
+        const outputs = parameters.outputs;
+        if (!(0, column_ref_utils_1.isRecord)(outputs))
+            return [];
+        const values = outputs.values;
+        if (!Array.isArray(values))
+            return [];
+        return values.flatMap((value) => {
+            if (!(0, column_ref_utils_1.isRecord)(value))
+                return [];
+            const outputName = value.outputName;
+            return typeof outputName === 'string' ? [outputName] : [];
+        });
+    }));
+}
+function pairsFromMetricNodes(nodes) {
+    const pairs = [];
+    for (const node of nodes) {
+        if (!(0, column_ref_utils_1.nodeTypeEndsWith)(node, 'evaluation') || readOperation(node) !== 'setMetrics')
+            continue;
+        const parameters = node.parameters;
+        if (!(0, column_ref_utils_1.isRecord)(parameters))
+            continue;
+        const expectedRaw = parameters.expectedAnswer;
+        const actualRaw = parameters.actualAnswer;
+        if (typeof expectedRaw !== 'string' || typeof actualRaw !== 'string')
+            continue;
+        const expectedRefs = (0, column_ref_utils_1.extractJsonColumnRefs)(expectedRaw).filter((ref) => ref.startsWith('expected'));
+        const actualRefs = (0, column_ref_utils_1.extractJsonColumnRefs)(actualRaw);
+        const len = Math.min(expectedRefs.length, actualRefs.length);
+        for (let i = 0; i < len; i++) {
+            pairs.push({ expectedColumn: expectedRefs[i], actualField: actualRefs[i] });
+        }
+    }
+    const map = new Map();
+    for (const p of pairs)
+        map.set(p.expectedColumn, p.actualField);
+    return [...map.entries()].map(([expectedColumn, actualField]) => ({
+        expectedColumn,
+        actualField,
+    }));
+}
+function firstReachableTargetNodeName(workflow, evalTriggerName) {
+    const reachableNodeNames = collectReachableNodeNames(workflow, evalTriggerName);
+    const reachableNameSet = new Set(reachableNodeNames);
+    const reachableNodes = (workflow.nodes ?? []).filter((node) => (0, column_ref_utils_1.nodeHasName)(node) && reachableNameSet.has(node.name));
+    const aiNodeNames = new Set((0, detect_ai_nodes_1.detectAiNodes)({ ...workflow, nodes: reachableNodes }).aiNodeNames);
+    return reachableNodeNames.find((name) => aiNodeNames.has(name));
+}
+function analyzeEvalDataRequirements(workflow) {
+    const evaluationTriggers = (workflow.nodes ?? []).filter((node) => (0, column_ref_utils_1.nodeHasName)(node) && (0, column_ref_utils_1.nodeTypeEndsWith)(node, 'evaluationTrigger'));
+    if (evaluationTriggers.length === 0) {
+        return { targets: [], reason: 'Workflow has no EvaluationTrigger nodes.' };
+    }
+    const targets = evaluationTriggers.flatMap((trigger) => {
+        const dataTableId = readDataTableId(trigger);
+        if (!dataTableId)
+            return [];
+        const reachableNames = new Set(collectReachableNodeNames(workflow, trigger.name));
+        const reachableNodes = (workflow.nodes ?? []).filter((node) => (0, column_ref_utils_1.nodeHasName)(node) && reachableNames.has(node.name));
+        const metricNodeNames = reachableNodes
+            .filter((node) => (0, column_ref_utils_1.nodeTypeEndsWith)(node, 'evaluation') && readOperation(node) === 'setMetrics')
+            .map((node) => node.name);
+        const targetNodeName = firstReachableTargetNodeName(workflow, trigger.name);
+        return [
+            {
+                dataTableId,
+                evaluationTriggerName: trigger.name,
+                targetNodeName,
+                inputColumns: targetNodeName
+                    ? (0, analyze_agent_input_columns_service_1.analyzeAgentInputColumns)(workflow, targetNodeName).inputColumns
+                    : [],
+                expectedOutputColumns: expectedColumnsFromMetricNodes(reachableNodes),
+                actualOutputColumns: actualColumnsFromSetOutputs(reachableNodes),
+                metricNodeNames,
+                expectedToActualPairs: pairsFromMetricNodes(reachableNodes),
+            },
+        ];
+    });
+    if (targets.length === 0) {
+        return {
+            targets: [],
+            reason: 'Workflow has EvaluationTrigger nodes, but none are wired to a DataTable id.',
+        };
+    }
+    return { targets };
+}
+//# sourceMappingURL=eval-data-requirements.service.js.map

package/dist/tools/evals/eval-data-requirements.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-data-requirements.service.js","sourceRoot":"","sources":["../../../src/tools/evals/eval-data-requirements.service.ts"],"names":[],"mappings":";;AAqKA,kEAkDC;AArND,+FAAiF;AACjF,yDAO4B;AAC5B,uDAAkD;AAwBlD,SAAS,aAAa,CAAC,IAAkB;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACvC,OAAO,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9D,CAAC;AAED,SAAS,eAAe,CAAC,IAAkB;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IAC3C,IAAI,OAAO,WAAW,KAAK,QAAQ;QAAE,OAAO,WAAW,CAAC;IACxD,IAAI,CAAC,IAAA,2BAAQ,EAAC,WAAW,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC;IAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,QAAsB,EAAE,UAAkB,EAAE,WAAoB;IACnF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,CAAC,iBAAiB,IAAI,OAAO,iBAAiB,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC3E,MAAM,IAAI,GAAI,iBAA6C,CAAC,IAAI,CAAC;IACjE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,KAAK,GACV,WAAW,KAAK,SAAS;QACxB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAqB,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,OAAO,IAAA,yBAAM,EACZ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;YAClC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;gBAAE,OAAO,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;YAC7B,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CACF,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,QAAsB,EAAE,SAAiB;IAC3E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,KAAK,GAAG,CAAC,SAAS,CAAC,CAAC;IAC1B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvB,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,8BAA8B,CAAC,KAAqB;IAC5D,OAAO,IAAA,yBAAM,EACZ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,IAAI,CAAC,IAAA,mCAAgB,EAAC,IAAI,EAAE,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,YAAY;YAAE,OAAO,EAAE,CAAC;QAC7F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC;QAC9C,OAAO,OAAO,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAA,wCAAqB,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClF,CAAC,CAAC,CACF,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,KAAqB;IACzD,OAAO,IAAA,yBAAM,EACZ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,IAAI,CAAC,IAAA,mCAAgB,EAAC,IAAI,EAAE,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,YAAY;YAAE,OAAO,EAAE,CAAC;QAC7F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,OAAO,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YAC/B,IAAI,CAAC,IAAA,2BAAQ,EAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAChC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;YACpC,OAAO,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CACF,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC5B,KAAqB;IAErB,MAAM,KAAK,GAA2D,EAAE,CAAC;IACzE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAA,mCAAgB,EAAC,IAAI,EAAE,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,YAAY;YAAE,SAAS;QAC5F,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC,IAAA,2BAAQ,EAAC,UAAU,CAAC;YAAE,SAAS;QACpC,MAAM,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC;QAC9C,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,CAAC;QAC1C,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS;QAC/E,MAAM,YAAY,GAAG,IAAA,wCAAqB,EAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACtE,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAC1B,CAAC;QACF,MAAM,UAAU,GAAG,IAAA,wCAAqB,EAAC,SAAS,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACF,CAAC;IAID,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;QACjE,cAAc;QACd,WAAW;KACX,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,4BAA4B,CACpC,QAAsB,EACtB,eAAuB;IAEvB,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAChF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CACnD,CAAC,IAAI,EAA2C,EAAE,CACjD,IAAA,8BAAW,EAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACrD,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAA,+BAAa,EAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;IAE/F,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,SAAgB,2BAA2B,CAAC,QAAsB;IACjE,MAAM,kBAAkB,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CACvD,CAAC,IAAI,EAA2C,EAAE,CACjD,IAAA,8BAAW,EAAC,IAAI,CAAC,IAAI,IAAA,mCAAgB,EAAC,IAAI,EAAE,mBAAmB,CAAC,CACjE,CAAC;IACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,0CAA0C,EAAE,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACtD,MAAM,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAE5B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,MAAM,cAAc,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,CACnD,CAAC,IAAI,EAA2C,EAAE,CACjD,IAAA,8BAAW,EAAC,IAAI,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACnD,CAAC;QACF,MAAM,eAAe,GAAG,cAAc;aACpC,MAAM,CACN,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,mCAAgB,EAAC,IAAI,EAAE,YAAY,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,YAAY,CACtF;aACA,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3B,MAAM,cAAc,GAAG,4BAA4B,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAE5E,OAAO;YACN;gBACC,WAAW;gBACX,qBAAqB,EAAE,OAAO,CAAC,IAAI;gBACnC,cAAc;gBACd,YAAY,EAAE,cAAc;oBAC3B,CAAC,CAAC,IAAA,8DAAwB,EAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,YAAY;oBACjE,CAAC,CAAC,EAAE;gBACL,qBAAqB,EAAE,8BAA8B,CAAC,cAAc,CAAC;gBACrE,mBAAmB,EAAE,2BAA2B,CAAC,cAAc,CAAC;gBAChE,eAAe;gBACf,qBAAqB,EAAE,oBAAoB,CAAC,cAAc,CAAC;aAC3D;SACD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO;YACN,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,6EAA6E;SACrF,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAC;AACpB,CAAC"}

package/dist/tools/evals/metric-catalog.d.ts

@@ -0,0 +1,57 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+export declare const METRIC_IDS: readonly ["correctness", "relevance", "tool_use", "helpfulness"];
+export type MetricId = (typeof METRIC_IDS)[number];
+export interface MetricCatalogEntry {
+    id: MetricId;
+    name: string;
+    kind: 'llm-judge';
+    cannedMetricKey: MetricId;
+    description: string;
+    prompt: string;
+    defaultEnabled: boolean;
+    requiresExpected: boolean;
+}
+export declare const METRIC_CATALOG: {
+    correctness: {
+        id: "correctness";
+        name: string;
+        kind: "llm-judge";
+        cannedMetricKey: "correctness";
+        description: string;
+        prompt: string;
+        defaultEnabled: true;
+        requiresExpected: true;
+    };
+    relevance: {
+        id: "relevance";
+        name: string;
+        kind: "llm-judge";
+        cannedMetricKey: "relevance";
+        description: string;
+        prompt: string;
+        defaultEnabled: false;
+        requiresExpected: false;
+    };
+    tool_use: {
+        id: "tool_use";
+        name: string;
+        kind: "llm-judge";
+        cannedMetricKey: "tool_use";
+        description: string;
+        prompt: string;
+        defaultEnabled: false;
+        requiresExpected: false;
+    };
+    helpfulness: {
+        id: "helpfulness";
+        name: string;
+        kind: "llm-judge";
+        cannedMetricKey: "helpfulness";
+        description: string;
+        prompt: string;
+        defaultEnabled: false;
+        requiresExpected: false;
+    };
+};
+export declare function getMetricsByIds(ids: string[]): MetricCatalogEntry[];
+export declare function proposeDefaultMetricIds(workflow: WorkflowJSON, agentNodeName: string): MetricId[];

package/dist/tools/evals/metric-catalog.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.METRIC_CATALOG = exports.METRIC_IDS = void 0;
+exports.getMetricsByIds = getMetricsByIds;
+exports.proposeDefaultMetricIds = proposeDefaultMetricIds;
+const column_ref_utils_1 = require("./column-ref-utils");
+exports.METRIC_IDS = ['correctness', 'relevance', 'tool_use', 'helpfulness'];
+exports.METRIC_CATALOG = {
+    correctness: {
+        id: 'correctness',
+        name: 'Correctness',
+        kind: 'llm-judge',
+        cannedMetricKey: 'correctness',
+        description: 'Judge whether the workflow output matches the expected ground-truth response.',
+        prompt: 'Given the input and expected output, rate from 0 to 1 how correct the actual output is.',
+        defaultEnabled: true,
+        requiresExpected: true,
+    },
+    relevance: {
+        id: 'relevance',
+        name: 'Relevance',
+        kind: 'llm-judge',
+        cannedMetricKey: 'relevance',
+        description: 'Judge whether retrieved context is relevant to the user query (RAG).',
+        prompt: 'Rate from 0 to 1 how relevant the retrieved documents are to the user query.',
+        defaultEnabled: false,
+        requiresExpected: false,
+    },
+    tool_use: {
+        id: 'tool_use',
+        name: 'Tool use',
+        kind: 'llm-judge',
+        cannedMetricKey: 'tool_use',
+        description: 'Judge whether the agent selected and used the correct tools for the input.',
+        prompt: 'Rate from 0 to 1 how appropriate the tool selection and arguments were for the user query.',
+        defaultEnabled: false,
+        requiresExpected: false,
+    },
+    helpfulness: {
+        id: 'helpfulness',
+        name: 'Helpfulness',
+        kind: 'llm-judge',
+        cannedMetricKey: 'helpfulness',
+        description: 'Judge whether the agent response is helpful to the user.',
+        prompt: 'Rate from 0 to 1 how helpful the response is to the user query.',
+        defaultEnabled: false,
+        requiresExpected: false,
+    },
+};
+function isMetricId(id) {
+    return exports.METRIC_IDS.includes(id);
+}
+function getMetricsByIds(ids) {
+    return ids.filter(isMetricId).map((id) => exports.METRIC_CATALOG[id]);
+}
+function agentHasTools(workflow, agentNodeName) {
+    const connections = workflow.connections ?? {};
+    for (const sourceConnections of Object.values(connections)) {
+        if (!(0, column_ref_utils_1.isRecord)(sourceConnections))
+            continue;
+        const aiTool = sourceConnections.ai_tool;
+        if (!Array.isArray(aiTool))
+            continue;
+        for (const slot of aiTool) {
+            if (!Array.isArray(slot))
+                continue;
+            for (const conn of slot) {
+                if ((0, column_ref_utils_1.isRecord)(conn) && conn.node === agentNodeName)
+                    return true;
+            }
+        }
+    }
+    return false;
+}
+function proposeDefaultMetricIds(workflow, agentNodeName) {
+    const ids = ['correctness'];
+    if (agentHasTools(workflow, agentNodeName))
+        ids.push('tool_use');
+    return ids;
+}
+//# sourceMappingURL=metric-catalog.js.map

package/dist/tools/evals/metric-catalog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-catalog.js","sourceRoot":"","sources":["../../../src/tools/evals/metric-catalog.ts"],"names":[],"mappings":";;;AAmEA,0CAEC;AAkBD,0DAIC;AAzFD,yDAA8C;AAEjC,QAAA,UAAU,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,CAAU,CAAC;AAc9E,QAAA,cAAc,GAAG;IAC7B,WAAW,EAAE;QACZ,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,WAAW;QACjB,eAAe,EAAE,aAAa;QAC9B,WAAW,EAAE,+EAA+E;QAC5F,MAAM,EACL,yFAAyF;QAC1F,cAAc,EAAE,IAAI;QACpB,gBAAgB,EAAE,IAAI;KACtB;IACD,SAAS,EAAE;QACV,EAAE,EAAE,WAAW;QACf,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,WAAW;QACjB,eAAe,EAAE,WAAW;QAC5B,WAAW,EAAE,sEAAsE;QACnF,MAAM,EAAE,8EAA8E;QACtF,cAAc,EAAE,KAAK;QACrB,gBAAgB,EAAE,KAAK;KACvB;IACD,QAAQ,EAAE;QACT,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,WAAW;QACjB,eAAe,EAAE,UAAU;QAC3B,WAAW,EAAE,4EAA4E;QACzF,MAAM,EACL,4FAA4F;QAC7F,cAAc,EAAE,KAAK;QACrB,gBAAgB,EAAE,KAAK;KACvB;IACD,WAAW,EAAE;QACZ,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,WAAW;QACjB,eAAe,EAAE,aAAa;QAC9B,WAAW,EAAE,0DAA0D;QACvE,MAAM,EAAE,iEAAiE;QACzE,cAAc,EAAE,KAAK;QACrB,gBAAgB,EAAE,KAAK;KACvB;CAC8C,CAAC;AAEjD,SAAS,UAAU,CAAC,EAAU;IAC7B,OAAQ,kBAAgC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,SAAgB,eAAe,CAAC,GAAa;IAC5C,OAAO,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,sBAAc,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,aAAa,CAAC,QAAsB,EAAE,aAAqB;IACnE,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;IAC/C,KAAK,MAAM,iBAAiB,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5D,IAAI,CAAC,IAAA,2BAAQ,EAAC,iBAAiB,CAAC;YAAE,SAAS;QAC3C,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,SAAS;YACnC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACzB,IAAI,IAAA,2BAAQ,EAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa;oBAAE,OAAO,IAAI,CAAC;YAChE,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAgB,uBAAuB,CAAC,QAAsB,EAAE,aAAqB;IACpF,MAAM,GAAG,GAAe,CAAC,aAAa,CAAC,CAAC;IACxC,IAAI,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACjE,OAAO,GAAG,CAAC;AACZ,CAAC"}

package/dist/tools/filesystem/create-tools-from-mcp-server.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createToolsFromLocalMcpServer = createToolsFromLocalMcpServer;
 const tools_1 = require("@mastra/core/tools");
 const api_types_1 = require("@n8n/api-types");
+const credential_1 = require("@n8n/mcp-browser/dist/tools/credential");
 const nanoid_1 = require("nanoid");
 const zod_1 = require("zod");
 const zod_from_json_schema_v3_1 = require("zod-from-json-schema-v3");
@@ -115,7 +116,12 @@ function createToolsFromLocalMcpServer(server, logger) {
         }
         let inputSchema;
         try {
-            inputSchema = (0, zod_from_json_schema_v3_1.convertJsonSchemaToZod)(mcpTool.inputSchema);
+            if (toolName === 'browser_create_credential') {
+                inputSchema = credential_1.browserCreateCredentialSchema;
+            }
+            else {
+                inputSchema = (0, zod_from_json_schema_v3_1.convertJsonSchemaToZod)(mcpTool.inputSchema);
+            }
         }
         catch {
             inputSchema = zod_1.z.record(zod_1.z.unknown());