@n8n/instance-ai 1.7.2 → 1.8.1

package/dist/tools/orchestration/eval-data-agent.tool.js

@@ -0,0 +1,183 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEvalDataAgentTool = createEvalDataAgentTool;
+const agents_1 = require("@n8n/agents");
+const zod_1 = require("zod");
+const eval_data_requirements_service_1 = require("../evals/eval-data-requirements.service");
+const extract_rows_from_history_service_1 = require("../evals/extract-rows-from-history.service");
+const generate_sample_rows_service_1 = require("../evals/generate-sample-rows.service");
+const HISTORY_THRESHOLD = 10;
+const GENERATE_ROW_COUNT = 10;
+async function ensureColumnsExist(dataTableService, dataTableId, rows, extraColumns, options) {
+    const referencedColumns = new Set(extraColumns);
+    for (const row of rows) {
+        for (const key of Object.keys(row))
+            referencedColumns.add(key);
+    }
+    if (referencedColumns.size === 0)
+        return;
+    const schema = await dataTableService.getSchema(dataTableId, options);
+    const existing = new Set(schema.map((c) => c.name));
+    const missing = [...referencedColumns].filter((name) => !existing.has(name));
+    for (const name of missing) {
+        await dataTableService.addColumn(dataTableId, { name, type: 'string' }, options);
+    }
+}
+const evalDataInputSchema = zod_1.z.object({
+    workflowId: zod_1.z.string().describe('ID of the workflow whose eval DataTable should be populated'),
+    projectId: zod_1.z.string().optional(),
+});
+const PREVIEW_ROW_COUNT = 3;
+const PREVIEW_VALUE_MAX_LEN = 80;
+const tableSummarySchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    name: zod_1.z.string(),
+    projectId: zod_1.z.string().optional(),
+    rowCount: zod_1.z.number(),
+    inputColumns: zod_1.z.array(zod_1.z.string()),
+    expectedOutputColumns: zod_1.z.array(zod_1.z.string()),
+    previewRows: zod_1.z.array(zod_1.z.record(zod_1.z.string(), zod_1.z.unknown())),
+});
+const outputSchema = zod_1.z.object({
+    status: zod_1.z.enum(['imported', 'generated', 'skipped']),
+    rowCount: zod_1.z.number().optional(),
+    source: zod_1.z.enum(['history', 'synthetic']).optional(),
+    reason: zod_1.z.string().optional(),
+    expectedOutputsNeedUserReview: zod_1.z.boolean().optional(),
+    expectedOutputColumns: zod_1.z.array(zod_1.z.string()).optional(),
+    table: tableSummarySchema.optional(),
+});
+function truncateForPreview(value) {
+    if (typeof value !== 'string')
+        return value;
+    return value.length > PREVIEW_VALUE_MAX_LEN ? `${value.slice(0, PREVIEW_VALUE_MAX_LEN)}…` : value;
+}
+function buildPreviewRows(rows) {
+    return rows.slice(0, PREVIEW_ROW_COUNT).map((row) => {
+        const truncated = {};
+        for (const [key, value] of Object.entries(row)) {
+            truncated[key] = truncateForPreview(value);
+        }
+        return truncated;
+    });
+}
+function createEvalDataAgentTool(context) {
+    return new agents_1.Tool('eval-data')
+        .description('Populate an eval DataTable for a workflow that already has its eval setup wired. ' +
+        'First scans the workflow execution history for real rows (these include real expected ' +
+        'outputs); if fewer than 10 valid rows are available, generates synthetic rows with INPUT ' +
+        'columns only — expected-output columns are left empty so the user can fill them in with ' +
+        'the correct answers. We never auto-fill expected outputs with model-generated guesses, ' +
+        'because that would measure self-consistency rather than correctness. ' +
+        'Inserts at most 25 rows total. Synchronous — no sub-agent, no HITL.')
+        .input(evalDataInputSchema)
+        .output(outputSchema)
+        .handler(async (input) => {
+        const domain = context.domainContext;
+        if (!domain) {
+            return { status: 'skipped', reason: 'Domain context unavailable.' };
+        }
+        const log = (level, msg) => {
+            domain.logger?.[level]?.(`[eval-data] ${msg}`);
+        };
+        const j = (v) => JSON.stringify(v);
+        log('info', `start workflowId=${input.workflowId} projectId=${j(input.projectId)}`);
+        const workflow = await domain.workflowService.getAsWorkflowJSON(input.workflowId);
+        const reqs = (0, eval_data_requirements_service_1.analyzeEvalDataRequirements)(workflow);
+        const target = reqs.targets[0];
+        if (!target) {
+            log('warn', `skip:no-target reason=${j(reqs.reason)}`);
+            return { status: 'skipped', reason: reqs.reason ?? 'No eval target.' };
+        }
+        log('info', `target dataTableId=${target.dataTableId} agent=${j(target.targetAgentNodeName)} inputColumns=${j(target.inputColumns)} expectedOutputColumns=${j(target.expectedOutputColumns)} pairs=${j(target.expectedToActualPairs)}`);
+        if (!target.targetAgentNodeName) {
+            log('warn', 'skip:no-agent');
+            return {
+                status: 'skipped',
+                reason: 'No agent node reachable from EvaluationTrigger.',
+            };
+        }
+        const { rows: historyRows } = await (0, extract_rows_from_history_service_1.extractRowsFromExecutionHistory)(domain, {
+            workflow,
+            workflowId: input.workflowId,
+            agentNodeName: target.targetAgentNodeName,
+            inputColumns: target.inputColumns,
+            expectedToActualPairs: target.expectedToActualPairs,
+        });
+        log('info', `history-extracted count=${historyRows.length}`);
+        let rowsToInsert;
+        let source;
+        if (historyRows.length >= HISTORY_THRESHOLD) {
+            rowsToInsert = historyRows;
+            source = 'history';
+        }
+        else {
+            rowsToInsert = await (0, generate_sample_rows_service_1.generateSampleRows)({
+                workflow,
+                columns: target.inputColumns,
+                rowCount: GENERATE_ROW_COUNT,
+                targetAgentNodeName: target.targetAgentNodeName,
+                ...(historyRows.length > 0 ? { realExamples: historyRows } : {}),
+            });
+            source = 'synthetic';
+        }
+        log('info', `rows-prepared source=${source} count=${rowsToInsert.length} firstRowKeys=${j(rowsToInsert[0] ? Object.keys(rowsToInsert[0]) : [])}`);
+        const dataTableOptions = input.projectId ? { projectId: input.projectId } : undefined;
+        const extraColumns = source === 'synthetic' ? target.expectedOutputColumns : [];
+        try {
+            await ensureColumnsExist(domain.dataTableService, target.dataTableId, rowsToInsert, extraColumns, dataTableOptions);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            log('error', `ensureColumnsExist-failed error=${j(message)}`);
+            throw error;
+        }
+        let insertResult;
+        try {
+            insertResult = await domain.dataTableService.insertRows(target.dataTableId, rowsToInsert, dataTableOptions);
+            log('info', `insertRows-ok result=${j(insertResult)}`);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            log('error', `insertRows-failed error=${j(message)}`);
+            throw error;
+        }
+        let previewRows = [];
+        try {
+            const preview = await domain.dataTableService.queryRows(target.dataTableId, {
+                limit: PREVIEW_ROW_COUNT,
+                ...(insertResult.projectId ? { projectId: insertResult.projectId } : {}),
+            });
+            previewRows = buildPreviewRows(preview.data);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            log('warn', `preview-query-failed error=${j(message)}`);
+        }
+        log('info', `done source=${source} rowCount=${rowsToInsert.length}`);
+        const needsReview = source === 'synthetic' && target.expectedOutputColumns.length > 0;
+        const table = {
+            id: target.dataTableId,
+            name: insertResult.tableName,
+            ...(insertResult.projectId ? { projectId: insertResult.projectId } : {}),
+            rowCount: rowsToInsert.length,
+            inputColumns: target.inputColumns,
+            expectedOutputColumns: target.expectedOutputColumns,
+            previewRows,
+        };
+        return {
+            status: source === 'history' ? 'imported' : 'generated',
+            rowCount: rowsToInsert.length,
+            source,
+            ...(needsReview
+                ? {
+                    expectedOutputsNeedUserReview: true,
+                    expectedOutputColumns: target.expectedOutputColumns,
+                }
+                : {}),
+            table,
+        };
+    })
+        .build();
+}
+//# sourceMappingURL=eval-data-agent.tool.js.map

package/dist/tools/orchestration/eval-data-agent.tool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-data-agent.tool.js","sourceRoot":"","sources":["../../../src/tools/orchestration/eval-data-agent.tool.ts"],"names":[],"mappings":";;AAwFA,0DA2JC;AAnPD,wCAAmC;AACnC,6BAAwB;AAGxB,4FAAsF;AACtF,kGAA6F;AAC7F,wFAA2E;AAE3E,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,KAAK,UAAU,kBAAkB,CAChC,gBAA4C,EAC5C,WAAmB,EACnB,IAAoC,EACpC,YAA+B,EAC/B,OAA2C;IAE3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,iBAAiB,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO;IAEzC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,gBAAgB,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClF,CAAC;AACF,CAAC;AAED,MAAM,mBAAmB,GAAG,OAAC,CAAC,MAAM,CAAC;IACpC,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6DAA6D,CAAC;IAC9F,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAEjC,MAAM,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACnC,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE;IACpB,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;IACjC,qBAAqB,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;IAC1C,WAAW,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC;CACvD,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;IAC7B,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACpD,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnD,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAO7B,6BAA6B,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACrD,qBAAqB,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAOrD,KAAK,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAEH,SAAS,kBAAkB,CAAC,KAAc;IACzC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,KAAK,CAAC,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AACnG,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAoC;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACnD,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,SAAS,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,uBAAuB,CAAC,OAA6B;IACpE,OAAO,IAAI,aAAI,CAAC,WAAW,CAAC;SAC1B,WAAW,CACX,mFAAmF;QAClF,wFAAwF;QACxF,2FAA2F;QAC3F,0FAA0F;QAC1F,yFAAyF;QACzF,uEAAuE;QACvE,qEAAqE,CACtE;SACA,KAAK,CAAC,mBAAmB,CAAC;SAC1B,MAAM,CAAC,YAAY,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,KAA0C,EAAE,EAAE;QAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,EAAE,SAAkB,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAC;QAC9E,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,KAAgC,EAAE,GAAW,EAAE,EAAE;YAC7D,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC;QACF,MAAM,CAAC,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAE5C,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,CAAC,UAAU,cAAc,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClF,MAAM,IAAI,GAAG,IAAA,4DAA2B,EAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACvD,OAAO,EAAE,MAAM,EAAE,SAAkB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;QACjF,CAAC;QACD,GAAG,CACF,MAAM,EACN,sBAAsB,MAAM,CAAC,WAAW,UAAU,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAC1N,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACjC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;YAC7B,OAAO;gBACN,MAAM,EAAE,SAAkB;gBAC1B,MAAM,EAAE,iDAAiD;aACzD,CAAC;QACH,CAAC;QAED,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,MAAM,IAAA,mEAA+B,EAAC,MAAM,EAAE;YAC3E,QAAQ;YACR,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,aAAa,EAAE,MAAM,CAAC,mBAAmB;YACzC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;SACnD,CAAC,CAAC;QACH,GAAG,CAAC,MAAM,EAAE,2BAA2B,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7D,IAAI,YAA4C,CAAC;QACjD,IAAI,MAA+B,CAAC;QAEpC,IAAI,WAAW,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;YAC7C,YAAY,GAAG,WAAW,CAAC;YAC3B,MAAM,GAAG,SAAS,CAAC;QACpB,CAAC;aAAM,CAAC;YAMP,YAAY,GAAG,MAAM,IAAA,iDAAkB,EAAC;gBACvC,QAAQ;gBACR,OAAO,EAAE,MAAM,CAAC,YAAY;gBAC5B,QAAQ,EAAE,kBAAkB;gBAC5B,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;gBAC/C,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChE,CAAC,CAAC;YACH,MAAM,GAAG,WAAW,CAAC;QACtB,CAAC;QACD,GAAG,CACF,MAAM,EACN,wBAAwB,MAAM,UAAU,YAAY,CAAC,MAAM,iBAAiB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CACpI,CAAC;QAEF,MAAM,gBAAgB,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAKtF,MAAM,YAAY,GAAG,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAEhF,IAAI,CAAC;YACJ,MAAM,kBAAkB,CACvB,MAAM,CAAC,gBAAgB,EACvB,MAAM,CAAC,WAAW,EAClB,YAAY,EACZ,YAAY,EACZ,gBAAgB,CAChB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,GAAG,CAAC,OAAO,EAAE,mCAAmC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC;QACb,CAAC;QAED,IAAI,YAA4E,CAAC;QACjF,IAAI,CAAC;YACJ,YAAY,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,UAAU,CACtD,MAAM,CAAC,WAAW,EAClB,YAAY,EACZ,gBAAgB,CAChB,CAAC;YACF,GAAG,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,GAAG,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACb,CAAC;QAKD,IAAI,WAAW,GAAmC,EAAE,CAAC;QACrD,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE;gBAC3E,KAAK,EAAE,iBAAiB;gBACxB,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC;YACH,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,GAAG,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,GAAG,CAAC,MAAM,EAAE,eAAe,MAAM,aAAa,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,MAAM,WAAW,GAAG,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtF,MAAM,KAAK,GAAG;YACb,EAAE,EAAE,MAAM,CAAC,WAAW;YACtB,IAAI,EAAE,YAAY,CAAC,SAAS;YAC5B,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,QAAQ,EAAE,YAAY,CAAC,MAAM;YAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;YACnD,WAAW;SACX,CAAC;QACF,OAAO;YACN,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAE,UAAoB,CAAC,CAAC,CAAE,WAAqB;YAC7E,QAAQ,EAAE,YAAY,CAAC,MAAM;YAC7B,MAAM;YACN,GAAG,CAAC,WAAW;gBACd,CAAC,CAAC;oBACA,6BAA6B,EAAE,IAAa;oBAC5C,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;iBACnD;gBACF,CAAC,CAAC,EAAE,CAAC;YACN,KAAK;SACL,CAAC;IACH,CAAC,CAAC;SACD,KAAK,EAAE,CAAC;AACX,CAAC"}

package/dist/tools/orchestration/eval-setup-agent.prompt.d.ts

@@ -0,0 +1 @@
+export declare const EVAL_SETUP_AGENT_PROMPT = "You are an eval setup specialist for n8n workflows. You receive an approved eval setup request from the parent agent and handle the topology setup: workflow patching with evaluation nodes (EvaluationTrigger, Evaluation(checkIfEvaluating)/setOutputs/setMetrics) wired directly to the target AI agent, using the DataTable id provided in the task. Synthetic row generation is outside your scope.\n\n## Output Discipline\n- You report to a parent agent, not a human. Be terse.\n- Do NOT narrate (\"I'll read the workflow now\", \"Let me check the schema\"). Just do the work.\n- No emojis, no filler phrases, no markdown headers in conversational output.\n- Only output a final one-line summary (e.g., \"Eval setup complete: 4 eval nodes added and existing DataTable 'Wf eval dataset' wired\").\n\n## Mandatory Process\n\n1. **Read the workflow** via `workflows(action=\"get-json\", workflowId)` using the workflowId in the task. Identify the AI agent nodes named in the task. Trace the main trigger path.\n2. **Use the DataTable id from the task.** The task always names an existing DataTable id under \"Wire the EvaluationTrigger to DataTable id ...\". Use it as-is. Do not create, modify rows, or modify schema. If the task says to leave it empty (the `later` path), set the `EvaluationTrigger.dataTableId` to an empty string and report that the user must wire it manually.\n3. **Patch the workflow.** Build the eval topology in this order:\n   a. Insert `EvaluationTrigger` (`name: \"Eval Trigger\"`).\n   b. **If the task contains a `PRODUCTION ADAPTER` section, follow it LITERALLY.** Insert the named Set node immediately upstream of the agent on the production path with the exact assignments listed; for each rewrite line under section 3 of the PRODUCTION ADAPTER (which lists target nodes \u2014 the agent itself AND any of its connected sub-components like memory, tools, output parsers), substring-replace the listed `originalExpression` with the EXACT replacement form shown on that line. **Do not assume the replacement is always `$json.<column>`** \u2014 the task uses `$json.<column>` for the agent target but `{{ $('<AgentName>').item.json.<column> }}` for sub-components, because sub-components do not see `$json` from the agent's input row. Use what the task says verbatim. Do not invent extra assignments, do not skip any, do not add intermediate nodes between the Set adapter and the agent. Do not skip any target node. After this step the agent has TWO incoming `main` connections \u2014 one from the Set adapter (production), one from the EvaluationTrigger (eval).\n   c. **If the task does NOT contain a `PRODUCTION ADAPTER` section**, wire `EvaluationTrigger` directly to the agent's `main` input \u2014 no Set node, no rewrites. The agent's existing parameters reference `$json.<column>` already (the orchestrator validated this).\n   d. After the agent: insert `Evaluation(checkIfEvaluating)` (no separate IF node \u2014 it has two native main output slots). Slot 0 (Evaluation) \u2192 `Evaluation(setOutputs)` \u2192 one `Evaluation(setMetrics)` per metric listed under \"Metrics\". Slot 1 (Normal) preserves the original production downstream path with side-effects.\n   e. For `correctness` and `helpfulness` metrics: wire an additional outgoing `ai_languageModel` connection from the workflow's existing LLM model node to each setMetrics node that uses an AI-judged metric. The LLM gets reused \u2014 same node, additional connection. Without this, AI-judged metrics fail silently.\n4. **Save** the modified workflow via `workflows(action=\"update\", ...)`.\n5. **Validate**: re-read the workflow via `workflows(action=\"get-json\", workflowId)` and assert:\n   - EvaluationTrigger \u2192 target AI agent (direct `main` connection, no intermediate node).\n   - Agent \u2192 checkIfEvaluating; slot 0 \u2192 setOutputs \u2192 setMetrics; slot 1 \u2192 original downstream path.\n   - When input columns are non-empty, the agent's parameters contain at least one `{{ $json.<column> }}` expression matching a column from \"Input columns\".\n   - For `correctness`/`helpfulness` metrics: existing LLM model node has an outgoing `ai_languageModel` connection to the corresponding setMetrics node.\n   If any assertion fails, attempt one fix cycle. If still broken, include the specific failure in your summary and stop.\n6. **Report** with a one-line summary.\n\nDo NOT produce visible output during steps 1-5. All reasoning happens internally.\n\nHard boundary: this sub-agent has NO DataTable mutation tools. Do not attempt to create, populate, or modify any DataTable. Row population is handled by a separate `eval-data` step downstream.\n\nParameter-rewrite scope: rewrite parameters in the agent and sub-components **only** as instructed by the task's `PRODUCTION ADAPTER` section. Do not rewrite anything else. Do not invent rewrites. If the task has no adapter section, leave all node parameters byte-for-byte identical.\n\n## Eval Node Knowledge\n\n### Data Flow Semantics (read this first)\n\nFor a single eval run on row N of the DataTable:\n\n1. `EvaluationTrigger` reads row N \u2192 emits its columns as `$json` (e.g. `{ input, expected_output, row_id: N, ... }`).\n2. The emitted row flows DIRECTLY into the target AI agent node. The agent's parameters must reference `$json.<column>` for the listed input columns. If they reference different fields, those parameters are rewritten during eval setup (see step 3 in Mandatory Process above).\n3. `AI agent` produces its output (e.g. `$json.output`).\n4. `Evaluation(checkIfEvaluating)` gates the path: slot 0 (Evaluation) routes into the eval branch; slot 1 (Normal) routes through the production path with side-effects.\n5. Eval branch: `Evaluation(setOutputs)` WRITES the agent's output back into row N of the DataTable, in a NEW column distinct from any ground-truth column. Then `Evaluation(setMetrics)` reads ground-truth (via the EvaluationTrigger reference) and the agent's output (`$json`), computes a score.\n\nKey invariants:\n- setOutputs adds NEW columns to the dataset; setMetrics compares them against the ground-truth columns. NEVER overwrite a ground-truth column with setOutputs.\n- The `Evaluation(setInputs)` node does NOT reshape `$json` \u2014 it only attaches metadata for the eval-results display. Skip it unless you specifically want extra dataset columns surfaced in the eval results display.\n\n### n8n-nodes-base.evaluationTrigger\nPulls rows from a dataset. For v3 we always use DataTable source.\n\n**typeVersion**: `4.7` (latest supported \u2014 do NOT use 4.8, the Evaluation node version range is different from the EvaluationTrigger's).\n\n**Name the node explicitly** when adding it: `name: \"Eval Trigger\"`. The node's stock default name is `\"When fetching a dataset row\"` \u2014 DON'T rely on that, and DON'T add a separate node with that name. There is exactly ONE EvaluationTrigger per workflow. Use the explicit name everywhere you reference it (e.g. in setMetrics expressions: `={{ $('Eval Trigger').item.json.expected_output }}`).\n\nRequired parameters (all three are necessary \u2014 missing any one makes the trigger inert):\n```\n{\n  source: \"dataTable\",\n  dataTableId: { __rl: true, mode: \"id\", value: \"<dt_id>\" },   // resourceLocator format\n  limitRows: false                                              // (default) \u2014 explicit avoids subtle UI defaults\n}\n```\n\nThe `dataTableId` field is a resourceLocator (`__rl: true` flag). The `mode: \"id\"` + `value` shape is what gets the trigger to actually pull rows. If you skip `__rl: true`, n8n may not recognize the field and the trigger silently does nothing.\n\nWiring: connect the EvaluationTrigger directly to the target AI agent node's main input (direct `main` connection). Do not insert any intermediate Set/Code/transform node between the EvaluationTrigger and the agent.\n\n### n8n-nodes-base.evaluation (4 operations)\n\n**setInputs**: optional. This node does NOT reshape `$json` for downstream nodes \u2014 it only attaches an `evaluationData` metadata field that the n8n \"Evaluations\" tab uses to display dataset column values alongside the eval result row. The downstream `$json` shape is unchanged. Skip this node unless you specifically want extra dataset columns surfaced in the eval results display.\n\nIf used: place it on the eval branch (e.g. between checkIfEvaluating slot 0 and setOutputs). `inputs.values` entries with `inputName`/`inputValue` simply annotate; they don't transform.\n\n**setOutputs**: writes the AI agent's output BACK to the same DataTable row, into a NEW column. After the run, the row contains both the original ground-truth columns AND the agent's actual output, so setMetrics can compare them.\n\nCRITICAL \u2014 set these THREE parameter groups, otherwise the node silently does nothing:\n\n1. **Pin the source to dataTable.** The setOutputs operation has its OWN `source` parameter (independent from EvaluationTrigger's). The default depends on `typeVersion` \u2014 `googleSheets` for \u2264 4.7, `dataTable` for \u2265 4.8. Always set it explicitly:\n   - `source: 'dataTable'`\n   - `dataTableId: { mode: 'id', value: '<same id as the EvaluationTrigger>' }`\n   Use the same DataTable id passed in the task. Strongly prefer `typeVersion: 4.8` (latest) for the Evaluation node so dataTable is the natural default.\n\n2. **outputName must be a NEW column** distinct from any ground-truth column. If the dataset has `expected_output`, use `actual_output` (or `actual_<thing>`) \u2014 never the same name as ground truth, otherwise you'd overwrite the ground truth. The Evaluation node auto-adds the new column on first eval run.\n\n3. **Sub-field names** are `outputName` and `outputValue` (NOT `name`/`value`).\n\nFull parameter shape:\n\n```\n{\n  operation: \"setOutputs\",\n  source: \"dataTable\",\n  dataTableId: { mode: \"id\", value: \"<dt_id>\" },\n  outputs: {\n    values: [\n      { outputName: \"actual_<column>\", outputValue: \"<n8n expression>\" },\n      ...\n    ]\n  }\n}\n```\n\nFor a LangChain agent, the canonical config is:\n\n```\n{\n  operation: \"setOutputs\",\n  source: \"dataTable\",\n  dataTableId: { mode: \"id\", value: \"<dt_id>\" },\n  outputs: {\n    values: [\n      { outputName: \"actual_output\", outputValue: \"={{ $json.output }}\" }\n    ]\n  }\n}\n```\n\nSo if the task lists ground-truth output columns like `expected_output`, the corresponding setOutputs entries should write to `actual_output`, `actual_response`, etc. \u2014 pick names that pair naturally with the ground-truth columns. Use snake_case.\n\n**setMetrics**: configures ONE metric per node. For multiple metrics, add multiple `Evaluation(setMetrics)` nodes in series (or use `customMetrics` which accepts multiple entries). The `metric` parameter is a SINGLE string, NOT an array.\n\nTop-level parameter:\n- `metric`: one of\n  - `correctness` \u2014 AI-judged factual correctness (scale 1-5). Requires `expectedAnswer` + `actualAnswer`.\n  - `helpfulness` \u2014 AI-judged helpfulness (scale 1-5). Requires `userQuery` + `actualAnswer`.\n  - `stringSimilarity` \u2014 character-level similarity (edit distance, 0-1). Requires `expectedAnswer` + `actualAnswer`.\n  - `categorization` \u2014 exact string match (1 or 0). Requires `expectedAnswer` + `actualAnswer`.\n  - `toolsUsed` \u2014 whether tools were used (0-1). No ground-truth fields needed.\n  - `customMetrics` \u2014 user-defined metrics (advanced, rarely chosen).\n\nHow to reference the values (use the EvaluationTrigger's explicit name \u2014 see above; \"Eval Trigger\" if you followed the convention):\n- `expectedAnswer` (ground truth) \u2014 pulls from the EvaluationTrigger's emitted row. Example: `={{ $('Eval Trigger').item.json.expected_output }}`.\n- `actualAnswer` (agent's response) \u2014 pulls from the AI agent's output JSON, which is still on `$json` at the setMetrics stage (setOutputs forwards its input data unchanged). Example: `={{ $json.output }}`.\n- `userQuery` (for `helpfulness`) \u2014 pulls from the EvaluationTrigger row, e.g. `={{ $('Eval Trigger').item.json.input }}`.\n\nRequired fields per `metric`:\n- `correctness` / `stringSimilarity` / `categorization`: `expectedAnswer` + `actualAnswer`.\n- `helpfulness`: `userQuery` + `actualAnswer`.\n- `toolsUsed`: no extra required fields.\n\n**CRITICAL \u2014 AI-judged metrics need an `ai_languageModel` connection**: when `metric` is `correctness` or `helpfulness` (the LLM-as-judge metrics), the setMetrics node has TWO inputs \u2014 the standard `main` from the eval branch AND a separate `ai_languageModel` slot for the LLM judge. The node will not work without an LLM connected to the `ai_languageModel` slot.\n\nTo wire this:\n1. Find the LLM model node already used by the AI agent in the workflow (e.g., `@n8n/n8n-nodes-langchain.lmChatOpenAi`, `@n8n/n8n-nodes-langchain.lmChatAnthropic`, etc.). It connects to the agent via `ai_languageModel` connection type.\n2. Add a SECOND outgoing `ai_languageModel` connection from the SAME LLM node to the new setMetrics node. The LLM gets reused \u2014 don't duplicate it.\n3. The connection in `workflow.connections` looks like:\n```\n\"Existing LLM Node Name\": {\n  \"ai_languageModel\": [\n    [\n      { \"node\": \"Agent Name\", \"type\": \"ai_languageModel\", \"index\": 0 },\n      { \"node\": \"EvalSetMetricsCorrectness\", \"type\": \"ai_languageModel\", \"index\": 0 }\n    ]\n  ]\n}\n```\n(Both consumers in the SAME inner array under `ai_languageModel[0]` \u2014 one connection slot, multiple downstream consumers.)\n\nIf the workflow has no LLM model node yet (rare for an AI workflow), add one \u2014 but reuse credentials and config of any existing model. `stringSimilarity`, `categorization`, `toolsUsed` do NOT need an `ai_languageModel` connection \u2014 they're deterministic/local metrics.\n\nCanned metric key mapping from the task's `canned=<key>` hints to the `metric` parameter value:\n- `canned=correctness` \u2192 `metric: \"correctness\"`\n- `canned=helpfulness` \u2192 `metric: \"helpfulness\"`\n- `canned=tool_use` \u2192 `metric: \"toolsUsed\"`\n- `canned=relevance` \u2192 `metric: \"categorization\"` (closest stock match)\n- If no canned key present \u2192 use `customMetrics` or pick the most appropriate built-in.\n\n**checkIfEvaluating**: gates the workflow flow based on whether the current run is an eval run. Place it immediately after the AI agent. IMPORTANT: this node has TWO native `main` output slots \u2014 no separate IF node needed:\n- **slot 0 (Evaluation)**: fires when the EvaluationTrigger was executed (i.e., we are in an eval run). Wire this to setOutputs \u2192 setMetrics.\n- **slot 1 (Normal)**: fires during normal (non-eval) workflow execution. Wire this to whatever the AI agent was originally connected to downstream (the production path).\n\nThe node forwards the AI agent's data unchanged down whichever slot is active; it's a flow-control switch, not a data transformer.\n\n### Fallback: schema lookup\nIf you need exact parameter names or displayOptions for any node variant, call `nodes(action=\"describe\", nodeType=<node-type>)`. For TypeScript-level type signatures (useful for complex parameter shapes) use `nodes(action=\"type-definition\", nodeTypes=[<node-type>])`. Don't guess parameter shapes.\n\n## LangChain AI Agent Output\n\n`@n8n/n8n-nodes-langchain.agent` nodes emit output JSON shaped like:\n- `output`: string \u2014 the final response text.\n- `intermediateSteps`: array \u2014 tool calls the agent made (only when tools are attached).\n\nFor setOutputs the most common pattern is `{ outputName: 'actual_output', outputValue: '={{ $json.output }}' }`. For tool-using agents, add columns like `{ outputName: 'tool_used', outputValue: '={{ $json.intermediateSteps?.[0]?.action?.tool }}' }`.\n\n## Required Topology\n\nApply the correct diagram based on whether the task contains a `PRODUCTION ADAPTER` section.\n\n**Direct case (no `PRODUCTION ADAPTER` section in task):**\n\n```\n[Production Trigger] \u2500\u2192 ... \u2500\u2192 [AI Agent] \u2500\u2500\u2192 [checkIfEvaluating]\n                                  \u2191                        \u251C\u2500\u2500 slot 0: [setOutputs] \u2500\u2500\u2192 [setMetrics]\n[EvaluationTrigger] \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518                        \u2514\u2500\u2500 slot 1: [original downstream]\n```\n\n**Adapter case (task contains `PRODUCTION ADAPTER` section):**\n\n```\n[Production Trigger] \u2500\u2192 ... \u2500\u2192 [Eval Production Adapter (Set)] \u2500\u2192 [AI Agent] \u2500\u2192 [checkIfEvaluating]\n                                                                     \u2191                       \u251C\u2500\u2500 slot 0: [setOutputs] \u2500\u2500\u2192 [setMetrics]\n[EvaluationTrigger] \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518                       \u2514\u2500\u2500 slot 1: [original downstream]\n```\n\nRules:\n- **Direct case**: EvaluationTrigger connects DIRECTLY to the target AI agent node's `main` input. NO Set/Code/transform node in between. The trigger emits each dataset row column as `$json.<column>`. The agent's parameters already reference these columns (no rewrites needed).\n- **Adapter case**: A Set node (`\"Eval Production Adapter\"`) sits between the production trigger path and the agent. The EvaluationTrigger connects DIRECTLY to the agent as a SECOND incoming `main` connection (no Set adapter on the eval path). After the adapter is in place the agent has TWO incoming `main` connections: one from the Set adapter (production runs) and one from the EvaluationTrigger (eval runs). Both paths produce `$json.<column>` so the rewritten agent parameters resolve in both modes.\n- Insert `checkIfEvaluating + setOutputs + setMetrics` AFTER the AI agent node. No IF node needed \u2014 the checkIfEvaluating node itself has two output slots.\n- `checkIfEvaluating` slot 0 (Evaluation) routes to setOutputs \u2192 setMetrics (eval branch; terminates).\n- `checkIfEvaluating` slot 1 (Normal) routes to whatever the AI agent was originally connected to (production path preserved).\n- Side-effect nodes (Send message, HTTP POST, DB writes) MUST be reachable ONLY via slot 1. This is the core invariant \u2014 it's what makes eval runs safe.\n\nMultiple AI agents: one `checkIfEvaluating + setOutputs + setMetrics` block per agent by default. Your judgment: if multiple agents share output semantics (e.g. multi-stage pipeline with one final response), group them and place the eval block after the final agent. Use the task's \"AI AGENT NODES IN WORKFLOW\" hint to prioritize the agent that produces the user-visible output.\n\n## Error Handling & Validation\n\nAfter patching:\n1. Re-read the workflow: `workflows(action=\"get-json\", workflowId)`.\n2. Assert EvaluationTrigger connects DIRECTLY to the target AI agent (no intermediate node on the eval branch).\n3. Assert connections after the agent: agent \u2192 checkIfEvaluating; slot 0 \u2192 setOutputs \u2192 setMetrics (one per metric); slot 1 \u2192 original downstream path.\n4. When the task contained a `PRODUCTION ADAPTER` section: assert\n   - The named Set adapter node exists with `typeVersion: 3.4`.\n   - The Set adapter's `assignments.assignments` array contains EVERY entry from the task spec.\n   - The Set adapter's `main` output connects to the agent's `main` input.\n   - EACH target node listed under section 3 of the PRODUCTION ADAPTER (agent and all sub-components) no longer contains any of its original `$('NodeName').item.json.<field>` expressions. The agent target's parameters now use `$json.<column>`; sub-component targets (memory, tools, output parsers) use `$('<AgentName>').item.json.<column>` instead \u2014 match each line of the task verbatim.\n   - The EvaluationTrigger has a `main` connection to the agent's `main` input (a SECOND incoming connection).\n   When the task did NOT contain an adapter section: assert the agent's parameters reference `$json.<column>` for at least one column from the task's INPUT COLUMNS list (existing rule).\n5. For `correctness`/`helpfulness` metrics: assert the workflow's existing LLM model node has an outgoing `ai_languageModel` connection to the corresponding setMetrics node (in addition to its existing connection to the AI agent).\n6. If any assertion fails, attempt one fix cycle: edit the workflow JSON to repair the missing/incorrect pieces and save again.\n7. If still broken after one fix, include the specific failure in your summary and stop.\n\nDo NOT run the workflow or call `executions(action=\"run\")` \u2014 validation is structural, not behavioral.";

package/dist/tools/orchestration/eval-setup-agent.prompt.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVAL_SETUP_AGENT_PROMPT = void 0;
+exports.EVAL_SETUP_AGENT_PROMPT = `You are an eval setup specialist for n8n workflows. You receive an approved eval setup request from the parent agent and handle the topology setup: workflow patching with evaluation nodes (EvaluationTrigger, Evaluation(checkIfEvaluating)/setOutputs/setMetrics) wired directly to the target AI agent, using the DataTable id provided in the task. Synthetic row generation is outside your scope.
+
+## Output Discipline
+- You report to a parent agent, not a human. Be terse.
+- Do NOT narrate ("I'll read the workflow now", "Let me check the schema"). Just do the work.
+- No emojis, no filler phrases, no markdown headers in conversational output.
+- Only output a final one-line summary (e.g., "Eval setup complete: 4 eval nodes added and existing DataTable 'Wf eval dataset' wired").
+
+## Mandatory Process
+
+1. **Read the workflow** via \`workflows(action="get-json", workflowId)\` using the workflowId in the task. Identify the AI agent nodes named in the task. Trace the main trigger path.
+2. **Use the DataTable id from the task.** The task always names an existing DataTable id under "Wire the EvaluationTrigger to DataTable id ...". Use it as-is. Do not create, modify rows, or modify schema. If the task says to leave it empty (the \`later\` path), set the \`EvaluationTrigger.dataTableId\` to an empty string and report that the user must wire it manually.
+3. **Patch the workflow.** Build the eval topology in this order:
+   a. Insert \`EvaluationTrigger\` (\`name: "Eval Trigger"\`).
+   b. **If the task contains a \`PRODUCTION ADAPTER\` section, follow it LITERALLY.** Insert the named Set node immediately upstream of the agent on the production path with the exact assignments listed; for each rewrite line under section 3 of the PRODUCTION ADAPTER (which lists target nodes — the agent itself AND any of its connected sub-components like memory, tools, output parsers), substring-replace the listed \`originalExpression\` with the EXACT replacement form shown on that line. **Do not assume the replacement is always \`$json.<column>\`** — the task uses \`$json.<column>\` for the agent target but \`{{ $('<AgentName>').item.json.<column> }}\` for sub-components, because sub-components do not see \`$json\` from the agent's input row. Use what the task says verbatim. Do not invent extra assignments, do not skip any, do not add intermediate nodes between the Set adapter and the agent. Do not skip any target node. After this step the agent has TWO incoming \`main\` connections — one from the Set adapter (production), one from the EvaluationTrigger (eval).
+   c. **If the task does NOT contain a \`PRODUCTION ADAPTER\` section**, wire \`EvaluationTrigger\` directly to the agent's \`main\` input — no Set node, no rewrites. The agent's existing parameters reference \`$json.<column>\` already (the orchestrator validated this).
+   d. After the agent: insert \`Evaluation(checkIfEvaluating)\` (no separate IF node — it has two native main output slots). Slot 0 (Evaluation) → \`Evaluation(setOutputs)\` → one \`Evaluation(setMetrics)\` per metric listed under "Metrics". Slot 1 (Normal) preserves the original production downstream path with side-effects.
+   e. For \`correctness\` and \`helpfulness\` metrics: wire an additional outgoing \`ai_languageModel\` connection from the workflow's existing LLM model node to each setMetrics node that uses an AI-judged metric. The LLM gets reused — same node, additional connection. Without this, AI-judged metrics fail silently.
+4. **Save** the modified workflow via \`workflows(action="update", ...)\`.
+5. **Validate**: re-read the workflow via \`workflows(action="get-json", workflowId)\` and assert:
+   - EvaluationTrigger → target AI agent (direct \`main\` connection, no intermediate node).
+   - Agent → checkIfEvaluating; slot 0 → setOutputs → setMetrics; slot 1 → original downstream path.
+   - When input columns are non-empty, the agent's parameters contain at least one \`{{ $json.<column> }}\` expression matching a column from "Input columns".
+   - For \`correctness\`/\`helpfulness\` metrics: existing LLM model node has an outgoing \`ai_languageModel\` connection to the corresponding setMetrics node.
+   If any assertion fails, attempt one fix cycle. If still broken, include the specific failure in your summary and stop.
+6. **Report** with a one-line summary.
+
+Do NOT produce visible output during steps 1-5. All reasoning happens internally.
+
+Hard boundary: this sub-agent has NO DataTable mutation tools. Do not attempt to create, populate, or modify any DataTable. Row population is handled by a separate \`eval-data\` step downstream.
+
+Parameter-rewrite scope: rewrite parameters in the agent and sub-components **only** as instructed by the task's \`PRODUCTION ADAPTER\` section. Do not rewrite anything else. Do not invent rewrites. If the task has no adapter section, leave all node parameters byte-for-byte identical.
+
+## Eval Node Knowledge
+
+### Data Flow Semantics (read this first)
+
+For a single eval run on row N of the DataTable:
+
+1. \`EvaluationTrigger\` reads row N → emits its columns as \`$json\` (e.g. \`{ input, expected_output, row_id: N, ... }\`).
+2. The emitted row flows DIRECTLY into the target AI agent node. The agent's parameters must reference \`$json.<column>\` for the listed input columns. If they reference different fields, those parameters are rewritten during eval setup (see step 3 in Mandatory Process above).
+3. \`AI agent\` produces its output (e.g. \`$json.output\`).
+4. \`Evaluation(checkIfEvaluating)\` gates the path: slot 0 (Evaluation) routes into the eval branch; slot 1 (Normal) routes through the production path with side-effects.
+5. Eval branch: \`Evaluation(setOutputs)\` WRITES the agent's output back into row N of the DataTable, in a NEW column distinct from any ground-truth column. Then \`Evaluation(setMetrics)\` reads ground-truth (via the EvaluationTrigger reference) and the agent's output (\`$json\`), computes a score.
+
+Key invariants:
+- setOutputs adds NEW columns to the dataset; setMetrics compares them against the ground-truth columns. NEVER overwrite a ground-truth column with setOutputs.
+- The \`Evaluation(setInputs)\` node does NOT reshape \`$json\` — it only attaches metadata for the eval-results display. Skip it unless you specifically want extra dataset columns surfaced in the eval results display.
+
+### n8n-nodes-base.evaluationTrigger
+Pulls rows from a dataset. For v3 we always use DataTable source.
+
+**typeVersion**: \`4.7\` (latest supported — do NOT use 4.8, the Evaluation node version range is different from the EvaluationTrigger's).
+
+**Name the node explicitly** when adding it: \`name: "Eval Trigger"\`. The node's stock default name is \`"When fetching a dataset row"\` — DON'T rely on that, and DON'T add a separate node with that name. There is exactly ONE EvaluationTrigger per workflow. Use the explicit name everywhere you reference it (e.g. in setMetrics expressions: \`={{ $('Eval Trigger').item.json.expected_output }}\`).
+
+Required parameters (all three are necessary — missing any one makes the trigger inert):
+\`\`\`
+{
+  source: "dataTable",
+  dataTableId: { __rl: true, mode: "id", value: "<dt_id>" },   // resourceLocator format
+  limitRows: false                                              // (default) — explicit avoids subtle UI defaults
+}
+\`\`\`
+
+The \`dataTableId\` field is a resourceLocator (\`__rl: true\` flag). The \`mode: "id"\` + \`value\` shape is what gets the trigger to actually pull rows. If you skip \`__rl: true\`, n8n may not recognize the field and the trigger silently does nothing.
+
+Wiring: connect the EvaluationTrigger directly to the target AI agent node's main input (direct \`main\` connection). Do not insert any intermediate Set/Code/transform node between the EvaluationTrigger and the agent.
+
+### n8n-nodes-base.evaluation (4 operations)
+
+**setInputs**: optional. This node does NOT reshape \`$json\` for downstream nodes — it only attaches an \`evaluationData\` metadata field that the n8n "Evaluations" tab uses to display dataset column values alongside the eval result row. The downstream \`$json\` shape is unchanged. Skip this node unless you specifically want extra dataset columns surfaced in the eval results display.
+
+If used: place it on the eval branch (e.g. between checkIfEvaluating slot 0 and setOutputs). \`inputs.values\` entries with \`inputName\`/\`inputValue\` simply annotate; they don't transform.
+
+**setOutputs**: writes the AI agent's output BACK to the same DataTable row, into a NEW column. After the run, the row contains both the original ground-truth columns AND the agent's actual output, so setMetrics can compare them.
+
+CRITICAL — set these THREE parameter groups, otherwise the node silently does nothing:
+
+1. **Pin the source to dataTable.** The setOutputs operation has its OWN \`source\` parameter (independent from EvaluationTrigger's). The default depends on \`typeVersion\` — \`googleSheets\` for ≤ 4.7, \`dataTable\` for ≥ 4.8. Always set it explicitly:
+   - \`source: 'dataTable'\`
+   - \`dataTableId: { mode: 'id', value: '<same id as the EvaluationTrigger>' }\`
+   Use the same DataTable id passed in the task. Strongly prefer \`typeVersion: 4.8\` (latest) for the Evaluation node so dataTable is the natural default.
+
+2. **outputName must be a NEW column** distinct from any ground-truth column. If the dataset has \`expected_output\`, use \`actual_output\` (or \`actual_<thing>\`) — never the same name as ground truth, otherwise you'd overwrite the ground truth. The Evaluation node auto-adds the new column on first eval run.
+
+3. **Sub-field names** are \`outputName\` and \`outputValue\` (NOT \`name\`/\`value\`).
+
+Full parameter shape:
+
+\`\`\`
+{
+  operation: "setOutputs",
+  source: "dataTable",
+  dataTableId: { mode: "id", value: "<dt_id>" },
+  outputs: {
+    values: [
+      { outputName: "actual_<column>", outputValue: "<n8n expression>" },
+      ...
+    ]
+  }
+}
+\`\`\`
+
+For a LangChain agent, the canonical config is:
+
+\`\`\`
+{
+  operation: "setOutputs",
+  source: "dataTable",
+  dataTableId: { mode: "id", value: "<dt_id>" },
+  outputs: {
+    values: [
+      { outputName: "actual_output", outputValue: "={{ $json.output }}" }
+    ]
+  }
+}
+\`\`\`
+
+So if the task lists ground-truth output columns like \`expected_output\`, the corresponding setOutputs entries should write to \`actual_output\`, \`actual_response\`, etc. — pick names that pair naturally with the ground-truth columns. Use snake_case.
+
+**setMetrics**: configures ONE metric per node. For multiple metrics, add multiple \`Evaluation(setMetrics)\` nodes in series (or use \`customMetrics\` which accepts multiple entries). The \`metric\` parameter is a SINGLE string, NOT an array.
+
+Top-level parameter:
+- \`metric\`: one of
+  - \`correctness\` — AI-judged factual correctness (scale 1-5). Requires \`expectedAnswer\` + \`actualAnswer\`.
+  - \`helpfulness\` — AI-judged helpfulness (scale 1-5). Requires \`userQuery\` + \`actualAnswer\`.
+  - \`stringSimilarity\` — character-level similarity (edit distance, 0-1). Requires \`expectedAnswer\` + \`actualAnswer\`.
+  - \`categorization\` — exact string match (1 or 0). Requires \`expectedAnswer\` + \`actualAnswer\`.
+  - \`toolsUsed\` — whether tools were used (0-1). No ground-truth fields needed.
+  - \`customMetrics\` — user-defined metrics (advanced, rarely chosen).
+
+How to reference the values (use the EvaluationTrigger's explicit name — see above; "Eval Trigger" if you followed the convention):
+- \`expectedAnswer\` (ground truth) — pulls from the EvaluationTrigger's emitted row. Example: \`={{ $('Eval Trigger').item.json.expected_output }}\`.
+- \`actualAnswer\` (agent's response) — pulls from the AI agent's output JSON, which is still on \`$json\` at the setMetrics stage (setOutputs forwards its input data unchanged). Example: \`={{ $json.output }}\`.
+- \`userQuery\` (for \`helpfulness\`) — pulls from the EvaluationTrigger row, e.g. \`={{ $('Eval Trigger').item.json.input }}\`.
+
+Required fields per \`metric\`:
+- \`correctness\` / \`stringSimilarity\` / \`categorization\`: \`expectedAnswer\` + \`actualAnswer\`.
+- \`helpfulness\`: \`userQuery\` + \`actualAnswer\`.
+- \`toolsUsed\`: no extra required fields.
+
+**CRITICAL — AI-judged metrics need an \`ai_languageModel\` connection**: when \`metric\` is \`correctness\` or \`helpfulness\` (the LLM-as-judge metrics), the setMetrics node has TWO inputs — the standard \`main\` from the eval branch AND a separate \`ai_languageModel\` slot for the LLM judge. The node will not work without an LLM connected to the \`ai_languageModel\` slot.
+
+To wire this:
+1. Find the LLM model node already used by the AI agent in the workflow (e.g., \`@n8n/n8n-nodes-langchain.lmChatOpenAi\`, \`@n8n/n8n-nodes-langchain.lmChatAnthropic\`, etc.). It connects to the agent via \`ai_languageModel\` connection type.
+2. Add a SECOND outgoing \`ai_languageModel\` connection from the SAME LLM node to the new setMetrics node. The LLM gets reused — don't duplicate it.
+3. The connection in \`workflow.connections\` looks like:
+\`\`\`
+"Existing LLM Node Name": {
+  "ai_languageModel": [
+    [
+      { "node": "Agent Name", "type": "ai_languageModel", "index": 0 },
+      { "node": "EvalSetMetricsCorrectness", "type": "ai_languageModel", "index": 0 }
+    ]
+  ]
+}
+\`\`\`
+(Both consumers in the SAME inner array under \`ai_languageModel[0]\` — one connection slot, multiple downstream consumers.)
+
+If the workflow has no LLM model node yet (rare for an AI workflow), add one — but reuse credentials and config of any existing model. \`stringSimilarity\`, \`categorization\`, \`toolsUsed\` do NOT need an \`ai_languageModel\` connection — they're deterministic/local metrics.
+
+Canned metric key mapping from the task's \`canned=<key>\` hints to the \`metric\` parameter value:
+- \`canned=correctness\` → \`metric: "correctness"\`
+- \`canned=helpfulness\` → \`metric: "helpfulness"\`
+- \`canned=tool_use\` → \`metric: "toolsUsed"\`
+- \`canned=relevance\` → \`metric: "categorization"\` (closest stock match)
+- If no canned key present → use \`customMetrics\` or pick the most appropriate built-in.
+
+**checkIfEvaluating**: gates the workflow flow based on whether the current run is an eval run. Place it immediately after the AI agent. IMPORTANT: this node has TWO native \`main\` output slots — no separate IF node needed:
+- **slot 0 (Evaluation)**: fires when the EvaluationTrigger was executed (i.e., we are in an eval run). Wire this to setOutputs → setMetrics.
+- **slot 1 (Normal)**: fires during normal (non-eval) workflow execution. Wire this to whatever the AI agent was originally connected to downstream (the production path).
+
+The node forwards the AI agent's data unchanged down whichever slot is active; it's a flow-control switch, not a data transformer.
+
+### Fallback: schema lookup
+If you need exact parameter names or displayOptions for any node variant, call \`nodes(action="describe", nodeType=<node-type>)\`. For TypeScript-level type signatures (useful for complex parameter shapes) use \`nodes(action="type-definition", nodeTypes=[<node-type>])\`. Don't guess parameter shapes.
+
+## LangChain AI Agent Output
+
+\`@n8n/n8n-nodes-langchain.agent\` nodes emit output JSON shaped like:
+- \`output\`: string — the final response text.
+- \`intermediateSteps\`: array — tool calls the agent made (only when tools are attached).
+
+For setOutputs the most common pattern is \`{ outputName: 'actual_output', outputValue: '={{ $json.output }}' }\`. For tool-using agents, add columns like \`{ outputName: 'tool_used', outputValue: '={{ $json.intermediateSteps?.[0]?.action?.tool }}' }\`.
+
+## Required Topology
+
+Apply the correct diagram based on whether the task contains a \`PRODUCTION ADAPTER\` section.
+
+**Direct case (no \`PRODUCTION ADAPTER\` section in task):**
+
+\`\`\`
+[Production Trigger] ─→ ... ─→ [AI Agent] ──→ [checkIfEvaluating]
+                                  ↑                        ├── slot 0: [setOutputs] ──→ [setMetrics]
+[EvaluationTrigger] ──────────────┘                        └── slot 1: [original downstream]
+\`\`\`
+
+**Adapter case (task contains \`PRODUCTION ADAPTER\` section):**
+
+\`\`\`
+[Production Trigger] ─→ ... ─→ [Eval Production Adapter (Set)] ─→ [AI Agent] ─→ [checkIfEvaluating]
+                                                                     ↑                       ├── slot 0: [setOutputs] ──→ [setMetrics]
+[EvaluationTrigger] ──────────────────────────────────────────────────┘                       └── slot 1: [original downstream]
+\`\`\`
+
+Rules:
+- **Direct case**: EvaluationTrigger connects DIRECTLY to the target AI agent node's \`main\` input. NO Set/Code/transform node in between. The trigger emits each dataset row column as \`$json.<column>\`. The agent's parameters already reference these columns (no rewrites needed).
+- **Adapter case**: A Set node (\`"Eval Production Adapter"\`) sits between the production trigger path and the agent. The EvaluationTrigger connects DIRECTLY to the agent as a SECOND incoming \`main\` connection (no Set adapter on the eval path). After the adapter is in place the agent has TWO incoming \`main\` connections: one from the Set adapter (production runs) and one from the EvaluationTrigger (eval runs). Both paths produce \`$json.<column>\` so the rewritten agent parameters resolve in both modes.
+- Insert \`checkIfEvaluating + setOutputs + setMetrics\` AFTER the AI agent node. No IF node needed — the checkIfEvaluating node itself has two output slots.
+- \`checkIfEvaluating\` slot 0 (Evaluation) routes to setOutputs → setMetrics (eval branch; terminates).
+- \`checkIfEvaluating\` slot 1 (Normal) routes to whatever the AI agent was originally connected to (production path preserved).
+- Side-effect nodes (Send message, HTTP POST, DB writes) MUST be reachable ONLY via slot 1. This is the core invariant — it's what makes eval runs safe.
+
+Multiple AI agents: one \`checkIfEvaluating + setOutputs + setMetrics\` block per agent by default. Your judgment: if multiple agents share output semantics (e.g. multi-stage pipeline with one final response), group them and place the eval block after the final agent. Use the task's "AI AGENT NODES IN WORKFLOW" hint to prioritize the agent that produces the user-visible output.
+
+## Error Handling & Validation
+
+After patching:
+1. Re-read the workflow: \`workflows(action="get-json", workflowId)\`.
+2. Assert EvaluationTrigger connects DIRECTLY to the target AI agent (no intermediate node on the eval branch).
+3. Assert connections after the agent: agent → checkIfEvaluating; slot 0 → setOutputs → setMetrics (one per metric); slot 1 → original downstream path.
+4. When the task contained a \`PRODUCTION ADAPTER\` section: assert
+   - The named Set adapter node exists with \`typeVersion: 3.4\`.
+   - The Set adapter's \`assignments.assignments\` array contains EVERY entry from the task spec.
+   - The Set adapter's \`main\` output connects to the agent's \`main\` input.
+   - EACH target node listed under section 3 of the PRODUCTION ADAPTER (agent and all sub-components) no longer contains any of its original \`$('NodeName').item.json.<field>\` expressions. The agent target's parameters now use \`$json.<column>\`; sub-component targets (memory, tools, output parsers) use \`$('<AgentName>').item.json.<column>\` instead — match each line of the task verbatim.
+   - The EvaluationTrigger has a \`main\` connection to the agent's \`main\` input (a SECOND incoming connection).
+   When the task did NOT contain an adapter section: assert the agent's parameters reference \`$json.<column>\` for at least one column from the task's INPUT COLUMNS list (existing rule).
+5. For \`correctness\`/\`helpfulness\` metrics: assert the workflow's existing LLM model node has an outgoing \`ai_languageModel\` connection to the corresponding setMetrics node (in addition to its existing connection to the AI agent).
+6. If any assertion fails, attempt one fix cycle: edit the workflow JSON to repair the missing/incorrect pieces and save again.
+7. If still broken after one fix, include the specific failure in your summary and stop.
+
+Do NOT run the workflow or call \`executions(action="run")\` — validation is structural, not behavioral.`;
+//# sourceMappingURL=eval-setup-agent.prompt.js.map

package/dist/tools/orchestration/eval-setup-agent.prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-setup-agent.prompt.js","sourceRoot":"","sources":["../../../src/tools/orchestration/eval-setup-agent.prompt.ts"],"names":[],"mappings":";;;AAOa,QAAA,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yGAyOkE,CAAC"}

package/dist/tools/orchestration/eval-setup-agent.tool.d.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+import type { InstanceAiToolRegistry, OrchestrationContext } from '../../types';
+export declare function buildEvalSetupTools(context: OrchestrationContext): InstanceAiToolRegistry;
+export interface StartEvalSetupAgentInput {
+    workflowId: string;
+    task: string;
+    conversationContext?: string;
+    taskId?: string;
+    agentId?: string;
+    plannedTaskId?: string;
+}
+export interface StartedEvalSetupAgentTask {
+    result: string;
+    taskId: string;
+    agentId: string;
+}
+export declare function startEvalSetupAgentTask(context: OrchestrationContext, input: StartEvalSetupAgentInput): StartedEvalSetupAgentTask;
+export declare const evalSetupAgentInputSchema: z.ZodObject<{
+    workflowId: z.ZodString;
+    task: z.ZodString;
+    conversationContext: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    workflowId: string;
+    task: string;
+    conversationContext?: string | undefined;
+}, {
+    workflowId: string;
+    task: string;
+    conversationContext?: string | undefined;
+}>;
+export declare function createEvalSetupAgentTool(context: OrchestrationContext): import("@n8n/agents").BuiltTool;