@n8n/instance-ai 1.7.0 → 1.8.0

package/dist/tools/evals/format-eval-setup-task.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatEvalSetupTask = formatEvalSetupTask;
+const column_ref_utils_1 = require("./column-ref-utils");
+const ensure_eval_data_table_service_1 = require("./ensure-eval-data-table.service");
+function taskString(value) {
+    return JSON.stringify(value) ?? '""';
+}
+function evalTriggerJsonRef(column) {
+    return `={{ $('Eval Trigger').item.json${(0, column_ref_utils_1.jsonFieldAccessor)(column)} }}`;
+}
+function formatProductionAdapter(namedRefs, agentNodeName, columnNameFor) {
+    if (namedRefs.length === 0)
+        return '';
+    const assignmentsByColumn = new Map();
+    for (const r of namedRefs) {
+        const column = columnNameFor(r.column);
+        if (!assignmentsByColumn.has(column)) {
+            assignmentsByColumn.set(column, {
+                column,
+                nodeName: r.nodeName,
+                field: r.field,
+            });
+        }
+    }
+    const assignments = [...assignmentsByColumn.values()]
+        .map((a) => `  - { name: ${taskString(a.column)}, value: ${taskString(`={{ ${(0, column_ref_utils_1.nodeItemJsonExpression)(a.nodeName, a.field)} }}`)}, type: "string" }`)
+        .join('\n');
+    const byTarget = new Map();
+    for (const r of namedRefs) {
+        const arr = byTarget.get(r.targetNodeName) ?? [];
+        arr.push(r);
+        byTarget.set(r.targetNodeName, arr);
+    }
+    const rewriteBlocks = [];
+    for (const [target, refs] of byTarget) {
+        const isAgent = target === agentNodeName;
+        const lines = refs
+            .map((r) => {
+            const column = columnNameFor(r.column);
+            const replacement = isAgent
+                ? `{{ ${(0, column_ref_utils_1.currentJsonExpression)(column)} }}`
+                : `{{ ${(0, column_ref_utils_1.nodeItemJsonExpression)(agentNodeName, column)} }}`;
+            return `    - Replace \`${r.originalExpression}\` with \`${replacement}\``;
+        })
+            .join('\n');
+        rewriteBlocks.push(`  In \`${target}\`:\n${lines}`);
+    }
+    const rewrites = rewriteBlocks.join('\n');
+    const sourceList = [...new Set(namedRefs.map((r) => r.nodeName))]
+        .map((n) => `\`${n}\``)
+        .join(', ');
+    const sourceLabel = assignmentsByColumn.size === 1 ? 'node' : 'nodes';
+    return `
+PRODUCTION ADAPTER (REQUIRED — the agent and/or its connected sub-components currently read input from named ${sourceLabel} ${sourceList}, which won't resolve in eval runs):
+
+1. Insert a new \`n8n-nodes-base.set\` node named \`"Eval Production Adapter"\` (\`typeVersion: 3.4\`) immediately upstream of the agent on the PRODUCTION path. The agent's existing \`main\` input parent on the production path becomes the Set adapter's \`main\` input parent. The Set adapter's \`main\` output goes to the agent.
+2. Configure the Set adapter's \`assignments.assignments\` array (one entry per unique dataset column):
+${assignments}
+3. Rewrite parameter expressions in each affected node — the agent and any sub-components (memory, tools, output parsers) that reference named source nodes. **The replacement form depends on the target:** the agent itself uses \`{{ $json.<col> }}\` (it receives \`$json\` directly), but sub-components must use the exact selected-agent expression listed below because their runtime context does not propagate \`$json\` from the agent's input row.
+${rewrites}
+4. The eval branch wires \`EvaluationTrigger\` directly to the agent's \`main\` input as a SECOND incoming connection (no Set adapter between them — the trigger row already has \`$json.<column>\` shape).
+
+After your edits the agent has TWO incoming \`main\` connections: one from the Eval Production Adapter (production runs) and one from the EvaluationTrigger (eval runs). Both produce \`$json.<column>\` for the agent. Sub-components reference the selected agent node by name, so they resolve to the agent's last input row in both modes.`;
+}
+function formatMetric(m, input, columnNameFor) {
+    const cannedSuffix = m.cannedMetricKey ? `, canned=${m.cannedMetricKey}` : '';
+    const promptSuffix = m.prompt ? `\n  Prompt: ${m.prompt}` : '';
+    const expectedOutputColumn = input.suggestedOutputColumns[0]
+        ? columnNameFor(input.suggestedOutputColumns[0])
+        : 'expected_output';
+    const inputColumn = input.suggestedInputColumns[0]
+        ? columnNameFor(input.suggestedInputColumns[0])
+        : 'input';
+    switch (m.cannedMetricKey) {
+        case 'correctness':
+            return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
+  Configure Evaluation(setMetrics) with \`metric: 'correctness'\`, \`expectedAnswer: ${evalTriggerJsonRef(expectedOutputColumn)}\`, \`actualAnswer: ={{ $json.output }}\`, and \`options.metricName: '${m.name}'\`. Connect an \`ai_languageModel\` input.${promptSuffix}`;
+        case 'helpfulness':
+            return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
+  Configure Evaluation(setMetrics) with \`metric: 'helpfulness'\`, \`userQuery: ${evalTriggerJsonRef(inputColumn)}\`, \`actualAnswer: ={{ $json.output }}\`, and \`options.metricName: '${m.name}'\`. Connect an \`ai_languageModel\` input.${promptSuffix}`;
+        case 'tool_use':
+            return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
+  Configure Evaluation(setMetrics) with \`metric: 'toolsUsed'\` (not \`'tool_use'\`), \`expectedTools\` from the dataset or the user's selected expected tool list, \`intermediateSteps: ={{ $json.intermediateSteps }}\`, and \`options.metricName: '${m.name}'\`. Enable returning intermediate steps on the agent.`;
+        case 'relevance':
+            return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
+  There is no native \`relevance\` metric option. Configure Evaluation(setMetrics) with \`metric: 'helpfulness'\`, \`userQuery: ${evalTriggerJsonRef(inputColumn)}\`, \`actualAnswer\` mapped to the retrieved context or response field being judged, \`options.metricName: '${m.name}'\`, and the relevance prompt below. Connect an \`ai_languageModel\` input.${promptSuffix}`;
+        default:
+            return `- ${m.name} (${m.kind}): ${m.description}${promptSuffix}`;
+    }
+}
+function formatDatasetSection(input, dataTableColumns) {
+    if (input.datasetChoice === 'link-existing') {
+        return `Wire the EvaluationTrigger to DataTable id \`${input.existingDataTableId}\`. This table already exists — do not modify its rows or schema.`;
+    }
+    if (input.datasetChoice === 'later') {
+        return "Do not create a DataTable. Leave the EvaluationTrigger's dataTableId empty — the user will wire it manually later.";
+    }
+    const tableName = (0, ensure_eval_data_table_service_1.formatEvalDataTableName)(input.workflowName);
+    const columns = dataTableColumns.map((c) => `- ${c}`).join('\n');
+    return `Create an empty DataTable named "${tableName}"${input.projectId ? ` in project id \`${input.projectId}\`` : ''} using only the \`create-empty-eval-data-table\` tool. Columns to create as strings:\n${columns}\n\nDo not insert rows, generate rows, or mutate row data. After creating the empty table, wire the EvaluationTrigger and setOutputs dataTableId to the created table id.`;
+}
+function formatSetOutputsDataTableInstruction(input) {
+    if (input.datasetChoice === 'link-existing') {
+        const id = input.existingDataTableId ?? '<existing DataTable id>';
+        return `The setOutputs node MUST have \`source: 'dataTable'\` and \`dataTableId: { mode: 'id', value: '${id}' }\` set explicitly (the node default for older typeVersions is googleSheets — that's a silent failure mode).`;
+    }
+    if (input.datasetChoice === 'later') {
+        return 'Leave setOutputs dataTableId empty until the user selects a DataTable. Do not write placeholder values into EvaluationTrigger or setOutputs.';
+    }
+    return "The setOutputs node MUST have `source: 'dataTable'` and `dataTableId` set explicitly using the id returned by `create-empty-eval-data-table` (the same id wired on the EvaluationTrigger). The node default for older typeVersions is googleSheets — that's a silent failure mode.";
+}
+function formatEvalSetupTask(input) {
+    const rawInputColumns = [
+        ...input.suggestedInputColumns,
+        ...(input.namedRefs ?? []).map((r) => r.column),
+    ];
+    const rawOutputColumns = input.suggestedOutputColumns;
+    const columnNameByRaw = (0, ensure_eval_data_table_service_1.formatEvalDataTableColumnNameMap)([
+        ...rawInputColumns,
+        ...rawOutputColumns,
+    ]);
+    const columnNameFor = (column) => columnNameByRaw.get(column) ?? (0, ensure_eval_data_table_service_1.formatEvalDataTableColumnName)(column);
+    const inputColumnNames = [...new Set(rawInputColumns.map(columnNameFor))];
+    const outputColumnNames = [...new Set(rawOutputColumns.map(columnNameFor))];
+    const dataTableColumns = [...new Set([...inputColumnNames, ...outputColumnNames])];
+    const outputColumns = outputColumnNames.map((c) => `- ${c}`).join('\n');
+    const inputColumns = inputColumnNames.map((c) => `- ${c}`).join('\n');
+    const metrics = input.enabledMetrics
+        .map((m) => formatMetric(m, input, columnNameFor))
+        .join('\n\n');
+    const datasetSection = formatDatasetSection(input, dataTableColumns);
+    const setOutputsDataTableInstruction = formatSetOutputsDataTableInstruction(input);
+    const agentNodeName = input.targetAgentNodeName ?? input.detectedAiNodes[0] ?? '';
+    const adapterSection = formatProductionAdapter(input.namedRefs ?? [], agentNodeName, columnNameFor);
+    return `Set up evaluations for workflow "${input.workflowName}" (id: ${input.workflowId}).
+
+AI AGENT NODES IN WORKFLOW: ${input.detectedAiNodes.join(', ')}
+
+DATASET:
+${datasetSection}
+
+INPUT COLUMNS (the AI agent's parameters MUST reference each of these via \`={{ $json.<column> }}\`. If the agent's existing parameters reference different fields, rewrite those parameter expressions to use these dataset columns. Only rewrite input-reading parameters — leave credentials, tools, model selection, and unrelated configuration untouched):
+${inputColumns}
+${adapterSection}
+
+GROUND-TRUTH OUTPUT COLUMNS (already in the dataset — these hold the EXPECTED values per row):
+${outputColumns}
+
+For setOutputs, write the agent's actual output to NEW columns derived from these ground-truth column names — convention: prefix with \`actual_\` (e.g. ground-truth \`expected_output\` → setOutputs writes to \`actual_output\`; ground-truth \`expected_response\` → setOutputs writes to \`actual_response\`). Never overwrite the ground-truth column itself. The Evaluation node auto-adds the new column on first eval run.
+
+${setOutputsDataTableInstruction} Use \`typeVersion: 4.8\` for the Evaluation node.
+
+METRICS TO CONFIGURE (on setMetrics):
+${metrics}
+
+Use the field mappings specified per metric above. Use the explicit name you assign to the EvaluationTrigger node (name it \`"Eval Trigger"\` when creating it, then reference exactly that name everywhere — never use the stock default \`"When fetching a dataset row"\` and never create a separate node for that purpose).
+
+For \`correctness\`, \`helpfulness\`, and \`relevance\` (configured with the native \`helpfulness\` metric): also wire an \`ai_languageModel\` connection from the workflow's existing LLM model node (the one already feeding the AI agent) to each setMetrics node that uses an AI-judged metric. The LLM is reused — same node, additional outgoing \`ai_languageModel\` connection. Without this, AI-judged metrics fail silently. \`stringSimilarity\`/\`categorization\`/\`toolsUsed\` don't need this.
+
+Apply the topology as described in your instructions:
+1. EvaluationTrigger → target AI agent node (direct \`main\` connection). No intermediate Set/Code/transform node. The trigger emits each dataset column as \`$json.<column>\`. If the agent's existing parameters reference fields that don't match the listed INPUT COLUMNS, rewrite those parameter expressions to use \`{{ $json.<column> }}\`. NOTE: \`Evaluation(setInputs)\` does NOT reshape data — it only attaches metadata for the eval-results display tab.
+2. After the AI agent: insert \`Evaluation(checkIfEvaluating)\` (no separate IF node — it has two native output slots). Slot 0 (Evaluation) routes to setOutputs → setMetrics. Slot 1 (Normal) preserves the original production path with side-effects.
+
+Report back with a one-line summary when done.`;
+}
+//# sourceMappingURL=format-eval-setup-task.js.map

package/dist/tools/evals/format-eval-setup-task.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format-eval-setup-task.js","sourceRoot":"","sources":["../../../src/tools/evals/format-eval-setup-task.ts"],"names":[],"mappings":";;AA4KA,kDA2DC;AAvOD,yDAI4B;AAE5B,qFAI0C;AAiB1C,SAAS,UAAU,CAAC,KAAa;IAChC,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;AACtC,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc;IACzC,OAAO,kCAAkC,IAAA,oCAAiB,EAAC,MAAM,CAAC,KAAK,CAAC;AACzE,CAAC;AAED,SAAS,uBAAuB,CAC/B,SAAqB,EACrB,aAAqB,EACrB,aAAyC;IAEzC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAGtC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAGhC,CAAC;IACJ,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,mBAAmB,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC/B,MAAM;gBACN,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;aACd,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,MAAM,WAAW,GAAG,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,CAAC;SACnD,GAAG,CACH,CAAC,CAAC,EAAE,EAAE,CACL,eAAe,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,UAAU,CAAC,OAAO,IAAA,yCAAsB,EAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,oBAAoB,CACvI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IAcb,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACjD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,KAAK,aAAa,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO;gBAC1B,CAAC,CAAC,MAAM,IAAA,wCAAqB,EAAC,MAAM,CAAC,KAAK;gBAC1C,CAAC,CAAC,MAAM,IAAA,yCAAsB,EAAC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC;YAC5D,OAAO,mBAAmB,CAAC,CAAC,kBAAkB,aAAa,WAAW,IAAI,CAAC;QAC5E,CAAC,CAAC;aACD,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,aAAa,CAAC,IAAI,CAAC,UAAU,MAAM,QAAQ,KAAK,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1C,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;SAC/D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SACtB,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IAEtE,OAAO;+GACuG,WAAW,IAAI,UAAU;;;;EAItI,WAAW;;EAEX,QAAQ;;;+UAGqU,CAAC;AAChV,CAAC;AAED,SAAS,YAAY,CACpB,CAAiB,EACjB,KAA+B,EAC/B,aAAyC;IAEzC,MAAM,YAAY,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9E,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,MAAM,oBAAoB,GAAG,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAC3D,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,iBAAiB,CAAC;IACrB,MAAM,WAAW,GAAG,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,OAAO,CAAC;IAEX,QAAQ,CAAC,CAAC,eAAe,EAAE,CAAC;QAC3B,KAAK,aAAa;YACjB,OAAO,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,YAAY,MAAM,CAAC,CAAC,WAAW;uFACqB,kBAAkB,CAAC,oBAAoB,CAAC,yEAAyE,CAAC,CAAC,IAAI,8CAA8C,YAAY,EAAE,CAAC;QACzQ,KAAK,aAAa;YACjB,OAAO,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,YAAY,MAAM,CAAC,CAAC,WAAW;kFACgB,kBAAkB,CAAC,WAAW,CAAC,yEAAyE,CAAC,CAAC,IAAI,8CAA8C,YAAY,EAAE,CAAC;QAC3P,KAAK,UAAU;YACd,OAAO,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,YAAY,MAAM,CAAC,CAAC,WAAW;wPACsL,CAAC,CAAC,IAAI,wDAAwD,CAAC;QACrT,KAAK,WAAW;YACf,OAAO,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,GAAG,YAAY,MAAM,CAAC,CAAC,WAAW;kIACgE,kBAAkB,CAAC,WAAW,CAAC,+GAA+G,CAAC,CAAC,IAAI,8EAA8E,YAAY,EAAE,CAAC;QACjX;YACC,OAAO,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,WAAW,GAAG,YAAY,EAAE,CAAC;IACpE,CAAC;AACF,CAAC;AAED,SAAS,oBAAoB,CAAC,KAA+B,EAAE,gBAA0B;IACxF,IAAI,KAAK,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;QAC7C,OAAO,gDAAgD,KAAK,CAAC,mBAAmB,mEAAmE,CAAC;IACrJ,CAAC;IACD,IAAI,KAAK,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QACrC,OAAO,oHAAoH,CAAC;IAC7H,CAAC;IAED,MAAM,SAAS,GAAG,IAAA,wDAAuB,EAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,OAAO,oCAAoC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,oBAAoB,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,yFAAyF,OAAO,2KAA2K,CAAC;AACnY,CAAC;AAED,SAAS,oCAAoC,CAAC,KAA+B;IAC5E,IAAI,KAAK,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;QAC7C,MAAM,EAAE,GAAG,KAAK,CAAC,mBAAmB,IAAI,yBAAyB,CAAC;QAClE,OAAO,kGAAkG,EAAE,gHAAgH,CAAC;IAC7N,CAAC;IACD,IAAI,KAAK,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QACrC,OAAO,8IAA8I,CAAC;IACvJ,CAAC;IACD,OAAO,oRAAoR,CAAC;AAC7R,CAAC;AAED,SAAgB,mBAAmB,CAAC,KAA+B;IAClE,MAAM,eAAe,GAAG;QACvB,GAAG,KAAK,CAAC,qBAAqB;QAC9B,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;KAC/C,CAAC;IACF,MAAM,gBAAgB,GAAG,KAAK,CAAC,sBAAsB,CAAC;IACtD,MAAM,eAAe,GAAG,IAAA,iEAAgC,EAAC;QACxD,GAAG,eAAe;QAClB,GAAG,gBAAgB;KACnB,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,CAAC,MAAc,EAAE,EAAE,CACxC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAA,8DAA6B,EAAC,MAAM,CAAC,CAAC;IACtE,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,gBAAgB,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;SACjD,IAAI,CAAC,MAAM,CAAC,CAAC;IACf,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IACrE,MAAM,8BAA8B,GAAG,oCAAoC,CAAC,KAAK,CAAC,CAAC;IACnF,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,cAAc,GAAG,uBAAuB,CAC7C,KAAK,CAAC,SAAS,IAAI,EAAE,EACrB,aAAa,EACb,aAAa,CACb,CAAC;IAEF,OAAO,oCAAoC,KAAK,CAAC,YAAY,UAAU,KAAK,CAAC,UAAU;;8BAE1D,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;;;EAG5D,cAAc;;;EAGd,YAAY;EACZ,cAAc;;;EAGd,aAAa;;;;EAIb,8BAA8B;;;EAG9B,OAAO;;;;;;;;;;+CAUsC,CAAC;AAChD,CAAC"}

package/dist/tools/evals/generate-sample-rows.service.d.ts

@@ -0,0 +1,35 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+import type { Logger } from '../../logger';
+export interface SampleRowFacet {
+    length: string;
+    edgeMode: string;
+    instructions: string;
+}
+export interface AgentContext {
+    workflowName: string;
+    agentNodeName: string;
+    systemPrompt: string | undefined;
+    promptTemplate: string | undefined;
+    connectedToolNames: string[];
+}
+export declare const SAMPLE_ROW_FACETS: readonly SampleRowFacet[];
+export declare function distributeRowCount(rowCount: number): number[];
+export declare function extractAgentContext(workflow: WorkflowJSON, agentNodeName: string): AgentContext | undefined;
+export interface RunBatchInput {
+    facet: SampleRowFacet;
+    rowCount: number;
+    context: AgentContext | undefined;
+    columns: string[];
+    realExamples?: ReadonlyArray<Record<string, unknown>>;
+    logger?: Pick<Logger, 'warn'>;
+}
+export declare function runBatch(input: RunBatchInput): Promise<Array<Record<string, string>>>;
+export interface GenerateSampleRowsInput {
+    workflow: WorkflowJSON;
+    columns: string[];
+    rowCount?: number;
+    targetAgentNodeName?: string;
+    realExamples?: ReadonlyArray<Record<string, unknown>>;
+    logger?: Pick<Logger, 'warn'>;
+}
+export declare function generateSampleRows(input: GenerateSampleRowsInput): Promise<Array<Record<string, string>>>;

package/dist/tools/evals/generate-sample-rows.service.js

@@ -0,0 +1,278 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SAMPLE_ROW_FACETS = void 0;
+exports.distributeRowCount = distributeRowCount;
+exports.extractAgentContext = extractAgentContext;
+exports.runBatch = runBatch;
+exports.generateSampleRows = generateSampleRows;
+const zod_1 = require("zod");
+const column_ref_utils_1 = require("./column-ref-utils");
+const detect_ai_nodes_1 = require("./detect-ai-nodes");
+const eval_agents_1 = require("../../utils/eval-agents");
+const FACET_COUNT = 5;
+const DEFAULT_ROW_COUNT = 25;
+const SYSTEM_PROMPT_MAX_CHARS = 2000;
+const REAL_EXAMPLES_MAX_COUNT = 10;
+const REAL_EXAMPLE_VALUE_MAX_CHARS = 300;
+exports.SAMPLE_ROW_FACETS = [
+    {
+        length: 'short (single sentence or one-line snippet)',
+        edgeMode: 'happy — typical, well-formed inputs the agent commonly receives',
+        instructions: 'Produce concise, realistic inputs the AI Agent would handle on a regular day. Each input should fit on one line. No edge cases.',
+    },
+    {
+        length: 'long (multi-paragraph or several lines)',
+        edgeMode: 'happy — typical, well-formed inputs',
+        instructions: 'Produce longer realistic inputs spanning multiple paragraphs or several lines. The content stays in-domain and well-structured.',
+    },
+    {
+        length: 'medium (2–3 sentences or comparable size)',
+        edgeMode: 'edge — ambiguity, truncations, typos, imperfect formatting',
+        instructions: 'Produce inputs that are messy but plausible: typos, ambiguous references, truncated phrasing, irregular casing or spacing. Still in-domain.',
+    },
+    {
+        length: 'long (multi-paragraph)',
+        edgeMode: 'adversarial — out-of-scope requests, contradictions, prompt injection',
+        instructions: 'Produce inputs that try to push the agent off task: out-of-scope demands, internally contradictory requirements, attempts at prompt injection ("ignore previous instructions"), or hostile framing.',
+    },
+    {
+        length: 'mixed extremes (empty, very short, or very long)',
+        edgeMode: 'robustness — empty inputs, multi-language fragments, malformed data',
+        instructions: 'Produce robustness probes: an empty input, an input of one or two characters, an input that mixes two languages, and an input with malformed structure (e.g., truncated JSON, broken markup). Use whatever the agent would plausibly stumble on in production.',
+    },
+];
+function distributeRowCount(rowCount) {
+    const safe = Math.max(0, Math.floor(rowCount));
+    const base = Math.floor(safe / FACET_COUNT);
+    const extra = safe % FACET_COUNT;
+    return Array.from({ length: FACET_COUNT }, (_, i) => base + (i < extra ? 1 : 0));
+}
+function readSystemPrompt(parameters) {
+    if (!parameters)
+        return undefined;
+    const direct = parameters.systemMessage;
+    if (typeof direct === 'string' && direct.length > 0) {
+        return direct.slice(0, SYSTEM_PROMPT_MAX_CHARS);
+    }
+    const options = parameters.options;
+    if ((0, column_ref_utils_1.isRecord)(options)) {
+        const nested = options.systemMessage;
+        if (typeof nested === 'string' && nested.length > 0) {
+            return nested.slice(0, SYSTEM_PROMPT_MAX_CHARS);
+        }
+    }
+    return undefined;
+}
+function readPromptTemplate(parameters) {
+    if (!parameters)
+        return undefined;
+    const text = parameters.text;
+    return typeof text === 'string' && text.length > 0 ? text : undefined;
+}
+function findConnectedTools(workflow, agentName) {
+    const tools = [];
+    const connections = workflow.connections ?? {};
+    for (const [sourceName, byType] of Object.entries(connections)) {
+        if (!(0, column_ref_utils_1.isRecord)(byType))
+            continue;
+        const aiTool = byType?.ai_tool;
+        if (!Array.isArray(aiTool))
+            continue;
+        const matches = aiTool.some((slot) => {
+            if (!Array.isArray(slot))
+                return false;
+            return slot.some((link) => (0, column_ref_utils_1.isRecord)(link) && link.node === agentName);
+        });
+        if (matches)
+            tools.push(sourceName);
+    }
+    return tools;
+}
+function extractAgentContext(workflow, agentNodeName) {
+    const node = (workflow.nodes ?? []).find((n) => n.name === agentNodeName);
+    if (!node)
+        return undefined;
+    const parameters = (0, column_ref_utils_1.isRecord)(node.parameters) ? node.parameters : undefined;
+    return {
+        workflowName: workflow.name ?? 'Untitled',
+        agentNodeName,
+        systemPrompt: readSystemPrompt(parameters),
+        promptTemplate: readPromptTemplate(parameters),
+        connectedToolNames: findConnectedTools(workflow, agentNodeName),
+    };
+}
+const batchRowSchema = zod_1.z.array(zod_1.z.record(zod_1.z.union([zod_1.z.string(), zod_1.z.number(), zod_1.z.boolean(), zod_1.z.null()])));
+function isExpectedOutputColumn(column) {
+    return column.startsWith('expected');
+}
+function buildAgentContextBlock(context) {
+    if (!context)
+        return 'Workflow context unavailable.';
+    const lines = [
+        `Workflow name: ${context.workflowName}`,
+        `AI Agent node: ${context.agentNodeName}`,
+    ];
+    if (context.systemPrompt) {
+        lines.push('Agent system prompt:', context.systemPrompt);
+    }
+    if (context.promptTemplate) {
+        lines.push(`Agent prompt template: ${context.promptTemplate}`);
+    }
+    if (context.connectedToolNames.length > 0) {
+        lines.push(`Connected tools: ${context.connectedToolNames.join(', ')}`);
+    }
+    return lines.join('\n');
+}
+const FORMAT_INFERENCE = "Inspect the agent's system prompt, prompt template, and connected tools to infer what kind of text this agent receives at runtime. It may be a user chat message, output from another tool, scraped web content, structured records (JSON/key-value), document chunks, log lines, code, etc. Generate inputs that look like what would arrive at the agent in production. Do not assume a human user when the agent suggests otherwise.";
+function truncateExampleValue(value) {
+    return value.length > REAL_EXAMPLE_VALUE_MAX_CHARS
+        ? `${value.slice(0, REAL_EXAMPLE_VALUE_MAX_CHARS)}…`
+        : value;
+}
+function buildRealExamplesBlock(examples, columns) {
+    if (!examples || examples.length === 0 || columns.length === 0)
+        return '';
+    const filtered = [];
+    for (const example of examples.slice(0, REAL_EXAMPLES_MAX_COUNT)) {
+        const row = {};
+        let hasValue = false;
+        for (const col of columns) {
+            const raw = example[col];
+            if (raw === undefined || raw === null)
+                continue;
+            const str = typeof raw === 'string' ? raw : JSON.stringify(raw);
+            if (str.length === 0)
+                continue;
+            row[col] = truncateExampleValue(str);
+            hasValue = true;
+        }
+        if (hasValue)
+            filtered.push(row);
+    }
+    if (filtered.length === 0)
+        return '';
+    const numbered = filtered.map((row, i) => `${i + 1}. ${JSON.stringify(row)}`).join('\n');
+    return [
+        '',
+        'Recent real inputs the agent has received in production (REFERENCE, not seeds):',
+        numbered,
+        'Use these as a hint about the actual domain, tone and shape of inputs the agent sees. Do NOT copy or paraphrase them — produce NEW inputs that fit the same setting.',
+    ].join('\n');
+}
+const BATCH_SYSTEM_INSTRUCTIONS = `You generate realistic test inputs for an n8n workflow evaluation dataset.
+
+Output: JSON array of objects. Keys = exactly the provided column names. Values = short strings. No prose outside the JSON.
+
+Generate only INPUT data — the inputs the agent would receive in production. Do not invent expected outputs, answers, or labels. Those columns are filled in by the user, who decides what the correct answer is. Generating both inputs and expected outputs with the same model would measure self-consistency, not correctness.
+
+Return only the JSON array.`;
+function normalizeBatchRow(rawRow, columns) {
+    const row = {};
+    for (const col of columns) {
+        if (isExpectedOutputColumn(col)) {
+            row[col] = '';
+            continue;
+        }
+        const v = rawRow[col];
+        if (v === undefined || v === null)
+            row[col] = '';
+        else if (typeof v === 'string')
+            row[col] = v;
+        else
+            row[col] = String(v);
+    }
+    return row;
+}
+function stripMarkdownFences(text) {
+    const trimmed = text.trim();
+    const fencedMatch = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
+    return fencedMatch ? fencedMatch[1].trim() : trimmed;
+}
+async function runBatch(input) {
+    const requestedRowCount = Math.max(0, Math.floor(input.rowCount));
+    if (requestedRowCount <= 0)
+        return [];
+    try {
+        const generatedColumns = input.columns.filter((column) => !isExpectedOutputColumn(column));
+        const agent = (0, eval_agents_1.createEvalAgent)('eval-sample-rows', {
+            model: eval_agents_1.HAIKU_MODEL,
+            instructions: BATCH_SYSTEM_INSTRUCTIONS,
+        });
+        const realExamplesBlock = buildRealExamplesBlock(input.realExamples, generatedColumns);
+        const sections = [buildAgentContextBlock(input.context)];
+        if (realExamplesBlock)
+            sections.push(realExamplesBlock);
+        sections.push(FORMAT_INFERENCE);
+        sections.push([
+            `Variation focus for this batch: length = ${input.facet.length}; mode = ${input.facet.edgeMode}.`,
+            input.facet.instructions,
+        ].join('\n'));
+        sections.push([
+            `Columns: ${generatedColumns.join(', ')}`,
+            `Generate exactly ${requestedRowCount} rows.`,
+        ].join('\n'));
+        const userText = sections.join('\n\n');
+        const result = await agent.generate(userText);
+        const text = (0, eval_agents_1.extractText)(result);
+        const parsed = JSON.parse(stripMarkdownFences(text));
+        const validated = batchRowSchema.safeParse(parsed);
+        if (!validated.success) {
+            input.logger?.warn('generate-sample-rows: invalid batch rows returned', {
+                rowCount: requestedRowCount,
+                facet: input.facet.edgeMode,
+                issues: validated.error.issues,
+            });
+            return [];
+        }
+        return validated.data
+            .slice(0, requestedRowCount)
+            .map((rawRow) => normalizeBatchRow(rawRow, input.columns));
+    }
+    catch (error) {
+        input.logger?.warn('generate-sample-rows: batch generation failed', {
+            rowCount: requestedRowCount,
+            facet: input.facet.edgeMode,
+            error,
+        });
+        return [];
+    }
+}
+function resolveAgentContext(workflow, targetAgentNodeName) {
+    if (targetAgentNodeName) {
+        return extractAgentContext(workflow, targetAgentNodeName);
+    }
+    const detected = (0, detect_ai_nodes_1.detectAiNodes)(workflow);
+    const firstAgent = detected.aiNodeNames[0];
+    if (!firstAgent)
+        return undefined;
+    return extractAgentContext(workflow, firstAgent);
+}
+function emptyRow(columns) {
+    return Object.fromEntries(columns.map((c) => [c, '']));
+}
+async function generateSampleRows(input) {
+    const rowCount = input.rowCount ?? DEFAULT_ROW_COUNT;
+    const counts = distributeRowCount(rowCount);
+    const context = resolveAgentContext(input.workflow, input.targetAgentNodeName);
+    const settled = await Promise.allSettled(exports.SAMPLE_ROW_FACETS.map(async (facet, i) => {
+        if (counts[i] <= 0)
+            return [];
+        return await runBatch({
+            facet,
+            rowCount: counts[i],
+            context,
+            columns: input.columns,
+            realExamples: input.realExamples,
+            logger: input.logger,
+        });
+    }));
+    const merged = [];
+    for (const r of settled) {
+        if (r.status === 'fulfilled')
+            merged.push(...r.value);
+    }
+    if (merged.length === 0)
+        return [emptyRow(input.columns)];
+    return merged;
+}
+//# sourceMappingURL=generate-sample-rows.service.js.map

package/dist/tools/evals/generate-sample-rows.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-sample-rows.service.js","sourceRoot":"","sources":["../../../src/tools/evals/generate-sample-rows.service.ts"],"names":[],"mappings":";;;AA6DA,gDAKC;AAwCD,kDAcC;AAoHD,4BAiDC;AAoCD,gDA2BC;AA3VD,6BAAwB;AAExB,yDAA8C;AAC9C,uDAAkD;AAElD,yDAAoF;AAEpF,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAgB5B,QAAA,iBAAiB,GAA8B;IAC3D;QACC,MAAM,EAAE,6CAA6C;QACrD,QAAQ,EAAE,iEAAiE;QAC3E,YAAY,EACX,iIAAiI;KAClI;IACD;QACC,MAAM,EAAE,yCAAyC;QACjD,QAAQ,EAAE,qCAAqC;QAC/C,YAAY,EACX,iIAAiI;KAClI;IACD;QACC,MAAM,EAAE,2CAA2C;QACnD,QAAQ,EAAE,4DAA4D;QACtE,YAAY,EACX,6IAA6I;KAC9I;IACD;QACC,MAAM,EAAE,wBAAwB;QAChC,QAAQ,EAAE,uEAAuE;QACjF,YAAY,EACX,qMAAqM;KACtM;IACD;QACC,MAAM,EAAE,kDAAkD;QAC1D,QAAQ,EAAE,qEAAqE;QAC/E,YAAY,EACX,gQAAgQ;KACjQ;CACD,CAAC;AAEF,SAAgB,kBAAkB,CAAC,QAAgB;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,WAAW,CAAC;IACjC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,gBAAgB,CAAC,UAA+C;IACxE,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,MAAM,MAAM,GAAG,UAAU,CAAC,aAAa,CAAC;IACxC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IACnC,IAAI,IAAA,2BAAQ,EAAC,OAAO,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;QACrC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACjD,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,UAA+C;IAC1E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;IAC7B,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AACvE,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAsB,EAAE,SAAiB;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,IAAA,2BAAQ,EAAC,MAAM,CAAC;YAAE,SAAS;QAChC,MAAM,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC;QAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC;YACvC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,2BAAQ,EAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QACH,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAgB,mBAAmB,CAClC,QAAsB,EACtB,aAAqB;IAErB,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC;IAC1E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,UAAU,GAAG,IAAA,2BAAQ,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,OAAO;QACN,YAAY,EAAE,QAAQ,CAAC,IAAI,IAAI,UAAU;QACzC,aAAa;QACb,YAAY,EAAE,gBAAgB,CAAC,UAAU,CAAC;QAC1C,cAAc,EAAE,kBAAkB,CAAC,UAAU,CAAC;QAC9C,kBAAkB,EAAE,kBAAkB,CAAC,QAAQ,EAAE,aAAa,CAAC;KAC/D,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,KAAK,CAAC,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,EAAE,OAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAEnG,SAAS,sBAAsB,CAAC,MAAc;IAC7C,OAAO,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAiC;IAChE,IAAI,CAAC,OAAO;QAAE,OAAO,+BAA+B,CAAC;IACrD,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,CAAC,YAAY,EAAE;QACxC,kBAAkB,OAAO,CAAC,aAAa,EAAE;KACzC,CAAC;IACF,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,OAAO,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,gBAAgB,GACrB,yaAAya,CAAC;AAE3a,SAAS,oBAAoB,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,MAAM,GAAG,4BAA4B;QACjD,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,4BAA4B,CAAC,GAAG;QACpD,CAAC,CAAC,KAAK,CAAC;AACV,CAAC;AAaD,SAAS,sBAAsB,CAC9B,QAA4D,EAC5D,OAAiB;IAEjB,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC1E,MAAM,QAAQ,GAAkC,EAAE,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,EAAE,CAAC;QAClE,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;gBAAE,SAAS;YAChD,MAAM,GAAG,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;YACrC,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC;QACD,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzF,OAAO;QACN,EAAE;QACF,iFAAiF;QACjF,QAAQ;QACR,sKAAsK;KACtK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG;;;;;;4BAMN,CAAC;AAW7B,SAAS,iBAAiB,CACzB,MAAwD,EACxD,OAAiB;IAEjB,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,sBAAsB,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACd,SAAS;QACV,CAAC;QACD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;aAC5C,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;;YACxC,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,WAAW,GAAG,oCAAoC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvE,OAAO,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACtD,CAAC;AAEM,KAAK,UAAU,QAAQ,CAAC,KAAoB;IAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClE,IAAI,iBAAiB,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAG,IAAA,6BAAe,EAAC,kBAAkB,EAAE;YACjD,KAAK,EAAE,yBAAW;YAClB,YAAY,EAAE,yBAAyB;SACvC,CAAC,CAAC;QACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,KAAK,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAG,CAAC,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QACzD,IAAI,iBAAiB;YAAE,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CACZ;YACC,4CAA4C,KAAK,CAAC,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG;YACjG,KAAK,CAAC,KAAK,CAAC,YAAY;SACxB,CAAC,IAAI,CAAC,IAAI,CAAC,CACZ,CAAC;QACF,QAAQ,CAAC,IAAI,CACZ;YACC,YAAY,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACzC,oBAAoB,iBAAiB,QAAQ;SAC7C,CAAC,IAAI,CAAC,IAAI,CAAC,CACZ,CAAC;QACF,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAA,yBAAW,EAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YACxB,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,mDAAmD,EAAE;gBACvE,QAAQ,EAAE,iBAAiB;gBAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ;gBAC3B,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;aAC9B,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACX,CAAC;QACD,OAAO,SAAS,CAAC,IAAI;aACnB,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC;aAC3B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,+CAA+C,EAAE;YACnE,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ;YAC3B,KAAK;SACL,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC;AAmBD,SAAS,mBAAmB,CAC3B,QAAsB,EACtB,mBAAuC;IAEvC,IAAI,mBAAmB,EAAE,CAAC;QACzB,OAAO,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,+BAAa,EAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,OAAO,mBAAmB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,QAAQ,CAAC,OAAiB;IAClC,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACvC,KAA8B;IAE9B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACrD,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE/E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACvC,yBAAiB,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;QACxC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9B,OAAO,MAAM,QAAQ,CAAC;YACrB,KAAK;YACL,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YACnB,OAAO;YACP,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAC;IACJ,CAAC,CAAC,CACF,CAAC;IAEF,MAAM,MAAM,GAAkC,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,OAAO,MAAM,CAAC;AACf,CAAC"}

package/dist/tools/evals/generate-tool-ref-pin-data.service.d.ts

@@ -0,0 +1,11 @@
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+import type { ToolRef } from './detect-tool-refs.service';
+export type ToolRefPinData = Record<string, Array<{
+    json: Record<string, unknown>;
+}>>;
+export interface GenerateToolRefPinDataInput {
+    workflow: WorkflowJSON;
+    agentNodeName: string;
+    refs: ToolRef[];
+}
+export declare function generateToolRefPinData(input: GenerateToolRefPinDataInput): Promise<ToolRefPinData>;

package/dist/tools/evals/generate-tool-ref-pin-data.service.js

@@ -0,0 +1,125 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateToolRefPinData = generateToolRefPinData;
+const zod_1 = require("zod");
+const column_ref_utils_1 = require("./column-ref-utils");
+const eval_agents_1 = require("../../utils/eval-agents");
+const PinDataResponseSchema = zod_1.z.record(zod_1.z.string(), zod_1.z.array(zod_1.z.object({ json: zod_1.z.record(zod_1.z.unknown()) })).min(1));
+const SYSTEM_INSTRUCTIONS = `You generate realistic mock output for n8n trigger and source nodes referenced inside an AI agent's tools/memory.
+
+Output: a single JSON object whose keys are node names and whose values are arrays of n8n pinData items in the form { "json": { ... } }. One item per node is enough.
+
+For each node, return data that matches what the node would naturally emit at runtime — based on its type and parameters. Make sure every requested field appears in your output, with a plausible value.
+
+Return only the JSON object. No prose, no markdown fences.`;
+function readAgentPurpose(agent) {
+    if (!agent)
+        return undefined;
+    const params = agent.parameters;
+    if (!params)
+        return undefined;
+    const direct = params.systemMessage;
+    if (typeof direct === 'string' && direct.length > 0)
+        return direct.slice(0, 1000);
+    const options = params.options;
+    if (options && typeof options === 'object') {
+        const nested = options.systemMessage;
+        if (typeof nested === 'string' && nested.length > 0)
+            return nested.slice(0, 1000);
+    }
+    const text = params.text;
+    if (typeof text === 'string' && text.length > 0)
+        return text.slice(0, 1000);
+    return undefined;
+}
+function buildNodeContexts(workflow, refs) {
+    const fieldsBySource = new Map();
+    for (const r of refs) {
+        const set = fieldsBySource.get(r.sourceNodeName) ?? new Set();
+        set.add(r.field);
+        fieldsBySource.set(r.sourceNodeName, set);
+    }
+    const nodesByName = new Map((workflow.nodes ?? []).filter(column_ref_utils_1.nodeHasName).map((n) => [n.name, n]));
+    const contexts = [];
+    for (const [name, fields] of fieldsBySource) {
+        const node = nodesByName.get(name);
+        if (!node)
+            continue;
+        contexts.push({
+            name,
+            type: node.type,
+            parameters: node.parameters,
+            requiredFields: [...fields],
+        });
+    }
+    return contexts;
+}
+function formatNodeContextBlock(ctx) {
+    const paramsLine = ctx.parameters
+        ? `Parameters: ${JSON.stringify(ctx.parameters).slice(0, 500)}`
+        : 'Parameters: (none)';
+    return [
+        `Node name: ${ctx.name}`,
+        `Node type: ${ctx.type}`,
+        paramsLine,
+        `Required fields on json: ${ctx.requiredFields.join(', ')}`,
+    ].join('\n');
+}
+function buildUserMessage(contexts, agentPurpose) {
+    const blocks = contexts.map(formatNodeContextBlock).join('\n\n');
+    const purpose = agentPurpose ? `\nAgent purpose / system prompt:\n${agentPurpose}\n` : '';
+    const nodeList = contexts.map((c) => `"${c.name}"`).join(', ');
+    return [
+        'Generate realistic mock output (pinData items) for the following n8n nodes.',
+        purpose,
+        blocks,
+        '',
+        `Output a single JSON object with exactly these keys: ${nodeList}.`,
+        'Each value: an array with one item shaped like { "json": { ...fields } }.',
+        'All required fields above MUST appear inside `json` for the corresponding node.',
+    ].join('\n');
+}
+function stripMarkdownFences(text) {
+    const trimmed = text.trim();
+    const fencedMatch = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
+    return fencedMatch ? fencedMatch[1].trim() : trimmed;
+}
+function filterToRequested(parsed, requested) {
+    const out = {};
+    for (const [name, items] of Object.entries(parsed)) {
+        if (!requested.has(name))
+            continue;
+        out[name] = items;
+    }
+    return out;
+}
+async function generateToolRefPinData(input) {
+    if (input.refs.length === 0)
+        return {};
+    const contexts = buildNodeContexts(input.workflow, input.refs);
+    if (contexts.length === 0)
+        return {};
+    const agent = (input.workflow.nodes ?? []).find((n) => (0, column_ref_utils_1.nodeHasName)(n) && n.name === input.agentNodeName);
+    const agentPurpose = readAgentPurpose(agent);
+    const userText = buildUserMessage(contexts, agentPurpose);
+    try {
+        const llm = (0, eval_agents_1.createEvalAgent)('eval-tool-ref-pin-data', {
+            model: eval_agents_1.HAIKU_MODEL,
+            instructions: SYSTEM_INSTRUCTIONS,
+        });
+        const result = await llm.generate([
+            { role: 'user', content: [{ type: 'text', text: userText }] },
+        ]);
+        const text = (0, eval_agents_1.extractText)(result);
+        const parsed = JSON.parse(stripMarkdownFences(text));
+        const validated = PinDataResponseSchema.safeParse(parsed);
+        if (!validated.success)
+            return {};
+        const requested = new Set(contexts.map((c) => c.name));
+        return filterToRequested(validated.data, requested);
+    }
+    catch {
+        return {};
+    }
+}
+//# sourceMappingURL=generate-tool-ref-pin-data.service.js.map