n8n-nodes-data-mapper 0.4.0 → 0.4.2

package/dist/nodes/DataMapper/DataMapper.node.js

@@ -7,80 +7,7 @@ const profile_1 = require("./profile");
 const visual_1 = require("./visual");
 const xml_1 = require("./xml");
 const show = (values) => ({ show: values });
-const mappingFields = [
-    {
-        displayName: 'Source',
-        name: 'sourceType',
-        type: 'options',
-        options: [
-            { name: 'Input Field', value: 'field', description: 'Read a value from the input by path' },
-            {
-                name: 'From Source Profile',
-                value: 'profile',
-                description: 'Pick a field from the source profile',
-            },
-            { name: 'Fixed Value', value: 'value', description: 'Write a fixed value or the result of an expression' },
-        ],
-        default: 'field',
-    },
-    {
-        displayName: 'Source Field Name or ID',
-        name: 'sourceField',
-        type: 'options',
-        typeOptions: { loadOptionsMethod: 'getSourceFields' },
-        default: '',
-        description: 'Field from the source profile. Fill the Source Profile field above first. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
-        displayOptions: show({ sourceType: ['profile'] }),
-    },
-    {
-        displayName: 'Source Path',
-        name: 'source',
-        type: 'string',
-        default: '',
-        placeholder: 'customer.addresses[*].city',
-        description: 'Dot path into the input. Use [0] for one array element, [*] for every element, or leave empty for the whole input.',
-        displayOptions: show({ sourceType: ['field'] }),
-    },
-    {
-        displayName: 'Value',
-        name: 'value',
-        type: 'string',
-        default: '',
-        description: 'Fixed text, or an expression such as {{ $now.toISO() }}',
-        displayOptions: show({ sourceType: ['value'] }),
-    },
-    {
-        displayName: 'Target',
-        name: 'targetMode',
-        type: 'options',
-        options: [
-            { name: 'Custom Path', value: 'custom', description: 'Type the target path yourself' },
-            {
-                name: 'From Target Profile',
-                value: 'profile',
-                description: 'Pick a field from the uploaded target profile',
-            },
-        ],
-        default: 'custom',
-    },
-    {
-        displayName: 'Target Field Name or ID',
-        name: 'targetField',
-        type: 'options',
-        typeOptions: { loadOptionsMethod: 'getTargetFields' },
-        default: '',
-        description: 'Field from the target profile. Fill the Target Profile field above first. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
-        displayOptions: show({ targetMode: ['profile'] }),
-    },
-    {
-        displayName: 'Target Path',
-        name: 'target',
-        type: 'string',
-        default: '',
-        placeholder: 'shipping.cities[*]',
-        description: 'Where to write the value in the output. Each [*] matches a [*] in the source path, in order.',
-        displayOptions: show({ targetMode: ['custom'] }),
-    },
+const transformFields = [
     {
         displayName: 'Transform',
         name: 'transform',
@@ -165,6 +92,82 @@ const mappingFields = [
         default: '',
         displayOptions: show({ transform: ['lookup'], lookupFallback: ['default'] }),
     },
+];
+const mappingFields = [
+    {
+        displayName: 'Source',
+        name: 'sourceType',
+        type: 'options',
+        options: [
+            { name: 'Input Field', value: 'field', description: 'Read a value from the input by path' },
+            {
+                name: 'From Source Profile',
+                value: 'profile',
+                description: 'Pick a field from the source profile',
+            },
+            { name: 'Fixed Value', value: 'value', description: 'Write a fixed value or the result of an expression' },
+        ],
+        default: 'field',
+    },
+    {
+        displayName: 'Source Field Name or ID',
+        name: 'sourceField',
+        type: 'options',
+        typeOptions: { loadOptionsMethod: 'getSourceFields' },
+        default: '',
+        description: 'Field from the source profile. Fill the Source Profile field above first. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+        displayOptions: show({ sourceType: ['profile'] }),
+    },
+    {
+        displayName: 'Source Path',
+        name: 'source',
+        type: 'string',
+        default: '',
+        placeholder: 'customer.addresses[*].city',
+        description: 'Dot path into the input. Use [0] for one array element, [*] for every element, or leave empty for the whole input.',
+        displayOptions: show({ sourceType: ['field'] }),
+    },
+    {
+        displayName: 'Value',
+        name: 'value',
+        type: 'string',
+        default: '',
+        description: 'Fixed text, or an expression such as {{ $now.toISO() }}',
+        displayOptions: show({ sourceType: ['value'] }),
+    },
+    {
+        displayName: 'Target',
+        name: 'targetMode',
+        type: 'options',
+        options: [
+            { name: 'Custom Path', value: 'custom', description: 'Type the target path yourself' },
+            {
+                name: 'From Target Profile',
+                value: 'profile',
+                description: 'Pick a field from the target profile',
+            },
+        ],
+        default: 'profile',
+    },
+    {
+        displayName: 'Target Field Name or ID',
+        name: 'targetField',
+        type: 'options',
+        typeOptions: { loadOptionsMethod: 'getTargetFields' },
+        default: '',
+        description: 'Field from the target profile. Fill the Target Profile field above first. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+        displayOptions: show({ targetMode: ['profile'] }),
+    },
+    {
+        displayName: 'Target Path',
+        name: 'target',
+        type: 'string',
+        default: '',
+        placeholder: 'shipping.cities[*]',
+        description: 'Where to write the value in the output. Each [*] matches a [*] in the source path, in order.',
+        displayOptions: show({ targetMode: ['custom'] }),
+    },
+    ...transformFields,
     {
         displayName: 'If Source Is Missing',
         name: 'ifMissing',
@@ -239,26 +242,6 @@ const v1Properties = [
         description: 'XML is read with attributes under "$" and text under "_", so a path looks like order.$.ID or order.total._',
         displayOptions: show({ inputSource: ['field', 'binary'] }),
     },
-    {
-        displayName: 'Define Mappings',
-        name: 'definitionMode',
-        type: 'options',
-        noDataExpression: true,
-        options: [
-            { name: 'Using Fields Below', value: 'fields' },
-            {
-                name: 'Auto-Map From Profile',
-                value: 'auto',
-                description: 'Match source fields to a target profile automatically by name',
-            },
-            {
-                name: 'Using JSON',
-                value: 'json',
-                description: 'Paste or reference a mapping definition, so one mapping can be reused across workflows',
-            },
-        ],
-        default: 'fields',
-    },
     {
         displayName: 'Source Profile',
         name: 'sourceProfile',
@@ -348,6 +331,91 @@ const v1Properties = [
         description: 'The kind of profile you provided. Detect Automatically works for most profiles.',
         displayOptions: show({ definitionMode: ['fields', 'auto'] }),
     },
+    {
+        displayName: 'Define Mappings',
+        name: 'definitionMode',
+        type: 'options',
+        noDataExpression: true,
+        options: [
+            { name: 'Map Each Column Manually', value: 'fields', description: 'Add a mapping per target field' },
+            {
+                name: 'Auto-Map From Profile',
+                value: 'auto',
+                description: 'Match source fields to a target profile automatically by name',
+            },
+            {
+                name: 'Using JSON',
+                value: 'json',
+                description: 'Paste or reference a mapping definition, so one mapping can be reused across workflows',
+            },
+        ],
+        default: 'fields',
+    },
+    {
+        displayName: 'Mapping Layout',
+        name: 'mappingLayout',
+        type: 'options',
+        noDataExpression: true,
+        options: [
+            { name: 'One Mapping at a Time', value: 'list', description: 'Add a mapping row for each field' },
+            {
+                name: 'Fill Target Fields',
+                value: 'fields',
+                description: 'List every target-profile field and set the source for each, like a visual mapper',
+            },
+        ],
+        default: 'list',
+        displayOptions: show({ definitionMode: ['fields'] }),
+    },
+    {
+        displayName: 'Target Fields',
+        name: 'columnMapping',
+        type: 'resourceMapper',
+        noDataExpression: true,
+        default: { mappingMode: 'defineBelow', value: null },
+        required: true,
+        typeOptions: {
+            loadOptionsDependsOn: ['targetProfile', 'targetProfileFormat', 'sourceProfile', 'sourceProfileFormat'],
+            resourceMapper: {
+                resourceMapperMethod: 'getProfileFields',
+                mode: 'add',
+                valuesLabel: 'Target Fields',
+                fieldWords: { singular: 'target field', plural: 'target fields' },
+                addAllFields: true,
+                multiKeyMatch: false,
+                supportAutoMap: false,
+                noFieldsError: 'No target fields found. Paste a target profile above.',
+            },
+        },
+        displayOptions: show({ definitionMode: ['fields'], mappingLayout: ['fields'] }),
+    },
+    {
+        displayName: 'Transform a Field',
+        name: 'fieldTransforms',
+        placeholder: 'Add Field Transform',
+        type: 'fixedCollection',
+        typeOptions: { multipleValues: true },
+        default: {},
+        description: 'Convert or reformat the target fields above (trim, number, date, lookup, split/join, ...). Pick a field and its transform. Number and boolean types from the profile already convert on their own.',
+        displayOptions: show({ definitionMode: ['fields'], mappingLayout: ['fields'] }),
+        options: [
+            {
+                displayName: 'Field',
+                name: 'item',
+                values: [
+                    {
+                        displayName: 'Target Field Name or ID',
+                        name: 'targetField',
+                        type: 'options',
+                        typeOptions: { loadOptionsMethod: 'getTargetFields' },
+                        default: '',
+                        description: 'Which target field to transform. Choose from the list, or specify an ID using an <a href="https://docs.n8n.io/code/expressions/">expression</a>.',
+                    },
+                    ...transformFields,
+                ],
+            },
+        ],
+    },
     {
         displayName: 'Mappings',
         name: 'mappings',
@@ -356,7 +424,7 @@ const v1Properties = [
         typeOptions: { multipleValues: true, sortable: true },
         default: {},
         description: 'Applied in order; a later mapping to the same target overwrites an earlier one',
-        displayOptions: show({ definitionMode: ['fields'] }),
+        displayOptions: show({ definitionMode: ['fields'], mappingLayout: ['list'] }),
         options: [{ displayName: 'Mapping', name: 'mapping', values: mappingFields }],
     },
     {
@@ -365,7 +433,7 @@ const v1Properties = [
         type: 'boolean',
         default: false,
         description: 'Whether to also fill any target-profile fields you have not mapped above, by matching them to source fields by name',
-        displayOptions: show({ definitionMode: ['fields'] }),
+        displayOptions: show({ definitionMode: ['fields'], mappingLayout: ['list'] }),
     },
     {
         displayName: 'Mapping Definition',
@@ -381,7 +449,7 @@ const v1Properties = [
         type: 'collection',
         placeholder: 'Add option',
         default: {},
-        displayOptions: show({ definitionMode: ['auto'] }),
+        displayOptions: show({ definitionMode: ['fields', 'auto'] }),
         options: [
             {
                 displayName: 'Add Auto-Map Report',
@@ -397,6 +465,15 @@ const v1Properties = [
                 default: false,
                 description: 'Whether field names must match in the same letter case',
             },
+            {
+                displayName: 'Custom Synonyms',
+                name: 'synonyms',
+                type: 'string',
+                typeOptions: { rows: 3 },
+                default: '',
+                placeholder: 'menge=quantity\nartikel=article',
+                description: 'Extra name matches for auto-map, one source=target per line (or a JSON object). Qty=quantity, Amt=amount and other common abbreviations are already built in.',
+            },
             {
                 displayName: 'Match',
                 name: 'strategy',
@@ -454,6 +531,8 @@ const v1Properties = [
         type: 'options',
         noDataExpression: true,
         options: [
+            { name: 'CSV', value: 'csv', description: 'Output the mapped data as CSV text in a field' },
+            { name: 'CSV File', value: 'csvFile', description: 'Output the mapped data as a CSV file' },
             { name: 'JSON', value: 'json', description: 'Output the mapped data as the item' },
             { name: 'JSON File', value: 'jsonFile', description: 'Output the mapped data as a JSON file' },
             { name: 'XML', value: 'xml', description: 'Output the mapped data as XML text in a field' },
@@ -465,9 +544,10 @@ const v1Properties = [
         displayName: 'Put Output in Field',
         name: 'outputField',
         type: 'string',
-        default: 'xml',
-        required: true,
-        displayOptions: show({ outputFormat: ['xml'] }),
+        default: '',
+        placeholder: 'xml',
+        description: 'Field for the XML or CSV text. Leave empty to use "xml" or "csv".',
+        displayOptions: show({ outputFormat: ['xml', 'csv'] }),
     },
     {
         displayName: 'Put Output File in Field',
@@ -475,7 +555,7 @@ const v1Properties = [
         type: 'string',
         default: 'data',
         required: true,
-        displayOptions: show({ outputFormat: ['xmlFile', 'jsonFile'] }),
+        displayOptions: show({ outputFormat: ['xmlFile', 'jsonFile', 'csvFile'] }),
     },
     {
         displayName: 'File Name',
@@ -484,7 +564,7 @@ const v1Properties = [
         default: '',
         placeholder: 'output.xml',
         description: 'Leave empty to name the file output with the matching extension',
-        displayOptions: show({ outputFormat: ['xmlFile', 'jsonFile'] }),
+        displayOptions: show({ outputFormat: ['xmlFile', 'jsonFile', 'csvFile'] }),
     },
     {
         displayName: 'Root Element',
@@ -503,6 +583,72 @@ const v1Properties = [
         description: 'Whether to start the XML with &lt;?xml version="1.0" encoding="UTF-8"?&gt;',
         displayOptions: show({ outputFormat: ['xml', 'xmlFile'] }),
     },
+    {
+        displayName: 'CSV Options',
+        name: 'csvOptions',
+        type: 'collection',
+        placeholder: 'Add option',
+        default: {},
+        displayOptions: show({ outputFormat: ['csv', 'csvFile'] }),
+        options: [
+            { displayName: 'Delimiter', name: 'csvDelimiter', type: 'string', default: ',' },
+            {
+                displayName: 'Escape Formulas',
+                name: 'csvEscapeFormulas',
+                type: 'boolean',
+                default: false,
+                description: "Whether to put ' before values starting with =, +, -, @, tab or CR so spreadsheets don't run them as formulas (CSV injection)",
+            },
+            {
+                displayName: 'Header Row',
+                name: 'csvHeader',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to start the CSV with a row of column names',
+            },
+            {
+                displayName: 'Line Ending',
+                name: 'csvLineEnding',
+                type: 'options',
+                options: [
+                    { name: 'LF', value: 'lf' },
+                    { name: 'CRLF', value: 'crlf' },
+                ],
+                default: 'lf',
+            },
+            {
+                displayName: 'One CSV for All Items',
+                name: 'csvCombine',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to put every item into one CSV document, instead of one CSV per item',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'fieldOptions',
+        type: 'collection',
+        placeholder: 'Add option',
+        default: {},
+        displayOptions: show({ definitionMode: ['fields'], mappingLayout: ['fields'] }),
+        options: [
+            {
+                displayName: 'Convert Types From Profile',
+                name: 'convertTypes',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to turn values into numbers or booleans where the profile says so. Values that cannot be converted are kept as they are.',
+            },
+            {
+                displayName: 'Keep Empty Values',
+                name: 'keepEmpty',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to write target fields whose value is empty, instead of leaving them out',
+            },
+        ],
+    },
 ];
 function forVersions(properties, versions) {
     return properties.map((property) => {
@@ -524,9 +670,9 @@ class DataMapper {
             icon: { light: 'file:mapper.svg', dark: 'file:mapper.dark.svg' },
             group: ['transform'],
             version: [1, 2],
-            defaultVersion: 2,
+            defaultVersion: 1,
             subtitle: '={{ $parameter["targetFormat"] ? "Map to " + $parameter["targetFormat"].toUpperCase() : $parameter["definitionMode"] === "auto" ? "Auto-map from profile" : $parameter["definitionMode"] === "json" ? "JSON definition" : "Field mappings" }}',
-            description: 'Drag and drop incoming data onto a target profile and output XML, JSON or CSV',
+            description: 'Map and transform data between structures and output XML, JSON or CSV',
             defaults: {
                 name: 'Data Mapper',
             },
@@ -599,18 +745,30 @@ class DataMapper {
         const items = this.getInputData();
         const returnData = [];
         const profileCache = new Map();
+        const format0 = this.getNodeParameter('outputFormat', 0, 'json');
+        const combineCsv = (format0 === 'csv' || format0 === 'csvFile') &&
+            this.getNodeParameter('csvOptions.csvCombine', 0, true) !== false;
+        const csvRecords = [];
+        let csvProfile;
         for (let i = 0; i < items.length; i++) {
             try {
                 const input = await readInput(this, items[i], i);
-                const { rules, report } = await buildRules(this, i, input, profileCache);
+                const { rules, report, profile, order } = await buildRules(this, i, input, profileCache);
                 const unmapped = this.getNodeParameter('unmapped', i);
                 const start = unmapped === 'keep' && input && typeof input === 'object' && !Array.isArray(input)
                     ? structuredClone(input)
                     : {};
-                const mapped = (0, mapping_1.applyMappings)(input, rules, start);
+                let mapped = (0, mapping_1.applyMappings)(input, rules, start);
+                if (order && profile)
+                    mapped = (0, visual_1.orderByProfile)(mapped, profile);
                 if (report)
                     mapped._autoMap = report;
-                returnData.push(await writeOutput(this, mapped, i));
+                if (combineCsv) {
+                    csvRecords.push({ mapped, index: i });
+                    csvProfile !== null && csvProfile !== void 0 ? csvProfile : (csvProfile = profile);
+                    continue;
+                }
+                returnData.push(await writeOutput(this, mapped, i, profile));
             }
             catch (error) {
                 if (this.continueOnFail()) {
@@ -629,6 +787,10 @@ class DataMapper {
                     });
             }
         }
+        if (combineCsv && csvRecords.length > 0) {
+            const output = await csvOutput(this, csvRecords.map((r) => r.mapped), csvProfile, 0, csvRecords.map((r) => ({ item: r.index })));
+            returnData.push(output);
+        }
         return [returnData];
     }
 }
@@ -713,6 +875,7 @@ function readAutoOptions(ctx, itemIndex) {
         caseSensitive: o.caseSensitive === true,
         reuseSources: o.reuseSources === true,
         unmatched: (_c = o.unmatched) !== null && _c !== void 0 ? _c : 'skip',
+        synonyms: (0, profile_1.parseSynonyms)(o.synonyms),
     };
 }
 function uiMappingToRule(m, ctx, itemIndex) {
@@ -750,7 +913,28 @@ function uiMappingToRule(m, ctx, itemIndex) {
         default: m.defaultValue,
     };
 }
+function applyFieldTransforms(ctx, itemIndex, rules) {
+    const items = ctx.getNodeParameter('fieldTransforms.item', itemIndex, []).filter(Boolean);
+    for (const it of items) {
+        if (!it.targetField || !it.transform || it.transform === 'none')
+            continue;
+        for (const rule of rules) {
+            if (rule.target !== it.targetField)
+                continue;
+            rule.transform = it.transform;
+            rule.separator = it.separator;
+            rule.decimals = it.decimals;
+            rule.dateInputFormat = it.dateInputFormat;
+            rule.dateOutputFormat = it.dateOutputFormat;
+            rule.lookup = it.lookupTable;
+            rule.lookupFallback = it.lookupFallback;
+            rule.lookupDefault = it.lookupDefault;
+            rule.lenient = true;
+        }
+    }
+}
 async function buildRules(ctx, itemIndex, input, cache) {
+    var _a, _b;
     const mode = ctx.getNodeParameter('definitionMode', itemIndex);
     if (mode === 'json')
         return { rules: (0, mapping_1.parseDefinition)(ctx.getNodeParameter('mappingJson', itemIndex)) };
@@ -770,27 +954,58 @@ async function buildRules(ctx, itemIndex, input, cache) {
         }
         return built;
     }
+    if (ctx.getNodeParameter('mappingLayout', itemIndex, 'list') === 'fields') {
+        const raw = ((_a = ctx.getNodeParameter('columnMapping.value', itemIndex, {}, {
+            rawExpressions: true,
+            skipValidation: true,
+        })) !== null && _a !== void 0 ? _a : {});
+        const rm = ctx.getNodeParameter('columnMapping', itemIndex, undefined, { skipValidation: true });
+        const resolved = ((_b = rm === null || rm === void 0 ? void 0 : rm.value) !== null && _b !== void 0 ? _b : {});
+        const fo = ctx.getNodeParameter('fieldOptions', itemIndex, {});
+        const rules = (0, visual_1.buildDropRules)(raw, resolved, profile, {
+            convertTypes: fo.convertTypes !== false,
+            keepEmpty: fo.keepEmpty === true,
+        });
+        applyFieldTransforms(ctx, itemIndex, rules);
+        return { rules, profile, order: true };
+    }
     const entries = ctx.getNodeParameter('mappings.mapping', itemIndex, []).filter(Boolean);
     const rules = entries.map((m) => uiMappingToRule(m, ctx, itemIndex));
     if (profile && ctx.getNodeParameter('autoMapRemaining', itemIndex, false) === true) {
         const covered = new Set(rules.map((r) => r.target));
         const remaining = { ...profile, fields: profile.fields.filter((f) => !covered.has(f.path)) };
-        const result = (0, profile_1.autoMap)((0, profile_1.flattenSource)(input), remaining, {
-            threshold: 0.6,
-            strategy: 'fuzzy',
-            caseSensitive: false,
-            reuseSources: false,
-            unmatched: 'skip',
-        });
+        const result = (0, profile_1.autoMap)((0, profile_1.flattenSource)(input), remaining, readAutoOptions(ctx, itemIndex));
         rules.push(...result.rules);
+        if (ctx.getNodeParameter('autoOptions.report', itemIndex, false) === true) {
+            return {
+                rules,
+                profile,
+                report: { matched: result.matched, unmatched: result.unmatched },
+            };
+        }
     }
     return { rules, profile };
 }
-async function writeOutput(ctx, mapped, itemIndex) {
+async function csvOutput(ctx, records, profile, itemIndex, pairedItem) {
+    const o = ctx.getNodeParameter('csvOptions', itemIndex, {});
+    const text = (0, visual_1.csvText)(records, profile, o);
+    if (ctx.getNodeParameter('outputFormat', itemIndex) === 'csvFile') {
+        const field = ctx.getNodeParameter('outputBinaryField', itemIndex);
+        const fileName = ctx.getNodeParameter('fileName', itemIndex, '') || 'output.csv';
+        const binary = await ctx.helpers.prepareBinaryData(Buffer.from(text, 'utf8'), fileName, 'text/csv');
+        return { json: {}, binary: { [field]: binary }, pairedItem };
+    }
+    const field = ctx.getNodeParameter('outputField', itemIndex, '') || 'csv';
+    return { json: { [field]: text }, pairedItem };
+}
+async function writeOutput(ctx, mapped, itemIndex, profile) {
     const format = ctx.getNodeParameter('outputFormat', itemIndex);
     const pairedItem = { item: itemIndex };
     if (format === 'json')
         return { json: mapped, pairedItem };
+    if (format === 'csv' || format === 'csvFile') {
+        return csvOutput(ctx, [mapped], profile, itemIndex, [pairedItem]);
+    }
     if (format === 'jsonFile') {
         const field = ctx.getNodeParameter('outputBinaryField', itemIndex);
         const fileName = ctx.getNodeParameter('fileName', itemIndex, '') || 'output.json';
@@ -804,7 +1019,7 @@ async function writeOutput(ctx, mapped, itemIndex) {
         textKey: '_',
     });
     if (format === 'xml') {
-        const field = ctx.getNodeParameter('outputField', itemIndex);
+        const field = ctx.getNodeParameter('outputField', itemIndex, '') || 'xml';
         return { json: { [field]: xml }, pairedItem };
     }
     const field = ctx.getNodeParameter('outputBinaryField', itemIndex);