n8n-nodes-base 1.20.0 → 1.21.0

package/dist/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.js

@@ -0,0 +1,221 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoveDuplicates = void 0;
+const get_1 = __importDefault(require("lodash/get"));
+const isEqual_1 = __importDefault(require("lodash/isEqual"));
+const lt_1 = __importDefault(require("lodash/lt"));
+const pick_1 = __importDefault(require("lodash/pick"));
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../utils/utils");
+const utils_2 = require("./utils");
+class RemoveDuplicates {
+    constructor() {
+        this.description = {
+            displayName: 'Remove Duplicates',
+            name: 'removeDuplicates',
+            icon: 'file:removeDuplicates.svg',
+            group: ['transform'],
+            subtitle: '',
+            version: 1,
+            description: 'Delete items with matching field values',
+            defaults: {
+                name: 'Remove Duplicates',
+            },
+            inputs: ['main'],
+            outputs: ['main'],
+            properties: [
+                {
+                    displayName: 'Compare',
+                    name: 'compare',
+                    type: 'options',
+                    options: [
+                        {
+                            name: 'All Fields',
+                            value: 'allFields',
+                        },
+                        {
+                            name: 'All Fields Except',
+                            value: 'allFieldsExcept',
+                        },
+                        {
+                            name: 'Selected Fields',
+                            value: 'selectedFields',
+                        },
+                    ],
+                    default: 'allFields',
+                    description: 'The fields of the input items to compare to see if they are the same',
+                },
+                {
+                    displayName: 'Fields To Exclude',
+                    name: 'fieldsToExclude',
+                    type: 'string',
+                    placeholder: 'e.g. email, name',
+                    requiresDataPath: 'multiple',
+                    description: 'Fields in the input to exclude from the comparison',
+                    default: '',
+                    displayOptions: {
+                        show: {
+                            compare: ['allFieldsExcept'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Fields To Compare',
+                    name: 'fieldsToCompare',
+                    type: 'string',
+                    placeholder: 'e.g. email, name',
+                    requiresDataPath: 'multiple',
+                    description: 'Fields in the input to add to the comparison',
+                    default: '',
+                    displayOptions: {
+                        show: {
+                            compare: ['selectedFields'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Options',
+                    name: 'options',
+                    type: 'collection',
+                    placeholder: 'Add Field',
+                    default: {},
+                    displayOptions: {
+                        show: {
+                            compare: ['allFieldsExcept', 'selectedFields'],
+                        },
+                    },
+                    options: [
+                        {
+                            displayName: 'Disable Dot Notation',
+                            name: 'disableDotNotation',
+                            type: 'boolean',
+                            default: false,
+                            description: 'Whether to disallow referencing child fields using `parent.child` in the field name',
+                        },
+                        {
+                            displayName: 'Remove Other Fields',
+                            name: 'removeOtherFields',
+                            type: 'boolean',
+                            default: false,
+                            description: 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.',
+                        },
+                    ],
+                },
+            ],
+        };
+    }
+    async execute() {
+        const items = this.getInputData();
+        const compare = this.getNodeParameter('compare', 0);
+        const disableDotNotation = this.getNodeParameter('options.disableDotNotation', 0, false);
+        const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false);
+        let keys = disableDotNotation
+            ? Object.keys(items[0].json)
+            : Object.keys((0, utils_2.flattenKeys)(items[0].json));
+        for (const item of items) {
+            for (const key of disableDotNotation
+                ? Object.keys(item.json)
+                : Object.keys((0, utils_2.flattenKeys)(item.json))) {
+                if (!keys.includes(key)) {
+                    keys.push(key);
+                }
+            }
+        }
+        if (compare === 'allFieldsExcept') {
+            const fieldsToExclude = (0, utils_1.prepareFieldsArray)(this.getNodeParameter('fieldsToExclude', 0, ''), 'Fields To Exclude');
+            if (!fieldsToExclude.length) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No fields specified. Please add a field to exclude from comparison');
+            }
+            if (!disableDotNotation) {
+                keys = Object.keys((0, utils_2.flattenKeys)(items[0].json));
+            }
+            keys = keys.filter((key) => !fieldsToExclude.includes(key));
+        }
+        if (compare === 'selectedFields') {
+            const fieldsToCompare = (0, utils_1.prepareFieldsArray)(this.getNodeParameter('fieldsToCompare', 0, ''), 'Fields To Compare');
+            if (!fieldsToCompare.length) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No fields specified. Please add a field to compare on');
+            }
+            if (!disableDotNotation) {
+                keys = Object.keys((0, utils_2.flattenKeys)(items[0].json));
+            }
+            keys = fieldsToCompare.map((key) => key.trim());
+        }
+        const newItems = items.map((item, index) => ({
+            json: { ...item.json, __INDEX: index },
+            pairedItem: { item: index },
+        }));
+        newItems.sort((a, b) => {
+            let result = 0;
+            for (const key of keys) {
+                let equal;
+                if (!disableDotNotation) {
+                    equal = (0, isEqual_1.default)((0, get_1.default)(a.json, key), (0, get_1.default)(b.json, key));
+                }
+                else {
+                    equal = (0, isEqual_1.default)(a.json[key], b.json[key]);
+                }
+                if (!equal) {
+                    let lessThan;
+                    if (!disableDotNotation) {
+                        lessThan = (0, lt_1.default)((0, get_1.default)(a.json, key), (0, get_1.default)(b.json, key));
+                    }
+                    else {
+                        lessThan = (0, lt_1.default)(a.json[key], b.json[key]);
+                    }
+                    result = lessThan ? -1 : 1;
+                    break;
+                }
+            }
+            return result;
+        });
+        for (const key of keys) {
+            let type = undefined;
+            for (const item of newItems) {
+                if (key === '') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Name of field to compare is blank');
+                }
+                const value = !disableDotNotation ? (0, get_1.default)(item.json, key) : item.json[key];
+                if (value === undefined && disableDotNotation && key.includes('.')) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `'${key}' field is missing from some input items`, {
+                        description: "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
+                    });
+                }
+                else if (value === undefined) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `'${key}' field is missing from some input items`);
+                }
+                if (type !== undefined && value !== undefined && type !== typeof value) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, {
+                        description: 'The type of this field varies between items',
+                    });
+                }
+                else {
+                    type = typeof value;
+                }
+            }
+        }
+        const removedIndexes = [];
+        let temp = newItems[0];
+        for (let index = 1; index < newItems.length; index++) {
+            if ((0, utils_2.compareItems)(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
+                removedIndexes.push(newItems[index].json.__INDEX);
+            }
+            else {
+                temp = newItems[index];
+            }
+        }
+        let returnData = items.filter((_, index) => !removedIndexes.includes(index));
+        if (removeOtherFields) {
+            returnData = returnData.map((item, index) => ({
+                json: (0, pick_1.default)(item.json, ...keys),
+                pairedItem: { item: index },
+            }));
+        }
+        return [returnData];
+    }
+}
+exports.RemoveDuplicates = RemoveDuplicates;
+//# sourceMappingURL=RemoveDuplicates.node.js.map

package/dist/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RemoveDuplicates.node.js","sourceRoot":"","sources":["../../../../nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts"],"names":[],"mappings":";;;;;;AAAA,qDAA6B;AAC7B,6DAAqC;AACrC,mDAA2B;AAC3B,uDAA+B;AAC/B,+CAMsB;AACtB,0CAAoD;AACpD,mCAAoD;AAEpD,MAAa,gBAAgB;IAA7B;QACC,gBAAW,GAAyB;YACnC,WAAW,EAAE,mBAAmB;YAChC,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,CAAC,WAAW,CAAC;YACpB,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,yCAAyC;YACtD,QAAQ,EAAE;gBACT,IAAI,EAAE,mBAAmB;aACzB;YACD,MAAM,EAAE,CAAC,MAAM,CAAC;YAChB,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,UAAU,EAAE;gBACX;oBACC,WAAW,EAAE,SAAS;oBACtB,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE;wBACR;4BACC,IAAI,EAAE,YAAY;4BAClB,KAAK,EAAE,WAAW;yBAClB;wBACD;4BACC,IAAI,EAAE,mBAAmB;4BACzB,KAAK,EAAE,iBAAiB;yBACxB;wBACD;4BACC,IAAI,EAAE,iBAAiB;4BACvB,KAAK,EAAE,gBAAgB;yBACvB;qBACD;oBACD,OAAO,EAAE,WAAW;oBACpB,WAAW,EAAE,sEAAsE;iBACnF;gBACD;oBACC,WAAW,EAAE,mBAAmB;oBAChC,IAAI,EAAE,iBAAiB;oBACvB,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kBAAkB;oBAC/B,gBAAgB,EAAE,UAAU;oBAC5B,WAAW,EAAE,oDAAoD;oBACjE,OAAO,EAAE,EAAE;oBACX,cAAc,EAAE;wBACf,IAAI,EAAE;4BACL,OAAO,EAAE,CAAC,iBAAiB,CAAC;yBAC5B;qBACD;iBACD;gBACD;oBACC,WAAW,EAAE,mBAAmB;oBAChC,IAAI,EAAE,iBAAiB;oBACvB,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kBAAkB;oBAC/B,gBAAgB,EAAE,UAAU;oBAC5B,WAAW,EAAE,8CAA8C;oBAC3D,OAAO,EAAE,EAAE;oBACX,cAAc,EAAE;wBACf,IAAI,EAAE;4BACL,OAAO,EAAE,CAAC,gBAAgB,CAAC;yBAC3B;qBACD;iBACD;gBACD;oBACC,WAAW,EAAE,SAAS;oBACtB,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,YAAY;oBAClB,WAAW,EAAE,WAAW;oBACxB,OAAO,EAAE,EAAE;oBACX,cAAc,EAAE;wBACf,IAAI,EAAE;4BACL,OAAO,EAAE,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;yBAC9C;qBACD;oBACD,OAAO,EAAE;wBACR;4BACC,WAAW,EAAE,sBAAsB;4BACnC,IAAI,EAAE,oBAAoB;4BAC1B,IAAI,EAAE,SAAS;4BACf,OAAO,EAAE,KAAK;4BACd,WAAW,EACV,qFAAqF;yBACtF;wBACD;4BACC,WAAW,EAAE,qBAAqB;4BAClC,IAAI,EAAE,mBAAmB;4BACzB,IAAI,EAAE,SAAS;4BACf,OAAO,EAAE,KAAK;4BACd,WAAW,EACV,+HAA+H;yBAChI;qBACD;iBACD;aACD;SACD,CAAC;IAwJH,CAAC;IAtJA,KAAK,CAAC,OAAO;QACZ,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,CAAW,CAAC;QAC9D,MAAM,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,CAC/C,4BAA4B,EAC5B,CAAC,EACD,KAAK,CACM,CAAC;QACb,MAAM,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAC9C,2BAA2B,EAC3B,CAAC,EACD,KAAK,CACM,CAAC;QAEb,IAAI,IAAI,GAAG,kBAAkB;YAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,kBAAkB;gBACnC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBACxB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,OAAO,KAAK,iBAAiB,EAAE,CAAC;YACnC,MAAM,eAAe,GAAG,IAAA,0BAAkB,EACzC,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAW,EACzD,mBAAmB,CACnB,CAAC;YAEF,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,oEAAoE,CACpE,CAAC;YACH,CAAC;YACD,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACzB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,KAAK,gBAAgB,EAAE,CAAC;YAClC,MAAM,eAAe,GAAG,IAAA,0BAAkB,EACzC,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAW,EACzD,mBAAmB,CACnB,CAAC;YACF,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,uDAAuD,CACvD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACzB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,mBAAW,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;QAID,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CACzB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACf,CAAC;YACA,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;YACtC,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;SAC3B,CAAuB,CACzB,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACtB,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,IAAI,KAAK,CAAC;gBACV,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBACzB,KAAK,GAAG,IAAA,iBAAO,EAAC,IAAA,aAAG,EAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,IAAA,aAAG,EAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;gBACrD,CAAC;qBAAM,CAAC;oBACP,KAAK,GAAG,IAAA,iBAAO,EAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,IAAI,CAAC,KAAK,EAAE,CAAC;oBACZ,IAAI,QAAQ,CAAC;oBACb,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACzB,QAAQ,GAAG,IAAA,YAAE,EAAC,IAAA,aAAG,EAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,IAAA,aAAG,EAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;oBACnD,CAAC;yBAAM,CAAC;wBACP,QAAQ,GAAG,IAAA,YAAE,EAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBACzC,CAAC;oBACD,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3B,MAAM;gBACP,CAAC;YACF,CAAC;YACD,OAAO,MAAM,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,GAAQ,SAAS,CAAC;YAC1B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;oBAChB,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;gBACnF,CAAC;gBACD,MAAM,KAAK,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAA,aAAG,EAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzE,IAAI,KAAK,KAAK,SAAS,IAAI,kBAAkB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpE,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,IAAI,GAAG,0CAA0C,EACjD;wBACC,WAAW,EACV,2GAA2G;qBAC5G,CACD,CAAC;gBACH,CAAC;qBAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBAChC,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,IAAI,GAAG,0CAA0C,CACjD,CAAC;gBACH,CAAC;gBACD,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO,KAAK,EAAE,CAAC;oBACxE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,GAAG,8BAA8B,EAAE;wBACnF,WAAW,EAAE,6CAA6C;qBAC1D,CAAC,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACP,IAAI,GAAG,OAAO,KAAK,CAAC;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;QAGD,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACtD,IAAI,IAAA,oBAAY,EAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBACnF,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAA4B,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACP,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACF,CAAC;QAED,IAAI,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QAE7E,IAAI,iBAAiB,EAAE,CAAC;YACvB,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC7C,IAAI,EAAE,IAAA,cAAI,EAAC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;gBAC9B,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;aAC3B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,UAAU,CAAC,CAAC;IACrB,CAAC;CACD;AAvPD,4CAuPC"}

package/dist/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.json

@@ -0,0 +1,29 @@
+{
+    "node": "n8n-nodes-base.removeDuplicates",
+    "nodeVersion": "1.0",
+    "codexVersion": "1.0",
+    "details": "",
+    "categories": ["Core Nodes"],
+    "resources": {
+        "primaryDocumentation": [
+            {
+                "url": "https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.removeduplicates/"
+            }
+        ],
+        "generic": []
+    },
+    "alias": [
+        "Dedupe",
+        "Deduplicate",
+        "Duplicates",
+        "Remove",
+        "Unique",
+        "Transform",
+        "Array",
+        "List",
+        "Item"
+    ],
+    "subcategories": {
+        "Core Nodes": ["Data Transformation"]
+    }
+}

package/dist/nodes/Transform/RemoveDuplicates/removeDuplicates.svg

@@ -0,0 +1,15 @@
+<svg width="512" height="512" viewBox="0 0 512 512" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_1156_1098)">
+<path d="M134.097 111H172.926V143.508H138.16V178.143H105.652V139.444C105.652 123.735 118.387 111 134.097 111Z" fill="#54B8C9"/>
+<path d="M211.755 143.508V111H289.412V143.508H211.755Z" fill="#54B8C9"/>
+<path d="M328.241 143.508V111H405.899V143.508H328.241Z" fill="#54B8C9"/>
+<path d="M444.728 143.508V111H483.557C499.267 111 512.002 123.735 512.002 139.444V178.143H479.494V143.508H444.728Z" fill="#54B8C9"/>
+<path d="M479.494 216.746H512.002V255.444C512.002 271.154 499.267 283.889 483.557 283.889H444.728V251.381H479.494V216.746Z" fill="#54B8C9"/>
+<path d="M0 244.537C0 229.329 12.735 217 28.4444 217H377.905C393.614 217 406.349 229.329 406.349 244.537V374.352C406.349 389.56 393.614 401.889 377.905 401.889H28.4444C12.735 401.889 0 389.56 0 374.352V244.537Z" fill="#54B8C9"/>
+</g>
+<defs>
+<clipPath id="clip0_1156_1098">
+<rect width="512" height="512" fill="white"/>
+</clipPath>
+</defs>
+</svg>

package/dist/nodes/Transform/RemoveDuplicates/test/workflow.removeDuplicates.json

@@ -0,0 +1,325 @@
+{
+    "name": "Remove Duplicates",
+    "nodes": [
+        {
+            "parameters": {},
+            "id": "a4da10da-991f-48ab-b873-9d633a11311f",
+            "name": "When clicking \"Execute Workflow\"",
+            "type": "n8n-nodes-base.manualTrigger",
+            "typeVersion": 1,
+            "position": [
+                760,
+                420
+            ]
+        },
+        {
+            "parameters": {
+                "jsCode": "return [{ id: 1, name: 'John Doe', age: 18 },{ id: 1, name: 'John Doe', age: 18 },\n  { id: 1, name: 'John Doe', age: 98 },\n  { id: 3, name: 'Bob Johnson', age:34 }]"
+            },
+            "id": "7ab7d5cd-0b1e-48bc-bdbd-57c91e201cf3",
+            "name": "Code",
+            "type": "n8n-nodes-base.code",
+            "typeVersion": 2,
+            "position": [
+                980,
+                420
+            ]
+        },
+        {
+            "parameters": {},
+            "id": "c336939c-062e-475e-ba7c-8601e3662e8c",
+            "name": "Remove Duplicates (All Fields)",
+            "type": "n8n-nodes-base.removeDuplicates",
+            "typeVersion": 1,
+            "position": [
+                1200,
+                260
+            ]
+        },
+        {
+            "parameters": {
+                "compare": "selectedFields",
+                "fieldsToCompare": "name",
+                "options": {}
+            },
+            "id": "d4343ffe-8a9e-4e34-a0a1-aa463afedd80",
+            "name": "Remove Duplicates (Selected Fields)",
+            "type": "n8n-nodes-base.removeDuplicates",
+            "typeVersion": 1,
+            "position": [
+                1200,
+                420
+            ]
+        },
+        {
+            "parameters": {
+                "compare": "allFieldsExcept",
+                "fieldsToExclude": "age",
+                "options": {}
+            },
+            "id": "b67daea4-4545-429e-9e2a-58f2d6a7df7b",
+            "name": "Remove Duplicates (Except Fields)",
+            "type": "n8n-nodes-base.removeDuplicates",
+            "typeVersion": 1,
+            "position": [
+                1200,
+                580
+            ]
+        },
+        {
+            "parameters": {},
+            "id": "813e690f-a83e-4a38-a64a-c3d72afcc9ba",
+            "name": "All Fields",
+            "type": "n8n-nodes-base.noOp",
+            "typeVersion": 1,
+            "position": [
+                1380,
+                260
+            ]
+        },
+        {
+            "parameters": {},
+            "id": "b5c5c946-2e96-451b-b9a6-78e478504d6c",
+            "name": "Selected Fields",
+            "type": "n8n-nodes-base.noOp",
+            "typeVersion": 1,
+            "position": [
+                1380,
+                420
+            ]
+        },
+        {
+            "parameters": {},
+            "id": "afb92bc5-beba-4b0a-aefb-b47cc708a125",
+            "name": "Except Fields",
+            "type": "n8n-nodes-base.noOp",
+            "typeVersion": 1,
+            "position": [
+                1380,
+                580
+            ]
+        },
+        {
+            "parameters": {
+                "compare": "allFieldsExcept",
+                "fieldsToExclude": "age",
+                "options": {
+                    "removeOtherFields": true
+                }
+            },
+            "id": "f92c5533-ac29-476c-aebb-4849ddd22110",
+            "name": "Remove Duplicates (Remove)",
+            "type": "n8n-nodes-base.removeDuplicates",
+            "typeVersion": 1,
+            "position": [
+                1200,
+                760
+            ]
+        },
+        {
+            "parameters": {},
+            "id": "1e142ab7-b32e-4f67-b5cc-5c9fb63fba89",
+            "name": "Remove",
+            "type": "n8n-nodes-base.noOp",
+            "typeVersion": 1,
+            "position": [
+                1380,
+                760
+            ]
+        }
+    ],
+    "pinData": {
+        "Code": [
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 18
+                }
+            },
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 18
+                }
+            },
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 98
+                }
+            },
+            {
+                "json": {
+                    "id": 3,
+                    "name": "Bob Johnson",
+                    "age": 34
+                }
+            }
+        ],
+        "All Fields": [
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 18
+                }
+            },
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 98
+                }
+            },
+            {
+                "json": {
+                    "id": 3,
+                    "name": "Bob Johnson",
+                    "age": 34
+                }
+            }
+        ],
+        "Selected Fields": [
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 18
+                }
+            },
+            {
+                "json": {
+                    "id": 3,
+                    "name": "Bob Johnson",
+                    "age": 34
+                }
+            }
+        ],
+        "Except Fields": [
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe",
+                    "age": 18
+                }
+            },
+            {
+                "json": {
+                    "id": 3,
+                    "name": "Bob Johnson",
+                    "age": 34
+                }
+            }
+        ],
+        "Remove": [
+            {
+                "json": {
+                    "id": 1,
+                    "name": "John Doe"
+                }
+            },
+            {
+                "json": {
+                    "id": 3,
+                    "name": "Bob Johnson"
+                }
+            }
+        ]
+    },
+    "connections": {
+        "When clicking \"Execute Workflow\"": {
+            "main": [
+                [
+                    {
+                        "node": "Code",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        },
+        "Code": {
+            "main": [
+                [
+                    {
+                        "node": "Remove Duplicates (All Fields)",
+                        "type": "main",
+                        "index": 0
+                    },
+                    {
+                        "node": "Remove Duplicates (Selected Fields)",
+                        "type": "main",
+                        "index": 0
+                    },
+                    {
+                        "node": "Remove Duplicates (Except Fields)",
+                        "type": "main",
+                        "index": 0
+                    },
+                    {
+                        "node": "Remove Duplicates (Remove)",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        },
+        "Remove Duplicates (All Fields)": {
+            "main": [
+                [
+                    {
+                        "node": "All Fields",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        },
+        "Remove Duplicates (Selected Fields)": {
+            "main": [
+                [
+                    {
+                        "node": "Selected Fields",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        },
+        "Remove Duplicates (Except Fields)": {
+            "main": [
+                [
+                    {
+                        "node": "Except Fields",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        },
+        "Remove Duplicates (Remove)": {
+            "main": [
+                [
+                    {
+                        "node": "Remove",
+                        "type": "main",
+                        "index": 0
+                    }
+                ]
+            ]
+        }
+    },
+    "active": false,
+    "settings": {
+        "executionOrder": "v1"
+    },
+    "versionId": "5bb09766-4c67-4fb4-ae53-89d8db4727e3",
+    "id": "74gMYOHjjPArZg4q",
+    "meta": {
+        "instanceId": "27cc9b56542ad45b38725555722c50a1c3fee1670bbb67980558314ee08517c4"
+    },
+    "tags": []
+}

package/dist/nodes/Transform/RemoveDuplicates/utils.d.ts

@@ -0,0 +1,3 @@
+import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
+export declare const compareItems: (obj: INodeExecutionData, obj2: INodeExecutionData, keys: string[], disableDotNotation: boolean, _node: INode) => boolean;
+export declare const flattenKeys: (obj: IDataObject, path?: string[]) => IDataObject;

package/dist/nodes/Transform/RemoveDuplicates/utils.js

@@ -0,0 +1,37 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.flattenKeys = exports.compareItems = void 0;
+const get_1 = __importDefault(require("lodash/get"));
+const isEqual_1 = __importDefault(require("lodash/isEqual"));
+const isObject_1 = __importDefault(require("lodash/isObject"));
+const merge_1 = __importDefault(require("lodash/merge"));
+const reduce_1 = __importDefault(require("lodash/reduce"));
+const compareItems = (obj, obj2, keys, disableDotNotation, _node) => {
+    let result = true;
+    for (const key of keys) {
+        if (!disableDotNotation) {
+            if (!(0, isEqual_1.default)((0, get_1.default)(obj.json, key), (0, get_1.default)(obj2.json, key))) {
+                result = false;
+                break;
+            }
+        }
+        else {
+            if (!(0, isEqual_1.default)(obj.json[key], obj2.json[key])) {
+                result = false;
+                break;
+            }
+        }
+    }
+    return result;
+};
+exports.compareItems = compareItems;
+const flattenKeys = (obj, path = []) => {
+    return !(0, isObject_1.default)(obj)
+        ? { [path.join('.')]: obj }
+        : (0, reduce_1.default)(obj, (cum, next, key) => (0, merge_1.default)(cum, (0, exports.flattenKeys)(next, [...path, key])), {});
+};
+exports.flattenKeys = flattenKeys;
+//# sourceMappingURL=utils.js.map

package/dist/nodes/Transform/RemoveDuplicates/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../nodes/Transform/RemoveDuplicates/utils.ts"],"names":[],"mappings":";;;;;;AAAA,qDAA6B;AAC7B,6DAAqC;AACrC,+DAAuC;AACvC,yDAAiC;AACjC,2DAAmC;AAG5B,MAAM,YAAY,GAAG,CAC3B,GAAuB,EACvB,IAAwB,EACxB,IAAc,EACd,kBAA2B,EAC3B,KAAY,EACX,EAAE;IACH,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzB,IAAI,CAAC,IAAA,iBAAO,EAAC,IAAA,aAAG,EAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,IAAA,aAAG,EAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;gBACvD,MAAM,GAAG,KAAK,CAAC;gBACf,MAAM;YACP,CAAC;QACF,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,IAAA,iBAAO,EAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC7C,MAAM,GAAG,KAAK,CAAC;gBACf,MAAM;YACP,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAtBW,QAAA,YAAY,gBAsBvB;AAEK,MAAM,WAAW,GAAG,CAAC,GAAgB,EAAE,OAAiB,EAAE,EAAe,EAAE;IACjF,OAAO,CAAC,IAAA,kBAAQ,EAAC,GAAG,CAAC;QACpB,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE;QAC3B,CAAC,CAAC,IAAA,gBAAM,EAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,IAAA,eAAK,EAAC,GAAG,EAAE,IAAA,mBAAW,EAAC,IAAmB,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACtG,CAAC,CAAC;AAJW,QAAA,WAAW,eAItB"}

package/dist/nodes/Transform/Sort/Sort.node.d.ts

@@ -0,0 +1,5 @@
+import { type IExecuteFunctions, type INodeExecutionData, type INodeType, type INodeTypeDescription } from 'n8n-workflow';
+export declare class Sort implements INodeType {
+    description: INodeTypeDescription;
+    execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]>;
+}