fuse-importer 0.1.3-testing → 0.1.5-testing

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/filterMethods.d.ts

@@ -1,5 +1,7 @@
-import { BatchValidationErrors, RecordDataSet } from "../../../../types";
+import { DuplicateRecordIds, Record } from "../../../../types";
 import { SpreadsheetFilter } from "./useSpreadsheetFilter";
-export declare const filterRecordsBySearchValue: (obj: RecordDataSet, value: string) => RecordDataSet;
-export declare const filterRecordsByType: (obj: RecordDataSet, filterType: SpreadsheetFilter["byRow"]) => RecordDataSet;
-export declare const filterRecordsByError: (obj: RecordDataSet, keyToFilter: string, errors: BatchValidationErrors) => RecordDataSet;
+type FilterCondition = (id: string) => boolean;
+export declare const getFilterConditions: (spreadsheetFilter: SpreadsheetFilter, getRecord: (recordId: string) => Record, errors: {
+    current: Record;
+}, filteredInputValue: string, duplicateRecordIds: DuplicateRecordIds) => FilterCondition[];
+export {};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/filterMethods.js

@@ -1,9 +1,30 @@
 import { rowIsEmpty } from "./utils";
-//Filter records by search value
-export var filterRecordsBySearchValue = function (obj, value) {
-    var isObject = function (value) {
-        return typeof value === "object" && value !== null && !Array.isArray(value);
-    };
+export var getFilterConditions = function (spreadsheetFilter, getRecord, errors, filteredInputValue, duplicateRecordIds) {
+    var filterConditions = [];
+    switch (spreadsheetFilter.type) {
+        case "rowBased":
+            if (spreadsheetFilter.byRow === "all") {
+                filterConditions.push(function (key) { return true; });
+            }
+            if (spreadsheetFilter.byRow === "invalid") {
+                filterConditions.push(function (key) {
+                    var _a, _b;
+                    return (((_b = (_a = getRecord(key)) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.isInvalid) && !rowIsEmpty(getRecord(key)));
+                });
+            }
+            if (spreadsheetFilter.byRow === "duplicates")
+                filterConditions.push(function (key) { return duplicateRecordIds.has(key); });
+            break;
+        case "columnBased":
+            if (spreadsheetFilter.byColumn.type === "fieldWithErrors") {
+                filterConditions.push(function (key) {
+                    var _a, _b, _c;
+                    return ((_b = (_a = getRecord(key)) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.isInvalid) &&
+                        ((_c = errors.current[key]) === null || _c === void 0 ? void 0 : _c[spreadsheetFilter.byColumn.field]);
+                });
+            }
+            break;
+    }
     var includesSearchValue = function (value, searchValue) {
         if (typeof value === "string" || typeof value === "number") {
             var stringValue = String(value).toLowerCase();
@@ -12,38 +33,17 @@ export var filterRecordsBySearchValue = function (obj, value) {
         }
         return false;
     };
-    var filteredObj = {};
-    for (var _i = 0, _a = Object.entries(obj); _i < _a.length; _i++) {
-        var _b = _a[_i], key = _b[0], currentObj = _b[1];
-        if (!isObject(currentObj)) {
-            continue;
-        }
-        var matches = Object.values(currentObj).some(function (propertyValue) {
-            return includesSearchValue(propertyValue, value);
+    if (filteredInputValue) {
+        filterConditions.push(function (key) {
+            var record = getRecord(key);
+            // check if any of the values in the record contain the search input
+            for (var fieldName in record) {
+                if (includesSearchValue(record[fieldName], filteredInputValue)) {
+                    return true;
+                }
+            }
+            return false;
         });
-        if (matches) {
-            filteredObj[key] = currentObj;
-        }
-    }
-    return filteredObj;
-};
-export var filterRecordsByType = function (obj, filterType) {
-    var _a, _b;
-    var filteredObj = {};
-    for (var key in obj) {
-        if (filterType === "all" ||
-            (((_b = (_a = obj[key]) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.isInvalid) && !rowIsEmpty(obj[key]))) {
-            filteredObj[key] = obj[key];
-        }
-    }
-    return filteredObj;
-};
-//Filter records by selected column error
-export var filterRecordsByError = function (obj, keyToFilter, errors) {
-    var filteredObj = {};
-    var filteredKeys = Object.keys(errors).filter(function (key) { var _a, _b; return ((_b = (_a = obj[key]) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.isInvalid) && errors[key][keyToFilter]; });
-    for (var i = 0; i < filteredKeys.length; i++) {
-        filteredObj[filteredKeys[i]] = obj[filteredKeys[i]];
     }
-    return filteredObj;
+    return filterConditions;
 };

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/hashCalculators.d.ts

@@ -0,0 +1,6 @@
+export declare const RecordHashCalculator: () => {
+    computeHashesForRecords: (dataSet: any, fields: any, _onHashesComputed: any) => void;
+};
+export declare const RecordValueHashCalculator: () => {
+    computeDuplicateValueHashes: (dataSet: any, _field: any, _onDuplicateHashesComputed: any) => Promise<void>;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/hashCalculators.js

@@ -0,0 +1,138 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+import { useBatchProcessing } from "./useBatchProcessing";
+import { rowIsEmpty } from "./utils";
+var findDuplicateHashes = function (sortedHashes) {
+    var _a;
+    var results = [];
+    for (var i = 0; i < sortedHashes.length - 1; i++) {
+        if (sortedHashes[i + 1].hash == sortedHashes[i].hash ||
+            ((_a = sortedHashes[i - 1]) === null || _a === void 0 ? void 0 : _a.hash) == sortedHashes[i].hash) {
+            results.push(sortedHashes[i]);
+        }
+    }
+    return results;
+};
+function compareHashes(a, b) {
+    if (a.hash < b.hash)
+        return -1;
+    if (a.hash > b.hash)
+        return 1;
+    return 0;
+}
+// generate hashes that can be used for comparing two records
+// for the sake of tracking duplicates
+// if the hash for one record equals the hash for another,
+// the two records are duplicates
+export var RecordHashCalculator = function () {
+    var computeHashesForBatch = function (dataSet, fields) {
+        var hashMap = [];
+        var _loop_1 = function (recordKey) {
+            var record = dataSet[recordKey];
+            var key = fields.map(function (field) { return record[field.name]; }).join("|");
+            if (!rowIsEmpty(record)) {
+                hashMap.push({ id: record._meta.id, hash: key });
+            }
+        };
+        for (var recordKey in dataSet) {
+            _loop_1(recordKey);
+        }
+        return hashMap;
+    };
+    var calculatingHashes = false;
+    var computeHashesForRecords = function (dataSet, fields, _onHashesComputed) {
+        if (!dataSet || calculatingHashes) {
+            return;
+        }
+        var processInBatches = useBatchProcessing().processInBatches;
+        calculatingHashes = true;
+        var results = [];
+        processInBatches(dataSet, function (batch) {
+            results = results.concat(computeHashesForBatch(batch, fields));
+        });
+        results.sort(compareHashes);
+        var duplicateHashes = findDuplicateHashes(results);
+        _onHashesComputed(duplicateHashes);
+        calculatingHashes = false;
+    };
+    return {
+        computeHashesForRecords: computeHashesForRecords,
+    };
+};
+// generate hashes that can be used for comparing the values of a specific field
+// for the sake of tracking uniqueness
+export var RecordValueHashCalculator = function () {
+    var columnsBeingCalculated = [];
+    var computeHashesForBatch = function (dataSet, field) {
+        var hashMap = [];
+        var isCaseInsensitive = field.validations.find(function (v) {
+            return v.name === "unique_case_insensitive";
+        }) !== undefined;
+        for (var recordKey in dataSet) {
+            var record = dataSet[recordKey];
+            var key = record[field.name];
+            if (isCaseInsensitive) {
+                key = key.toLowerCase();
+            }
+            if (key) {
+                hashMap.push({ id: record._meta.id, hash: key });
+            }
+        }
+        return hashMap;
+    };
+    var computeDuplicateValueHashes = function (dataSet, _field, _onDuplicateHashesComputed) { return __awaiter(void 0, void 0, void 0, function () {
+        var processInBatches, results;
+        return __generator(this, function (_a) {
+            if (!dataSet || columnsBeingCalculated[_field.name]) {
+                return [2 /*return*/];
+            }
+            columnsBeingCalculated[_field.name] = true;
+            processInBatches = useBatchProcessing().processInBatches;
+            results = [];
+            processInBatches(dataSet, function (batch) {
+                results = results.concat(computeHashesForBatch(batch, _field));
+            });
+            results.sort(compareHashes);
+            _onDuplicateHashesComputed(results);
+            columnsBeingCalculated[_field.name] = false;
+            return [2 /*return*/];
+        });
+    }); };
+    return {
+        computeDuplicateValueHashes: computeDuplicateValueHashes,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/index.d.ts

@@ -1,47 +1,25 @@
 import React from "react";
 import { SpreadsheetOptions } from "../../../..";
-import { BatchValidationErrors, DuplicateRecords, Field, Record, RecordDataSet } from "../../../../types";
-import { SortBy, SpreadsheetProps } from "../common/types";
-import { ValidationState } from "../hooks";
-import { SpreadsheetFilter } from "./useSpreadsheetFilter";
+import { Field, RecordDataSet } from "../../../../types";
+import { SpreadsheetProps } from "../common/types";
 export type OnDataSetChanged = (dataSet: RecordDataSet) => void;
-export type OnValidationStateChanged = (state: ValidationState) => void;
 type State = {
-    data: Record[];
+    filteredDataIds: string[];
     fields: Field[];
     isReadOnly?: boolean;
-    errors: BatchValidationErrors;
-    warnings: BatchValidationErrors;
     options: SpreadsheetOptions;
-    sortByFieldErrors: string;
-    sortByFieldWarnings: string;
-    updateRecord: (record: Record) => void;
-    setSortByFieldErrors: React.Dispatch<React.SetStateAction<string>>;
     selectedRows: string[];
     setSelectedRows: React.Dispatch<React.SetStateAction<string[]>>;
-    setSortByFieldWarnings: React.Dispatch<React.SetStateAction<string>>;
-    deleteRecords: (selectedRecords: string[]) => void;
     lastClickedIndexRef: React.MutableRefObject<number | null>;
-    filteredInputValue: string;
-    setFilteredInputValue: React.Dispatch<React.SetStateAction<string>>;
-    addNewRowsToSpreadsheet: () => void;
-    spreadsheetFilter: SpreadsheetFilter;
-    filterRecordsByRowType: (filterByRow: SpreadsheetFilter["byRow"]) => void;
-    filterRecordsByFieldErrors: (filterByColumn: string) => void;
-    resetSpreadsheetFilter: () => void;
-    recordsWithValuesCount: number;
-    filteredDataCount: number;
     checkboxColumnWidth: number;
-    isFiltering: boolean;
-    sortBy: SortBy | null;
-    setSortBy: React.Dispatch<React.SetStateAction<SortBy>>;
     setIsTransformTooltipOpened: React.Dispatch<React.SetStateAction<boolean>>;
     isTransformTooltipOpened: boolean;
-    duplicateRecords: DuplicateRecords;
-} & ValidationState;
+    inefficientRerender: boolean;
+    setInefficientRerender: React.Dispatch<React.SetStateAction<boolean>>;
+};
 type Props = SpreadsheetProps & {
     children: JSX.Element;
 };
-export declare const SpreadsheetContextProvider: ({ initialDataSet, fields, options, isReadOnly, children, onValidateRecord, onDataSetChanged, onValidationStateChanged, selectedRows, setSelectedRows, getSortCompareFn, }: Props) => JSX.Element;
+export declare const SpreadsheetContextProvider: ({ fields, options, isReadOnly, children, onValidateRecord, selectedRows, setSelectedRows, }: Props) => JSX.Element;
 export declare const useSpreadsheetContext: () => State;
 export {};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/index.js

@@ -21,207 +21,25 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 import { jsx as _jsx } from "react/jsx-runtime";
-import { get } from "lodash";
-import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, } from "react";
-import { addMeta } from "../../../data";
-import { useValidationState } from "../hooks";
-import { useSpreadsheetFilter, } from "./useSpreadsheetFilter";
-import { filterRecordsByType, filterRecordsByError, filterRecordsBySearchValue, } from "./filterMethods";
-import { hasValueInAnyField, rowIsEmpty } from "./utils";
-var getDefaultSortCompareFn = function (sortBy) {
-    return function (a, b) {
-        var aField = get(a, sortBy.field);
-        var bField = get(b, sortBy.field);
-        if (aField === bField)
-            return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-        if (!aField)
-            return 1;
-        if (!bField)
-            return -1;
-        if (sortBy.direction === "asc")
-            return aField > bField ? 1 : -1;
-        return aField < bField ? 1 : -1;
-    };
-};
+import { createContext, useContext, useMemo, useRef, useState, } from "react";
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
 var SpreadsheetContext = createContext({});
 export var SpreadsheetContextProvider = function (_a) {
-    var initialDataSet = _a.initialDataSet, fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, onDataSetChanged = _a.onDataSetChanged, onValidationStateChanged = _a.onValidationStateChanged, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows, _b = _a.getSortCompareFn, getSortCompareFn = _b === void 0 ? getDefaultSortCompareFn : _b;
-    var _c = useState(initialDataSet), dataSet = _c[0], setDataSet = _c[1];
-    var _d = useState(null), sortByFieldErrors = _d[0], setSortByFieldErrors = _d[1];
-    var _e = useState(null), sortByFieldWarnings = _e[0], setSortByFieldWarnings = _e[1];
-    var _f = useState(""), filteredInputValue = _f[0], setFilteredInputValue = _f[1];
-    var _g = useState(null), sortBy = _g[0], setSortBy = _g[1];
-    var addRowTimeoutRef = useRef(null);
-    var _h = useState(false), isTransformTooltipOpened = _h[0], setIsTransformTooltipOpened = _h[1];
-    var _j = useSpreadsheetFilter(), spreadsheetFilter = _j.spreadsheetFilter, filterRecordsByRowType = _j.filterRecordsByRowType, filterRecordsByFieldErrors = _j.filterRecordsByFieldErrors, resetSpreadsheetFilter = _j.resetSpreadsheetFilter;
-    var _k = useValidationState({
-        dataSet: dataSet,
-        initialDataSet: initialDataSet,
-        fields: fields,
-        options: options,
-        isReadOnly: isReadOnly,
-        setDataSet: setDataSet,
-        onValidateRecord: onValidateRecord,
-    }), updateRecordAndRevalidate = _k.updateRecordAndRevalidate, deleteRecords = _k.deleteRecords, warnings = _k.warnings, errors = _k.errors, validationState = __rest(_k, ["updateRecordAndRevalidate", "deleteRecords", "warnings", "errors"]);
-    var addNewRowsToSpreadsheet = function () {
-        addRowTimeoutRef.current = setTimeout(function () {
-            var _a, _b;
-            var highestRowIndex = -1;
-            for (var key in dataSet)
-                if (((_b = (_a = dataSet[key]) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
-                    highestRowIndex = dataSet[key]._meta.rowIndex;
-            var newRow = addMeta({}, highestRowIndex + 1);
-            setDataSet(function (oldValues) {
-                var _a;
-                return (__assign(__assign({}, oldValues), (_a = {}, _a[newRow._meta.id] = newRow, _a)));
-            });
-            addRowTimeoutRef.current = null;
-        }, 300);
-    };
-    var handleDeleteErrors = function (selectedRows) {
-        deleteRecords(selectedRows, errors, warnings);
-    };
-    var duplicateRecords = useMemo(function () {
-        if (isReadOnly)
-            return { records: {}, count: 0 };
-        var uniqueRecords = new Map();
-        var duplicateGroups = [];
-        var _loop_1 = function (recordKey) {
-            if (Object.prototype.hasOwnProperty.call(dataSet, recordKey)) {
-                var record_1 = dataSet[recordKey];
-                if (rowIsEmpty(record_1))
-                    return "continue";
-                var key = fields.map(function (field) { return record_1[field.name]; }).join("|");
-                if (uniqueRecords.has(key)) {
-                    var saved_1 = uniqueRecords.get(key);
-                    var existingGroup = duplicateGroups.find(function (group) {
-                        return group.some(function (item) { return item._meta.id === saved_1._meta.id; });
-                    });
-                    if (existingGroup) {
-                        existingGroup.push(record_1);
-                    }
-                    else {
-                        duplicateGroups.push([saved_1, record_1]);
-                    }
-                }
-                else {
-                    uniqueRecords.set(key, record_1);
-                }
-            }
-        };
-        for (var recordKey in dataSet) {
-            _loop_1(recordKey);
-        }
-        var count = duplicateGroups.length;
-        var records = {};
-        for (var i = 0; i < duplicateGroups.length; i++) {
-            var group = duplicateGroups[i];
-            for (var j = 0; j < group.length; j++) {
-                var record = group[j];
-                records[record._meta.id] = __assign({}, record);
-            }
-        }
-        return { records: records, count: count };
-    }, [dataSet]);
-    var dataSnapshot = useMemo(function () {
-        var filteredData = dataSet;
-        switch (spreadsheetFilter.type) {
-            case "rowBased":
-                if (spreadsheetFilter.byRow === "duplicates") {
-                    filteredData = duplicateRecords.records;
-                    break;
-                }
-                filteredData = filterRecordsByType(dataSet, spreadsheetFilter.byRow);
-                break;
-            case "columnBased":
-                filteredData = filterRecordsByError(dataSet, spreadsheetFilter.byColumn.field, errors);
-                break;
-        }
-        var filteredDatasetByString = filteredInputValue
-            ? filterRecordsBySearchValue(filteredData, filteredInputValue)
-            : filteredData;
-        var dataSetValues = Object.values(filteredDatasetByString);
-        if (sortBy) {
-            return dataSetValues.sort(getSortCompareFn(sortBy));
-        }
-        else if (!!sortByFieldErrors || !!sortByFieldWarnings)
-            return dataSetValues.sort(function (a, b) {
-                var aHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(a, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                var bHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(b, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                if (aHasErrors === bHasErrors) {
-                    return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-                }
-                return bHasErrors ? 1 : -1;
-            });
-        return dataSetValues;
-    }, [
-        filteredInputValue,
-        spreadsheetFilter,
-        sortBy,
-        Object.keys(dataSet).length,
-        duplicateRecords.count,
-        sortByFieldErrors,
-        sortByFieldWarnings,
-    ]);
-    var recordsWithValuesCount = useMemo(function () {
-        return Object.values(dataSet).filter(hasValueInAnyField).length;
-    }, [dataSet]);
-    var filteredDataCount = useMemo(function () {
-        return Object.values(dataSnapshot).filter(hasValueInAnyField).length;
-    }, [dataSnapshot]);
-    var isFiltering = useMemo(function () { return !!filteredInputValue; }, [filteredInputValue]);
-    useEffect(function () {
-        onDataSetChanged && onDataSetChanged(dataSet);
-        if (addRowTimeoutRef.current) {
-            // avoid adding new rows when the user is typing
-            clearTimeout(addRowTimeoutRef.current);
-        }
-        var dataSetKeys = Object.keys(dataSet);
-        var latestDataSet = dataSet[dataSetKeys[dataSetKeys.length - 1]];
-        var latestDataSetValuesWithoutMeta = Object.values(latestDataSet).filter(function (key) { return key !== "_meta"; });
-        var hasValueInTheLastObject = latestDataSetValuesWithoutMeta
-            .filter(function (value) {
-            return value !== null && value !== undefined && typeof value !== "object";
-        })
-            .some(function (value) { return value !== "null" && value !== "undefined"; });
-        if (hasValueInTheLastObject) {
-            addNewRowsToSpreadsheet();
-        }
-    }, [dataSet]);
-    // Review this later, this probably isn't the best approach
-    useEffect(function () {
-        setDataSet(initialDataSet);
-    }, [initialDataSet]);
-    useEffect(function () {
-        onValidationStateChanged &&
-            onValidationStateChanged(__assign(__assign({}, validationState), { errors: errors, warnings: warnings }));
-    }, [validationState, onValidationStateChanged]);
-    var handleSortByFieldErrors = useCallback(function (field) {
-        setSortByFieldErrors(field === sortByFieldErrors ? null : field);
-        setSortByFieldWarnings(null);
-        setSortBy(null);
-    }, [sortByFieldErrors]);
-    var handleSortByFieldWarnings = useCallback(function (field) {
-        setSortByFieldWarnings(field === sortByFieldWarnings ? null : field);
-        setSortByFieldErrors(null);
-        setSortBy(null);
-    }, [sortByFieldWarnings]);
+    var fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows;
+    var _b = useState(false), isTransformTooltipOpened = _b[0], setIsTransformTooltipOpened = _b[1];
+    var _c = useState(false), inefficientRerender = _c[0], setInefficientRerender = _c[1];
+    var filteredDataIds = useFilteredDataContext().filteredDataIds;
+    var _d = useValidationContext(), calculateColumnErrorCounts = _d.calculateColumnErrorCounts, calculateColumnWarningCounts = _d.calculateColumnWarningCounts, warnings = _d.warnings, errors = _d.errors, validationState = __rest(_d, ["calculateColumnErrorCounts", "calculateColumnWarningCounts", "warnings", "errors"]);
     var lastClickedIndexRef = useRef(null);
     var checkboxColumnWidth = useMemo(function () {
-        var lastRowIndexLength = dataSnapshot.length.toString().length;
+        var lastRowIndexLength = filteredDataIds.length.toString().length;
         var singleLetterWidth = 15;
         var horizontalSpacing = 8;
         var baseCheckboxColumnWidth = lastRowIndexLength * singleLetterWidth + horizontalSpacing;
         return baseCheckboxColumnWidth < 40 ? 40 : baseCheckboxColumnWidth;
-    }, [dataSnapshot]);
-    var updatedDataSnapshot = useMemo(function () {
-        return dataSnapshot.map(function (data) {
-            var _a;
-            var updatedData = dataSet[(_a = data._meta) === null || _a === void 0 ? void 0 : _a.id];
-            return updatedData;
-        });
-    }, [dataSet, dataSnapshot]);
-    var value = __assign(__assign({ data: updatedDataSnapshot, fields: fields }, validationState), { isReadOnly: isReadOnly, sortByFieldErrors: sortByFieldErrors, sortByFieldWarnings: sortByFieldWarnings, options: options, recordsWithValuesCount: recordsWithValuesCount, filteredDataCount: filteredDataCount, sortBy: sortBy, warnings: warnings, errors: errors, setSortByFieldErrors: handleSortByFieldErrors, setSortByFieldWarnings: handleSortByFieldWarnings, onValidateRecord: onValidateRecord, updateRecord: updateRecordAndRevalidate, selectedRows: selectedRows, setSelectedRows: setSelectedRows, deleteRecords: handleDeleteErrors, filteredInputValue: filteredInputValue, setFilteredInputValue: setFilteredInputValue, lastClickedIndexRef: lastClickedIndexRef, addNewRowsToSpreadsheet: addNewRowsToSpreadsheet, spreadsheetFilter: spreadsheetFilter, resetSpreadsheetFilter: resetSpreadsheetFilter, filterRecordsByRowType: filterRecordsByRowType, filterRecordsByFieldErrors: filterRecordsByFieldErrors, checkboxColumnWidth: checkboxColumnWidth, setSortBy: setSortBy, isFiltering: isFiltering, setIsTransformTooltipOpened: setIsTransformTooltipOpened, isTransformTooltipOpened: isTransformTooltipOpened, duplicateRecords: duplicateRecords });
+    }, [filteredDataIds.length]);
+    var value = __assign(__assign({ filteredDataIds: filteredDataIds, fields: fields }, validationState), { isReadOnly: isReadOnly, options: options, onValidateRecord: onValidateRecord, selectedRows: selectedRows, setSelectedRows: setSelectedRows, lastClickedIndexRef: lastClickedIndexRef, checkboxColumnWidth: checkboxColumnWidth, setIsTransformTooltipOpened: setIsTransformTooltipOpened, isTransformTooltipOpened: isTransformTooltipOpened, inefficientRerender: inefficientRerender, setInefficientRerender: setInefficientRerender });
     return (_jsx(SpreadsheetContext.Provider, __assign({ value: value }, { children: children })));
 };
 export var useSpreadsheetContext = function () { return useContext(SpreadsheetContext); };

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.d.ts

@@ -0,0 +1,5 @@
+import { Record, RecordDataSet } from "../../../../types";
+export declare const useBatchProcessing: (batchPageSize?: number) => {
+    processRecordsInBatches: (records: Record[], handleBatch: any) => Promise<void>;
+    processInBatches: (dataSet: RecordDataSet, handleBatch: any) => void;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.js

@@ -0,0 +1,72 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+export var useBatchProcessing = function (batchPageSize) {
+    if (batchPageSize === void 0) { batchPageSize = 10000; }
+    var processRecordsInBatches = function (records, handleBatch) { return __awaiter(void 0, void 0, void 0, function () {
+        var dataSetLength, batchSize, batchCount, i, idx, batch;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    dataSetLength = records.length;
+                    batchSize = Math.min(Math.floor(batchPageSize), dataSetLength) || dataSetLength;
+                    batchCount = Math.ceil(dataSetLength / batchSize);
+                    i = 0;
+                    _a.label = 1;
+                case 1:
+                    if (!(i < batchCount)) return [3 /*break*/, 4];
+                    idx = i * batchSize;
+                    batch = records.slice(idx, idx + batchSize);
+                    return [4 /*yield*/, handleBatch(batch)];
+                case 2:
+                    _a.sent();
+                    _a.label = 3;
+                case 3:
+                    i++;
+                    return [3 /*break*/, 1];
+                case 4: return [2 /*return*/];
+            }
+        });
+    }); };
+    var processInBatches = function (dataSet, handleBatch) {
+        var records = Object.values(dataSet);
+        processRecordsInBatches(records, handleBatch);
+    };
+    return {
+        processRecordsInBatches: processRecordsInBatches,
+        processInBatches: processInBatches,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.d.ts

@@ -0,0 +1,4 @@
+export declare const useDataSetManipulation: () => {
+    appendNewRecordToSpreadsheet: () => void;
+    deleteRecordsFromSpreadsheet: (recordIds: string[]) => void;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.js

@@ -0,0 +1,46 @@
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useDataSetContext } from "../DataSetContextProvider";
+import { hasValueInAnyField } from "./utils";
+import { addMeta } from "../../../../Importer/data";
+import { useEventManagerContext } from "../../../../common/EventManagerContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
+import { useDuplicateDataContext } from "../DuplicateDataContextProvider";
+export var useDataSetManipulation = function () {
+    var _a = useFilteredDataContext(), lastFilteredRecordId = _a.lastFilteredRecordId, setRecalculateFilters = _a.setRecalculateFilters;
+    var _b = useDataSetContext(), getRecord = _b.getRecord, setRecord = _b.setRecord, dataSet = _b.dataSet, deleteRecord = _b.deleteRecord, emptyRecordIds = _b.emptyRecordIds;
+    var _c = useEventManagerContext(), emit = _c.emit, EVENTS = _c.EVENTS;
+    var _d = useValidationContext(), errors = _d.errors, warnings = _d.warnings, calculateColumnErrorCounts = _d.calculateColumnErrorCounts, calculateColumnWarningCounts = _d.calculateColumnWarningCounts;
+    var updateDuplicateValues = useDuplicateDataContext().updateDuplicateValues;
+    var updateRecordAndRevalidate = useValidationContext().updateRecordAndRevalidate;
+    var appendNewRecordToSpreadsheet = function () {
+        var _a, _b;
+        var _record = getRecord(lastFilteredRecordId.current);
+        if (!hasValueInAnyField(_record)) {
+            return;
+        }
+        var highestRowIndex = -1;
+        for (var recordId in dataSet.current)
+            if (((_b = (_a = getRecord(recordId)) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
+                highestRowIndex = getRecord(recordId)._meta.rowIndex;
+        var newRow = addMeta({}, highestRowIndex + 1);
+        setRecord(newRow._meta.id, newRow);
+        emit(EVENTS.ON_LAST_ROW_CHANGED);
+    };
+    var deleteRecordsFromSpreadsheet = function (recordIds) {
+        recordIds.forEach(function (recordId) {
+            updateDuplicateValues(getRecord(recordId), {}, false, updateRecordAndRevalidate);
+            deleteRecord(recordId);
+            calculateColumnErrorCounts(errors.current[recordId], {});
+            calculateColumnWarningCounts(warnings.current[recordId], {});
+            emptyRecordIds.current.delete(recordId);
+        });
+        setRecalculateFilters(function (prev) { return !prev; });
+        emit(EVENTS.UPDATE_COLUMN_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ROW_COUNT);
+    };
+    return {
+        appendNewRecordToSpreadsheet: appendNewRecordToSpreadsheet,
+        deleteRecordsFromSpreadsheet: deleteRecordsFromSpreadsheet,
+    };
+};