fuse-importer 0.27.0 → 0.30.0

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/hashCalculators.js

@@ -0,0 +1,152 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+import { useBatchProcessing } from "./useBatchProcessing";
+import { rowIsEmpty } from "./utils";
+var findDuplicateHashes = function (sortedHashes) {
+    var _a;
+    var results = [];
+    for (var i = 0; i < sortedHashes.length - 1; i++) {
+        if (sortedHashes[i + 1].hash == sortedHashes[i].hash ||
+            ((_a = sortedHashes[i - 1]) === null || _a === void 0 ? void 0 : _a.hash) == sortedHashes[i].hash) {
+            results.push(sortedHashes[i]);
+        }
+    }
+    return results;
+};
+function compareHashes(a, b) {
+    if (a.hash < b.hash)
+        return -1;
+    if (a.hash > b.hash)
+        return 1;
+    return 0;
+}
+// generate hashes that can be used for comparing two records
+// for the sake of tracking duplicates
+// if the hash for one record equals the hash for another,
+// the two records are duplicates
+export var RecordHashCalculator = function () {
+    var computeHashesForBatch = function (dataSet, fields) {
+        var hashMap = [];
+        var _loop_1 = function (recordKey) {
+            var record = dataSet[recordKey];
+            var key = fields.map(function (field) { return record[field.name]; }).join("|");
+            if (!rowIsEmpty(record)) {
+                hashMap.push({ id: record._meta.id, hash: key });
+            }
+        };
+        for (var recordKey in dataSet) {
+            _loop_1(recordKey);
+        }
+        return hashMap;
+    };
+    var calculatingHashes = false;
+    var computeHashesForRecords = function (dataSet, fields, _onHashesComputed) { return __awaiter(void 0, void 0, void 0, function () {
+        var processInBatches, results, duplicateHashes;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    if (!dataSet || calculatingHashes) {
+                        return [2 /*return*/];
+                    }
+                    processInBatches = useBatchProcessing().processInBatches;
+                    calculatingHashes = true;
+                    results = [];
+                    return [4 /*yield*/, processInBatches(dataSet, function (batch) {
+                            results = results.concat(computeHashesForBatch(batch, fields));
+                        })];
+                case 1:
+                    _a.sent();
+                    results.sort(compareHashes);
+                    duplicateHashes = findDuplicateHashes(results);
+                    _onHashesComputed(duplicateHashes);
+                    calculatingHashes = false;
+                    return [2 /*return*/];
+            }
+        });
+    }); };
+    return {
+        computeHashesForRecords: computeHashesForRecords,
+    };
+};
+// generate hashes that can be used for comparing the values of a specific field
+// for the sake of tracking uniqueness
+export var RecordValueHashCalculator = function () {
+    var columnsBeingCalculated = [];
+    var computeHashesForBatch = function (dataSet, field) {
+        var hashMap = [];
+        var isCaseInsensitive = field.validations.find(function (v) {
+            return v.name === "unique_case_insensitive";
+        }) !== undefined;
+        for (var recordKey in dataSet) {
+            var record = dataSet[recordKey];
+            var key = record[field.name];
+            if (isCaseInsensitive) {
+                key = key.toLowerCase();
+            }
+            if (key) {
+                hashMap.push({ id: record._meta.id, hash: key });
+            }
+        }
+        return hashMap;
+    };
+    var computeDuplicateValueHashes = function (dataSet, _field, _onDuplicateHashesComputed) { return __awaiter(void 0, void 0, void 0, function () {
+        var processInBatches, results;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    if (!dataSet || columnsBeingCalculated[_field.name]) {
+                        return [2 /*return*/];
+                    }
+                    columnsBeingCalculated[_field.name] = true;
+                    processInBatches = useBatchProcessing().processInBatches;
+                    results = [];
+                    return [4 /*yield*/, processInBatches(dataSet, function (batch) {
+                            results = results.concat(computeHashesForBatch(batch, _field));
+                        })];
+                case 1:
+                    _a.sent();
+                    results.sort(compareHashes);
+                    _onDuplicateHashesComputed(results);
+                    columnsBeingCalculated[_field.name] = false;
+                    return [2 /*return*/];
+            }
+        });
+    }); };
+    return {
+        computeDuplicateValueHashes: computeDuplicateValueHashes,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/index.d.ts

@@ -1,46 +1,23 @@
 import React from "react";
 import { SpreadsheetOptions } from "../../../..";
-import { BatchValidationErrors, Field, Record, RecordDataSet } from "../../../../types";
-import { SortBy, SpreadsheetProps } from "../common/types";
-import { ValidationState } from "../hooks";
-import { SpreadsheetFilter } from "./useSpreadsheetFilter";
+import { Field, RecordDataSet } from "../../../../types";
+import { SpreadsheetProps } from "../common/types";
 export type OnDataSetChanged = (dataSet: RecordDataSet) => void;
-export type OnValidationStateChanged = (state: ValidationState) => void;
 type State = {
-    data: Record[];
+    filteredDataIds: string[];
     fields: Field[];
-    isLoading: boolean;
     isReadOnly?: boolean;
-    errors: BatchValidationErrors;
-    warnings: BatchValidationErrors;
     options: SpreadsheetOptions;
-    sortByFieldErrors: string;
-    sortByFieldWarnings: string;
-    updateRecord: (record: Record) => void;
-    setIsLoading: React.Dispatch<React.SetStateAction<boolean>>;
-    setSortByFieldErrors: React.Dispatch<React.SetStateAction<string>>;
     selectedRows: string[];
     setSelectedRows: React.Dispatch<React.SetStateAction<string[]>>;
-    setSortByFieldWarnings: React.Dispatch<React.SetStateAction<string>>;
-    deleteRecords: (selectedRecords: string[]) => void;
     lastClickedIndexRef: React.MutableRefObject<number | null>;
-    filteredInputValue: string;
-    setFilteredInputValue: React.Dispatch<React.SetStateAction<string>>;
-    addNewRowsToSpreadsheet: () => void;
-    spreadsheetFilter: SpreadsheetFilter;
-    filterRecordsByRowType: (filterByRow: SpreadsheetFilter["byRow"]) => void;
-    filterRecordsByFieldErrors: (filterByColumn: string) => void;
-    resetSpreadsheetFilter: () => void;
-    recordsWithValuesCount: number;
-    filteredDataCount: number;
     checkboxColumnWidth: number;
-    isFiltering: boolean;
-    sortBy: SortBy | null;
-    setSortBy: React.Dispatch<React.SetStateAction<SortBy>>;
-} & ValidationState;
+    inefficientRerender: boolean;
+    setInefficientRerender: React.Dispatch<React.SetStateAction<boolean>>;
+};
 type Props = SpreadsheetProps & {
     children: JSX.Element;
 };
-export declare const SpreadsheetContextProvider: ({ initialDataSet, fields, options, isReadOnly, children, onValidateRecord, onDataSetChanged, onValidationStateChanged, selectedRows, setSelectedRows, getSortCompareFn, }: Props) => JSX.Element;
+export declare const SpreadsheetContextProvider: ({ fields, options, isReadOnly, children, onValidateRecord, selectedRows, setSelectedRows, }: Props) => JSX.Element;
 export declare const useSpreadsheetContext: () => State;
 export {};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/index.js

@@ -21,155 +21,24 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 import { jsx as _jsx } from "react/jsx-runtime";
-import { get } from "lodash";
-import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, } from "react";
-import { addMeta } from "../../../data";
-import { useValidationState } from "../hooks";
-import { useSpreadsheetFilter, } from "./useSpreadsheetFilter";
-import { filterRecordsByType, filterRecordsByError, filterRecordsBySearchValue, } from "./filterMethods";
-import { hasValueInAnyField } from "./utils";
-var getDefaultSortCompareFn = function (sortBy) {
-    return function (a, b) {
-        var aField = get(a, sortBy.field);
-        var bField = get(b, sortBy.field);
-        if (aField === bField)
-            return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-        if (!aField)
-            return 1;
-        if (!bField)
-            return -1;
-        if (sortBy.direction === "asc")
-            return aField > bField ? 1 : -1;
-        return aField < bField ? 1 : -1;
-    };
-};
+import { createContext, useContext, useMemo, useRef, useState, } from "react";
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
 var SpreadsheetContext = createContext({});
 export var SpreadsheetContextProvider = function (_a) {
-    var initialDataSet = _a.initialDataSet, fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, onDataSetChanged = _a.onDataSetChanged, onValidationStateChanged = _a.onValidationStateChanged, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows, _b = _a.getSortCompareFn, getSortCompareFn = _b === void 0 ? getDefaultSortCompareFn : _b;
-    var _c = useState(initialDataSet), dataSet = _c[0], setDataSet = _c[1];
-    var _d = useState(false), isLoading = _d[0], setIsLoading = _d[1];
-    var _e = useState(null), sortByFieldErrors = _e[0], setSortByFieldErrors = _e[1];
-    var _f = useState(null), sortByFieldWarnings = _f[0], setSortByFieldWarnings = _f[1];
-    var _g = useState(""), filteredInputValue = _g[0], setFilteredInputValue = _g[1];
-    var _h = useState(null), sortBy = _h[0], setSortBy = _h[1];
-    var addRowTimeoutRef = useRef(null);
-    var _j = useSpreadsheetFilter(), spreadsheetFilter = _j.spreadsheetFilter, filterRecordsByRowType = _j.filterRecordsByRowType, filterRecordsByFieldErrors = _j.filterRecordsByFieldErrors, resetSpreadsheetFilter = _j.resetSpreadsheetFilter;
-    var _k = useValidationState({
-        dataSet: dataSet,
-        initialDataSet: initialDataSet,
-        fields: fields,
-        options: options,
-        isReadOnly: isReadOnly,
-        setDataSet: setDataSet,
-        setIsLoading: setIsLoading,
-        onValidateRecord: onValidateRecord,
-    }), updateRecordAndRevalidate = _k.updateRecordAndRevalidate, deleteRecords = _k.deleteRecords, warnings = _k.warnings, errors = _k.errors, validationState = __rest(_k, ["updateRecordAndRevalidate", "deleteRecords", "warnings", "errors"]);
-    var addNewRowsToSpreadsheet = function () {
-        addRowTimeoutRef.current = setTimeout(function () {
-            var _a, _b;
-            var highestRowIndex = -1;
-            for (var key in dataSet)
-                if (((_b = (_a = dataSet[key]) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
-                    highestRowIndex = dataSet[key]._meta.rowIndex;
-            var newRow = addMeta({}, highestRowIndex + 1);
-            setDataSet(function (oldValues) {
-                var _a;
-                return (__assign(__assign({}, oldValues), (_a = {}, _a[newRow._meta.id] = newRow, _a)));
-            });
-            addRowTimeoutRef.current = null;
-        }, 300);
-    };
-    var handleDeleteErrors = function (selectedRows) {
-        deleteRecords(selectedRows, errors, warnings);
-    };
-    var dataSnapshot = useMemo(function () {
-        var filteredData = spreadsheetFilter.type === "rowBased"
-            ? filterRecordsByType(dataSet, spreadsheetFilter.byRow)
-            : filterRecordsByError(dataSet, spreadsheetFilter.byColumn.field, errors);
-        var filteredDatasetByString = filteredInputValue
-            ? filterRecordsBySearchValue(filteredData, filteredInputValue)
-            : filteredData;
-        var dataSetValues = Object.values(filteredDatasetByString);
-        if (sortBy) {
-            return dataSetValues.sort(getSortCompareFn(sortBy));
-        }
-        else if (!!sortByFieldErrors || !!sortByFieldWarnings)
-            return dataSetValues.sort(function (a, b) {
-                var aHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(a, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                var bHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(b, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                if (aHasErrors === bHasErrors) {
-                    return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-                }
-                return bHasErrors ? 1 : -1;
-            });
-        return dataSetValues;
-    }, [
-        filteredInputValue,
-        spreadsheetFilter,
-        sortBy,
-        Object.keys(dataSet).length,
-        sortByFieldErrors,
-        sortByFieldWarnings,
-    ]);
-    var recordsWithValuesCount = useMemo(function () {
-        return Object.values(dataSet).filter(hasValueInAnyField).length;
-    }, [dataSet]);
-    var filteredDataCount = useMemo(function () {
-        return Object.values(dataSnapshot).filter(hasValueInAnyField).length;
-    }, [dataSnapshot]);
-    var isFiltering = useMemo(function () { return !!filteredInputValue; }, [filteredInputValue]);
-    useEffect(function () {
-        onDataSetChanged && onDataSetChanged(dataSet);
-        if (addRowTimeoutRef.current) {
-            // avoid adding new rows when the user is typing
-            clearTimeout(addRowTimeoutRef.current);
-        }
-        var dataSetKeys = Object.keys(dataSet);
-        var latestDataSet = dataSet[dataSetKeys[dataSetKeys.length - 1]];
-        var latestDataSetValuesWithoutMeta = Object.values(latestDataSet).filter(function (key) { return key !== "_meta"; });
-        var hasValueInTheLastObject = latestDataSetValuesWithoutMeta
-            .filter(function (value) {
-            return value !== null && value !== undefined && typeof value !== "object";
-        })
-            .some(function (value) { return value !== "null" && value !== "undefined"; });
-        if (hasValueInTheLastObject) {
-            addNewRowsToSpreadsheet();
-        }
-    }, [dataSet]);
-    // Review this later, this probably isn't the best approach
-    useEffect(function () {
-        setDataSet(initialDataSet);
-    }, [initialDataSet]);
-    useEffect(function () {
-        onValidationStateChanged &&
-            onValidationStateChanged(__assign(__assign({}, validationState), { errors: errors, warnings: warnings }));
-    }, [validationState, onValidationStateChanged]);
-    var handleSortByFieldErrors = useCallback(function (field) {
-        setSortByFieldErrors(field === sortByFieldErrors ? null : field);
-        setSortByFieldWarnings(null);
-        setSortBy(null);
-    }, [sortByFieldErrors]);
-    var handleSortByFieldWarnings = useCallback(function (field) {
-        setSortByFieldWarnings(field === sortByFieldWarnings ? null : field);
-        setSortByFieldErrors(null);
-        setSortBy(null);
-    }, [sortByFieldWarnings]);
+    var fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows;
+    var _b = useState(false), inefficientRerender = _b[0], setInefficientRerender = _b[1];
+    var filteredDataIds = useFilteredDataContext().filteredDataIds;
+    var _c = useValidationContext(), calculateColumnErrorCounts = _c.calculateColumnErrorCounts, calculateColumnWarningCounts = _c.calculateColumnWarningCounts, warnings = _c.warnings, errors = _c.errors, validationState = __rest(_c, ["calculateColumnErrorCounts", "calculateColumnWarningCounts", "warnings", "errors"]);
     var lastClickedIndexRef = useRef(null);
     var checkboxColumnWidth = useMemo(function () {
-        var lastRowIndexLength = dataSnapshot.length.toString().length;
+        var lastRowIndexLength = filteredDataIds.length.toString().length;
         var singleLetterWidth = 15;
         var horizontalSpacing = 8;
         var baseCheckboxColumnWidth = lastRowIndexLength * singleLetterWidth + horizontalSpacing;
         return baseCheckboxColumnWidth < 40 ? 40 : baseCheckboxColumnWidth;
-    }, [dataSnapshot]);
-    var updatedDataSnapshot = useMemo(function () {
-        return dataSnapshot.map(function (data) {
-            var _a;
-            var updatedData = dataSet[(_a = data._meta) === null || _a === void 0 ? void 0 : _a.id];
-            return updatedData;
-        });
-    }, [dataSet, dataSnapshot]);
-    var value = __assign(__assign({ data: updatedDataSnapshot, fields: fields, isLoading: isLoading }, validationState), { isReadOnly: isReadOnly, sortByFieldErrors: sortByFieldErrors, sortByFieldWarnings: sortByFieldWarnings, options: options, recordsWithValuesCount: recordsWithValuesCount, filteredDataCount: filteredDataCount, sortBy: sortBy, warnings: warnings, errors: errors, setIsLoading: setIsLoading, setSortByFieldErrors: handleSortByFieldErrors, setSortByFieldWarnings: handleSortByFieldWarnings, onValidateRecord: onValidateRecord, updateRecord: updateRecordAndRevalidate, selectedRows: selectedRows, setSelectedRows: setSelectedRows, deleteRecords: handleDeleteErrors, filteredInputValue: filteredInputValue, setFilteredInputValue: setFilteredInputValue, lastClickedIndexRef: lastClickedIndexRef, addNewRowsToSpreadsheet: addNewRowsToSpreadsheet, spreadsheetFilter: spreadsheetFilter, resetSpreadsheetFilter: resetSpreadsheetFilter, filterRecordsByRowType: filterRecordsByRowType, filterRecordsByFieldErrors: filterRecordsByFieldErrors, checkboxColumnWidth: checkboxColumnWidth, setSortBy: setSortBy, isFiltering: isFiltering });
+    }, [filteredDataIds.length]);
+    var value = __assign(__assign({ filteredDataIds: filteredDataIds, fields: fields }, validationState), { isReadOnly: isReadOnly, options: options, onValidateRecord: onValidateRecord, selectedRows: selectedRows, setSelectedRows: setSelectedRows, lastClickedIndexRef: lastClickedIndexRef, checkboxColumnWidth: checkboxColumnWidth, inefficientRerender: inefficientRerender, setInefficientRerender: setInefficientRerender });
     return (_jsx(SpreadsheetContext.Provider, __assign({ value: value }, { children: children })));
 };
 export var useSpreadsheetContext = function () { return useContext(SpreadsheetContext); };

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.d.ts

@@ -0,0 +1,5 @@
+import { Record, RecordDataSet } from "../../../../types";
+export declare const useBatchProcessing: (batchPageSize?: number) => {
+    processRecordsInBatches: (records: Record[], handleBatch: (x: any) => void) => Promise<void>;
+    processInBatches: (dataSet: RecordDataSet, handleBatch: (x: any) => void) => Promise<void>;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.js

@@ -0,0 +1,81 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+export var useBatchProcessing = function (batchPageSize) {
+    if (batchPageSize === void 0) { batchPageSize = 10000; }
+    var processRecordsInBatches = function (records, handleBatch) { return __awaiter(void 0, void 0, void 0, function () {
+        var dataSetLength, batchSize, batchCount, i, idx, batch;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    dataSetLength = records.length;
+                    batchSize = Math.min(Math.floor(batchPageSize), dataSetLength) || dataSetLength;
+                    batchCount = Math.ceil(dataSetLength / batchSize);
+                    i = 0;
+                    _a.label = 1;
+                case 1:
+                    if (!(i < batchCount)) return [3 /*break*/, 4];
+                    idx = i * batchSize;
+                    batch = records.slice(idx, idx + batchSize);
+                    return [4 /*yield*/, handleBatch(batch)];
+                case 2:
+                    _a.sent();
+                    _a.label = 3;
+                case 3:
+                    i++;
+                    return [3 /*break*/, 1];
+                case 4: return [2 /*return*/];
+            }
+        });
+    }); };
+    var processInBatches = function (dataSet, handleBatch) { return __awaiter(void 0, void 0, void 0, function () {
+        var records;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    records = Object.values(dataSet);
+                    return [4 /*yield*/, processRecordsInBatches(records, handleBatch)];
+                case 1:
+                    _a.sent();
+                    return [2 /*return*/];
+            }
+        });
+    }); };
+    return {
+        processRecordsInBatches: processRecordsInBatches,
+        processInBatches: processInBatches,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.d.ts

@@ -0,0 +1,5 @@
+export declare const useDataSetManipulation: () => {
+    appendNewRecordToSpreadsheet: () => void;
+    deleteRecordsFromSpreadsheet: (recordIds: string[]) => void;
+    findAndReplaceInSpreadsheet: (searchText: string, replaceText: string, matchCase: boolean, useRegex: boolean) => Promise<any[]>;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.js

@@ -0,0 +1,119 @@
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useDataSetContext } from "../DataSetContextProvider";
+import { hasValueInAnyField } from "./utils";
+import { addMeta } from "../../../../Importer/data";
+import { useEventManagerContext } from "../../../../common/EventManagerContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
+import { useDuplicateDataContext } from "../DuplicateDataContextProvider";
+import { useBatchProcessing } from "./useBatchProcessing";
+import useFindAndReplace from "../../../../common/Table/TableActions/useFindAndReplace";
+export var useDataSetManipulation = function () {
+    var _a = useFilteredDataContext(), lastFilteredRecordId = _a.lastFilteredRecordId, setRecalculateFilters = _a.setRecalculateFilters;
+    var _b = useDataSetContext(), getRecord = _b.getRecord, setRecord = _b.setRecord, dataSet = _b.dataSet, deleteRecord = _b.deleteRecord, emptyRecordIds = _b.emptyRecordIds;
+    var _c = useEventManagerContext(), emit = _c.emit, EVENTS = _c.EVENTS;
+    var _d = useValidationContext(), errors = _d.errors, warnings = _d.warnings, calculateColumnErrorCounts = _d.calculateColumnErrorCounts, calculateColumnWarningCounts = _d.calculateColumnWarningCounts;
+    var updateDuplicateValues = useDuplicateDataContext().updateDuplicateValues;
+    var updateRecordAndRevalidate = useValidationContext().updateRecordAndRevalidate;
+    var processInBatches = useBatchProcessing().processInBatches;
+    var replaceInRecord = useFindAndReplace().replaceInRecord;
+    var appendNewRecordToSpreadsheet = function () {
+        var _a, _b;
+        var _record = getRecord(lastFilteredRecordId.current);
+        if (!hasValueInAnyField(_record)) {
+            return;
+        }
+        var highestRowIndex = -1;
+        for (var recordId in dataSet.current)
+            if (((_b = (_a = getRecord(recordId)) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
+                highestRowIndex = getRecord(recordId)._meta.rowIndex;
+        var newRow = addMeta({}, highestRowIndex + 1);
+        setRecord(newRow._meta.id, newRow);
+        emit(EVENTS.ON_LAST_ROW_CHANGED);
+    };
+    var deleteRecordsFromSpreadsheet = function (recordIds) {
+        recordIds.forEach(function (recordId) {
+            updateDuplicateValues(getRecord(recordId), {}, false, updateRecordAndRevalidate);
+            deleteRecord(recordId);
+            calculateColumnErrorCounts(errors.current[recordId], {});
+            calculateColumnWarningCounts(warnings.current[recordId], {});
+            emptyRecordIds.current.delete(recordId);
+        });
+        setRecalculateFilters(function (prev) { return !prev; });
+        emit(EVENTS.UPDATE_COLUMN_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ROW_COUNT);
+    };
+    var findAndReplaceInSpreadsheet = function (searchText, replaceText, matchCase, useRegex) { return __awaiter(void 0, void 0, void 0, function () {
+        var cellsWithText;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    cellsWithText = [];
+                    return [4 /*yield*/, processInBatches(dataSet.current, function (batch) {
+                            for (var recordKey in batch) {
+                                var record = __assign({}, batch[recordKey]);
+                                if (replaceInRecord(record, searchText, replaceText, matchCase, useRegex)) {
+                                    cellsWithText.push(record);
+                                }
+                            }
+                        })];
+                case 1:
+                    _a.sent();
+                    updateRecordAndRevalidate(cellsWithText);
+                    return [2 /*return*/, cellsWithText];
+            }
+        });
+    }); };
+    return {
+        appendNewRecordToSpreadsheet: appendNewRecordToSpreadsheet,
+        deleteRecordsFromSpreadsheet: deleteRecordsFromSpreadsheet,
+        findAndReplaceInSpreadsheet: findAndReplaceInSpreadsheet,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useSpreadsheetFilter.d.ts

@@ -1,6 +1,6 @@
 export type SpreadsheetFilter = {
     type: "rowBased" | "columnBased";
-    byRow?: "all" | "invalid" | "valid";
+    byRow?: "all" | "invalid" | "duplicates";
     byColumn?: {
         type: "fieldWithErrors";
         field: string;

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/utils.d.ts

@@ -1,2 +1,3 @@
 import { Record } from "../../../../types";
 export declare const hasValueInAnyField: (record: Record) => boolean;
+export declare const rowIsEmpty: (row: Record) => boolean;

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/utils.js

@@ -1,3 +1,4 @@
+import { isEmpty } from "../common";
 export var hasValueInAnyField = function (record) {
     return Object.entries(record)
         .filter(function (_a) {
@@ -12,3 +13,10 @@ export var hasValueInAnyField = function (record) {
         return value !== null && value !== undefined;
     });
 };
+export var rowIsEmpty = function (row) {
+    return Object.keys(row).every(function (key) {
+        if (key === "_meta")
+            return true;
+        return isEmpty(row[key]);
+    });
+};

package/lib/esm/Importer/common/Spreadsheet/ValidationContextProvider/index.d.ts

@@ -0,0 +1,31 @@
+import { MutableRefObject } from "react";
+import { SpreadsheetOptions } from "../../../..";
+import { BatchValidationErrors, CalculateColumnCount, ColumnWithCount, Field, OnValidateRecord, Record } from "../../../../types";
+type UseValidationStateArgs = {
+    onValidateRecord?: OnValidateRecord;
+    fields: Field[];
+    options: SpreadsheetOptions;
+    isReadOnly: boolean;
+    children: any;
+};
+export type ValidationState = {
+    isValid: boolean;
+    isValidationCompleted: boolean;
+    validationPercentage: number;
+    errors: MutableRefObject<BatchValidationErrors>;
+    warnings: MutableRefObject<BatchValidationErrors>;
+    totalErrorCount: MutableRefObject<number>;
+    invalidRowsCountRef: MutableRefObject<number>;
+    columnsWithErrorCounts: MutableRefObject<ColumnWithCount>;
+    columnsWithWarningCounts: MutableRefObject<ColumnWithCount>;
+    calculateColumnErrorCounts?: CalculateColumnCount;
+    calculateColumnWarningCounts?: CalculateColumnCount;
+};
+type UseValidationStateReturnValue = ValidationState & {
+    updateRecordAndRevalidate: (record: Record | Record[], initialValidation?: boolean, loadingIndicator?: boolean) => Promise<void>;
+    spreadsheetInitialized: boolean;
+};
+export declare const ValidationContext: import("react").Context<UseValidationStateReturnValue>;
+export declare const ValidationContextProvider: ({ fields, isReadOnly, onValidateRecord, children, }: UseValidationStateArgs) => JSX.Element;
+export declare const useValidationContext: () => UseValidationStateReturnValue;
+export {};