fuse-importer 0.1.4-testing → 0.1.6-testing

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/index.js

@@ -21,207 +21,25 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 import { jsx as _jsx } from "react/jsx-runtime";
-import { get } from "lodash";
-import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, } from "react";
-import { addMeta } from "../../../data";
-import { useValidationState } from "../hooks";
-import { useSpreadsheetFilter, } from "./useSpreadsheetFilter";
-import { filterRecordsByType, filterRecordsByError, filterRecordsBySearchValue, } from "./filterMethods";
-import { hasValueInAnyField, rowIsEmpty } from "./utils";
-var getDefaultSortCompareFn = function (sortBy) {
-    return function (a, b) {
-        var aField = get(a, sortBy.field);
-        var bField = get(b, sortBy.field);
-        if (aField === bField)
-            return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-        if (!aField)
-            return 1;
-        if (!bField)
-            return -1;
-        if (sortBy.direction === "asc")
-            return aField > bField ? 1 : -1;
-        return aField < bField ? 1 : -1;
-    };
-};
+import { createContext, useContext, useMemo, useRef, useState, } from "react";
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
 var SpreadsheetContext = createContext({});
 export var SpreadsheetContextProvider = function (_a) {
-    var initialDataSet = _a.initialDataSet, fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, onDataSetChanged = _a.onDataSetChanged, onValidationStateChanged = _a.onValidationStateChanged, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows, _b = _a.getSortCompareFn, getSortCompareFn = _b === void 0 ? getDefaultSortCompareFn : _b;
-    var _c = useState(initialDataSet), dataSet = _c[0], setDataSet = _c[1];
-    var _d = useState(null), sortByFieldErrors = _d[0], setSortByFieldErrors = _d[1];
-    var _e = useState(null), sortByFieldWarnings = _e[0], setSortByFieldWarnings = _e[1];
-    var _f = useState(""), filteredInputValue = _f[0], setFilteredInputValue = _f[1];
-    var _g = useState(null), sortBy = _g[0], setSortBy = _g[1];
-    var addRowTimeoutRef = useRef(null);
-    var _h = useState(false), isTransformTooltipOpened = _h[0], setIsTransformTooltipOpened = _h[1];
-    var _j = useSpreadsheetFilter(), spreadsheetFilter = _j.spreadsheetFilter, filterRecordsByRowType = _j.filterRecordsByRowType, filterRecordsByFieldErrors = _j.filterRecordsByFieldErrors, resetSpreadsheetFilter = _j.resetSpreadsheetFilter;
-    var _k = useValidationState({
-        dataSet: dataSet,
-        initialDataSet: initialDataSet,
-        fields: fields,
-        options: options,
-        isReadOnly: isReadOnly,
-        setDataSet: setDataSet,
-        onValidateRecord: onValidateRecord,
-    }), updateRecordAndRevalidate = _k.updateRecordAndRevalidate, deleteRecords = _k.deleteRecords, warnings = _k.warnings, errors = _k.errors, validationState = __rest(_k, ["updateRecordAndRevalidate", "deleteRecords", "warnings", "errors"]);
-    var addNewRowsToSpreadsheet = function () {
-        addRowTimeoutRef.current = setTimeout(function () {
-            var _a, _b;
-            var highestRowIndex = -1;
-            for (var key in dataSet)
-                if (((_b = (_a = dataSet[key]) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
-                    highestRowIndex = dataSet[key]._meta.rowIndex;
-            var newRow = addMeta({}, highestRowIndex + 1);
-            setDataSet(function (oldValues) {
-                var _a;
-                return (__assign(__assign({}, oldValues), (_a = {}, _a[newRow._meta.id] = newRow, _a)));
-            });
-            addRowTimeoutRef.current = null;
-        }, 300);
-    };
-    var handleDeleteErrors = function (selectedRows) {
-        deleteRecords(selectedRows, errors, warnings);
-    };
-    var duplicateRecords = useMemo(function () {
-        if (isReadOnly)
-            return { records: {}, count: 0 };
-        var uniqueRecords = new Map();
-        var duplicateGroups = [];
-        var _loop_1 = function (recordKey) {
-            if (Object.prototype.hasOwnProperty.call(dataSet, recordKey)) {
-                var record_1 = dataSet[recordKey];
-                if (rowIsEmpty(record_1))
-                    return "continue";
-                var key = fields.map(function (field) { return record_1[field.name]; }).join("|");
-                if (uniqueRecords.has(key)) {
-                    var saved_1 = uniqueRecords.get(key);
-                    var existingGroup = duplicateGroups.find(function (group) {
-                        return group.some(function (item) { return item._meta.id === saved_1._meta.id; });
-                    });
-                    if (existingGroup) {
-                        existingGroup.push(record_1);
-                    }
-                    else {
-                        duplicateGroups.push([saved_1, record_1]);
-                    }
-                }
-                else {
-                    uniqueRecords.set(key, record_1);
-                }
-            }
-        };
-        for (var recordKey in dataSet) {
-            _loop_1(recordKey);
-        }
-        var count = duplicateGroups.length;
-        var records = {};
-        for (var i = 0; i < duplicateGroups.length; i++) {
-            var group = duplicateGroups[i];
-            for (var j = 0; j < group.length; j++) {
-                var record = group[j];
-                records[record._meta.id] = __assign({}, record);
-            }
-        }
-        return { records: records, count: count };
-    }, [dataSet]);
-    var dataSnapshot = useMemo(function () {
-        var filteredData = dataSet;
-        switch (spreadsheetFilter.type) {
-            case "rowBased":
-                if (spreadsheetFilter.byRow === "duplicates") {
-                    filteredData = duplicateRecords.records;
-                    break;
-                }
-                filteredData = filterRecordsByType(dataSet, spreadsheetFilter.byRow);
-                break;
-            case "columnBased":
-                filteredData = filterRecordsByError(dataSet, spreadsheetFilter.byColumn.field, errors);
-                break;
-        }
-        var filteredDatasetByString = filteredInputValue
-            ? filterRecordsBySearchValue(filteredData, filteredInputValue)
-            : filteredData;
-        var dataSetValues = Object.values(filteredDatasetByString);
-        if (sortBy) {
-            return dataSetValues.sort(getSortCompareFn(sortBy));
-        }
-        else if (!!sortByFieldErrors || !!sortByFieldWarnings)
-            return dataSetValues.sort(function (a, b) {
-                var aHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(a, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                var bHasErrors = !!get(sortByFieldErrors ? errors : warnings, "".concat(get(b, "_meta.id"), ".").concat(sortByFieldErrors || sortByFieldWarnings), false);
-                if (aHasErrors === bHasErrors) {
-                    return a._meta.rowIndex > b._meta.rowIndex ? 1 : -1;
-                }
-                return bHasErrors ? 1 : -1;
-            });
-        return dataSetValues;
-    }, [
-        filteredInputValue,
-        spreadsheetFilter,
-        sortBy,
-        Object.keys(dataSet).length,
-        duplicateRecords.count,
-        sortByFieldErrors,
-        sortByFieldWarnings,
-    ]);
-    var recordsWithValuesCount = useMemo(function () {
-        return Object.values(dataSet).filter(hasValueInAnyField).length;
-    }, [dataSet]);
-    var filteredDataCount = useMemo(function () {
-        return Object.values(dataSnapshot).filter(hasValueInAnyField).length;
-    }, [dataSnapshot]);
-    var isFiltering = useMemo(function () { return !!filteredInputValue; }, [filteredInputValue]);
-    useEffect(function () {
-        onDataSetChanged && onDataSetChanged(dataSet);
-        if (addRowTimeoutRef.current) {
-            // avoid adding new rows when the user is typing
-            clearTimeout(addRowTimeoutRef.current);
-        }
-        var dataSetKeys = Object.keys(dataSet);
-        var latestDataSet = dataSet[dataSetKeys[dataSetKeys.length - 1]];
-        var latestDataSetValuesWithoutMeta = Object.values(latestDataSet).filter(function (key) { return key !== "_meta"; });
-        var hasValueInTheLastObject = latestDataSetValuesWithoutMeta
-            .filter(function (value) {
-            return value !== null && value !== undefined && typeof value !== "object";
-        })
-            .some(function (value) { return value !== "null" && value !== "undefined"; });
-        if (hasValueInTheLastObject) {
-            addNewRowsToSpreadsheet();
-        }
-    }, [dataSet]);
-    // Review this later, this probably isn't the best approach
-    useEffect(function () {
-        setDataSet(initialDataSet);
-    }, [initialDataSet]);
-    useEffect(function () {
-        onValidationStateChanged &&
-            onValidationStateChanged(__assign(__assign({}, validationState), { errors: errors, warnings: warnings }));
-    }, [validationState, onValidationStateChanged]);
-    var handleSortByFieldErrors = useCallback(function (field) {
-        setSortByFieldErrors(field === sortByFieldErrors ? null : field);
-        setSortByFieldWarnings(null);
-        setSortBy(null);
-    }, [sortByFieldErrors]);
-    var handleSortByFieldWarnings = useCallback(function (field) {
-        setSortByFieldWarnings(field === sortByFieldWarnings ? null : field);
-        setSortByFieldErrors(null);
-        setSortBy(null);
-    }, [sortByFieldWarnings]);
+    var fields = _a.fields, options = _a.options, isReadOnly = _a.isReadOnly, children = _a.children, onValidateRecord = _a.onValidateRecord, selectedRows = _a.selectedRows, setSelectedRows = _a.setSelectedRows;
+    var _b = useState(false), isTransformTooltipOpened = _b[0], setIsTransformTooltipOpened = _b[1];
+    var _c = useState(false), inefficientRerender = _c[0], setInefficientRerender = _c[1];
+    var filteredDataIds = useFilteredDataContext().filteredDataIds;
+    var _d = useValidationContext(), calculateColumnErrorCounts = _d.calculateColumnErrorCounts, calculateColumnWarningCounts = _d.calculateColumnWarningCounts, warnings = _d.warnings, errors = _d.errors, validationState = __rest(_d, ["calculateColumnErrorCounts", "calculateColumnWarningCounts", "warnings", "errors"]);
     var lastClickedIndexRef = useRef(null);
     var checkboxColumnWidth = useMemo(function () {
-        var lastRowIndexLength = dataSnapshot.length.toString().length;
+        var lastRowIndexLength = filteredDataIds.length.toString().length;
         var singleLetterWidth = 15;
         var horizontalSpacing = 8;
         var baseCheckboxColumnWidth = lastRowIndexLength * singleLetterWidth + horizontalSpacing;
         return baseCheckboxColumnWidth < 40 ? 40 : baseCheckboxColumnWidth;
-    }, [dataSnapshot]);
-    var updatedDataSnapshot = useMemo(function () {
-        return dataSnapshot.map(function (data) {
-            var _a;
-            var updatedData = dataSet[(_a = data._meta) === null || _a === void 0 ? void 0 : _a.id];
-            return updatedData;
-        });
-    }, [dataSet, dataSnapshot]);
-    var value = __assign(__assign({ data: updatedDataSnapshot, fields: fields }, validationState), { isReadOnly: isReadOnly, sortByFieldErrors: sortByFieldErrors, sortByFieldWarnings: sortByFieldWarnings, options: options, recordsWithValuesCount: recordsWithValuesCount, filteredDataCount: filteredDataCount, sortBy: sortBy, warnings: warnings, errors: errors, setSortByFieldErrors: handleSortByFieldErrors, setSortByFieldWarnings: handleSortByFieldWarnings, onValidateRecord: onValidateRecord, updateRecord: updateRecordAndRevalidate, selectedRows: selectedRows, setSelectedRows: setSelectedRows, deleteRecords: handleDeleteErrors, filteredInputValue: filteredInputValue, setFilteredInputValue: setFilteredInputValue, lastClickedIndexRef: lastClickedIndexRef, addNewRowsToSpreadsheet: addNewRowsToSpreadsheet, spreadsheetFilter: spreadsheetFilter, resetSpreadsheetFilter: resetSpreadsheetFilter, filterRecordsByRowType: filterRecordsByRowType, filterRecordsByFieldErrors: filterRecordsByFieldErrors, checkboxColumnWidth: checkboxColumnWidth, setSortBy: setSortBy, isFiltering: isFiltering, setIsTransformTooltipOpened: setIsTransformTooltipOpened, isTransformTooltipOpened: isTransformTooltipOpened, duplicateRecords: duplicateRecords });
+    }, [filteredDataIds.length]);
+    var value = __assign(__assign({ filteredDataIds: filteredDataIds, fields: fields }, validationState), { isReadOnly: isReadOnly, options: options, onValidateRecord: onValidateRecord, selectedRows: selectedRows, setSelectedRows: setSelectedRows, lastClickedIndexRef: lastClickedIndexRef, checkboxColumnWidth: checkboxColumnWidth, setIsTransformTooltipOpened: setIsTransformTooltipOpened, isTransformTooltipOpened: isTransformTooltipOpened, inefficientRerender: inefficientRerender, setInefficientRerender: setInefficientRerender });
     return (_jsx(SpreadsheetContext.Provider, __assign({ value: value }, { children: children })));
 };
 export var useSpreadsheetContext = function () { return useContext(SpreadsheetContext); };

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.d.ts

@@ -0,0 +1,5 @@
+import { Record, RecordDataSet } from "../../../../types";
+export declare const useBatchProcessing: (batchPageSize?: number) => {
+    processRecordsInBatches: (records: Record[], handleBatch: any) => Promise<void>;
+    processInBatches: (dataSet: RecordDataSet, handleBatch: any) => void;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useBatchProcessing.js

@@ -0,0 +1,72 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+export var useBatchProcessing = function (batchPageSize) {
+    if (batchPageSize === void 0) { batchPageSize = 10000; }
+    var processRecordsInBatches = function (records, handleBatch) { return __awaiter(void 0, void 0, void 0, function () {
+        var dataSetLength, batchSize, batchCount, i, idx, batch;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    dataSetLength = records.length;
+                    batchSize = Math.min(Math.floor(batchPageSize), dataSetLength) || dataSetLength;
+                    batchCount = Math.ceil(dataSetLength / batchSize);
+                    i = 0;
+                    _a.label = 1;
+                case 1:
+                    if (!(i < batchCount)) return [3 /*break*/, 4];
+                    idx = i * batchSize;
+                    batch = records.slice(idx, idx + batchSize);
+                    return [4 /*yield*/, handleBatch(batch)];
+                case 2:
+                    _a.sent();
+                    _a.label = 3;
+                case 3:
+                    i++;
+                    return [3 /*break*/, 1];
+                case 4: return [2 /*return*/];
+            }
+        });
+    }); };
+    var processInBatches = function (dataSet, handleBatch) {
+        var records = Object.values(dataSet);
+        processRecordsInBatches(records, handleBatch);
+    };
+    return {
+        processRecordsInBatches: processRecordsInBatches,
+        processInBatches: processInBatches,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.d.ts

@@ -0,0 +1,4 @@
+export declare const useDataSetManipulation: () => {
+    appendNewRecordToSpreadsheet: () => void;
+    deleteRecordsFromSpreadsheet: (recordIds: string[]) => void;
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/useDataSetManipulation.js

@@ -0,0 +1,46 @@
+import { useFilteredDataContext } from "../FilteredDataContextProvider";
+import { useDataSetContext } from "../DataSetContextProvider";
+import { hasValueInAnyField } from "./utils";
+import { addMeta } from "../../../../Importer/data";
+import { useEventManagerContext } from "../../../../common/EventManagerContextProvider";
+import { useValidationContext } from "../ValidationContextProvider";
+import { useDuplicateDataContext } from "../DuplicateDataContextProvider";
+export var useDataSetManipulation = function () {
+    var _a = useFilteredDataContext(), lastFilteredRecordId = _a.lastFilteredRecordId, setRecalculateFilters = _a.setRecalculateFilters;
+    var _b = useDataSetContext(), getRecord = _b.getRecord, setRecord = _b.setRecord, dataSet = _b.dataSet, deleteRecord = _b.deleteRecord, emptyRecordIds = _b.emptyRecordIds;
+    var _c = useEventManagerContext(), emit = _c.emit, EVENTS = _c.EVENTS;
+    var _d = useValidationContext(), errors = _d.errors, warnings = _d.warnings, calculateColumnErrorCounts = _d.calculateColumnErrorCounts, calculateColumnWarningCounts = _d.calculateColumnWarningCounts;
+    var updateDuplicateValues = useDuplicateDataContext().updateDuplicateValues;
+    var updateRecordAndRevalidate = useValidationContext().updateRecordAndRevalidate;
+    var appendNewRecordToSpreadsheet = function () {
+        var _a, _b;
+        var _record = getRecord(lastFilteredRecordId.current);
+        if (!hasValueInAnyField(_record)) {
+            return;
+        }
+        var highestRowIndex = -1;
+        for (var recordId in dataSet.current)
+            if (((_b = (_a = getRecord(recordId)) === null || _a === void 0 ? void 0 : _a._meta) === null || _b === void 0 ? void 0 : _b.rowIndex) > highestRowIndex)
+                highestRowIndex = getRecord(recordId)._meta.rowIndex;
+        var newRow = addMeta({}, highestRowIndex + 1);
+        setRecord(newRow._meta.id, newRow);
+        emit(EVENTS.ON_LAST_ROW_CHANGED);
+    };
+    var deleteRecordsFromSpreadsheet = function (recordIds) {
+        recordIds.forEach(function (recordId) {
+            updateDuplicateValues(getRecord(recordId), {}, false, updateRecordAndRevalidate);
+            deleteRecord(recordId);
+            calculateColumnErrorCounts(errors.current[recordId], {});
+            calculateColumnWarningCounts(warnings.current[recordId], {});
+            emptyRecordIds.current.delete(recordId);
+        });
+        setRecalculateFilters(function (prev) { return !prev; });
+        emit(EVENTS.UPDATE_COLUMN_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ERROR_COUNT);
+        emit(EVENTS.UPDATE_TOTAL_ROW_COUNT);
+    };
+    return {
+        appendNewRecordToSpreadsheet: appendNewRecordToSpreadsheet,
+        deleteRecordsFromSpreadsheet: deleteRecordsFromSpreadsheet,
+    };
+};

package/lib/esm/Importer/common/Spreadsheet/SpreadsheetContextProvider/utils.js

@@ -1,3 +1,4 @@
+import { isEmpty } from "../common";
 export var hasValueInAnyField = function (record) {
     return Object.entries(record)
         .filter(function (_a) {
@@ -16,6 +17,6 @@ export var rowIsEmpty = function (row) {
     return Object.keys(row).every(function (key) {
         if (key === "_meta")
             return true;
-        return !row[key];
+        return isEmpty(row[key]);
     });
 };

package/lib/esm/Importer/common/Spreadsheet/ValidationContextProvider/index.d.ts

@@ -0,0 +1,31 @@
+import { MutableRefObject } from "react";
+import { SpreadsheetOptions } from "../../../..";
+import { BatchValidationErrors, CalculateColumnCount, ColumnWithCount, Field, OnValidateRecord, Record } from "../../../../types";
+type UseValidationStateArgs = {
+    onValidateRecord?: OnValidateRecord;
+    fields: Field[];
+    options: SpreadsheetOptions;
+    isReadOnly: boolean;
+    children: any;
+};
+export type ValidationState = {
+    isValid: boolean;
+    isValidationCompleted: boolean;
+    validationPercentage: number;
+    errors: MutableRefObject<BatchValidationErrors>;
+    warnings: MutableRefObject<BatchValidationErrors>;
+    totalErrorCount: MutableRefObject<number>;
+    invalidRowsCountRef: MutableRefObject<number>;
+    columnsWithErrorCounts: MutableRefObject<ColumnWithCount>;
+    columnsWithWarningCounts: MutableRefObject<ColumnWithCount>;
+    calculateColumnErrorCounts?: CalculateColumnCount;
+    calculateColumnWarningCounts?: CalculateColumnCount;
+};
+type UseValidationStateReturnValue = ValidationState & {
+    updateRecordAndRevalidate: (record: Record | Record[], initialValidation?: boolean) => Promise<void>;
+    spreadsheetInitialized: boolean;
+};
+export declare const ValidationContext: import("react").Context<UseValidationStateReturnValue>;
+export declare const ValidationContextProvider: ({ fields, isReadOnly, onValidateRecord, children, }: UseValidationStateArgs) => JSX.Element;
+export declare const useValidationContext: () => UseValidationStateReturnValue;
+export {};