@lavalogic/scoria 0.12.3 → 0.13.2

package/dist/Components/Table/Types/Context/TableContext.svelte.js

@@ -1,6 +1,6 @@
 import { browser, dev } from '$app/environment';
-import { asyncAll, asyncFilterIndices, asyncSleep, devCatch, getCanvasContext, getErrorMessage } from '../../../../Helpers/Helpers.svelte.js';
-import { tick } from 'svelte';
+import { asyncAll, asyncFilterIndices, asyncForEach, asyncMap, asyncSleep, devCatch, getCanvasContext, getErrorMessage } from '../../../../Helpers/Helpers.svelte.js';
+import { tick, untrack } from 'svelte';
 import { SvelteMap, SvelteSet } from 'svelte/reactivity';
 import { ColumnType } from '../Columns/ColumnType.js';
 import { AccessorDef } from '../Columns/Definitions/Accessors/AccessorDef.svelte.js';
@@ -39,15 +39,16 @@ export class TableContext {
     selectionExtractor;
     getUserId;
     _onEditCell;
+    debug;
     static identifier = Symbol();
     static init(defaultColumnDefs, tableName, dataRepo, options) {
-        const context = new this(InternalSymbol, defaultColumnDefs, tableName, dataRepo, options.allowCopy ?? true, options.enableResize ?? true, options.allowColumnReordering ?? true, options.allowQuickFiltering ?? true, options.showQuickFilterByDefault ?? true, options.allowAdvancedFiltering ?? true, options.toolbar ?? ToolbarPosition.Right, options.displayFooter ?? true, options.selectionExtractor, options.getUserId, options.onEditCell);
+        const context = new this(InternalSymbol, defaultColumnDefs, tableName, dataRepo, options.allowCopy ?? true, options.enableResize ?? true, options.allowColumnReordering ?? true, options.allowQuickFiltering ?? true, options.showQuickFilterByDefault ?? true, options.allowAdvancedFiltering ?? true, options.toolbar ?? ToolbarPosition.Right, options.displayFooter ?? true, options.selectionExtractor, options.getUserId, options.onEditCell, options.debug ?? false);
         return context;
     }
     // #region constructor
     constructor(INTERNAL_ONLY, defaultColumnDefs, tableName, dataRepo, allowCopy, enableResize, allowColumnReordering, allowQuickFiltering, showQuickFilterByDefault, allowAdvancedFiltering, toolbarPosition, displayFooter, selectionExtractor, getUserId, _onEditCell = () => {
         console.warn('Cannot edit immutable table');
-    }) {
+    }, debug) {
         this.defaultColumnDefs = defaultColumnDefs;
         this.tableName = tableName;
         this.allowCopy = allowCopy;
@@ -61,11 +62,24 @@ export class TableContext {
         this.selectionExtractor = selectionExtractor;
         this.getUserId = getUserId;
         this._onEditCell = _onEditCell;
+        this.debug = debug;
         if (INTERNAL_ONLY != InternalSymbol) {
             throw new Error('This class may only be instantiated through the static init() method');
         }
         this._dataRepo = $state(dataRepo);
         // this.dataRepo.forceRefresh = this.forceVisibilityUpdate;
+        $effect(() => {
+            void this.currentPageData;
+            void this.selectedItems;
+            untrack(async () => {
+                await this.currentPageData;
+                if (this.paginationRepo?.paginationState.pageIndex != undefined) {
+                    await this.updateSelectedItems();
+                    this.resetHighlightedRows();
+                    this.recalculateAllSelectedInCurrentPage();
+                }
+            }).catch(devCatch);
+        });
         if (defaultColumnDefs.remoteFilters) {
             this.remoteFilters = defaultColumnDefs.remoteFilters;
         }
@@ -142,11 +156,13 @@ export class TableContext {
         $effect(() => {
             void this.paginationRepo?.paginationType;
             void this.dataRepo.filtering;
-            void this.dataRepo.filterModes.values();
+            void this.dataRepo.filterModes;
             void this.dataRepo.data;
+            void this.paginationRepo?.paginationState;
+            // console.log('massive effect start');
             (async () => {
                 const startTime = Date.now();
-                let currentTimerPromise;
+                let currentTimer;
                 const abortController = new AbortController();
                 try {
                     if (this.paginationRepo?.paginationType == PaginationType.Local) {
@@ -156,77 +172,75 @@ export class TableContext {
                             const snapshot = $state.snapshot(filter.value);
                             checkedIds.add(filter.id);
                             const filterMode = this.dataRepo.filterModes.get(filter.id);
-                            return (!!filterMode &&
-                                (allowedEmptyFilterModes.has(filterMode) ||
-                                    (snapshot !== '' && (!Array.isArray(snapshot) || snapshot[0] !== ''))));
+                            return ((snapshot !== '' &&
+                                snapshot != null &&
+                                (!Array.isArray(snapshot) || !!snapshot[0])) ||
+                                (!!filterMode && allowedEmptyFilterModes.has(filterMode)));
                         });
                         const data = this.dataRepo.data;
-                        // HACK add filter state for filters that don't require a value to function
-                        // this.dataRepo?.filterModes?.entries().forEach(([key, value]) => {
-                        // 	if (!checkedIds.has(key)) {
-                        // 		checkedIds.add(key);
-                        // 		if (allowedEmptyFilterModes.has(value)) {
-                        // 			filterState.push({
-                        // 				id: key,
-                        // 				value: ''
-                        // 			});
-                        // 		}
-                        // 	} else {
-                        // 		console.warn(`key ${key} was already checked, new value:`, value);
-                        // 	}
-                        // });
-                        if (!filterState.length) {
-                            this.filteredIndices = data.map((_, index) => index);
-                        }
-                        currentTimerPromise = this.filterTimer = this.filterTimer + 1;
+                        currentTimer = ++this.filterTimer;
                         await asyncSleep(150);
-                        const currentTimer = currentTimerPromise;
-                        if (currentTimerPromise != this.filterTimer) {
+                        if (currentTimer != this.filterTimer) {
                             throw new Error('stale filter');
                         }
-                        let settled = false;
-                        const filterPromise = asyncFilterIndices(data, async (item, index) => {
-                            const row = this.createRow(item, index, index);
-                            return asyncAll(filterState, async (filter) => {
-                                abortController.signal.throwIfAborted();
-                                const def = this.columnDefsById.get(filter.id);
-                                if (!def) {
-                                    console.warn(`Could not find column def for filter with id ${filter.id}`);
+                        if (!filterState.length) {
+                            if (dev) {
+                                console.log('no filter state was found');
+                            }
+                            this.filteredIndices = data.map((_item, index) => {
+                                return index;
+                            });
+                        }
+                        else {
+                            this.dataRepo.isLoading = true;
+                            if (this.debug) {
+                                console.log('getting filter promise');
+                            }
+                            const settled = $state({ value: false });
+                            const filterPromise = asyncFilterIndices(data, async (item, index) => {
+                                const row = this.createRow(item, index, index);
+                                return asyncAll(filterState, async (filter) => {
+                                    abortController.signal.throwIfAborted();
+                                    const def = this.columnDefsById.get(filter.id);
+                                    if (!def) {
+                                        console.warn(`Could not find column def for filter with id ${filter.id}`);
+                                        return false;
+                                    }
+                                    if (def.filterFn) {
+                                        return def.filterFn(row, filter.id, filter.value);
+                                    }
+                                    if (this.debug) {
+                                        console.warn(`column ${def.id} did not have a filter function`);
+                                    }
                                     return false;
+                                });
+                            });
+                            filterPromise
+                                .then((indices) => {
+                                if (this.debug) {
+                                    console.log(`iteration ${currentTimer}: got indices from filter promise:`, indices);
+                                }
+                                this.filteredIndices = indices;
+                            })
+                                .catch((e) => {
+                                if (this.debug) {
+                                    console.log(`caught filter rejection:`, e);
+                                    console.log(`iteration ${currentTimer} cancelled after ${Date.now() - startTime}ms`);
+                                }
+                            })
+                                .finally(() => {
+                                if (this.debug) {
+                                    console.log(`timer ${currentTimer} is settled`);
                                 }
-                                if (def.filterFn) {
-                                    return def.filterFn(row, filter.id, filter.value);
+                                settled.value = true;
+                                if (currentTimer === this.filterTimer) {
+                                    this.dataRepo.isLoading = false;
                                 }
-                                if (dev) {
-                                    console.warn(`column ${def.id} did not have a filter function`);
+                                else if (this.debug) {
+                                    console.warn(`skipping disabling load overlay because current timer is ${currentTimer} but newest is ${this.filterTimer}`);
                                 }
-                                return false;
                             });
-                        });
-                        filterPromise
-                            .then((indices) => {
-                            this.filteredIndices = indices;
-                        })
-                            .catch((e) => {
-                            if (dev) {
-                                console.log(`caught filter rejection:`, e);
-                                console.log(`iteration ${currentTimer} cancelled after ${Date.now() - startTime}ms`);
-                            }
-                        })
-                            .finally(() => {
-                            settled = true;
-                        });
-                        for (;;) {
-                            if (currentTimerPromise !== this.filterTimer) {
-                                abortController.abort('stale data');
-                            }
-                            // HACK this warning is wrong. We're in an async context and settled is being changed in the finally callback above
-                            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-                            if (settled) {
-                                console.log(`iteration ${currentTimerPromise} finished ${Date.now() - startTime}ms`);
-                                return;
-                            }
-                            await tick();
+                            await this.filteringLoop(currentTimer, abortController, settled, startTime);
                         }
                     }
                 }
@@ -234,9 +248,11 @@ export class TableContext {
                     if (dev) {
                         const message = getErrorMessage(e);
                         if (!message.toLocaleLowerCase().startsWith('stale')) {
-                            console.error(e);
+                            console.error(`Iteration ${currentTimer}`, e);
+                        }
+                        else {
+                            console.log(`iteration ${currentTimer} cancelled after ${Date.now() - startTime}ms`);
                         }
-                        console.log(`iteration ${currentTimerPromise} cancelled after ${Date.now() - startTime}ms`);
                     }
                 }
             })().catch(devCatch);
@@ -258,6 +274,14 @@ export class TableContext {
         return this._showQuickFiltering;
     }
     set showQuickFiltering(v) {
+        if (this.debug) {
+            if (!this.allowQuickFiltering) {
+                console.log(`attempted to toggle quick filtering when allowQuickFiltering = false`);
+            }
+            else {
+                console.log(`setting quick filtering = ${v}`);
+            }
+        }
         this._showQuickFiltering = this.allowQuickFiltering && v;
     }
     _dataRepo;
@@ -293,6 +317,8 @@ export class TableContext {
     set limitFooter(v) {
         this._limitFooter = v;
     }
+    /* eslint-disable @typescript-eslint/no-explicit-any */
+    // type here set to any because users of this property should not access the filter type, but typescript doesn't support that
     _dragTarget = $state(null);
     get dragTarget() {
         return this._dragTarget;
@@ -300,17 +326,13 @@ export class TableContext {
     set dragTarget(v) {
         this._dragTarget = v;
     }
+    /* eslint-enable @typescript-eslint/no-explicit-any */
     _columnDefs = $state([]);
     get columnDefs() {
         return this._columnDefs;
     }
     set columnDefs(v) {
-        // if (this._columnDefs.length > 0 && v != this._columnDefs) {
-        // 	debugger;
-        // }
         this._columnDefs = v;
-        // this causes an error...?
-        // this.table = createTable(this.tableOptions);
     }
     _columnPresets = $state([]);
     get columnPresets() {
@@ -358,7 +380,7 @@ export class TableContext {
     }
     set focusStart(v) {
         this._focusStart = v;
-        this._recalculateFocusedElements();
+        this._recalculateFocusedElements().catch(devCatch);
     }
     _focusStartIsBeingEdited = $state(false);
     get focusStartIsBeingEdited() {
@@ -373,7 +395,7 @@ export class TableContext {
     }
     set focusEnd(v) {
         this._focusEnd = v;
-        this._recalculateFocusedElements();
+        this._recalculateFocusedElements().catch(devCatch);
     }
     _singleClickEditing = $state(false);
     get singleClickEditing() {
@@ -445,34 +467,38 @@ export class TableContext {
     set openModalCoordinates(v) {
         this._openModalCoordinates = v;
     }
-    _allSelectedInCurrentPage = $derived.by(() => {
+    _allSelectedInCurrentPage = $derived.by(async () => {
         if (this.selectedItems.size === 0) {
             return Promise.resolve(false);
         }
+        void this.paginationRepo?.pageStartIndex;
+        void this.columnDefs;
+        const sortedData = await this.sortedData;
         const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
         const pageEndIndexExclusive = this.paginationRepo
-            ? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-            : this.sortedData.length;
+            ? Math.min(this.paginationRepo.pageEndIndexExclusive, sortedData.length)
+            : sortedData.length;
         const selectableMethod = this.columnDefs.find((it) => it instanceof RowSelectionDef)?.isSelectable;
-        return this.allSelectedBySelectableMethod(pageStartIndex, pageEndIndexExclusive, selectableMethod);
+        return await this.allSelectedBySelectableMethod(pageStartIndex, pageEndIndexExclusive, selectableMethod);
     });
     get allSelectedInCurrentPage() {
         return this._allSelectedInCurrentPage;
     }
     async allSelectedBySelectableMethod(pageStartIndex, pageEndIndexExclusive, selectableMethod) {
+        const sortedData = await this.sortedData;
         if (selectableMethod) {
             for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                const item = this.sortedData[index];
+                const item = sortedData[index];
                 if ((await selectableMethod(item, index)) &&
-                    !this.selectedItems.has(this.selectionExtractor(item))) {
+                    !this.selectedItems.has(await this.selectionExtractor(item))) {
                     return Promise.resolve(false);
                 }
             }
         }
         else {
             for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                const item = this.sortedData[index];
-                if (!this.selectedItems.has(this.selectionExtractor(item))) {
+                const item = sortedData[index];
+                if (!this.selectedItems.has(await this.selectionExtractor(item))) {
                     return Promise.resolve(false);
                 }
             }
@@ -480,48 +506,28 @@ export class TableContext {
         return Promise.resolve(true);
     }
     async _areAllHighlighted(pageStartIndex, pageEndIndexExclusive) {
+        const sortedData = await this.sortedData;
         for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-            const item = this.sortedData[index];
-            if (!this.highlightedItems.has(this.selectionExtractor(item))) {
+            const item = sortedData[index];
+            if (!this.highlightedItems.has(await this.selectionExtractor(item))) {
                 return Promise.resolve(false);
             }
         }
         return Promise.resolve(true);
     }
-    _allHighlighted = $derived.by(() => {
+    _allHighlighted = $derived.by(async () => {
         if (this.highlightedItems.size === 0) {
             return Promise.resolve(false);
         }
         const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
         const pageEndIndexExclusive = this.paginationRepo
-            ? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-            : this.sortedData.length;
-        return this._areAllHighlighted(pageStartIndex, pageEndIndexExclusive);
+            ? Math.min(this.paginationRepo.pageEndIndexExclusive, (await this.sortedData).length)
+            : (await this.sortedData).length;
+        return await this._areAllHighlighted(pageStartIndex, pageEndIndexExclusive);
     });
     get allHighlighted() {
         return this._allHighlighted;
     }
-    // private _allHighlighted: boolean = $derived.by(() => {
-    // 	if (!this.dataRepo || this.highlightedItems.size === 0) {
-    // 		return false;
-    // 	}
-    // 	const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
-    // 	const pageEndIndexExclusive = this.paginationRepo
-    // 		? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-    // 		: this.sortedData.length;
-    // 	if (pageStartIndex == undefined || pageEndIndexExclusive == undefined) {
-    // 		return false;
-    // 	}
-    // 	for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-    // 		if (!this.highlightedItems.has(index)) {
-    // 			return false;
-    // 		}
-    // 	}
-    // 	return true;
-    // });
-    // public get allHighlighted(): boolean {
-    // 	return this._allHighlighted;
-    // }
     _hasHighlightedItems = $derived(this.highlightedItems.size > 0);
     get hasHighlightedItems() {
         return this._hasHighlightedItems;
@@ -534,18 +540,6 @@ export class TableContext {
     set selectedItems(v) {
         this._selectedItems = v;
     }
-    // HACK this is lazily loaded. it will not be used before initialisation.
-    // public getSelectedItemsAsObjects = (): T[] => {
-    // 	if (!this.selectionExtractor) {
-    // 		throw new Error("No selection extractor was supplied.")
-    // 	};
-    // 	return this.sortedData.filter((item) => {
-    // 		return this._selectedIdentifiers.has(this.selectionExtractor(item));
-    // 	}) ?? []
-    // }
-    // public get selectedItemsAsObjects(): T[] {
-    // 	return this._selectedItemsAsObjects;
-    // }
     _invalidValues = $state(new SvelteMap());
     get invalidValues() {
         return this._invalidValues;
@@ -591,10 +585,6 @@ export class TableContext {
         // }
         return comparableSortingFn(id, desc);
     };
-    originalRowIndices = $derived(
-    // TODO decide
-    // eslint-disable-next-line svelte/prefer-svelte-reactivity
-    new Map(this.dataRepo.data.map((item, index) => [item, index])));
     // private _filterTimer: ReturnType<typeof setTimeout> | undefined;
     _filterTimer = 1;
     get filterTimer() {
@@ -608,31 +598,43 @@ export class TableContext {
         return this._filteredIndices;
     }
     set filteredIndices(v) {
+        if (this.debug) {
+            console.log('updating filtered indices');
+        }
         this._filteredIndices = v;
     }
     filteredData = $derived.by(() => {
+        void this.dataRepo.data;
+        void this.filteredIndices;
+        void this.showSelectedOnly;
+        const dataRepo = this.dataRepo;
         if (this.paginationRepo?.paginationType == PaginationType.Local) {
             const data = this.showSelectedOnly
                 ? // FIXME this will show the selected items by order of insertion to the selectedItems map.
                     [...this.selectedItems.values()]
                 : this.filteredIndices.map((index) => {
-                    return this.dataRepo.data[index];
+                    return dataRepo.data[index];
                 });
             return data;
         }
-        return this.showSelectedOnly ? [...this.selectedItems.values()] : this.dataRepo.data;
+        if (this.debug) {
+            console.log(`updating filtered data. Selected only: ${this.showSelectedOnly}`);
+        }
+        return this.showSelectedOnly ? [...this.selectedItems.values()] : dataRepo.data;
     });
-    sortedData = $derived.by(() => {
+    // eslint-disable-next-line @typescript-eslint/require-await
+    sortedData = $derived.by(async () => {
+        if (this.debug) {
+            console.log('sorted data');
+        }
+        const fd = this.filteredData;
+        const sorting = this.dataRepo.sorting;
         if (this.paginationRepo?.paginationType == PaginationType.Local) {
-            const state = this.dataRepo.sorting;
-            const sortedData = this.showSelectedOnly
-                ? // FIXME this will show the selected items by order of insertion to the selectedItems map.
-                    [...this.selectedItems.values()]
-                : this.filteredData.concat();
-            sortedData.sort((a, b) => {
-                const end = state.length;
+            const dataToSort = fd.concat();
+            dataToSort.sort((a, b) => {
+                const end = sorting.length;
                 for (let i = 0; i < end; i++) {
-                    const { id, desc } = state[i];
+                    const { id, desc } = sorting[i];
                     const sortingFn = this.getSortingFn(id, desc);
                     if (sortingFn) {
                         const result = sortingFn(a, b);
@@ -640,19 +642,27 @@ export class TableContext {
                             return result;
                         }
                     }
-                    else {
+                    else if (dev) {
                         console.warn('no sorting fn found for ' + id);
                     }
                 }
                 return 0;
             });
-            return sortedData;
+            return dataToSort;
         }
-        return this.filteredData;
+        return fd;
     });
-    sortedRowModel = $derived.by(() => {
-        const rows = this.sortedData.map((item, index) => {
-            const unsortedIndex = this.originalRowIndices.get(item) ?? index;
+    sortedRowModel = $derived.by(async () => {
+        void this.originalRowIndices;
+        void this.sortedData;
+        const sd = await this.sortedData;
+        const ori = await this.originalRowIndices;
+        const rows = await asyncMap(sd, async (item, index) => {
+            const originalIndex = ori.get(await this.selectionExtractor(item));
+            const unsortedIndex = originalIndex ?? index;
+            if (originalIndex == null) {
+                throw new Error(`Sorted Data not found in Original Row Indices: original index ${originalIndex}, index ${index}`);
+            }
             return this.createRow(item, index, unsortedIndex);
         });
         return {
@@ -661,13 +671,33 @@ export class TableContext {
             rowsById: new Map(rows.map((it, index) => [index.toString(), it])) //FIXME should this be by sorted id or by original id?
         };
     });
-    _currentPageData = $derived(this.dataRepo.paginationType == PaginationType.Local
-        ? this.sortedData.slice(this.dataRepo.pageStartIndex, this.dataRepo.pageEndIndexExclusive)
-        : this.sortedData);
-    paginationRowModel = $derived.by(() => {
-        const sortedData = this._currentPageData;
-        const rows = sortedData.map((item, index) => {
-            const unsortedIndex = this.originalRowIndices.get(item) ?? index;
+    currentPageData = $derived.by(async () => {
+        void this.sortedData;
+        const dataRepo = this.dataRepo;
+        const sd = await this.sortedData;
+        if (dataRepo.paginationType != PaginationType.Local) {
+            return sd;
+        }
+        const x = sd.slice(dataRepo.pageStartIndex, dataRepo.pageEndIndexExclusive);
+        return x;
+    });
+    // FIXME this is most likely the cuplrit for add pod not working on receipt. It may be one tick behind the other values
+    originalRowIndices = $derived.by(async () => {
+        const dataRepo = this.dataRepo;
+        const awaited = await asyncMap(dataRepo.data, async (item, index) => [await this.selectionExtractor(item), index]);
+        // eslint-disable-next-line svelte/prefer-svelte-reactivity
+        return new Map(awaited);
+    });
+    paginationRowModel = $derived.by(async () => {
+        void this.originalRowIndices;
+        void this.currentPageData;
+        void this.columnDefsById;
+        void this.columnVisibility;
+        void this.paginationRepo?.sorting;
+        void this.columnPinning;
+        const currentPageData = await this.currentPageData;
+        const rows = await asyncMap(currentPageData, async (item, index) => {
+            const unsortedIndex = (await this.originalRowIndices).get(item) ?? index;
             const row = {
                 getValue: async (key) => {
                     const gotValue = await this.columnDefsById.get(key)?.accessorFn?.(item, index);
@@ -739,44 +769,25 @@ export class TableContext {
             rowsById: new Map(rows.map((it, index) => [index.toString(), it])) //FIXME should this be by sorted id or by original id?
         };
     });
-    get currentPageData() {
-        return this._currentPageData;
-    }
     // #endregion
     // #region public methods
-    selectionContains = (item) => {
+    selectionContains = async (item) => {
         if (!this.isSelectable) {
             return true;
         }
-        return this.selectedItems.has(this.selectionExtractor(item));
+        return this.selectedItems.has(await this.selectionExtractor(item));
     };
-    // public readonly HACK__reselectItem = (item: T) => {
-    // 	if (!this.selectionExtractor) {
-    // 		throw new Error('No selection extractor was supplied');
-    // 	}
-    // 	if (!this.isSelectable) {
-    // 		return true;
-    // 	}
-    // 	const key = this.selectionExtractor(item);
-    // 	if (this.selectedItems.has(key)) {
-    // 		this.selectedItems.delete(key);
-    // 		this.selectedItems.set(key, item);
-    // 	} else {
-    // 		this.selectedItems.set(key, item);
-    // 		this.selectedItems.delete(key);
-    // 	}
-    // };
-    addInvalidValue = (item) => {
-        this.invalidValues.set(this.selectionExtractor(item), item);
+    addInvalidValue = async (item) => {
+        this.invalidValues.set(await this.selectionExtractor(item), item);
     };
-    removeInvalidValue = (item) => {
-        this.invalidValues.delete(this.selectionExtractor(item));
+    removeInvalidValue = async (item) => {
+        this.invalidValues.delete(await this.selectionExtractor(item));
     };
     setOpenModalCoordinates = (coordinates) => {
         this.openModalCoordinates = coordinates ? { ...coordinates } : null;
     };
-    toggleHighlightedRow = (item) => {
-        const key = this.selectionExtractor(item);
+    toggleHighlightedRow = async (item) => {
+        const key = await this.selectionExtractor(item);
         if (this.highlightedItems.has(key)) {
             this.highlightedItems.delete(key);
         }
@@ -784,58 +795,49 @@ export class TableContext {
             this.highlightedItems.set(key, item);
         }
     };
-    // public readonly toggleHighlightedRow = (rowIndex: number) => {
-    // 	if (this.highlightedRowIndexIds.has(rowIndex)) {
-    // 		this.highlightedRowIndexIds.delete(rowIndex);
-    // 	} else {
-    // 		this.highlightedRowIndexIds.add(rowIndex);
-    // 	}
-    // };
     resetHighlightedRows = () => {
         this.highlightedItems.clear();
     };
-    toggleSelected = (item, selected) => {
-        const key = this.selectionExtractor(item);
+    toggleSelected = async (item, selected) => {
+        const key = await this.selectionExtractor(item);
         if (selected) {
             this.selectedItems.set(key, item);
         }
         else {
             this.selectedItems.delete(key);
         }
-        this.recalculateAllSelectedInCurrentPage();
     };
     setSelectionMap = (set) => {
         this.selectedItems = set;
     };
     updateSelectedItems = async () => {
         const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
+        const sortedData = await this.sortedData;
         const pageEndIndexExclusive = this.paginationRepo
-            ? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-            : this.sortedData.length;
+            ? Math.min(this.paginationRepo.pageEndIndexExclusive, sortedData.length)
+            : sortedData.length;
         const selectableMethod = this.columnDefs.find((it) => it instanceof RowSelectionDef)?.isSelectable;
+        const updateSelectedItem = async (item) => {
+            const key = await this.selectionExtractor(item);
+            if (this.selectedItems.has(key)) {
+                //replace selected item with newer version of the same identity
+                this.selectedItems.set(key, item);
+            }
+        };
         if (selectableMethod) {
             for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                const item = this.sortedData[index];
+                const item = sortedData[index];
                 if (await selectableMethod(item, index)) {
-                    const key = this.selectionExtractor(item);
-                    if (this.selectedItems.has(key)) {
-                        //replace selected item with newer version of the same identity
-                        this.selectedItems.set(key, item);
-                    }
+                    await updateSelectedItem(item);
                 }
             }
         }
         else {
             for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                const item = this.sortedData[index];
-                const key = this.selectionExtractor(item);
-                if (this.selectedItems.has(key)) {
-                    //replace selected item with newer version of the same identity
-                    this.selectedItems.set(key, item);
-                }
+                const item = sortedData[index];
+                await updateSelectedItem(item);
             }
         }
-        this.recalculateAllSelectedInCurrentPage();
     };
     resetSelectedItems = () => {
         for (const key of this.selectedItems.keys()) {
@@ -843,25 +845,37 @@ export class TableContext {
         }
         this.recalculateAllSelectedInCurrentPage();
     };
+    lastIncrement = Number.MIN_SAFE_INTEGER;
     recalculateAllSelectedInCurrentPage = () => {
-        tick()
+        const invalidValues = this._invalidValues;
+        const allSelectedInCurrentPage = [...this.selectedItems.keys()];
+        const currentIncrement = ++this.lastIncrement;
+        this.allSelectedInCurrentPage
             .then(() => {
-            return this.allSelectedInCurrentPage.then(() => {
-                this.hasInvalidSelectedItems = [...this.selectedItems.keys()].some((key) => this._invalidValues.has(key));
-            });
+            if (this.lastIncrement == currentIncrement) {
+                this.hasInvalidSelectedItems = allSelectedInCurrentPage.some((key) => {
+                    return invalidValues.has(key);
+                });
+            }
+            else {
+                if (dev) {
+                    console.log('skipping recalculation due to stale data');
+                }
+            }
         })
             .catch(devCatch);
     };
     selectAllInCurrentPage = async () => {
+        void this.paginationRepo?.pageStartIndex;
+        const sortedData = await this.sortedData;
         if (await this.allSelectedInCurrentPage) {
-            // this.resetSelectedItems();
             const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
             const pageEndIndexExclusive = this.paginationRepo
-                ? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-                : this.sortedData.length;
+                ? Math.min(this.paginationRepo.pageEndIndexExclusive, sortedData.length)
+                : sortedData.length;
             for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                const item = this.sortedData[index];
-                this.selectedItems.delete(this.selectionExtractor(item));
+                const item = sortedData[index];
+                this.selectedItems.delete(await this.selectionExtractor(item));
             }
         }
         else {
@@ -869,50 +883,51 @@ export class TableContext {
             // const selectedItems = new SvelteSet<number>();
             const pageStartIndex = this.paginationRepo ? this.paginationRepo.pageStartIndex : 0;
             const pageEndIndexExclusive = this.paginationRepo
-                ? Math.min(this.paginationRepo.pageEndIndexExclusive, this.sortedData.length)
-                : this.sortedData.length;
+                ? Math.min(this.paginationRepo.pageEndIndexExclusive, sortedData.length)
+                : sortedData.length;
             const selectableMethod = this.columnDefs.find((it) => it instanceof RowSelectionDef)?.isSelectable;
             if (selectableMethod) {
                 for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                    const item = this.sortedData[index];
+                    const item = sortedData[index];
                     if (await selectableMethod(item, index)) {
-                        this.selectedItems.set(this.selectionExtractor(item), item);
+                        this.selectedItems.set(await this.selectionExtractor(item), item);
                     }
                 }
             }
             else {
                 for (let index = pageStartIndex; index < pageEndIndexExclusive; index++) {
-                    const item = this.sortedData[index];
-                    this.selectedItems.set(this.selectionExtractor(item), item);
+                    const item = sortedData[index];
+                    this.selectedItems.set(await this.selectionExtractor(item), item);
                 }
             }
         }
-        this.recalculateAllSelectedInCurrentPage();
     };
     onHighlightAll = () => {
         this.allHighlighted
             .then((areAllHighlighted) => {
             if (areAllHighlighted) {
                 this.highlightedItems.clear();
+                return Promise.resolve();
             }
             else {
-                this._currentPageData.forEach((item) => {
-                    this.highlightedItems.set(this.selectionExtractor(item), item);
+                return this.currentPageData.then((items) => {
+                    return asyncForEach(items, async (item) => {
+                        this.highlightedItems.set(await this.selectionExtractor(item), item);
+                    });
                 });
             }
         })
             .catch(devCatch);
     };
     onEditCell = async (item, coords, value) => {
-        const oldKey = this.selectionExtractor(item);
-        const wasSelected = oldKey != null && this.selectedItems.has(oldKey);
-        // debugger;
+        const oldKey = await this.selectionExtractor(item);
+        const wasSelected = this.selectedItems.has(oldKey);
         if (wasSelected) {
             this.selectedItems.delete(oldKey);
         }
         await this._onEditCell?.(item, coords, value);
         if (wasSelected && this.isSelectable) {
-            const newKey = this.selectionExtractor(item);
+            const newKey = await this.selectionExtractor(item);
             if (newKey) {
                 this.selectedItems.set(newKey, item);
             }
@@ -941,13 +956,16 @@ export class TableContext {
                 this.tabBackward();
             }
             else {
-                this.tabForward();
+                this.tabForward().catch(devCatch);
             }
         }
         if (e.target.tagName === 'INPUT' &&
             e.target.attributes.getNamedItem('type')?.value !== 'checkbox') {
             return;
         }
+        this.handleKeystroke(e).catch(devCatch);
+    };
+    async handleKeystroke(e) {
         switch (e.key) {
             case 'Enter':
             case 'F2':
@@ -955,7 +973,6 @@ export class TableContext {
                     break;
                 }
                 e.preventDefault();
-                e.stopPropagation();
                 e.target?.dispatchEvent(new CustomEvent('edit', { detail: { selectAll: e.key === 'Enter' } }));
                 // focusStartIsBeingEdited.set(true);
                 break;
@@ -966,20 +983,20 @@ export class TableContext {
                         if (!this.allowCopy) {
                             return;
                         }
-                        this.focusToTop();
+                        await this.focusToTop();
                     }
                     else {
-                        this.moveToTop();
+                        await this.moveToTop();
                     }
                 }
                 else if (e.getModifierState('Shift')) {
                     if (!this.allowCopy) {
                         return;
                     }
-                    this.moveFocusUp();
+                    await this.moveFocusUp();
                 }
                 else {
-                    this.setFocusUp();
+                    await this.setFocusUp();
                 }
                 return;
             }
@@ -990,20 +1007,20 @@ export class TableContext {
                         if (!this.allowCopy) {
                             return;
                         }
-                        this.focusToBottom();
+                        await this.focusToBottom();
                     }
                     else {
-                        this.moveToBottom();
+                        await this.moveToBottom();
                     }
                 }
                 else if (e.getModifierState('Shift')) {
                     if (!this.allowCopy) {
                         return;
                     }
-                    this.moveFocusDown();
+                    await this.moveFocusDown();
                 }
                 else {
-                    this.setFocusDown();
+                    await this.setFocusDown();
                 }
                 return;
             }
@@ -1058,9 +1075,9 @@ export class TableContext {
                 case 'a': {
                     if (e.ctrlKey) {
                         e.preventDefault();
-                        this.moveToTop();
+                        await this.moveToTop();
                         this.moveToStart();
-                        this.focusToBottom();
+                        await this.focusToBottom();
                         this.focusToEnd();
                     }
                     break;
@@ -1069,12 +1086,12 @@ export class TableContext {
                 case 'C':
                     if (e.getModifierState('Control')) {
                         e.preventDefault();
-                        this.copyDataToClipboard().catch(devCatch);
+                        await this.copyDataToClipboard();
                     }
                     break;
             }
         }
-    };
+    }
     toggleQuickFiltering = () => {
         if (this.allowQuickFiltering) {
             const fs = this.focusStart;
@@ -1147,12 +1164,12 @@ export class TableContext {
         if (!browser) {
             return;
         }
-        const ctx = getCanvasContext();
-        const currentPageContents = this.currentPageData;
         const colId = columnDef.id;
         if (!colId) {
             return;
         }
+        const currentPageContents = await this.currentPageData;
+        const ctx = getCanvasContext();
         let maxWidth = 0;
         if (columnDef.accessorFn) {
             for (let i = currentPageContents.length - 1; i > -1; i--) {
@@ -1325,41 +1342,41 @@ export class TableContext {
         this.focusStart = null;
         this.focusEnd = null;
     };
-    focusToTop = () => {
+    focusToTop = async () => {
         if (!this.allowCopy) {
             return;
         }
-        this._moveFocusEndRow(-Infinity);
+        return this._moveFocusEndRow(-Infinity);
     };
     focusToBottom = () => {
         if (!this.allowCopy) {
             return;
         }
-        this._moveFocusEndRow(Infinity);
+        return this._moveFocusEndRow(Infinity);
     };
-    moveToBottom = () => {
-        this._moveFocusRow(Infinity);
+    moveToBottom = async () => {
+        return this._moveFocusRow(Infinity);
     };
-    moveToTop = () => {
-        this._moveFocusRow(-Infinity);
+    moveToTop = async () => {
+        return this._moveFocusRow(-Infinity);
     };
-    moveFocusUp = () => {
+    moveFocusUp = async () => {
         if (!this.allowCopy) {
             return;
         }
-        this._moveFocusEndRow(-1);
+        return this._moveFocusEndRow(-1);
     };
-    setFocusUp = () => {
-        this._moveFocusRow(-1);
+    setFocusUp = async () => {
+        return this._moveFocusRow(-1);
     };
-    tabForward = () => {
+    tabForward = async () => {
         if (!this.allowCopy) {
             return;
         }
         const defs = this.columnDefs;
         const visibility = this.columnVisibility;
         const indexBounds = defs.length;
-        const rowBounds = this.sortedData.length;
+        const rowBounds = (await this.sortedData).length;
         if (!this.focusStart || !rowBounds) {
             return;
         }
@@ -1480,14 +1497,14 @@ export class TableContext {
         }
         //otherwise return without doing anything
     };
-    moveFocusDown = () => {
+    moveFocusDown = async () => {
         if (!this.allowCopy) {
             return;
         }
-        this._moveFocusEndRow(1);
+        return this._moveFocusEndRow(1);
     };
-    setFocusDown = () => {
-        this._moveFocusRow(1);
+    setFocusDown = async () => {
+        return this._moveFocusRow(1);
     };
     moveFocusLeft = () => {
         if (!this.allowCopy) {
@@ -1611,6 +1628,9 @@ export class TableContext {
     // 	}
     // };
     setSelectedPreset = (preset) => {
+        if (this.debug) {
+            console.log('setting selected preset:', preset);
+        }
         this.columnDefs = preset.columnDefs;
         if (!this.paginationRepo?.updateSorting) {
             console.warn('no sorting state found');
@@ -1751,7 +1771,7 @@ export class TableContext {
         };
         localStorage.setItem(`${this.userId}-${this.tableName}-settings`, JSON.stringify(settingsObject));
     }
-    _moveFocusEndRow(offset) {
+    async _moveFocusEndRow(offset) {
         if (!this.allowCopy) {
             return;
         }
@@ -1760,27 +1780,27 @@ export class TableContext {
             return;
         }
         const focusEnd = this.focusEnd;
-        const rows = this.paginationRowModel.rows;
+        const rows = (await this.paginationRowModel).rows;
         const endRowIndex = rows.findIndex((row) => row.visibleIndex === focusEnd.row);
         this.focusEnd = {
             ...this.focusEnd,
             row: rows[Math.max(Math.min(endRowIndex + offset, rows.length - 1), 0)].visibleIndex
         };
     }
-    _moveFocusRow(offset) {
+    async _moveFocusRow(offset) {
         if (!this.focusStart) {
             console.warn('no focusStart found');
             return;
         }
         const focusStart = this.focusStart;
-        const newEndRowIndex = (() => {
-            const rows = this.paginationRowModel.rows;
+        const newEndRowIndex = await (async () => {
+            const rows = (await this.paginationRowModel).rows;
             const endRowIndex = rows.findIndex((row) => row.visibleIndex === focusStart.row);
             return rows[Math.max(Math.min(endRowIndex + offset, rows.length - 1), 0)].visibleIndex;
         })();
         this.moveFocus({ ...this.focusStart, row: newEndRowIndex });
     }
-    _recalculateFocusedElements() {
+    async _recalculateFocusedElements() {
         if (!this.focusStart || !this.focusEnd) {
             this.focusedRowIndexIds = new SvelteSet();
             this.focusedColumnIds = new SvelteSet();
@@ -1788,7 +1808,7 @@ export class TableContext {
         }
         const { row: startRowIndexId, column: startColumnIndex } = this.focusStart;
         const { row: endRowIndexId, column: endColumnIndex } = this.focusEnd;
-        const model = this.sortedRowModel;
+        const model = await this.sortedRowModel;
         const startRowIndex = model.rows.findIndex((row) => row.visibleIndex === startRowIndexId);
         const endRowIndex = model.rows.findIndex((row) => row.visibleIndex === endRowIndexId);
         if (startRowIndex == -1 || endRowIndex == -1) {
@@ -1822,11 +1842,12 @@ export class TableContext {
         // eslint-disable-next-line svelte/prefer-svelte-reactivity
         const indices = new Map();
         const selectedRows = [...this.highlightedItems.values()];
+        const sortedData = await this.sortedData;
         for (const item of selectedRows) {
-            const rowIndex = this.sortedData.indexOf(item);
+            const rowIndex = sortedData.indexOf(item);
             if (rowIndex == -1) {
-                // eslint-disable-next-line no-debugger
-                debugger;
+                console.warn(`row index for highlighted row data not found`);
+                continue;
             }
             indices.set(item, rowIndex);
         }
@@ -1835,22 +1856,22 @@ export class TableContext {
         const rowLength = selectedRows.length;
         for (let rowIndex = 0; rowIndex < rowLength; rowIndex++) {
             const item = selectedRows[rowIndex];
-            const key = this.selectionExtractor(item);
+            const key = await this.selectionExtractor(item);
             selectedCells[rowIndex] = [];
             const targetRow = selectedCells[rowIndex];
             const colLength = affectedColumns.length;
             for (let colIndex = 0; colIndex < colLength; colIndex++) {
                 const columnDef = affectedColumns[colIndex];
                 if (columnDef instanceof RowSelectionDef) {
-                    targetRow.push(key !== undefined && this.selectedItems.has(key) ? 'true' : 'false');
+                    targetRow.push(this.selectedItems.has(key) ? 'true' : 'false');
                     continue;
                 }
                 if (uncopyableColumns.has(columnDef.columnType)) {
                     continue;
                 }
-                const x = (await columnDef.getDisplayValue?.(item, this.sortedData.lastIndexOf(item))) ||
+                const x = (await columnDef.getDisplayValue?.(item, sortedData.lastIndexOf(item))) ||
                     ('accessorFn' in columnDef &&
-                        (await columnDef.accessorFn?.(item, this.sortedData.indexOf(item))));
+                        (await columnDef.accessorFn?.(item, sortedData.indexOf(item))));
                 if (x && typeof x === 'string') {
                     targetRow.push(x);
                     continue;
@@ -1878,10 +1899,10 @@ export class TableContext {
             const { row: startRowIndexId, column: startColumnIndex } = this.focusStart;
             const { row: endRowIndexId, column: endColumnIndex } = this.focusEnd;
             // const model = this._currentPageData;
-            const model = this.paginationRowModel;
+            const model = await this.paginationRowModel;
             const startRowIndex = model.rows.findIndex((row) => row.visibleIndex === startRowIndexId);
             const endRowIndex = model.rows.findIndex((row) => row.visibleIndex === endRowIndexId);
-            if (!this._currentPageData.length || startRowIndex == -1 || endRowIndex == -1) {
+            if (!(await this.currentPageData).length || startRowIndex == -1 || endRowIndex == -1) {
                 throw new Error(`copy was missing one of the two indices. First: ${startRowIndex}, second: ${endRowIndex}`);
             }
             const minRowIndex = Math.min(startRowIndex, endRowIndex);
@@ -1906,24 +1927,25 @@ export class TableContext {
             // TODO somewhere between here and the ending of the copy functionality, we end up with issues where instead of it using the accessor fn, it tries to pick up the select option, and ends up using the value ("1", etc)
             const selectedCells = [];
             const rowLength = selectedRows.length;
+            const sortedData = await this.sortedData;
             for (let rowIndex = 0; rowIndex < rowLength; rowIndex++) {
                 const item = selectedRows[rowIndex];
-                const key = this.selectionExtractor(item);
+                const key = await this.selectionExtractor(item);
                 selectedCells[rowIndex] = [];
                 const targetRow = selectedCells[rowIndex];
                 const colLength = affectedColumns.length;
                 for (let colIndex = 0; colIndex < colLength; colIndex++) {
                     const columnDef = affectedColumns[colIndex];
                     if (columnDef instanceof RowSelectionDef) {
-                        targetRow.push(key !== undefined && this.selectedItems.has(key) ? 'true' : 'false');
+                        targetRow.push(this.selectedItems.has(key) ? 'true' : 'false');
                         continue;
                     }
                     if (uncopyableColumns.has(columnDef.columnType)) {
                         continue;
                     }
-                    const x = (await columnDef.getDisplayValue?.(item, this.sortedData.lastIndexOf(item))) ||
+                    const x = (await columnDef.getDisplayValue?.(item, sortedData.lastIndexOf(item))) ||
                         ('accessorFn' in columnDef &&
-                            (await columnDef.accessorFn?.(item, this.sortedData.indexOf(item))));
+                            (await columnDef.accessorFn?.(item, sortedData.indexOf(item))));
                     if (x && typeof x === 'string') {
                         targetRow.push(x);
                         continue;
@@ -2207,6 +2229,22 @@ export class TableContext {
             };
         });
     }
+    async filteringLoop(currentTimer, abortController, settled, startTime) {
+        for (;;) {
+            if (currentTimer !== this.filterTimer) {
+                if (this.debug) {
+                    console.log(`aborting due to stale data`);
+                }
+                abortController.abort('stale data');
+                return;
+            }
+            if (settled.value) {
+                console.log(`iteration ${currentTimer} finished ${Date.now() - startTime}ms`);
+                return;
+            }
+            await tick();
+        }
+    }
 }
 // #region static functions
 function areDefsEquivalent(newColumnDefs, oldColumnDefs) {
@@ -2222,7 +2260,8 @@ function areDefsEquivalent(newColumnDefs, oldColumnDefs) {
         const oldDef = oldColumnDefs.find((d) => d.id === newDef.id);
         if (oldDef === undefined ||
             newDef.header !== oldDef.header ||
-            newDef.columnType !== oldDef.columnType
+            newDef.columnType !== oldDef.columnType ||
+            newDef.filterValueType !== oldDef.filterValueType
         // ||
         // newDef.defaultHidden !== oldDef.defaultHidden ||
         // newDef.enableHiding !== oldDef.enableHiding ||