@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-memory/binary-search-bounds.ts

@@ -0,0 +1,106 @@
+/**
+ * Everything in this file was copied and adapted from
+ * @link https://github.com/mikolalysenko/binary-search-bounds
+ *
+ * We should use the original npm module instead when this bug is fixed:
+ * @link https://github.com/mikolalysenko/binary-search-bounds/pull/14
+ */
+
+
+
+type Compare<T> = ((a: T, b: T) => number | null | undefined);
+
+function ge<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any): number {
+    let i: number = h + 1;
+    while (l <= h) {
+        const m = (l + h) >>> 1;
+        const x: any = a[m];
+        const p: any = (c !== undefined) ? c(x, y) : (x - (y as any));
+        if (p >= 0) {
+            i = m; h = m - 1;
+        } else {
+            l = m + 1;
+        }
+    }
+    return i;
+}
+
+function gt<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any): number {
+    let i = h + 1;
+    while (l <= h) {
+        const m = (l + h) >>> 1;
+        const x = a[m];
+        const p: any = (c !== undefined) ? c(x, y) : ((x as any) - (y as any));
+        if (p > 0) {
+            i = m; h = m - 1;
+        } else {
+            l = m + 1;
+        }
+    }
+    return i;
+}
+
+function lt<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any): number {
+    let i = l - 1;
+    while (l <= h) {
+        const m = (l + h) >>> 1, x = a[m];
+        const p: any = (c !== undefined) ? c(x, y) : ((x as any) - (y as any));
+        if (p < 0) {
+            i = m; l = m + 1;
+        } else {
+            h = m - 1;
+        }
+    }
+    return i;
+}
+
+function le<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any): number {
+    let i = l - 1;
+    while (l <= h) {
+        const m = (l + h) >>> 1, x = a[m];
+        const p: any = (c !== undefined) ? c(x, y) : ((x as any) - (y as any));
+        if (p <= 0) {
+            i = m; l = m + 1;
+        } else {
+            h = m - 1;
+        }
+    }
+    return i;
+}
+
+function eq<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any): number {
+    while (l <= h) {
+        const m = (l + h) >>> 1, x = a[m];
+        const p: any = (c !== undefined) ? c(x, y) : ((x as any) - (y as any));
+        if (p === 0) {
+            return m;
+        }
+        if (p <= 0) {
+            l = m + 1;
+        } else {
+            h = m - 1;
+        }
+    }
+    return -1;
+}
+
+function norm<T>(a: T[], y: T, c: Compare<T>, l: any, h: any, f: any) {
+    return f(a, y, c, (l === undefined) ? 0 : l | 0, (h === undefined) ? a.length - 1 : h | 0);
+}
+
+
+export function boundGE<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any) {
+    return norm(a, y, c, l, h, ge);
+}
+export function boundGT<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any) {
+    return norm(a, y, c, l, h, gt);
+}
+export function boundLT<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any) {
+    return norm(a, y, c, l, h, lt);
+}
+export function boundLE<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any) {
+    return norm(a, y, c, l, h, le);
+}
+export function boundEQ<T>(a: T[], y: T, c: Compare<T>, l?: any, h?: any) {
+    return norm(a, y, c, l, h, eq);
+}

package/src/plugins/storage-memory/index.ts

@@ -0,0 +1,52 @@
+import { ensureRxStorageInstanceParamsAreCorrect } from '../../rx-storage-helper.ts';
+import type { RxStorageInstanceCreationParams } from '../../types/index.d.ts';
+import { RXDB_VERSION } from '../utils/utils-rxdb-version.ts';
+import type {
+    RxStorageMemory,
+    RxStorageMemoryInstanceCreationOptions,
+    RxStorageMemorySettings
+} from './memory-types.ts';
+import {
+    createMemoryStorageInstance,
+    RxStorageInstanceMemory
+} from './rx-storage-instance-memory.ts';
+
+/**
+ * Keep the state even when the storage instance is closed.
+ * This makes it easier to use the memory storage
+ * to test filesystem-like and multiInstance behaviors.
+ */
+const COLLECTION_STATES = new Map();
+
+export function getRxStorageMemory(
+    settings: RxStorageMemorySettings = {}
+): RxStorageMemory {
+
+    const storage: RxStorageMemory = {
+        name: 'memory',
+        rxdbVersion: RXDB_VERSION,
+        collectionStates: COLLECTION_STATES,
+        createStorageInstance<RxDocType>(
+            params: RxStorageInstanceCreationParams<RxDocType, RxStorageMemoryInstanceCreationOptions>
+        ): Promise<RxStorageInstanceMemory<RxDocType>> {
+            ensureRxStorageInstanceParamsAreCorrect(params);
+            const useSettings = Object.assign(
+                {},
+                settings,
+                params.options
+            );
+
+
+            return createMemoryStorageInstance(this, params, useSettings);
+        }
+    };
+
+    return storage;
+}
+
+
+export * from './memory-helper.ts';
+export * from './binary-search-bounds.ts';
+export * from './memory-types.ts';
+export * from './memory-indexes.ts';
+export * from './rx-storage-instance-memory.ts';

package/src/plugins/storage-memory/memory-helper.ts

@@ -0,0 +1,259 @@
+import type {
+    BulkWriteRow,
+    RxDocumentData,
+    RxJsonSchema
+} from '../../types/index.d.ts';
+import type {
+    DocWithIndexString,
+    MemoryStorageInternals,
+    MemoryStorageInternalsByIndex
+} from './memory-types.ts';
+import type { RxStorageInstanceMemory } from './rx-storage-instance-memory.ts';
+import {
+    pushAtSortPosition
+} from 'array-push-at-sort-position';
+import { newRxError } from '../../rx-error.ts';
+import { boundEQ } from './binary-search-bounds.ts';
+
+
+export function getMemoryCollectionKey(
+    databaseName: string,
+    collectionName: string,
+    schemaVersion: number
+): string {
+    return [
+        databaseName,
+        collectionName,
+        schemaVersion
+    ].join('--memory--');
+}
+
+
+export function ensureNotRemoved(
+    instance: RxStorageInstanceMemory<any>
+) {
+    if (instance.internals.removed) {
+        throw new Error(
+            'removed already ' +
+            instance.databaseName + ' - ' + instance.collectionName +
+            ' - ' + instance.schema.version
+        );
+    }
+}
+
+export function attachmentMapKey(documentId: string, attachmentId: string): string {
+    return documentId + '||' + attachmentId;
+}
+
+
+function sortByIndexStringComparator<RxDocType>(a: DocWithIndexString<RxDocType>, b: DocWithIndexString<RxDocType>) {
+    if (a[0] < b[0]) {
+        return -1;
+    } else {
+        return 1;
+    }
+}
+
+
+
+/**
+ * @hotPath
+ */
+export function putWriteRowToState<RxDocType>(
+    docId: string,
+    state: MemoryStorageInternals<RxDocType>,
+    stateByIndex: MemoryStorageInternalsByIndex<RxDocType>[],
+    document: RxDocumentData<RxDocType>,
+    docInState?: RxDocumentData<RxDocType>
+) {
+    state.documents.set(docId, document as any);
+    for (let i = 0; i < stateByIndex.length; ++i) {
+        const byIndex = stateByIndex[i];
+        const docsWithIndex = byIndex.docsWithIndex;
+        const getIndexableString = byIndex.getIndexableString;
+        const newIndexString = getIndexableString(document as any);
+        const insertPosition = pushAtSortPosition(
+            docsWithIndex,
+            [
+                newIndexString,
+                document,
+                docId,
+            ],
+            sortByIndexStringComparator,
+            0
+        );
+
+        /**
+         * Remove previous if it was in the state
+         */
+        if (docInState) {
+            const previousIndexString = getIndexableString(docInState);
+            if (previousIndexString === newIndexString) {
+                /**
+                 * Performance shortcut.
+                 * If index was not changed -> The old doc must be before or after the new one.
+                 */
+                const prev = docsWithIndex[insertPosition - 1];
+                if (prev && prev[2] === docId) {
+                    docsWithIndex.splice(insertPosition - 1, 1);
+                } else {
+                    const next = docsWithIndex[insertPosition + 1];
+                    if (next[2] === docId) {
+                        docsWithIndex.splice(insertPosition + 1, 1);
+                    } else {
+                        throw newRxError('SNH', {
+                            document,
+                            args: {
+                                byIndex
+                            }
+                        });
+                    }
+                }
+            } else {
+                /**
+                 * Index changed, we must search for the old one and remove it.
+                 */
+                const indexBefore = boundEQ(
+                    docsWithIndex,
+                    [
+                        previousIndexString
+                    ] as any,
+                    compareDocsWithIndex
+                );
+                docsWithIndex.splice(indexBefore, 1);
+            }
+        }
+    }
+}
+
+
+/**
+ * @hotPath
+ * Efficiently inserts multiple documents into the state at once.
+ * Instead of inserting one-by-one with Array.splice() (O(n) per insert),
+ * this pre-computes all index entries, sorts them, and merges them into
+ * the existing sorted arrays in a single pass (O(n log n + n + m)).
+ */
+export function bulkInsertToState<RxDocType>(
+    primaryPath: string,
+    state: MemoryStorageInternals<RxDocType>,
+    stateByIndex: MemoryStorageInternalsByIndex<RxDocType>[],
+    docs: { document: RxDocumentData<RxDocType> }[]
+) {
+    const docsLength = docs.length;
+
+    // Extract documents and docIds once, store in Map
+    const documents: RxDocumentData<RxDocType>[] = new Array(docsLength);
+    const docIds: string[] = new Array(docsLength);
+    for (let i = 0; i < docsLength; ++i) {
+        const doc = docs[i].document;
+        const docId: string = (doc as any)[primaryPath];
+        documents[i] = doc;
+        docIds[i] = docId;
+        state.documents.set(docId, doc as any);
+    }
+
+    // For each index, batch-compute entries, sort, and merge
+    for (let indexI = 0; indexI < stateByIndex.length; ++indexI) {
+        const byIndex = stateByIndex[indexI];
+        const docsWithIndex = byIndex.docsWithIndex;
+        const getIndexableString = byIndex.getIndexableString;
+
+        // Build new entries
+        const newEntries: DocWithIndexString<RxDocType>[] = new Array(docsLength);
+        for (let i = 0; i < docsLength; ++i) {
+            const doc = documents[i];
+            newEntries[i] = [
+                getIndexableString(doc as any),
+                doc,
+                docIds[i]
+            ];
+        }
+
+        // Sort by index string
+        newEntries.sort(sortByIndexStringComparator);
+
+        if (docsWithIndex.length === 0) {
+            // Index is empty, just assign sorted entries
+            byIndex.docsWithIndex = newEntries;
+        } else {
+            // Merge sorted arrays
+            byIndex.docsWithIndex = mergeSortedArrays(docsWithIndex, newEntries, sortByIndexStringComparator);
+        }
+    }
+}
+
+
+/**
+ * Merges two sorted arrays into a single sorted array.
+ * Runs in O(n + m) where n and m are the lengths of the input arrays.
+ */
+function mergeSortedArrays<T>(
+    a: T[],
+    b: T[],
+    comparator: (x: T, y: T) => number
+): T[] {
+    const aLen = a.length;
+    const bLen = b.length;
+    const result: T[] = new Array(aLen + bLen);
+    let ai = 0;
+    let bi = 0;
+    let ri = 0;
+
+    while (ai < aLen && bi < bLen) {
+        if (comparator(a[ai], b[bi]) <= 0) {
+            result[ri++] = a[ai++];
+        } else {
+            result[ri++] = b[bi++];
+        }
+    }
+
+    while (ai < aLen) {
+        result[ri++] = a[ai++];
+    }
+    while (bi < bLen) {
+        result[ri++] = b[bi++];
+    }
+
+    return result;
+}
+
+export function removeDocFromState<RxDocType>(
+    primaryPath: string,
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    state: MemoryStorageInternals<RxDocType>,
+    doc: RxDocumentData<RxDocType>
+) {
+    const docId: string = (doc as any)[primaryPath];
+    state.documents.delete(docId);
+
+    Object.values(state.byIndex).forEach(byIndex => {
+        const docsWithIndex = byIndex.docsWithIndex;
+        const indexString = byIndex.getIndexableString(doc);
+
+        const positionInIndex = boundEQ(
+            docsWithIndex,
+            [
+                indexString
+            ] as any,
+            compareDocsWithIndex
+        );
+        docsWithIndex.splice(positionInIndex, 1);
+    });
+}
+
+
+export function compareDocsWithIndex<RxDocType>(
+    a: DocWithIndexString<RxDocType>,
+    b: DocWithIndexString<RxDocType>
+): 1 | 0 | -1 {
+    const indexStringA = a[0];
+    const indexStringB = b[0];
+    if (indexStringA < indexStringB) {
+        return -1;
+    } else if (indexStringA === indexStringB) {
+        return 0;
+    } else {
+        return 1;
+    }
+}

package/src/plugins/storage-memory/memory-indexes.ts

@@ -0,0 +1,34 @@
+import { getIndexableStringMonad } from '../../custom-index.ts';
+import { getPrimaryFieldOfPrimaryKey } from '../../rx-schema-helper.ts';
+import type { RxDocumentData, RxJsonSchema } from '../../types/index.d.ts';
+import { toArray } from '../../plugins/utils/index.ts';
+import type { MemoryStorageInternals } from './memory-types.ts';
+
+export function addIndexesToInternalsState<RxDocType>(
+    state: MemoryStorageInternals<RxDocType>,
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>
+) {
+    const primaryPath = getPrimaryFieldOfPrimaryKey(schema.primaryKey);
+    const useIndexes: string[][] = !schema.indexes ? [] : schema.indexes.map(row => toArray(row)) as any;
+
+    // we need this index for running cleanup()
+    useIndexes.push([
+        '_deleted',
+        '_meta.lwt',
+        primaryPath
+    ]);
+
+
+    useIndexes.forEach(indexAr => {
+        state.byIndex[getMemoryIndexName(indexAr)] = {
+            index: indexAr,
+            docsWithIndex: [],
+            getIndexableString: getIndexableStringMonad(schema, indexAr)
+        };
+    });
+}
+
+
+export function getMemoryIndexName(index: string[]): string {
+    return index.join(',');
+}

package/src/plugins/storage-memory/memory-types.ts

@@ -0,0 +1,90 @@
+import { Subject } from 'rxjs';
+import type {
+    CategorizeBulkWriteRowsOutput,
+    EventBulk,
+    RxAttachmentWriteData,
+    RxDocumentData,
+    RxJsonSchema,
+    RxStorage,
+    RxStorageChangeEvent,
+    RxStorageDefaultCheckpoint
+} from '../../types/index.d.ts';
+
+export type RxStorageMemorySettings = {};
+export type RxStorageMemoryInstanceCreationOptions = {};
+export type RxStorageMemory = RxStorage<MemoryStorageInternals<any>, RxStorageMemoryInstanceCreationOptions> & {
+    /**
+     * State by collectionKey
+     */
+    collectionStates: Map<string, MemoryStorageInternals<any>>;
+};
+
+export type MemoryStorageInternalsByIndex<RxDocType> = {
+    index: string[];
+    docsWithIndex: DocWithIndexString<RxDocType>[];
+    getIndexableString: (docData: RxDocumentData<RxDocType>) => string;
+};
+
+/**
+ * The internals are shared between multiple storage instances
+ * that have been created with the same [databaseName+collectionName] combination.
+ */
+export type MemoryStorageInternals<RxDocType> = {
+    // used to debug stuff and identify instances
+    id: string;
+
+    /**
+     * Schema of the first instance created with the given settings.
+     * Used to ensure that the same storage is not re-created with
+     * a different schema.
+     */
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>;
+
+    /**
+     * We reuse the memory state when multiple instances
+     * are created with the same params.
+     * If refCount becomes 0, we can delete the state.
+     */
+    refCount: number;
+    /**
+     * If this becomes true,
+     * it means that an instance has called remove()
+     * so all other instances should also not work anymore.
+     */
+    removed: boolean;
+    documents: Map<string, RxDocumentData<RxDocType>>;
+    /**
+     * Attachments data, indexed by a combined string
+     * consisting of [documentId + '||' + attachmentId]
+     */
+    attachments: Map<string, {
+        writeData: RxAttachmentWriteData;
+        digest: string;
+    }>;
+    byIndex: {
+        /**
+         * Because RxDB requires a deterministic sorting
+         * on all indexes, we can be sure that the composed index key
+         * of each document is unique, because it contains the primaryKey
+         * as last index part.
+         * So we do not have to store the index-position when we want to do fast
+         * writes. Instead we can do a binary search over the existing array
+         * because RxDB also knows the previous state of the document when we do a bulkWrite().
+         */
+        [indexName: string]: MemoryStorageInternalsByIndex<RxDocType>;
+    };
+
+    /**
+     * We need these to do lazy writes.
+     */
+    ensurePersistenceTask?: CategorizeBulkWriteRowsOutput<RxDocType>;
+    ensurePersistenceIdlePromise?: Promise<void>;
+
+    changes$: Subject<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, RxStorageDefaultCheckpoint>>;
+};
+
+export type DocWithIndexString<RxDocType> = [
+    string, // indexString, must be first because often we only need that one.
+    RxDocumentData<RxDocType>, // document
+    string, // id
+];