@connecttomahdi/rxdb 17.0.0-beta.17 → 17.1.0

package/src/plugins/storage-memory/memory-helper.ts

@@ -1,5 +1,4 @@
 import type {
-    BulkWriteRow,
     RxDocumentData,
     RxJsonSchema
 } from '../../types/index.d.ts';
@@ -13,7 +12,7 @@ import {
     pushAtSortPosition
 } from 'array-push-at-sort-position';
 import { newRxError } from '../../rx-error.ts';
-import { boundEQ } from './binary-search-bounds.ts';
+import { boundEQByIndexString } from './binary-search-bounds.ts';
 
 
 export function getMemoryCollectionKey(
@@ -46,6 +45,14 @@ export function attachmentMapKey(documentId: string, attachmentId: string): stri
 }
 
 
+/**
+ * @performance
+ * Threshold for using in-place splice vs. full merge-sort when inserting
+ * documents into indexes. Below this batch size, in-place binary search + splice
+ * is faster because it avoids allocating a new full-size array and copying all elements.
+ */
+const IN_PLACE_INSERT_THRESHOLD = 64;
+
 function sortByIndexStringComparator<RxDocType>(a: DocWithIndexString<RxDocType>, b: DocWithIndexString<RxDocType>) {
     if (a[0] < b[0]) {
         return -1;
@@ -67,38 +74,65 @@ export function putWriteRowToState<RxDocType>(
     docInState?: RxDocumentData<RxDocType>
 ) {
     state.documents.set(docId, document as any);
-    for (let i = 0; i < stateByIndex.length; ++i) {
+    const stateByIndexLength = stateByIndex.length;
+    for (let i = 0; i < stateByIndexLength; ++i) {
         const byIndex = stateByIndex[i];
         const docsWithIndex = byIndex.docsWithIndex;
         const getIndexableString = byIndex.getIndexableString;
         const newIndexString = getIndexableString(document as any);
-        const insertPosition = pushAtSortPosition(
-            docsWithIndex,
-            [
-                newIndexString,
-                document,
-                docId,
-            ],
-            sortByIndexStringComparator,
-            0
-        );
 
         /**
-         * Remove previous if it was in the state
+         * @performance
+         * When updating a document, first compute whether the index changed.
+         * If it did not change, we only need to update the document reference
+         * in-place without any splice operations.
          */
         if (docInState) {
             const previousIndexString = getIndexableString(docInState);
             if (previousIndexString === newIndexString) {
                 /**
                  * Performance shortcut.
-                 * If index was not changed -> The old doc must be before or after the new one.
+                 * Index did not change, so the old entry is at the same position.
+                 * We can find it by string-specialized binary search and update in-place.
                  */
-                const prev = docsWithIndex[insertPosition - 1];
-                if (prev && prev[2] === docId) {
+                const eqPos = boundEQByIndexString(
+                    docsWithIndex,
+                    previousIndexString
+                );
+                if (eqPos !== -1) {
+                    /**
+                     * There might be multiple entries with the same index string
+                     * (e.g. different documents). Search around eqPos for ours.
+                     */
+                    if (docsWithIndex[eqPos][2] === docId) {
+                        docsWithIndex[eqPos][1] = document;
+                        continue;
+                    }
+                    // Check neighbors
+                    const prev = docsWithIndex[eqPos - 1];
+                    if (prev && prev[0] === previousIndexString && prev[2] === docId) {
+                        docsWithIndex[eqPos - 1][1] = document;
+                        continue;
+                    }
+                    const next = docsWithIndex[eqPos + 1];
+                    if (next && next[0] === previousIndexString && next[2] === docId) {
+                        docsWithIndex[eqPos + 1][1] = document;
+                        continue;
+                    }
+                }
+                // Fallback: use the old insert+remove approach
+                const insertPosition = pushAtSortPosition(
+                    docsWithIndex,
+                    [newIndexString, document, docId],
+                    sortByIndexStringComparator,
+                    0
+                );
+                const prevEntry = docsWithIndex[insertPosition - 1];
+                if (prevEntry && prevEntry[2] === docId) {
                     docsWithIndex.splice(insertPosition - 1, 1);
                 } else {
-                    const next = docsWithIndex[insertPosition + 1];
-                    if (next[2] === docId) {
+                    const nextEntry = docsWithIndex[insertPosition + 1];
+                    if (nextEntry[2] === docId) {
                         docsWithIndex.splice(insertPosition + 1, 1);
                     } else {
                         throw newRxError('SNH', {
@@ -109,20 +143,27 @@ export function putWriteRowToState<RxDocType>(
                         });
                     }
                 }
+                continue;
             } else {
                 /**
-                 * Index changed, we must search for the old one and remove it.
+                 * Index changed, we must remove the old entry and insert the new one.
                  */
-                const indexBefore = boundEQ(
+                const indexBefore = boundEQByIndexString(
                     docsWithIndex,
-                    [
-                        previousIndexString
-                    ] as any,
-                    compareDocsWithIndex
+                    previousIndexString
                 );
-                docsWithIndex.splice(indexBefore, 1);
+                if (indexBefore !== -1) {
+                    docsWithIndex.splice(indexBefore, 1);
+                }
             }
         }
+
+        pushAtSortPosition(
+            docsWithIndex,
+            [newIndexString, document, docId],
+            sortByIndexStringComparator,
+            0
+        );
     }
 }
 
@@ -130,9 +171,13 @@ export function putWriteRowToState<RxDocType>(
 /**
  * @hotPath
  * Efficiently inserts multiple documents into the state at once.
- * Instead of inserting one-by-one with Array.splice() (O(n) per insert),
- * this pre-computes all index entries, sorts them, and merges them into
- * the existing sorted arrays in a single pass (O(n log n + n + m)).
+ *
+ * Uses two strategies based on batch size:
+ * - For small batches (relative to existing index size), uses in-place
+ *   binary search + splice per document. This avoids allocating a new
+ *   full-size array and copying all elements, reducing GC pressure.
+ * - For large batches (or empty indexes), pre-computes all index entries,
+ *   sorts them, and merges into the existing sorted arrays in a single pass.
  */
 export function bulkInsertToState<RxDocType>(
     primaryPath: string,
@@ -141,6 +186,7 @@ export function bulkInsertToState<RxDocType>(
     docs: { document: RxDocumentData<RxDocType> }[]
 ) {
     const docsLength = docs.length;
+    const stateByIndexLength = stateByIndex.length;
 
     // Extract documents and docIds once, store in Map
     const documents: RxDocumentData<RxDocType>[] = new Array(docsLength);
@@ -153,55 +199,99 @@ export function bulkInsertToState<RxDocType>(
         state.documents.set(docId, doc as any);
     }
 
-    // For each index, batch-compute entries, sort, and merge
-    for (let indexI = 0; indexI < stateByIndex.length; ++indexI) {
-        const byIndex = stateByIndex[indexI];
-        const docsWithIndex = byIndex.docsWithIndex;
-        const getIndexableString = byIndex.getIndexableString;
+    /**
+     * @performance
+     * For small batch sizes, use in-place binary search + splice
+     * instead of creating a full merged array copy. This is faster
+     * for serial inserts and small bulk inserts because it avoids:
+     * - Allocating a new array of size n+m
+     * - Copying all n existing elements
+     * - GC pressure from discarding the old array
+     *
+     * The threshold is based on when the merge approach becomes more
+     * efficient than individual splices.
+     */
+    const useInPlaceInsert = docsLength < IN_PLACE_INSERT_THRESHOLD;
+
+    if (useInPlaceInsert) {
+        for (let indexI = 0; indexI < stateByIndexLength; ++indexI) {
+            const byIndex = stateByIndex[indexI];
+            const docsWithIndex = byIndex.docsWithIndex;
+            const getIndexableString = byIndex.getIndexableString;
 
-        // Build new entries
-        const newEntries: DocWithIndexString<RxDocType>[] = new Array(docsLength);
-        for (let i = 0; i < docsLength; ++i) {
-            const doc = documents[i];
-            newEntries[i] = [
-                getIndexableString(doc as any),
-                doc,
-                docIds[i]
-            ];
+            if (docsWithIndex.length === 0) {
+                for (let i = 0; i < docsLength; ++i) {
+                    const doc = documents[i];
+                    const indexString = getIndexableString(doc as any);
+                    docsWithIndex.push([indexString, doc, docIds[i]]);
+                }
+                docsWithIndex.sort(sortByIndexStringComparator);
+            } else {
+                for (let i = 0; i < docsLength; ++i) {
+                    const doc = documents[i];
+                    const indexString = getIndexableString(doc as any);
+                    const newEntry: DocWithIndexString<RxDocType> = [indexString, doc, docIds[i]];
+                    pushAtSortPosition(
+                        docsWithIndex,
+                        newEntry,
+                        sortByIndexStringComparator,
+                        0
+                    );
+                }
+            }
         }
+    } else {
+        // For each index, batch-compute entries, sort, and merge
+        for (let indexI = 0; indexI < stateByIndexLength; ++indexI) {
+            const byIndex = stateByIndex[indexI];
+            const docsWithIndex = byIndex.docsWithIndex;
+            const getIndexableString = byIndex.getIndexableString;
 
-        // Sort by index string
-        newEntries.sort(sortByIndexStringComparator);
+            // Build new entries
+            const newEntries: DocWithIndexString<RxDocType>[] = new Array(docsLength);
+            for (let i = 0; i < docsLength; ++i) {
+                const doc = documents[i];
+                newEntries[i] = [
+                    getIndexableString(doc as any),
+                    doc,
+                    docIds[i]
+                ];
+            }
 
-        if (docsWithIndex.length === 0) {
-            // Index is empty, just assign sorted entries
-            byIndex.docsWithIndex = newEntries;
-        } else {
-            // Merge sorted arrays
-            byIndex.docsWithIndex = mergeSortedArrays(docsWithIndex, newEntries, sortByIndexStringComparator);
+            // Sort by index string
+            newEntries.sort(sortByIndexStringComparator);
+
+            if (docsWithIndex.length === 0) {
+                // Index is empty, just assign sorted entries
+                byIndex.docsWithIndex = newEntries;
+            } else {
+                // Merge sorted arrays
+                byIndex.docsWithIndex = mergeSortedArrays(docsWithIndex, newEntries);
+            }
         }
     }
 }
 
 
 /**
- * Merges two sorted arrays into a single sorted array.
+ * Merges two sorted DocWithIndexString arrays into a single sorted array.
  * Runs in O(n + m) where n and m are the lengths of the input arrays.
+ * @performance Comparator is inlined to avoid function call overhead
+ * per comparison, which is significant for large arrays.
  */
-function mergeSortedArrays<T>(
-    a: T[],
-    b: T[],
-    comparator: (x: T, y: T) => number
-): T[] {
+function mergeSortedArrays<RxDocType>(
+    a: DocWithIndexString<RxDocType>[],
+    b: DocWithIndexString<RxDocType>[]
+): DocWithIndexString<RxDocType>[] {
     const aLen = a.length;
     const bLen = b.length;
-    const result: T[] = new Array(aLen + bLen);
+    const result: DocWithIndexString<RxDocType>[] = new Array(aLen + bLen);
     let ai = 0;
     let bi = 0;
     let ri = 0;
 
     while (ai < aLen && bi < bLen) {
-        if (comparator(a[ai], b[bi]) <= 0) {
+        if (a[ai][0] <= b[bi][0]) {
             result[ri++] = a[ai++];
         } else {
             result[ri++] = b[bi++];
@@ -227,19 +317,20 @@ export function removeDocFromState<RxDocType>(
     const docId: string = (doc as any)[primaryPath];
     state.documents.delete(docId);
 
-    Object.values(state.byIndex).forEach(byIndex => {
+    const stateByIndex = state.byIndexArray;
+    for (let i = 0; i < stateByIndex.length; ++i) {
+        const byIndex = stateByIndex[i];
         const docsWithIndex = byIndex.docsWithIndex;
         const indexString = byIndex.getIndexableString(doc);
 
-        const positionInIndex = boundEQ(
+        const positionInIndex = boundEQByIndexString(
             docsWithIndex,
-            [
-                indexString
-            ] as any,
-            compareDocsWithIndex
+            indexString
         );
-        docsWithIndex.splice(positionInIndex, 1);
-    });
+        if (positionInIndex !== -1) {
+            docsWithIndex.splice(positionInIndex, 1);
+        }
+    }
 }
 
 

package/src/plugins/storage-memory/memory-indexes.ts

@@ -26,6 +26,7 @@ export function addIndexesToInternalsState<RxDocType>(
             getIndexableString: getIndexableStringMonad(schema, indexAr)
         };
     });
+    state.byIndexArray = Object.values(state.byIndex);
 }
 
 

package/src/plugins/storage-memory/memory-types.ts

@@ -73,6 +73,11 @@ export type MemoryStorageInternals<RxDocType> = {
          */
         [indexName: string]: MemoryStorageInternalsByIndex<RxDocType>;
     };
+    /**
+     * Cached array of all MemoryStorageInternalsByIndex values.
+     * Avoids calling Object.values(byIndex) on every write operation.
+     */
+    byIndexArray: MemoryStorageInternalsByIndex<RxDocType>[];
 
     /**
      * We need these to do lazy writes.

package/src/plugins/storage-memory/rx-storage-instance-memory.ts

@@ -37,15 +37,14 @@ import {
     requestIdlePromiseNoQueue
 } from '../../plugins/utils/index.ts';
 import {
-    boundGE,
-    boundGT,
-    boundLE,
-    boundLT
+    boundGEByIndexString,
+    boundGTByIndexString,
+    boundLEByIndexString,
+    boundLTByIndexString
 } from './binary-search-bounds.ts';
 import {
     attachmentMapKey,
     bulkInsertToState,
-    compareDocsWithIndex,
     ensureNotRemoved,
     getMemoryCollectionKey,
     putWriteRowToState,
@@ -170,7 +169,6 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
             return;
         }
         const internals = this.internals;
-        const documentsById = this.internals.documents;
         const primaryPath = this.primaryPath;
 
         const categorized = this.internals.ensurePersistenceTask;
@@ -178,8 +176,9 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
 
         /**
          * Do inserts/updates
+         * @performance Use cached byIndexArray instead of Object.values()
          */
-        const stateByIndex = Object.values(this.internals.byIndex);
+        const stateByIndex = internals.byIndexArray;
 
         /**
          * @performance Use batch insert for bulk inserts to avoid
@@ -202,12 +201,19 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
             const writeRow = bulkUpdateDocs[i];
             const doc = writeRow.document;
             const docId = doc[primaryPath];
+            /**
+             * @performance
+             * Pass writeRow.previous directly as the old document state
+             * instead of re-looking it up from the documents Map.
+             * This is safe because categorizeBulkWriteRows already verified
+             * that previous._rev matches the document in the Map (conflict check).
+             */
             putWriteRowToState(
                 docId as any,
                 internals,
                 stateByIndex,
                 doc,
-                documentsById.get(docId as any)
+                writeRow.previous
             );
         }
 
@@ -254,17 +260,23 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
         if (documentsById.size === 0) {
             return Promise.resolve(ret);
         }
-        for (let i = 0; i < docIds.length; ++i) {
-            const docId = docIds[i];
-            const docInDb = documentsById.get(docId);
-            if (
-                docInDb &&
-                (
-                    !docInDb._deleted ||
-                    withDeleted
-                )
-            ) {
-                ret.push(docInDb);
+        /**
+         * @performance
+         * Split into two paths to avoid checking withDeleted on every iteration.
+         */
+        if (withDeleted) {
+            for (let i = 0; i < docIds.length; ++i) {
+                const docInDb = documentsById.get(docIds[i]);
+                if (docInDb) {
+                    ret.push(docInDb);
+                }
+            }
+        } else {
+            for (let i = 0; i < docIds.length; ++i) {
+                const docInDb = documentsById.get(docIds[i]);
+                if (docInDb && !docInDb._deleted) {
+                    ret.push(docInDb);
+                }
             }
         }
         return Promise.resolve(ret);
@@ -314,22 +326,17 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
         const docsWithIndex = this.internals.byIndex[indexName].docsWithIndex;
 
 
+        /**
+         * @performance Use string-specialized binary search to avoid
+         * temporary array allocations on every query.
+         */
+        let indexOfLower = queryPlan.inclusiveStart
+            ? boundGEByIndexString(docsWithIndex, lowerBoundString)
+            : boundGTByIndexString(docsWithIndex, lowerBoundString);
 
-        let indexOfLower = (queryPlan.inclusiveStart ? boundGE : boundGT)(
-            docsWithIndex,
-            [
-                lowerBoundString
-            ] as any,
-            compareDocsWithIndex
-        );
-
-        const indexOfUpper = (queryPlan.inclusiveEnd ? boundLE : boundLT)(
-            docsWithIndex,
-            [
-                upperBoundString
-            ] as any,
-            compareDocsWithIndex
-        );
+        const indexOfUpper = queryPlan.inclusiveEnd
+            ? boundLEByIndexString(docsWithIndex, upperBoundString)
+            : boundLTByIndexString(docsWithIndex, upperBoundString);
 
         let rows: RxDocumentData<RxDocType>[] = [];
 
@@ -405,6 +412,7 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
          * If the selector is satisfied by the index,
          * we can compute the count directly from the index range
          * without extracting document data into an array.
+         * Uses string-specialized binary search to avoid allocations.
          */
         if (queryPlan.selectorSatisfiedByIndex) {
             const queryPlanFields: string[] = queryPlan.index;
@@ -428,17 +436,13 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
             }
             const docsWithIndex = this.internals.byIndex[indexName].docsWithIndex;
 
-            const indexOfLower = (queryPlan.inclusiveStart ? boundGE : boundGT)(
-                docsWithIndex,
-                [lowerBoundString] as any,
-                compareDocsWithIndex
-            );
+            const indexOfLower = queryPlan.inclusiveStart
+                ? boundGEByIndexString(docsWithIndex, lowerBoundString)
+                : boundGTByIndexString(docsWithIndex, lowerBoundString);
 
-            const indexOfUpper = (queryPlan.inclusiveEnd ? boundLE : boundLT)(
-                docsWithIndex,
-                [upperBoundString] as any,
-                compareDocsWithIndex
-            );
+            const indexOfUpper = queryPlan.inclusiveEnd
+                ? boundLEByIndexString(docsWithIndex, upperBoundString)
+                : boundLTByIndexString(docsWithIndex, upperBoundString);
 
             const count = Math.max(0, indexOfUpper - indexOfLower + 1);
             return Promise.resolve({
@@ -447,10 +451,54 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
             });
         }
 
-        return this.query(preparedQuery).then(result => ({
-            count: result.documents.length,
+        const queryMatcher = getQueryMatcher(
+            this.schema,
+            preparedQuery.query
+        );
+        const queryPlanFields: string[] = queryPlan.index;
+        const index: string[] = queryPlanFields;
+        const lowerBound: any[] = queryPlan.startKeys;
+        const lowerBoundString = getStartIndexStringFromLowerBound(
+            this.schema,
+            index,
+            lowerBound
+        );
+        const upperBound: any[] = queryPlan.endKeys;
+        const upperBoundString = getStartIndexStringFromUpperBound(
+            this.schema,
+            index,
+            upperBound
+        );
+        const indexName = getMemoryIndexName(index);
+        if (!this.internals.byIndex[indexName]) {
+            throw newRxError('SNH', { args: { indexName } });
+        }
+        const docsWithIndex = this.internals.byIndex[indexName].docsWithIndex;
+
+        let indexOfLower = queryPlan.inclusiveStart
+            ? boundGEByIndexString(docsWithIndex, lowerBoundString)
+            : boundGTByIndexString(docsWithIndex, lowerBoundString);
+
+        const indexOfUpper = queryPlan.inclusiveEnd
+            ? boundLEByIndexString(docsWithIndex, upperBoundString)
+            : boundLTByIndexString(docsWithIndex, upperBoundString);
+
+        let count = 0;
+        while (indexOfLower <= indexOfUpper) {
+            const currentRow = docsWithIndex[indexOfLower];
+            if (!currentRow) {
+                break;
+            }
+            if (queryMatcher(currentRow[1])) {
+                count++;
+            }
+            indexOfLower++;
+        }
+
+        return Promise.resolve({
+            count,
             mode: 'fast' as const
-        }));
+        });
     }
 
     cleanup(minimumDeletedTime: number): Promise<boolean> {
@@ -470,12 +518,9 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
             ]
         );
 
-        let indexOfLower = boundGT(
+        let indexOfLower = boundGTByIndexString(
             docsWithIndex,
-            [
-                lowerBoundString
-            ] as any,
-            compareDocsWithIndex
+            lowerBoundString
         );
 
         let done = false;
@@ -490,7 +535,12 @@ export class RxStorageInstanceMemory<RxDocType> implements RxStorageInstance<
                     this.internals,
                     currentDoc[1]
                 );
-                indexOfLower++;
+                /**
+                 * Do NOT increment indexOfLower after removal.
+                 * removeDocFromState() splices the element out of the array,
+                 * so the next element shifts into the current position.
+                 * Incrementing would skip it.
+                 */
             }
         }
         return PROMISE_RESOLVE_TRUE;
@@ -574,6 +624,7 @@ export function createMemoryStorageInstance<RxDocType>(
             documents: new Map(),
             attachments: params.schema.attachments ? new Map() : undefined as any,
             byIndex: {},
+            byIndexArray: [],
             changes$: new Subject()
         };
         addIndexesToInternalsState(internals, params.schema);

package/src/plugins/storage-mongodb/mongodb-helper.ts

@@ -6,7 +6,8 @@ import type {
     RxJsonSchema
 } from '../../types/index.d.ts';
 import {
-    Sort as MongoSort
+    Sort as MongoSort,
+    MongoClient
 } from 'mongodb';
 import { RXDB_VERSION, flatClone } from '../utils/index.ts';
 import { MongoDBPreparedQuery, MongoQuerySelector } from './mongodb-types.ts';
@@ -117,3 +118,44 @@ export function swapToMongoSort<RxDocType>(
 export function getMongoDBIndexName(index: string[]): string {
     return index.join('|');
 }
+
+export const MONGO_CLIENT_CACHE = new Map<string, { client: MongoClient, refCount: number, promise: Promise<MongoClient>, closeTimeout?: ReturnType<typeof setTimeout> }>();
+
+export async function getMongoDBClient(connection: string): Promise<MongoClient> {
+    let cached = MONGO_CLIENT_CACHE.get(connection);
+    if (!cached) {
+        const client = new MongoClient(connection, MONGO_OPTIONS_DRIVER_INFO);
+        cached = {
+            client,
+            refCount: 1,
+            promise: client.connect().then(() => client)
+        };
+        MONGO_CLIENT_CACHE.set(connection, cached);
+        return cached.promise;
+    }
+    cached.refCount++;
+    if (cached.closeTimeout) {
+        clearTimeout(cached.closeTimeout);
+        cached.closeTimeout = undefined;
+    }
+    return cached.promise;
+}
+
+export async function closeMongoDBClient(connection: string): Promise<void> {
+    const cached = MONGO_CLIENT_CACHE.get(connection);
+    if (cached) {
+        cached.refCount--;
+        if (cached.refCount === 0) {
+            if (cached.closeTimeout) {
+                clearTimeout(cached.closeTimeout);
+            }
+            cached.closeTimeout = setTimeout(() => {
+                const innerCached = MONGO_CLIENT_CACHE.get(connection);
+                if (innerCached && innerCached.refCount === 0) {
+                    MONGO_CLIENT_CACHE.delete(connection);
+                    innerCached.client.close().catch(() => { });
+                }
+            }, 2000); // 2 second delay to allow reuse
+        }
+    }
+}