@connecttomahdi/rxdb 17.0.0-beta.17

package/src/custom-index.ts

@@ -0,0 +1,376 @@
+/**
+ * For some RxStorage implementations,
+ * we need to use our custom crafted indexes
+ * so we can easily iterate over them. And sort plain arrays of document data.
+ *
+ * We really often have to craft an index string for a given document.
+ * Performance of everything in this file is very important
+ * which is why the code sometimes looks strange.
+ * Run performance tests before and after you touch anything here!
+ */
+
+import {
+    getSchemaByObjectPath
+} from './rx-schema-helper.ts';
+import type {
+    JsonSchema,
+    RxDocumentData,
+    RxJsonSchema
+} from './types/index.d.ts';
+import {
+    ensureNotFalsy,
+    objectPathMonad,
+    ObjectPathMonadFunction
+} from './plugins/utils/index.ts';
+import {
+    INDEX_MAX,
+    INDEX_MIN
+} from './query-planner.ts';
+
+
+/**
+ * Prepare all relevant information
+ * outside of the returned function
+ * from getIndexableStringMonad()
+ * to save performance when the returned
+ * function is called many times.
+ */
+type IndexMetaField<RxDocType> = {
+    fieldName: string;
+    schemaPart: JsonSchema;
+    /*
+     * Only in number fields.
+     */
+    parsedLengths?: ParsedLengths;
+    getValue: ObjectPathMonadFunction<RxDocType>;
+    getIndexStringPart: (docData: RxDocumentData<RxDocType>) => string;
+};
+
+export function getIndexMeta<RxDocType>(
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    index: string[]
+): IndexMetaField<RxDocType>[] {
+    const fieldNameProperties: IndexMetaField<RxDocType>[] = index.map(fieldName => {
+        const schemaPart = getSchemaByObjectPath(
+            schema,
+            fieldName
+        );
+        if (!schemaPart) {
+            throw new Error('not in schema: ' + fieldName);
+        }
+        const type = schemaPart.type;
+        let parsedLengths: ParsedLengths | undefined;
+        if (type === 'number' || type === 'integer') {
+            parsedLengths = getStringLengthOfIndexNumber(
+                schemaPart
+            );
+        }
+
+        const getValue = objectPathMonad(fieldName);
+        const maxLength = schemaPart.maxLength ? schemaPart.maxLength : 0;
+
+        let getIndexStringPart: (docData: RxDocumentData<RxDocType>) => string;
+        if (type === 'string') {
+            getIndexStringPart = docData => {
+                let fieldValue = getValue(docData);
+                if (!fieldValue) {
+                    fieldValue = '';
+                }
+                return fieldValue.padEnd(maxLength, ' ');
+            };
+        } else if (type === 'boolean') {
+            getIndexStringPart = docData => {
+                const fieldValue = getValue(docData);
+                return fieldValue ? '1' : '0';
+            };
+        } else { // number
+            getIndexStringPart = docData => {
+                const fieldValue = getValue(docData);
+                return getNumberIndexString(
+                    parsedLengths as any,
+                    fieldValue
+                );
+            };
+        }
+
+        const ret: IndexMetaField<RxDocType> = {
+            fieldName,
+            schemaPart,
+            parsedLengths,
+            getValue,
+            getIndexStringPart
+        };
+        return ret;
+    });
+    return fieldNameProperties;
+}
+
+
+/**
+ * Crafts an indexable string that can be used
+ * to check if a document would be sorted below or above
+ * another documents, dependent on the index values.
+ * @monad for better performance
+ *
+ * IMPORTANT: Performance is really important here
+ * which is why we code so 'strange'.
+ * Always run performance tests when you want to
+ * change something in this method.
+ */
+export function getIndexableStringMonad<RxDocType>(
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    index: string[]
+): (docData: RxDocumentData<RxDocType>) => string {
+    const fieldNameProperties = getIndexMeta(schema, index);
+    const fieldNamePropertiesAmount = fieldNameProperties.length;
+    const indexPartsFunctions = fieldNameProperties.map(r => r.getIndexStringPart);
+
+
+    /**
+     * @hotPath Performance of this function is very critical!
+     */
+    const ret = function (docData: RxDocumentData<RxDocType>): string {
+        let str = '';
+        for (let i = 0; i < fieldNamePropertiesAmount; ++i) {
+            str += indexPartsFunctions[i](docData);
+        }
+        return str;
+    };
+    return ret;
+}
+
+
+declare type ParsedLengths = {
+    minimum: number;
+    maximum: number;
+    nonDecimals: number;
+    decimals: number;
+    roundedMinimum: number;
+};
+export function getStringLengthOfIndexNumber(
+    schemaPart: JsonSchema
+): ParsedLengths {
+    const minimum = Math.floor(schemaPart.minimum as number);
+    const maximum = Math.ceil(schemaPart.maximum as number);
+    const multipleOf: number = schemaPart.multipleOf as number;
+
+    const valueSpan = maximum - minimum;
+    const nonDecimals = valueSpan.toString().length;
+
+    const multipleOfParts = multipleOf.toString().split('.');
+    let decimals = 0;
+    if (multipleOfParts.length > 1) {
+        decimals = multipleOfParts[1].length;
+    }
+    return {
+        minimum,
+        maximum,
+        nonDecimals,
+        decimals,
+        roundedMinimum: minimum
+    };
+}
+
+export function getIndexStringLength<RxDocType>(
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    index: string[]
+): number {
+    const fieldNameProperties = getIndexMeta(schema, index);
+    let length = 0;
+    fieldNameProperties.forEach(props => {
+        const schemaPart = props.schemaPart;
+        const type = schemaPart.type;
+
+        if (type === 'string') {
+            length += schemaPart.maxLength as number;
+        } else if (type === 'boolean') {
+            length += 1;
+        } else {
+            const parsedLengths = props.parsedLengths as ParsedLengths;
+            length = length + parsedLengths.nonDecimals + parsedLengths.decimals;
+        }
+
+    });
+    return length;
+}
+
+
+export function getPrimaryKeyFromIndexableString(
+    indexableString: string,
+    primaryKeyLength: number
+): string {
+    const paddedPrimaryKey = indexableString.slice(primaryKeyLength * -1);
+    // we can safely trim here because the primary key is not allowed to start or end with a space char.
+    const primaryKey = paddedPrimaryKey.trim();
+    return primaryKey;
+}
+
+
+export function getNumberIndexString(
+    parsedLengths: ParsedLengths,
+    fieldValue: number
+): string {
+    /**
+     * Ensure that the given value is in the boundaries
+     * of the schema, otherwise it would create a broken index string.
+     * This can happen for example if you have a minimum of 0
+     * and run a query like
+     * selector {
+     *  numField: { $gt: -1000 }
+     * }
+     */
+    if (typeof fieldValue === 'undefined') {
+        fieldValue = 0;
+    }
+    if (fieldValue < parsedLengths.minimum) {
+        fieldValue = parsedLengths.minimum;
+    }
+    if (fieldValue > parsedLengths.maximum) {
+        fieldValue = parsedLengths.maximum;
+    }
+
+    const nonDecimalsValueAsString = (Math.floor(fieldValue) - parsedLengths.roundedMinimum).toString();
+    let str = nonDecimalsValueAsString.padStart(parsedLengths.nonDecimals, '0');
+
+    if (parsedLengths.decimals > 0) {
+        const splitByDecimalPoint = fieldValue.toString().split('.');
+        const decimalValueAsString = splitByDecimalPoint.length > 1 ? splitByDecimalPoint[1] : '0';
+        str += decimalValueAsString.padEnd(parsedLengths.decimals, '0');
+    }
+    return str;
+}
+
+export function getStartIndexStringFromLowerBound(
+    schema: RxJsonSchema<any>,
+    index: string[],
+    lowerBound: (string | boolean | number | null | undefined)[]
+): string {
+    let str = '';
+    index.forEach((fieldName, idx) => {
+        const schemaPart = getSchemaByObjectPath(
+            schema,
+            fieldName
+        );
+        const bound = lowerBound[idx];
+        const type = schemaPart.type;
+
+        switch (type) {
+            case 'string':
+                const maxLength = ensureNotFalsy(schemaPart.maxLength, 'maxLength not set');
+                if (typeof bound === 'string') {
+                    str += (bound as string).padEnd(maxLength, ' ');
+                } else {
+                    // str += ''.padStart(maxLength, inclusiveStart ? ' ' : INDEX_MAX);
+                    str += ''.padEnd(maxLength, ' ');
+                }
+                break;
+            case 'boolean':
+                if (bound === null) {
+                    str += '0';
+                } else if (bound === INDEX_MIN) {
+                    str += '0';
+                } else if (bound === INDEX_MAX) {
+                    str += '1';
+                } else {
+                    const boolToStr = bound ? '1' : '0';
+                    str += boolToStr;
+                }
+                break;
+            case 'number':
+            case 'integer':
+                const parsedLengths = getStringLengthOfIndexNumber(
+                    schemaPart
+                );
+                if (bound === null || bound === INDEX_MIN) {
+                    const fillChar = '0';
+                    str += fillChar.repeat(parsedLengths.nonDecimals + parsedLengths.decimals);
+                } else if (bound === INDEX_MAX) {
+                    str += getNumberIndexString(
+                        parsedLengths,
+                        parsedLengths.maximum
+                    );
+                } else {
+                    const add = getNumberIndexString(
+                        parsedLengths,
+                        bound as number
+                    );
+                    str += add;
+                }
+                break;
+            default:
+                throw new Error('unknown index type ' + type);
+        }
+    });
+    return str;
+}
+
+
+export function getStartIndexStringFromUpperBound(
+    schema: RxJsonSchema<any>,
+    index: string[],
+    upperBound: (string | boolean | number | null | undefined)[]
+): string {
+    let str = '';
+    index.forEach((fieldName, idx) => {
+        const schemaPart = getSchemaByObjectPath(
+            schema,
+            fieldName
+        );
+        const bound = upperBound[idx];
+        const type = schemaPart.type;
+
+        switch (type) {
+            case 'string':
+                const maxLength = ensureNotFalsy(schemaPart.maxLength, 'maxLength not set');
+                if (typeof bound === 'string' && bound !== INDEX_MAX) {
+                    str += (bound as string).padEnd(maxLength, ' ');
+                } else if (bound === INDEX_MIN) {
+                    str += ''.padEnd(maxLength, ' ');
+                } else {
+                    str += ''.padEnd(maxLength, INDEX_MAX);
+                }
+                break;
+            case 'boolean':
+                if (bound === null) {
+                    str += '1';
+                } else {
+                    const boolToStr = bound ? '1' : '0';
+                    str += boolToStr;
+                }
+                break;
+            case 'number':
+            case 'integer':
+                const parsedLengths = getStringLengthOfIndexNumber(
+                    schemaPart
+                );
+                if (bound === null || bound === INDEX_MAX) {
+                    const fillChar = '9';
+                    str += fillChar.repeat(parsedLengths.nonDecimals + parsedLengths.decimals);
+                } else if (bound === INDEX_MIN) {
+                    const fillChar = '0';
+                    str += fillChar.repeat(parsedLengths.nonDecimals + parsedLengths.decimals);
+                } else {
+                    str += getNumberIndexString(
+                        parsedLengths,
+                        bound as number
+                    );
+                }
+                break;
+            default:
+                throw new Error('unknown index type ' + type);
+        }
+    });
+    return str;
+}
+
+/**
+ * Used in storages where it is not possible
+ * to define inclusiveEnd/inclusiveStart
+ */
+export function changeIndexableStringByOneQuantum(str: string, direction: 1 | -1): string {
+    const lastChar = str.slice(-1);
+    let charCode = lastChar.charCodeAt(0);
+    charCode = charCode + direction;
+    const withoutLastChar = str.slice(0, -1);
+    return withoutLastChar + String.fromCharCode(charCode);
+}

package/src/doc-cache.ts

@@ -0,0 +1,294 @@
+import type {
+    RxDocument,
+    RxDocumentData,
+    RxStorageChangeEvent,
+    WeakRef,
+    FinalizationRegistry
+} from './types/index.d.ts';
+import {
+    getFromMapOrThrow,
+    getHeightOfRevision,
+    overwriteGetterForCaching,
+    requestIdlePromiseNoQueue
+} from './plugins/utils/index.ts';
+import {
+    overwritable
+} from './overwritable.ts';
+import { Observable } from 'rxjs';
+
+/**
+ * Because we have to create many cache items,
+ * we use an array instead of an object with properties
+ * for better performance and less memory usage.
+ * @link https://stackoverflow.com/questions/17295056/array-vs-object-efficiency-in-javascript
+ */
+declare type CacheItem<RxDocType, OrmMethods> = [
+    /**
+     * Store the different document states of time
+     * based on their [revisionHeight+_meta.lwt] .
+     * We store WeakRefs so that we can later clean up
+     * document states that are no longer needed.
+     *
+     * Notice that we can not only rely on the revisionHeight
+     * because when cleanup is used, two document states can end up with
+     * the same revision but different _meta.lwt.
+     */
+    Map<string, WeakRef<RxDocument<RxDocType, OrmMethods>>>,
+
+    /**
+     * Store the latest known document state.
+     * As long as any state of the document is in the cache,
+     * we observe the changestream and update the latestDoc accordingly.
+     * This makes it easier to optimize performance on other parts
+     * because for each known document we can always get the current state
+     * in the storage.
+     * Also it makes it possible to call RxDocument.latest() in a non-async way
+     * to retrieve the latest document state or to observe$ some property.
+     *
+     * To not prevent the whole cacheItem from being garbage collected,
+     * we store only the document data here, but not the RxDocument.
+     */
+    RxDocumentData<RxDocType>
+];
+
+
+/**
+ * @link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry
+ */
+declare type FinalizationRegistryValue = {
+    docId: string;
+    revisionHeight: number;
+    lwt: number;
+};
+
+/**
+ * The DocumentCache stores RxDocument objects
+ * by their primary key and revision.
+ * This is useful on client side applications where
+ * it is not known how much memory can be used, so
+ * we de-duplicate RxDocument states to save memory.
+ * To not fill up the memory with old document states, the DocumentCache
+ * only contains weak references to the RxDocuments themself.
+ * @link https://caniuse.com/?search=weakref
+ */
+export class DocumentCache<RxDocType, OrmMethods> {
+    public readonly cacheItemByDocId = new Map<string, CacheItem<RxDocType, OrmMethods>>();
+
+    /**
+     * Process stuff lazy to not block the CPU
+     * on critical paths.
+     */
+    public readonly tasks = new Set<Function>();
+
+    /**
+     * Some JavaScript runtimes like QuickJS,
+     * so not have a FinalizationRegistry or WeakRef.
+     * Therefore we need a workaround which might waste a lot of memory,
+     * but at least works.
+     */
+    public readonly registry?: FinalizationRegistry<FinalizationRegistryValue> = typeof FinalizationRegistry === 'function' ?
+        new FinalizationRegistry<FinalizationRegistryValue>(docMeta => {
+            const docId = docMeta.docId;
+            const cacheItem = this.cacheItemByDocId.get(docId);
+            if (cacheItem) {
+                cacheItem[0].delete(docMeta.revisionHeight + docMeta.lwt + '');
+                if (cacheItem[0].size === 0) {
+                    /**
+                     * No state of the document is cached anymore,
+                     * so we can clean up.
+                     */
+                    this.cacheItemByDocId.delete(docId);
+                }
+            }
+        }) as any :
+        undefined;
+
+    constructor(
+        public readonly primaryPath: string,
+        public readonly changes$: Observable<RxStorageChangeEvent<RxDocType>[]>,
+        /**
+         * A method that can create a RxDocument by the given document data.
+         */
+        public documentCreator: (docData: RxDocumentData<RxDocType>) => RxDocument<RxDocType, OrmMethods>
+    ) {
+        changes$.subscribe((events: RxStorageChangeEvent<RxDocType>[]) => {
+            this.tasks.add(() => {
+                const cacheItemByDocId = this.cacheItemByDocId;
+                for (let index = 0; index < events.length; index++) {
+                    const event = events[index];
+                    const cacheItem = cacheItemByDocId.get(event.documentId);
+                    if (cacheItem) {
+                        let documentData = event.documentData;
+                        if (!documentData) {
+                            documentData = event.previousDocumentData as any;
+                        }
+                        cacheItem[1] = documentData;
+                    }
+                }
+            });
+            if (this.tasks.size <= 1) {
+                requestIdlePromiseNoQueue().then(() => {
+                    this.processTasks();
+                });
+            }
+        });
+    }
+
+    public processTasks() {
+        if (this.tasks.size === 0) {
+            return;
+        }
+        const tasks = Array.from(this.tasks);
+        tasks.forEach(task => task());
+        this.tasks.clear();
+    }
+
+    /**
+     * Get the RxDocument from the cache
+     * and create a new one if not exits before.
+     * @overwrites itself with the actual function
+     * because this is @performance relevant.
+     * It is called on each document row for each write and read.
+     */
+    get getCachedRxDocuments(): (docsData: RxDocumentData<RxDocType>[]) => RxDocument<RxDocType, OrmMethods>[] {
+        const fn = getCachedRxDocumentMonad(this);
+        return overwriteGetterForCaching(
+            this,
+            'getCachedRxDocuments',
+            fn
+        );
+    }
+
+    get getCachedRxDocument(): (docData: RxDocumentData<RxDocType>) => RxDocument<RxDocType, OrmMethods> {
+        const fn = getCachedRxDocumentMonad(this);
+        return overwriteGetterForCaching(
+            this,
+            'getCachedRxDocument',
+            doc => fn([doc])[0]
+        );
+    }
+
+    /**
+     * Throws if not exists
+     */
+    public getLatestDocumentData(docId: string): RxDocumentData<RxDocType> {
+        this.processTasks();
+        const cacheItem = getFromMapOrThrow(this.cacheItemByDocId, docId);
+        return cacheItem[1];
+    }
+
+    public getLatestDocumentDataIfExists(docId: string): RxDocumentData<RxDocType> | undefined {
+        this.processTasks();
+        const cacheItem = this.cacheItemByDocId.get(docId);
+        if (cacheItem) {
+            return cacheItem[1];
+        }
+    }
+}
+
+/**
+ * This function is called very very often.
+ * @hotPath This is one of the most important methods for performance.
+ * It is used in many places to transform the raw document data into RxDocuments.
+ */
+function getCachedRxDocumentMonad<RxDocType, OrmMethods>(
+    docCache: DocumentCache<RxDocType, OrmMethods>
+): (docsData: RxDocumentData<RxDocType>[]) => RxDocument<RxDocType, OrmMethods>[] {
+    const primaryPath = docCache.primaryPath;
+    const cacheItemByDocId = docCache.cacheItemByDocId;
+    const registry = docCache.registry;
+    const deepFreezeWhenDevMode = overwritable.deepFreezeWhenDevMode;
+    const documentCreator = docCache.documentCreator;
+    const fn: (docsData: RxDocumentData<RxDocType>[]) => RxDocument<RxDocType, OrmMethods>[] = (docsData: RxDocumentData<RxDocType>[]) => {
+        const ret: RxDocument<RxDocType, OrmMethods>[] = new Array(docsData.length);
+        const registryTasks: RxDocument<RxDocType, OrmMethods>[] = [];
+        for (let index = 0; index < docsData.length; index++) {
+            let docData = docsData[index];
+            const docId: string = (docData as any)[primaryPath];
+
+            const revisionHeight = getHeightOfRevision(docData._rev);
+
+            /**
+             * @performance
+             * Compute the byRev cache key once and reuse it
+             * for both the Map.get() and Map.set() calls.
+             */
+            const cacheKey = revisionHeight + docData._meta.lwt + '';
+
+            let byRev: Map<string, WeakRef<RxDocument<RxDocType, OrmMethods>>>;
+            let cachedRxDocumentWeakRef: WeakRef<RxDocument<RxDocType, OrmMethods>> | undefined;
+            let cacheItem = cacheItemByDocId.get(docId);
+            if (!cacheItem) {
+                byRev = new Map();
+                cacheItem = [
+                    byRev,
+                    docData
+                ];
+                cacheItemByDocId.set(docId, cacheItem);
+            } else {
+                byRev = cacheItem[0];
+                cachedRxDocumentWeakRef = byRev.get(cacheKey);
+            }
+            let cachedRxDocument = cachedRxDocumentWeakRef ? cachedRxDocumentWeakRef.deref() : undefined;
+            if (!cachedRxDocument) {
+                docData = deepFreezeWhenDevMode(docData) as any;
+                cachedRxDocument = documentCreator(docData) as RxDocument<RxDocType, OrmMethods>;
+                byRev.set(cacheKey, createWeakRefWithFallback(cachedRxDocument));
+                if (registry) {
+                    registryTasks.push(cachedRxDocument);
+                }
+            }
+            ret[index] = cachedRxDocument;
+        }
+        if (registryTasks.length > 0 && registry) {
+            /**
+             * Calling registry.register() has shown to have
+             * really bad performance. So we add the cached documents
+             * lazily.
+             */
+            docCache.tasks.add(() => {
+                for (let index = 0; index < registryTasks.length; index++) {
+                    const doc = registryTasks[index];
+                    registry.register(doc, {
+                        docId: doc.primary,
+                        revisionHeight: getHeightOfRevision(doc.revision),
+                        lwt: doc._data._meta.lwt
+                    });
+                }
+            });
+            if (docCache.tasks.size <= 1) {
+                requestIdlePromiseNoQueue().then(() => {
+                    docCache.processTasks();
+                });
+            }
+        }
+        return ret;
+    };
+    return fn;
+}
+
+export function mapDocumentsDataToCacheDocs<RxDocType, OrmMethods>(
+    docCache: DocumentCache<RxDocType, OrmMethods>,
+    docsData: RxDocumentData<RxDocType>[]
+) {
+    const getCachedRxDocuments = docCache.getCachedRxDocuments;
+    return getCachedRxDocuments(docsData);
+}
+
+/**
+ * Fallback for JavaScript runtimes that do not support WeakRef.
+ * The fallback will keep the items in cache forever,
+ * but at least works.
+ */
+const HAS_WEAK_REF = typeof WeakRef === 'function';
+const createWeakRefWithFallback = HAS_WEAK_REF ? createWeakRef : createWeakRefFallback;
+function createWeakRef<T extends object>(obj: T): WeakRef<T> {
+    return new WeakRef(obj) as any;
+}
+function createWeakRefFallback<T extends object>(obj: T): WeakRef<T> {
+    return {
+        deref() {
+            return obj;
+        }
+    } as any;
+}