@connecttomahdi/rxdb 17.0.0-beta.17

package/src/event-reduce.ts

@@ -0,0 +1,159 @@
+import {
+    ActionName,
+    calculateActionName,
+    runAction,
+    QueryParams,
+    StateResolveFunctionInput,
+    ChangeEvent
+} from 'event-reduce-js';
+import type {
+    RxQuery,
+    MangoQuery,
+    StringKeys,
+    RxDocumentData,
+    RxStorageChangeEvent
+} from './types/index.d.ts';
+import { rxChangeEventToEventReduceChangeEvent } from './rx-change-event.ts';
+import {
+    clone,
+    ensureNotFalsy,
+    getFromMapOrCreate
+} from './plugins/utils/index.ts';
+import { getQueryMatcher, getSortComparator, normalizeMangoQuery } from './rx-query-helper.ts';
+
+export type EventReduceResultNeg = {
+    runFullQueryAgain: true;
+};
+export type EventReduceResultPos<RxDocumentType> = {
+    runFullQueryAgain: false;
+    changed: boolean;
+    newResults: RxDocumentType[];
+};
+export type EventReduceResult<RxDocumentType> = EventReduceResultNeg | EventReduceResultPos<RxDocumentType>;
+
+
+export function getSortFieldsOfQuery<RxDocType>(
+    primaryKey: StringKeys<RxDocumentData<RxDocType>>,
+    query: MangoQuery<RxDocType>
+): (string | StringKeys<RxDocType>)[] {
+    if (!query.sort || query.sort.length === 0) {
+        return [primaryKey];
+    } else {
+        return query.sort.map(part => Object.keys(part)[0]);
+    }
+}
+
+
+
+export const RXQUERY_QUERY_PARAMS_CACHE: WeakMap<RxQuery, QueryParams<any>> = new WeakMap();
+export function getQueryParams<RxDocType>(
+    rxQuery: RxQuery<RxDocType>
+): QueryParams<RxDocType> {
+    return getFromMapOrCreate(
+        RXQUERY_QUERY_PARAMS_CACHE,
+        rxQuery,
+        () => {
+            const collection = rxQuery.collection;
+            const normalizedMangoQuery = normalizeMangoQuery(
+                collection.storageInstance.schema,
+                clone(rxQuery.mangoQuery)
+            );
+            const primaryKey = collection.schema.primaryPath;
+
+            /**
+             * Create a custom sort comparator
+             * that uses the hooks to ensure
+             * we send for example compressed documents to be sorted by compressed queries.
+             *
+             * @performance
+             * Avoid creating intermediate wrapper objects on every comparison call.
+             * The sortComparator and queryMatcher are called directly.
+             */
+            const useSortComparator = getSortComparator(
+                collection.schema.jsonSchema,
+                normalizedMangoQuery
+            );
+
+            /**
+             * Create a custom query matcher
+             * that uses the hooks to ensure
+             * we send for example compressed documents to match compressed queries.
+             */
+            const useQueryMatcher = getQueryMatcher(
+                collection.schema.jsonSchema,
+                normalizedMangoQuery
+            );
+
+            const ret: QueryParams<any> = {
+                primaryKey: rxQuery.collection.schema.primaryPath as any,
+                skip: normalizedMangoQuery.skip,
+                limit: normalizedMangoQuery.limit,
+                sortFields: getSortFieldsOfQuery(primaryKey, normalizedMangoQuery) as string[],
+                sortComparator: useSortComparator,
+                queryMatcher: useQueryMatcher
+            };
+            return ret;
+        }
+    );
+}
+
+
+export function calculateNewResults<RxDocumentType>(
+    rxQuery: RxQuery<RxDocumentType>,
+    rxChangeEvents: RxStorageChangeEvent<RxDocumentType>[]
+): EventReduceResult<RxDocumentType> {
+    if (!rxQuery.collection.database.eventReduce) {
+        return {
+            runFullQueryAgain: true
+        };
+    }
+    const queryParams = getQueryParams(rxQuery);
+    const previousResults: RxDocumentType[] = ensureNotFalsy(rxQuery._result).docsData.slice(0);
+    const previousResultsMap: Map<string, RxDocumentType> = ensureNotFalsy(rxQuery._result).docsDataMap;
+    let changed: boolean = false;
+
+
+    const eventReduceEvents: ChangeEvent<RxDocumentType>[] = [];
+    for (let index = 0; index < rxChangeEvents.length; index++) {
+        const cE = rxChangeEvents[index];
+        const eventReduceEvent = rxChangeEventToEventReduceChangeEvent(cE);
+        if (eventReduceEvent) {
+            eventReduceEvents.push(eventReduceEvent);
+        }
+    }
+
+    const foundNonOptimizeable = eventReduceEvents.find(eventReduceEvent => {
+        const stateResolveFunctionInput: StateResolveFunctionInput<RxDocumentType> = {
+            queryParams,
+            changeEvent: eventReduceEvent,
+            previousResults,
+            keyDocumentMap: previousResultsMap
+        };
+
+        const actionName: ActionName = calculateActionName(stateResolveFunctionInput);
+        if (actionName === 'runFullQueryAgain') {
+            return true;
+        } else if (actionName !== 'doNothing') {
+            changed = true;
+            runAction(
+                actionName,
+                queryParams,
+                eventReduceEvent,
+                previousResults,
+                previousResultsMap
+            );
+            return false;
+        }
+    });
+    if (foundNonOptimizeable) {
+        return {
+            runFullQueryAgain: true,
+        };
+    } else {
+        return {
+            runFullQueryAgain: false,
+            changed,
+            newResults: previousResults
+        };
+    }
+}

package/src/hooks.ts

@@ -0,0 +1,128 @@
+
+/**
+ * hook-functions that can be extended by the plugin
+ */
+export const HOOKS = {
+    /**
+     * Runs before a plugin is added.
+     * Use this to block the usage of non-compatible plugins.
+     */
+    preAddRxPlugin: [],
+    /**
+     * functions that run before the database is created
+     */
+    preCreateRxDatabase: [],
+    /**
+     * runs after the database is created and prepared
+     * but before the instance is returned to the user
+     * @async
+     */
+    createRxDatabase: [],
+    preCreateRxCollection: [],
+    createRxCollection: [],
+    createRxState: [],
+    /**
+    * runs at the end of the close-process of a collection
+    * @async
+    */
+    postCloseRxCollection: [],
+    /**
+     * Runs after a collection is removed.
+     * @async
+     */
+    postRemoveRxCollection: [],
+    /**
+      * functions that get the json-schema as input
+      * to do additionally checks/manipulation
+      */
+    preCreateRxSchema: [],
+    /**
+     * functions that run after the RxSchema is created
+     * gets RxSchema as attribute
+     */
+    createRxSchema: [],
+    prePrepareRxQuery: [],
+    preCreateRxQuery: [],
+    /**
+     * Runs before a query is sent to the
+     * prepareQuery function of the storage engine.
+     */
+    prePrepareQuery: [],
+    createRxDocument: [],
+    /**
+     * runs after a RxDocument is created,
+     * cannot be async
+     */
+    postCreateRxDocument: [],
+    /**
+     * Runs before a RxStorageInstance is created
+     * gets the params of createStorageInstance()
+     * as attribute so you can manipulate them.
+     * Notice that you have to clone stuff before mutating the inputs.
+     */
+    preCreateRxStorageInstance: [],
+    preStorageWrite: [],
+    /**
+     * runs on the document-data before the document is migrated
+     * {
+     *   doc: Object, // original doc-data
+     *   migrated: // migrated doc-data after run through migration-strategies
+     * }
+     */
+    preMigrateDocument: [],
+    /**
+     * runs after the migration of a document has been done
+     */
+    postMigrateDocument: [],
+    /**
+     * runs at the beginning of the close-process of a database
+     */
+    preCloseRxDatabase: [],
+    /**
+     * runs after a database has been removed
+     * @async
+     */
+    postRemoveRxDatabase: [],
+
+
+    postCleanup: [],
+
+    /**
+     * runs before the replication writes the rows to master
+     * but before the rows have been modified
+     * @async
+     */
+    preReplicationMasterWrite: [],
+
+    /**
+     * runs after the replication has been sent to the server
+     * but before the new documents have been handled
+     * @async
+     */
+    preReplicationMasterWriteDocumentsHandle: [],
+};
+
+export function runPluginHooks(hookKey: keyof typeof HOOKS, obj: any) {
+    if (HOOKS[hookKey].length > 0) {
+        HOOKS[hookKey].forEach(fun => (fun as any)(obj));
+    }
+}
+
+
+/**
+ * We do intentionally not run the hooks in parallel
+ * because that makes stuff unpredictable and we use runAsyncPluginHooks()
+ * only in places that are not that relevant for performance.
+ */
+export async function runAsyncPluginHooks(hookKey: keyof typeof HOOKS, obj: any): Promise<any> {
+    for (const fn of HOOKS[hookKey]) {
+        await (fn as any)(obj);
+    }
+}
+
+/**
+ * used in tests to remove hooks
+ */
+export function _clearHook(type: keyof typeof HOOKS, fun: Function) {
+    HOOKS[type] = HOOKS[type].filter(h => h !== fun);
+}

package/src/incremental-write.ts

@@ -0,0 +1,227 @@
+import {
+    isBulkWriteConflictError,
+    rxStorageWriteErrorToRxError
+} from './rx-error.ts';
+import type {
+    ModifyFunction,
+    BulkWriteRow,
+    MaybePromise,
+    RxDocumentData,
+    RxDocumentWriteData,
+    RxError,
+    RxStorageBulkWriteResponse,
+    RxStorageInstance,
+    StringKeys,
+    WithDeleted
+} from './types/index.d.ts';
+import {
+    clone,
+    ensureNotFalsy,
+    getFromMapOrCreate,
+    getFromMapOrThrow,
+    getHeightOfRevision,
+    stripMetaDataFromDocument
+} from './plugins/utils/index.ts';
+import { getWrittenDocumentsFromBulkWriteResponse } from './rx-storage-helper.ts';
+
+
+
+export type IncrementalWriteModifier<RxDocType> = (
+    doc: RxDocumentData<RxDocType>
+) => MaybePromise<RxDocumentData<RxDocType>> | MaybePromise<RxDocumentWriteData<RxDocType>>;
+
+
+type IncrementalWriteQueueItem<RxDocType> = {
+    lastKnownDocumentState: RxDocumentData<RxDocType>;
+    modifier: IncrementalWriteModifier<RxDocType>;
+    resolve: (d: RxDocumentData<RxDocType>) => void;
+    reject: (error: RxError) => void;
+};
+
+/**
+ * The incremental write queue
+ * batches up all incremental writes to a collection
+ * so that performance can be improved by:
+ * - Running only one write even when there are multiple modifications to the same document.
+ * - Run all writes ins a single bulkWrite() call even when there are writes to many documents.
+ */
+export class IncrementalWriteQueue<RxDocType> {
+    public queueByDocId = new Map<string, IncrementalWriteQueueItem<RxDocType>[]>();
+    public isRunning: boolean = false;
+
+    constructor(
+        public readonly storageInstance: RxStorageInstance<RxDocType, any, any>,
+        public readonly primaryPath: StringKeys<RxDocumentData<RxDocType>>,
+        // can be used to run hooks etc.
+        public readonly preWrite: (newData: RxDocumentData<RxDocType>, oldData: RxDocumentData<RxDocType>) => MaybePromise<void>,
+        public readonly postWrite: (docData: RxDocumentData<RxDocType>) => void
+
+    ) { }
+
+    addWrite(
+        lastKnownDocumentState: RxDocumentData<RxDocType>,
+        modifier: IncrementalWriteModifier<RxDocType>
+    ): Promise<RxDocumentData<RxDocType>> {
+        const docId: string = lastKnownDocumentState[this.primaryPath] as any;
+        const ar = getFromMapOrCreate(this.queueByDocId, docId, () => []);
+        const ret = new Promise<RxDocumentData<RxDocType>>((resolve, reject) => {
+            const item: IncrementalWriteQueueItem<RxDocType> = {
+                lastKnownDocumentState,
+                modifier,
+                resolve,
+                reject
+            };
+            ensureNotFalsy(ar).push(item);
+            this.triggerRun();
+        });
+        return ret;
+    }
+
+    async triggerRun(): Promise<void> {
+        if (
+            this.isRunning === true ||
+            this.queueByDocId.size === 0
+        ) {
+            // already running
+            return;
+        }
+        this.isRunning = true;
+        const writeRows: BulkWriteRow<RxDocType>[] = [];
+
+        /**
+         * 'take over' so that while the async functions runs,
+         * new incremental updates could be added from the outside.
+         */
+        const itemsById = this.queueByDocId;
+        this.queueByDocId = new Map();
+        await Promise.all(
+            Array.from(itemsById.entries())
+                .map(async ([_docId, items]) => {
+                    const oldData = findNewestOfDocumentStates(
+                        items.map(i => i.lastKnownDocumentState)
+                    );
+                    let newData = oldData;
+                    for (const item of items) {
+                        try {
+                            newData = await item.modifier(
+                                /**
+                                 * We have to clone() each time because the modifier
+                                 * might throw while it already changed some properties
+                                 * of the document.
+                                 */
+                                clone(newData)
+                            ) as any;
+                        } catch (err: any) {
+                            item.reject(err);
+                            item.reject = () => { };
+                            item.resolve = () => { };
+                        }
+                    }
+
+                    try {
+                        await this.preWrite(newData, oldData);
+                    } catch (err: any) {
+                        /**
+                         * If the before-hooks fail,
+                         * we reject all of the writes because it is
+                         * not possible to determine which one is to blame.
+                         */
+                        items.forEach(item => item.reject(err));
+                        return;
+                    }
+                    writeRows.push({
+                        previous: oldData,
+                        document: newData
+                    });
+                })
+        );
+        const writeResult: RxStorageBulkWriteResponse<RxDocType> = writeRows.length > 0 ?
+            await this.storageInstance.bulkWrite(writeRows, 'incremental-write') :
+            { error: [] };
+
+        // process success
+        await Promise.all(
+            getWrittenDocumentsFromBulkWriteResponse(this.primaryPath, writeRows, writeResult).map(result => {
+                const docId = result[this.primaryPath] as string;
+                this.postWrite(result);
+                const items = getFromMapOrThrow(itemsById, docId);
+                items.forEach(item => item.resolve(result));
+            })
+        );
+
+        // process errors
+        writeResult.error
+            .forEach(error => {
+                const docId = error.documentId;
+                const items = getFromMapOrThrow(itemsById, docId);
+                const isConflict = isBulkWriteConflictError<RxDocType>(error);
+                if (isConflict) {
+                    // had conflict -> retry afterwards
+                    const ar = getFromMapOrCreate(this.queueByDocId, docId, () => []);
+                    /**
+                     * Add the items back to this.queueByDocId
+                     * by maintaining the original order.
+                     */
+                    items
+                        .reverse()
+                        .forEach(item => {
+                            item.lastKnownDocumentState = ensureNotFalsy(isConflict.documentInDb);
+                            ensureNotFalsy(ar).unshift(item);
+                        });
+                } else {
+                    // other error -> must be thrown
+                    const rxError = rxStorageWriteErrorToRxError(error);
+                    items.forEach(item => item.reject(rxError));
+                }
+            });
+        this.isRunning = false;
+
+        /**
+         * Always trigger another run
+         * because in between there might be new items
+         * been added to the queue.
+         */
+        return this.triggerRun();
+    }
+}
+
+
+export function modifierFromPublicToInternal<RxDocType>(
+    publicModifier: ModifyFunction<RxDocType>
+): IncrementalWriteModifier<RxDocType> {
+    const ret = async (docData: RxDocumentData<RxDocType>) => {
+        const withoutMeta: WithDeleted<RxDocType> = stripMetaDataFromDocument(docData) as any;
+        withoutMeta._deleted = docData._deleted;
+        const modified = await publicModifier(withoutMeta);
+        const reattachedMeta: RxDocumentData<RxDocType> = Object.assign({}, modified, {
+            _meta: docData._meta,
+            _attachments: docData._attachments,
+            _rev: docData._rev,
+            _deleted: typeof (modified as WithDeleted<RxDocType>)._deleted !== 'undefined' ?
+                (modified as WithDeleted<RxDocType>)._deleted :
+                docData._deleted
+        });
+        if (typeof reattachedMeta._deleted === 'undefined') {
+            reattachedMeta._deleted = false;
+        }
+        return reattachedMeta;
+    };
+    return ret;
+}
+
+
+export function findNewestOfDocumentStates<RxDocType>(
+    docs: RxDocumentData<RxDocType>[]
+): RxDocumentData<RxDocType> {
+
+    let newest = docs[0];
+    let newestRevisionHeight = getHeightOfRevision(newest._rev);
+    docs.forEach(doc => {
+        const height = getHeightOfRevision(doc._rev);
+        if (height > newestRevisionHeight) {
+            newest = doc;
+            newestRevisionHeight = height;
+        }
+    });
+    return newest;
+}

package/src/index.ts

@@ -0,0 +1,30 @@
+/**
+ * this is the main entry-point
+ * for when the you call "import from 'rxdb'".
+ */
+
+export * from './plugin.ts';
+export * from './rx-database.ts';
+export * from './rx-error.ts';
+export * from './rx-database-internal-store.ts';
+export * from './overwritable.ts';
+export * from './rx-collection.ts';
+export * from './rx-collection-helper.ts';
+export * from './rx-document.ts';
+export * from './rx-change-event.ts';
+export * from './rx-document-prototype-merge.ts';
+export * from './rx-query.ts';
+export * from './rx-query-single-result.ts';
+export * from './rx-query-helper.ts';
+export * from './rx-schema.ts';
+export * from './rx-schema-helper.ts';
+export * from './rx-storage-helper.ts';
+export * from './replication-protocol/index.ts';
+export * from './rx-storage-multiinstance.ts';
+export * from './custom-index.ts';
+export * from './query-planner.ts';
+export * from './plugin-helpers.ts';
+export * from './plugins/utils/index.ts';
+export * from './hooks.ts';
+export * from './query-cache.ts';
+export type * from './types/index.d.ts';

package/src/overwritable.ts

@@ -0,0 +1,41 @@
+/**
+ * functions that can or should be overwritten by plugins
+ * IMPORTANT: Do not import any big stuff from RxDB here!
+ * An 'overwritable' can be used inside WebWorkers for RxStorage only,
+ * and we do not want to have the full RxDB lib bundled in them.
+ */
+
+import type { DeepReadonly } from './types/util.d.ts';
+
+export const overwritable = {
+    /**
+     * if this method is overwritten with one
+     * that returns true, we do additional checks
+     * which help the developer but have bad performance
+     */
+    isDevMode(): boolean {
+        return false;
+    },
+
+    /**
+     * Deep freezes and object when in dev-mode.
+     * Deep-Freezing has the same performance as deep-cloning, so we only do that in dev-mode.
+     * Also, we can ensure the readonly state via typescript
+     * @link https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze
+     */
+    deepFreezeWhenDevMode<T>(obj: T): DeepReadonly<T> {
+        return obj as any;
+    },
+
+    /**
+     * overwritten to map error-codes to text-messages
+     */
+    tunnelErrorMessage(message: string): string {
+        return `
+        RxDB Error-Code: ${message}.
+        Hint: Error messages are not included in RxDB core to reduce build size.
+        To show the full error messages and to ensure that you do not make any mistakes when using RxDB,
+        use the dev-mode plugin when you are in development mode: https://rxdb.info/dev-mode.html?console=error
+        `;
+    }
+};