@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/crdt/index.ts

@@ -0,0 +1,505 @@
+import { newRxError } from '../../rx-error.ts';
+import type {
+    CRDTDocumentField,
+    CRDTEntry,
+    CRDTOperation,
+    FilledMangoQuery,
+    HashFunction,
+    JsonSchema,
+    RxConflictHandler,
+    RxConflictHandlerInput,
+    RxDocument,
+    RxDocumentData,
+    RxJsonSchema,
+    RxPlugin,
+    WithDeleted
+} from '../../types/index.d.ts';
+import {
+    clone,
+    deepEqual,
+    ensureNotFalsy,
+    getProperty,
+    now,
+    objectPathMonad,
+    setProperty,
+    toArray
+} from '../../plugins/utils/index.ts';
+import {
+    getQueryMatcher,
+    META_LWT_UNIX_TIME_MAX,
+    overwritable,
+    RxCollection,
+    RxDocumentWriteData,
+    RxError
+} from '../../index.ts';
+import { mingoUpdater } from '../update/mingo-updater.ts';
+
+
+
+export async function updateCRDT<RxDocType>(
+    this: RxDocument<RxDocType>,
+    entry: CRDTEntry<RxDocType> | CRDTEntry<RxDocType>[]
+) {
+    entry = overwritable.deepFreezeWhenDevMode(entry) as any;
+
+    const jsonSchema = this.collection.schema.jsonSchema;
+    if (!jsonSchema.crdt) {
+        throw newRxError('CRDT1', {
+            schema: jsonSchema,
+            queryObj: entry
+        });
+    }
+    const crdtOptions = ensureNotFalsy(jsonSchema.crdt);
+    const storageToken = await this.collection.database.storageToken;
+
+    return this.incrementalModify(async (docData) => {
+        const crdtDocField: CRDTDocumentField<RxDocType> = clone(getProperty(docData as any, crdtOptions.field));
+        const operation: CRDTOperation<RxDocType> = {
+            body: toArray(entry),
+            creator: storageToken,
+            time: now()
+        };
+
+        /**
+         * A new write will ALWAYS be an operation in the last
+         * array which was non existing before.
+         */
+        const lastAr: CRDTOperation<RxDocType>[] = [operation];
+        crdtDocField.operations.push(lastAr);
+        crdtDocField.hash = await hashCRDTOperations(this.collection.database.hashFunction, crdtDocField);
+
+        docData = runOperationOnDocument(
+            this.collection.schema.jsonSchema,
+            docData,
+            operation
+        );
+        setProperty(docData, crdtOptions.field, crdtDocField);
+        return docData;
+    }, RX_CRDT_CONTEXT);
+}
+
+
+export async function insertCRDT<RxDocType>(
+    this: RxCollection<RxDocType>,
+    entry: CRDTEntry<RxDocType> | CRDTEntry<RxDocType>[]
+) {
+    entry = overwritable.deepFreezeWhenDevMode(entry) as any;
+
+    const jsonSchema = this.schema.jsonSchema;
+    if (!jsonSchema.crdt) {
+        throw newRxError('CRDT1', {
+            schema: jsonSchema,
+            queryObj: entry
+        });
+    }
+    const crdtOptions = ensureNotFalsy(jsonSchema.crdt);
+    const storageToken = await this.database.storageToken;
+    const operation: CRDTOperation<RxDocType> = {
+        body: Array.isArray(entry) ? entry : [entry],
+        creator: storageToken,
+        time: now()
+    };
+
+    let insertData: RxDocumentWriteData<RxDocType> = {} as any;
+    insertData = runOperationOnDocument(
+        this.schema.jsonSchema,
+        insertData as any,
+        operation
+    ) as any;
+    const crdtDocField: CRDTDocumentField<RxDocType> = {
+        operations: [],
+        hash: ''
+    };
+    setProperty(insertData as any, crdtOptions.field, crdtDocField);
+
+    const lastAr: CRDTOperation<RxDocType>[] = [operation];
+    crdtDocField.operations.push(lastAr);
+    crdtDocField.hash = await hashCRDTOperations(this.database.hashFunction, crdtDocField);
+
+    const result = await this.insert(insertData).catch(async (err: RxError) => {
+        if (err.code === 'CONFLICT') {
+            // was a conflict, update document instead of inserting
+            const doc = await this.findOne(err.parameters.id).exec(true);
+            return doc.updateCRDT(entry);
+        } else {
+            throw err;
+        }
+    });
+    return result;
+}
+
+
+export function sortOperationComparator<RxDocType>(a: CRDTOperation<RxDocType>, b: CRDTOperation<RxDocType>) {
+    return a.creator > b.creator ? 1 : -1;
+}
+
+
+function runOperationOnDocument<RxDocType>(
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    docData: WithDeleted<RxDocType>,
+    operation: CRDTOperation<RxDocType>
+): WithDeleted<RxDocType> {
+    const entryParts = operation.body;
+    entryParts.forEach(entryPart => {
+        let isMatching: boolean;
+        if (entryPart.selector) {
+            const query: FilledMangoQuery<RxDocType> = {
+                selector: ensureNotFalsy(entryPart.selector as any),
+                sort: [],
+                skip: 0
+            };
+            const matcher = getQueryMatcher(schema, query);
+            isMatching = matcher(docData as any);
+        } else {
+            isMatching = true;
+        }
+        if (isMatching) {
+            if (entryPart.ifMatch) {
+                docData = mingoUpdater<WithDeleted<RxDocType>>(docData, entryPart.ifMatch);
+            }
+        } else {
+            if (entryPart.ifNotMatch) {
+                docData = mingoUpdater<WithDeleted<RxDocType>>(docData, entryPart.ifNotMatch);
+            }
+        }
+    });
+    return docData;
+}
+
+export async function hashCRDTOperations(
+    hashFunction: HashFunction,
+    crdts: CRDTDocumentField<any>
+): Promise<string> {
+    const hashObj = crdts.operations.map((operations) => {
+        return operations.map(op => op.creator);
+    });
+    const hash = await hashFunction(JSON.stringify(hashObj));
+    return hash;
+}
+
+export function getCRDTSchemaPart<RxDocType>(): JsonSchema<CRDTDocumentField<RxDocType>> {
+    const operationSchema: JsonSchema<CRDTOperation<RxDocType>> = {
+        type: 'object',
+        properties: {
+            body: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        selector: {
+                            type: 'object'
+                        },
+                        ifMatch: {
+                            type: 'object'
+                        },
+                        ifNotMatch: {
+                            type: 'object'
+                        }
+                    },
+                    additionalProperties: false
+                },
+                minItems: 1
+            },
+            creator: {
+                type: 'string'
+            },
+            time: {
+                type: 'number',
+                minimum: 1,
+                maximum: META_LWT_UNIX_TIME_MAX,
+                multipleOf: 0.01
+            }
+        },
+        additionalProperties: false,
+        required: [
+            'body',
+            'creator',
+            'time'
+        ]
+    };
+    return {
+        type: 'object',
+        properties: {
+            operations: {
+                type: 'array',
+                items: {
+                    type: 'array',
+                    items: operationSchema
+                }
+            },
+            hash: {
+                type: 'string',
+                // set a minLength to not accidentally store an empty string
+                minLength: 2
+            }
+        },
+        additionalProperties: false,
+        required: ['operations', 'hash']
+    };
+}
+
+
+export async function mergeCRDTFields<RxDocType>(
+    hashFunction: HashFunction,
+    crdtsA: CRDTDocumentField<RxDocType>,
+    crdtsB: CRDTDocumentField<RxDocType>
+): Promise<CRDTDocumentField<RxDocType>> {
+
+    // the value with most operations must be A to
+    // ensure we not miss out rows when iterating over both fields.
+    if (crdtsA.operations.length < crdtsB.operations.length) {
+        [crdtsA, crdtsB] = [crdtsB, crdtsA];
+    }
+
+    const ret: CRDTDocumentField<RxDocType> = {
+        operations: [],
+        hash: ''
+    };
+    crdtsA.operations.forEach((row, index) => {
+        let mergedOps: CRDTOperation<RxDocType>[] = [];
+        const ids = new Set<string>(); // used to deduplicate
+
+        row.forEach(op => {
+            ids.add(op.creator);
+            mergedOps.push(op);
+        });
+        if (crdtsB.operations[index]) {
+            crdtsB.operations[index].forEach(op => {
+                if (!ids.has(op.creator)) {
+                    mergedOps.push(op);
+                }
+            });
+        }
+        mergedOps = mergedOps.sort(sortOperationComparator);
+        ret.operations[index] = mergedOps;
+    });
+
+
+    ret.hash = await hashCRDTOperations(hashFunction, ret);
+    return ret;
+}
+
+export function rebuildFromCRDT<RxDocType>(
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    docData: WithDeleted<RxDocType> | RxDocType,
+    crdts: CRDTDocumentField<RxDocType>
+): WithDeleted<RxDocType> {
+    let base: WithDeleted<RxDocType> = {
+        _deleted: false
+    } as any;
+    setProperty(base, ensureNotFalsy(schema.crdt).field, crdts);
+    crdts.operations.forEach(operations => {
+        operations.forEach(op => {
+            base = runOperationOnDocument(
+                schema,
+                base,
+                op
+            );
+        });
+    });
+    return base;
+}
+
+
+export function getCRDTConflictHandler<RxDocType>(
+    hashFunction: HashFunction,
+    schema: RxJsonSchema<RxDocumentData<RxDocType>>
+): RxConflictHandler<RxDocType> {
+    const crdtOptions = ensureNotFalsy(schema.crdt);
+    const crdtField = crdtOptions.field;
+    const getCRDTValue = objectPathMonad<WithDeleted<RxDocType> | RxDocType, CRDTDocumentField<RxDocType>>(crdtField);
+
+    const conflictHandler: RxConflictHandler<RxDocType> = {
+        isEqual(a, b, ctx) {
+            return getCRDTValue(a).hash === getCRDTValue(b).hash;
+        },
+        async resolve(i) {
+            const newDocCrdt = getCRDTValue(i.newDocumentState);
+            const masterDocCrdt = getCRDTValue(i.realMasterState);
+            const mergedCrdt = await mergeCRDTFields(hashFunction, newDocCrdt, masterDocCrdt);
+            const mergedDoc = rebuildFromCRDT(
+                schema,
+                i.newDocumentState,
+                mergedCrdt
+            );
+            return mergedDoc;
+        }
+    }
+    return conflictHandler;
+}
+
+
+export const RX_CRDT_CONTEXT = 'rx-crdt';
+
+export const RxDBcrdtPlugin: RxPlugin = {
+    name: 'crdt',
+    rxdb: true,
+    prototypes: {
+        RxDocument: (proto: any) => {
+            proto.updateCRDT = updateCRDT;
+
+            const oldRemove = proto.remove;
+            proto.remove = function (this: RxDocument) {
+                if (!this.collection.schema.jsonSchema.crdt) {
+                    return oldRemove.bind(this)();
+                }
+                return this.updateCRDT({
+                    ifMatch: {
+                        $set: {
+                            _deleted: true
+                        }
+                    }
+                });
+            };
+
+            const oldincrementalPatch = proto.incrementalPatch;
+            proto.incrementalPatch = function (this: RxDocument, patch: any) {
+                if (!this.collection.schema.jsonSchema.crdt) {
+                    return oldincrementalPatch.bind(this)(patch);
+                }
+                return this.updateCRDT({
+                    ifMatch: {
+                        $set: patch
+                    }
+                });
+            };
+            const oldincrementalModify = proto.incrementalModify;
+            proto.incrementalModify = function (fn: any, context: string) {
+                if (!this.collection.schema.jsonSchema.crdt) {
+                    return oldincrementalModify.bind(this)(fn);
+                }
+                if (context === RX_CRDT_CONTEXT) {
+                    return oldincrementalModify.bind(this)(fn);
+                } else {
+                    throw newRxError('CRDT2', {
+                        id: this.primary,
+                        args: { context }
+                    });
+                }
+            };
+        },
+        RxCollection: (proto: any) => {
+            proto.insertCRDT = insertCRDT;
+        }
+    },
+    overwritable: {},
+    hooks: {
+        preCreateRxCollection: {
+            after: (data) => {
+                if (!data.schema.crdt) {
+                    return;
+                }
+                if (data.conflictHandler) {
+                    throw newRxError('CRDT3', {
+                        collection: data.name,
+                        schema: data.schema
+                    });
+                }
+                data.conflictHandler = getCRDTConflictHandler(
+                    data.database.hashFunction,
+                    data.schema
+                );
+            }
+        },
+        createRxCollection: {
+            after: ({ collection }) => {
+                if (!collection.schema.jsonSchema.crdt) {
+                    return;
+                }
+
+                const crdtOptions = ensureNotFalsy(collection.schema.jsonSchema.crdt);
+                const crdtField = crdtOptions.field;
+                const getCrdt = objectPathMonad<any, CRDTDocumentField<any>>(crdtOptions.field);
+
+                /**
+                 * In dev-mode we have to ensure that all document writes
+                 * have the correct crdt state so that nothing is missed out
+                 * or could accidentally do non-crdt writes to the document.
+                 */
+                if (overwritable.isDevMode()) {
+                    const bulkWriteBefore = collection.storageInstance.bulkWrite.bind(collection.storageInstance);
+                    collection.storageInstance.bulkWrite = async function (writes, context) {
+
+                        await Promise.all(
+                            writes.map(async (write) => {
+                                const newDocState: typeof write.document = clone(write.document);
+                                const crdts = getCrdt(newDocState);
+
+                                const rebuild = rebuildFromCRDT(
+                                    collection.schema.jsonSchema,
+                                    newDocState,
+                                    crdts
+                                );
+
+                                function docWithoutMeta(doc: any) {
+                                    const ret: any = {};
+                                    Object.entries(doc).forEach(([k, v]) => {
+                                        if (
+                                            !k.startsWith('_') &&
+                                            typeof v !== 'undefined'
+                                        ) {
+                                            ret[k] = v;
+                                        }
+                                    });
+                                    return ret;
+                                }
+                                if (!deepEqual(docWithoutMeta(newDocState), docWithoutMeta(rebuild))) {
+                                    throw newRxError('SNH', {
+                                        document: newDocState
+                                    });
+                                }
+                                const recalculatedHash = await hashCRDTOperations(collection.database.hashFunction, crdts);
+                                if (crdts.hash !== recalculatedHash) {
+                                    throw newRxError('SNH', {
+                                        document: newDocState,
+                                        args: { hash: crdts.hash, recalculatedHash }
+                                    });
+                                }
+                            })
+                        );
+
+                        return bulkWriteBefore(writes, context);
+                    };
+                }
+
+
+                const bulkInsertBefore = collection.bulkInsert.bind(collection);
+                collection.bulkInsert = async function (docsData: any[]) {
+                    const storageToken = await collection.database.storageToken;
+                    const useDocsData = await Promise.all(
+                        docsData.map(async (docData) => {
+                            const setMe: Partial<RxDocumentData<any>> = {};
+                            Object.entries(docData).forEach(([key, value]) => {
+                                if (
+                                    !key.startsWith('_') &&
+                                    key !== crdtField
+                                ) {
+                                    setMe[key] = value;
+                                }
+                            });
+
+                            const crdtOperations: CRDTDocumentField<any> = {
+                                operations: [
+                                    [{
+                                        creator: storageToken,
+                                        body: [{
+                                            ifMatch: {
+                                                $set: setMe
+                                            }
+                                        }],
+                                        time: now()
+                                    }]
+                                ],
+                                hash: ''
+                            };
+                            crdtOperations.hash = await hashCRDTOperations(collection.database.hashFunction, crdtOperations);
+                            setProperty(docData, crdtOptions.field, crdtOperations);
+                            return docData;
+                        })
+                    );
+                    return bulkInsertBefore(useDocsData);
+                };
+            }
+        }
+    }
+};

package/src/plugins/dev-mode/check-document.ts

@@ -0,0 +1,176 @@
+import { newRxError } from '../../rx-error.ts';
+import { fillPrimaryKey, getPrimaryFieldOfPrimaryKey } from '../../rx-schema-helper.ts';
+import type { BulkWriteRow, RxDocumentData, RxStorageInstance } from '../../types/index.d.ts';
+
+/**
+ * Strip Blob data from _attachments so that
+ * the structured-clone / JSON check does not fail on Blob objects.
+ */
+function stripBlobsFromAttachments(
+    attachments: Record<string, any> | undefined
+): Record<string, any> {
+    if (!attachments) {
+        return {};
+    }
+    const ret: Record<string, any> = {};
+    for (const [key, val] of Object.entries(attachments)) {
+        const clone = Object.assign({}, val);
+        delete clone.data;
+        ret[key] = clone;
+    }
+    return ret;
+}
+
+export function ensurePrimaryKeyValid(
+    primaryKey: string,
+    docData: RxDocumentData<any>
+) {
+    if (!primaryKey) {
+        throw newRxError('DOC20', {
+            primaryKey,
+            document: docData
+        });
+    }
+
+
+    /**
+     * This is required so that we can left-pad
+     * the primaryKey and we are still able to de-left-pad
+     * it to get again the original key.
+     */
+    if (
+        primaryKey !== primaryKey.trim()
+    ) {
+        throw newRxError('DOC21', {
+            primaryKey,
+            document: docData
+        });
+    }
+    if (
+        primaryKey.includes('\r') ||
+        primaryKey.includes('\n')
+    ) {
+        throw newRxError('DOC22', {
+            primaryKey,
+            document: docData
+        });
+    }
+    if (
+        primaryKey.includes('"')
+    ) {
+        throw newRxError('DOC23', {
+            primaryKey,
+            document: docData
+        });
+    }
+}
+
+/**
+ * Deeply checks if the object contains an
+ * instance of the JavaScript Date class.
+ * @recursive
+ */
+export function containsDateInstance(obj: any): boolean {
+    if (typeof obj !== 'object' || obj === null) {
+        return false;
+    }
+    for (let key in obj) {
+        if (Object.prototype.hasOwnProperty.call(obj, key)) {
+            if (obj[key] instanceof Date) {
+                return true;
+            }
+            if (typeof obj[key] === 'object' && containsDateInstance(obj[key])) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+
+export function checkWriteRows<RxDocType>(
+    storageInstance: RxStorageInstance<RxDocType, any, any, any>,
+    rows: BulkWriteRow<RxDocType>[]
+) {
+    const primaryPath = getPrimaryFieldOfPrimaryKey(storageInstance.schema.primaryKey);
+    for (const writeRow of rows) {
+        // ensure that the primary key has not been changed
+        writeRow.document = fillPrimaryKey(
+            primaryPath,
+            storageInstance.schema,
+            writeRow.document
+        );
+
+
+
+        /**
+         * Ensure that _meta fields have been merged
+         * and not replaced.
+         * This is important so that when one plugin A
+         * sets a _meta field and another plugin B does a write
+         * to the document, it must be ensured that the
+         * field of plugin A was not removed.
+         */
+        if (writeRow.previous) {
+            Object.keys(writeRow.previous._meta)
+                .forEach(metaFieldName => {
+                    if (!Object.prototype.hasOwnProperty.call(writeRow.document._meta, metaFieldName)) {
+                        throw newRxError('SNH', {
+                            dataBefore: writeRow.previous,
+                            dataAfter: writeRow.document,
+                            args: {
+                                metaFieldName
+                            }
+                        });
+                    }
+                });
+        }
+
+        /**
+         * Ensure it can be structured cloned.
+         */
+        try {
+            /**
+             * Notice that structuredClone() is not available
+             * in ReactNative, so we test for JSON.stringify() instead
+             * @link https://github.com/pubkey/rxdb/issues/5046#issuecomment-1827374498
+             */
+            if (typeof structuredClone === 'function') {
+                structuredClone(writeRow);
+            } else {
+                /**
+                 * JSON.stringify cannot handle Blob objects,
+                 * so we must strip them from _attachments before checking.
+                 */
+                const checkRow = Object.assign({}, writeRow, {
+                    document: Object.assign({}, writeRow.document, {
+                        _attachments: stripBlobsFromAttachments(writeRow.document._attachments)
+                    })
+                });
+                if (writeRow.previous) {
+                    checkRow.previous = Object.assign({}, writeRow.previous, {
+                        _attachments: stripBlobsFromAttachments(writeRow.previous._attachments)
+                    });
+                }
+                JSON.parse(JSON.stringify(checkRow));
+            }
+        } catch (err) {
+            throw newRxError('DOC24', {
+                collection: storageInstance.collectionName,
+                document: writeRow.document
+            });
+        }
+
+
+        /**
+         * Ensure it does not contain a Date() object
+         */
+        if (containsDateInstance(writeRow.document)) {
+            throw newRxError('DOC24', {
+                collection: storageInstance.collectionName,
+                document: writeRow.document
+            });
+        }
+    }
+
+}