@connecttomahdi/rxdb 17.0.0-beta.17

package/src/replication-protocol/helper.ts

@@ -0,0 +1,96 @@
+import type {
+    BulkWriteRow,
+    RxDocumentData,
+    RxDocumentWriteData,
+    RxStorageInstance,
+    RxStorageInstanceReplicationState,
+    RxStorageReplicationMeta,
+    WithDeletedAndAttachments
+} from '../types/index.d.ts';
+import {
+    clone,
+    createRevision,
+    flatClone,
+    getDefaultRevision,
+    now
+} from '../plugins/utils/index.ts';
+import { stripAttachmentsDataFromDocument } from '../rx-storage-helper.ts';
+
+export function docStateToWriteDoc<RxDocType>(
+    databaseInstanceToken: string,
+    hasAttachments: boolean,
+    keepMeta: boolean,
+    docState: WithDeletedAndAttachments<RxDocType>,
+    previous?: RxDocumentData<RxDocType>
+): RxDocumentWriteData<RxDocType> {
+    const docData: RxDocumentWriteData<RxDocType> = Object.assign(
+        {},
+        docState,
+        {
+            _attachments: hasAttachments && docState._attachments ? docState._attachments : {},
+            _meta: keepMeta ? (docState as any)._meta : Object.assign(
+                {},
+                previous ? previous._meta : {},
+                {
+                    lwt: now()
+                }
+            ),
+            _rev: keepMeta ? (docState as any)._rev : getDefaultRevision()
+        }
+    );
+    if (!docData._rev) {
+        docData._rev = createRevision(
+            databaseInstanceToken,
+            previous
+        );
+    }
+
+    return docData;
+}
+
+export function writeDocToDocState<RxDocType>(
+    writeDoc: RxDocumentData<RxDocType>,
+    keepAttachments: boolean,
+    keepMeta: boolean
+): WithDeletedAndAttachments<RxDocType> {
+    const ret = flatClone(writeDoc);
+
+    if (!keepAttachments) {
+        delete (ret as any)._attachments;
+    }
+    if (!keepMeta) {
+        delete (ret as any)._meta;
+        delete (ret as any)._rev;
+    }
+    return ret;
+}
+
+
+export function stripAttachmentsDataFromMetaWriteRows<RxDocType>(
+    state: RxStorageInstanceReplicationState<any>,
+    rows: BulkWriteRow<RxStorageReplicationMeta<RxDocType, any>>[]
+): BulkWriteRow<RxStorageReplicationMeta<RxDocType, any>>[] {
+    if (!state.hasAttachments) {
+        return rows;
+    }
+    return rows.map(row => {
+        const document = clone(row.document);
+        document.docData = stripAttachmentsDataFromDocument(document.docData);
+        return {
+            document,
+            previous: row.previous
+        };
+    });
+}
+
+export function getUnderlyingPersistentStorage<RxDocType>(
+    instance: RxStorageInstance<RxDocType, any, any, any>
+): RxStorageInstance<RxDocType, any, any, any> {
+    while (true) {
+        if (instance.underlyingPersistentStorage) {
+            instance = instance.underlyingPersistentStorage;
+        } else {
+            return instance;
+        }
+    }
+}

package/src/replication-protocol/index.ts

@@ -0,0 +1,348 @@
+/**
+ * These files contain the replication protocol.
+ * It can be used to replicated RxStorageInstances or RxCollections
+ * or even to do a client(s)-server replication.
+ */
+
+
+import {
+    BehaviorSubject,
+    combineLatest,
+    filter,
+    firstValueFrom,
+    mergeMap,
+    Subject
+} from 'rxjs';
+import {
+    getPrimaryFieldOfPrimaryKey
+} from '../rx-schema-helper.ts';
+import type {
+    BulkWriteRow,
+    ById,
+    DocumentsWithCheckpoint,
+    EventBulk,
+    RxConflictHandler,
+    RxDocumentData,
+    RxReplicationHandler,
+    RxReplicationWriteToMasterRow,
+    RxStorageChangeEvent,
+    RxStorageInstance,
+    RxStorageInstanceReplicationInput,
+    RxStorageInstanceReplicationState,
+    WithDeleted
+} from '../types/index.d.ts';
+import {
+    clone,
+    ensureNotFalsy,
+    flatClone,
+    PROMISE_RESOLVE_VOID
+} from '../plugins/utils/index.ts';
+import {
+    getCheckpointKey
+} from './checkpoint.ts';
+import { startReplicationDownstream } from './downstream.ts';
+import {
+    docStateToWriteDoc,
+    getUnderlyingPersistentStorage,
+    writeDocToDocState
+} from './helper.ts';
+import { startReplicationUpstream } from './upstream.ts';
+import { fillWriteDataForAttachmentsChange } from '../plugins/attachments/index.ts';
+import { getChangedDocumentsSince } from '../rx-storage-helper.ts';
+import { newRxError } from '../rx-error.ts';
+
+
+export * from './checkpoint.ts';
+export * from './downstream.ts';
+export * from './upstream.ts';
+export * from './meta-instance.ts';
+export * from './conflicts.ts';
+export * from './helper.ts';
+export * from './default-conflict-handler.ts';
+
+
+/**
+ * Used in tests to ensure all states are cleaned up
+ * and we have no memory leak.
+ */
+export const OPEN_REPLICATION_STATES = new Set<RxStorageInstanceReplicationState<any>>();
+
+export function replicateRxStorageInstance<RxDocType>(
+    input: RxStorageInstanceReplicationInput<RxDocType>
+): RxStorageInstanceReplicationState<RxDocType> {
+    input = flatClone(input);
+    input.forkInstance = getUnderlyingPersistentStorage(input.forkInstance);
+    input.metaInstance = getUnderlyingPersistentStorage(input.metaInstance);
+    const checkpointKeyPromise = getCheckpointKey(input);
+    const state: RxStorageInstanceReplicationState<RxDocType> = {
+        primaryPath: getPrimaryFieldOfPrimaryKey(input.forkInstance.schema.primaryKey),
+        hasAttachments: !!input.forkInstance.schema.attachments,
+        input,
+        skipStoringPullMeta: input.skipStoringPullMeta,
+        checkpointKey: checkpointKeyPromise,
+        downstreamBulkWriteFlag: checkpointKeyPromise.then(checkpointKey => 'replication-downstream-' + checkpointKey),
+        events: {
+            canceled: new BehaviorSubject<boolean>(false),
+            paused: new BehaviorSubject<boolean>(false),
+            active: {
+                down: new BehaviorSubject<boolean>(true),
+                up: new BehaviorSubject<boolean>(true)
+            },
+            processed: {
+                down: new Subject(),
+                up: new Subject()
+            },
+            resolvedConflicts: new Subject(),
+            error: new Subject()
+        },
+        stats: {
+            down: {
+                addNewTask: 0,
+                downstreamProcessChanges: 0,
+                downstreamResyncOnce: 0,
+                masterChangeStreamEmit: 0,
+                persistFromMaster: 0
+            },
+            up: {
+                forkChangeStreamEmit: 0,
+                persistToMaster: 0,
+                persistToMasterConflictWrites: 0,
+                persistToMasterHadConflicts: 0,
+                processTasks: 0,
+                upstreamInitialSync: 0
+            }
+        },
+        firstSyncDone: {
+            down: new BehaviorSubject<boolean>(false),
+            up: new BehaviorSubject<boolean>(false)
+        },
+        streamQueue: {
+            down: PROMISE_RESOLVE_VOID,
+            up: PROMISE_RESOLVE_VOID
+        },
+        checkpointQueue: PROMISE_RESOLVE_VOID,
+        lastCheckpointDoc: {}
+    };
+    OPEN_REPLICATION_STATES.add(state);
+
+    startReplicationDownstream(state);
+    startReplicationUpstream(state);
+    return state;
+}
+
+export function awaitRxStorageReplicationFirstInSync(
+    state: RxStorageInstanceReplicationState<any>
+): Promise<void> {
+    return firstValueFrom(
+        combineLatest([
+            state.firstSyncDone.down.pipe(
+                filter((v: boolean) => !!v)
+            ),
+            state.firstSyncDone.up.pipe(
+                filter((v: boolean) => !!v)
+            )
+        ])
+    ).then(() => { });
+}
+
+export function awaitRxStorageReplicationInSync(
+    replicationState: RxStorageInstanceReplicationState<any>
+) {
+    return Promise.all([
+        replicationState.streamQueue.up,
+        replicationState.streamQueue.down,
+        replicationState.checkpointQueue
+    ]);
+}
+
+
+export async function awaitRxStorageReplicationIdle(
+    state: RxStorageInstanceReplicationState<any>
+) {
+    await awaitRxStorageReplicationFirstInSync(state);
+    while (true) {
+        const { down, up } = state.streamQueue;
+        await Promise.all([
+            up,
+            down
+        ]);
+        /**
+         * If the Promises have not been reassigned
+         * after awaiting them, we know that the replication
+         * is in idle state at this point in time.
+         */
+        if (
+            down === state.streamQueue.down &&
+            up === state.streamQueue.up
+        ) {
+            return;
+        }
+    }
+}
+
+
+export function rxStorageInstanceToReplicationHandler<RxDocType, MasterCheckpointType>(
+    instance: RxStorageInstance<RxDocType, any, any, MasterCheckpointType>,
+    conflictHandler: RxConflictHandler<RxDocType>,
+    databaseInstanceToken: string,
+    /**
+     * If set to true,
+     * the _meta.lwt from the pushed documents is kept.
+     * (Used in the migration to ensure checkpoints are still valid)
+     */
+    keepMeta: boolean = false
+): RxReplicationHandler<RxDocType, MasterCheckpointType> {
+    instance = getUnderlyingPersistentStorage(instance);
+
+    const hasAttachments = !!instance.schema.attachments;
+    const primaryPath = getPrimaryFieldOfPrimaryKey(instance.schema.primaryKey);
+    const replicationHandler: RxReplicationHandler<RxDocType, MasterCheckpointType> = {
+        masterChangeStream$: instance.changeStream().pipe(
+            mergeMap(async (eventBulk: EventBulk<RxStorageChangeEvent<RxDocType>, MasterCheckpointType>) => {
+                const ret: DocumentsWithCheckpoint<RxDocType, MasterCheckpointType> = {
+                    checkpoint: eventBulk.checkpoint,
+                    documents: await Promise.all(
+                        eventBulk.events.map(async (event: RxStorageChangeEvent<RxDocType>) => {
+                            let docData = writeDocToDocState(event.documentData, hasAttachments, keepMeta);
+                            if (hasAttachments) {
+                                docData = await fillWriteDataForAttachmentsChange(
+                                    primaryPath,
+                                    instance,
+                                    clone(docData),
+                                    /**
+                                     * Notice that the master never knows
+                                     * the client state of the document.
+                                     * Therefore we always send all attachments data.
+                                     */
+                                    undefined
+                                );
+                            }
+                            return docData;
+                        })
+                    )
+                };
+                return ret;
+            })
+        ),
+        masterChangesSince(
+            checkpoint,
+            batchSize
+        ) {
+            return getChangedDocumentsSince(
+                instance,
+                batchSize,
+                checkpoint
+            ).then(async (result) => {
+                return {
+                    checkpoint: result.documents.length > 0 ? result.checkpoint : checkpoint,
+                    documents: await Promise.all(
+                        result.documents.map(async (plainDocumentData) => {
+                            let docData = writeDocToDocState(plainDocumentData, hasAttachments, keepMeta);
+                            if (hasAttachments) {
+                                docData = await fillWriteDataForAttachmentsChange(
+                                    primaryPath,
+                                    instance,
+                                    clone(docData),
+                                    /**
+                                     * Notice that the master never knows
+                                     * the client state of the document.
+                                     * Therefore we always send all attachments data.
+                                     */
+                                    undefined
+                                );
+                            }
+                            return docData;
+                        })
+                    )
+                };
+            });
+        },
+        async masterWrite(
+            rows
+        ) {
+            const rowById: ById<RxReplicationWriteToMasterRow<RxDocType>> = {};
+            rows.forEach(row => {
+                const docId: string = (row.newDocumentState as any)[primaryPath];
+                rowById[docId] = row;
+            });
+            const ids = Object.keys(rowById);
+
+            const masterDocsStateList = await instance.findDocumentsById(
+                ids,
+                true
+            );
+            const masterDocsState = new Map<string, RxDocumentData<RxDocType>>();
+            masterDocsStateList.forEach(doc => masterDocsState.set((doc as any)[primaryPath], doc));
+            const conflicts: WithDeleted<RxDocType>[] = [];
+            const writeRows: BulkWriteRow<RxDocType>[] = [];
+            await Promise.all(
+                Object.entries(rowById)
+                    .map(([id, row]) => {
+                        const masterState = masterDocsState.get(id);
+                        if (!masterState) {
+                            writeRows.push({
+                                document: docStateToWriteDoc(databaseInstanceToken, hasAttachments, keepMeta, row.newDocumentState)
+                            });
+                        } else if (
+                            masterState &&
+                            !row.assumedMasterState
+                        ) {
+                            conflicts.push(writeDocToDocState(masterState, hasAttachments, keepMeta));
+                        } else if (
+                            conflictHandler.isEqual(
+                                writeDocToDocState(masterState, hasAttachments, keepMeta),
+                                ensureNotFalsy(row.assumedMasterState),
+                                'rxStorageInstanceToReplicationHandler-masterWrite'
+                            ) === true
+                        ) {
+                            writeRows.push({
+                                previous: masterState,
+                                document: docStateToWriteDoc(databaseInstanceToken, hasAttachments, keepMeta, row.newDocumentState, masterState)
+                            });
+                        } else {
+                            conflicts.push(writeDocToDocState(masterState, hasAttachments, keepMeta));
+                        }
+                    })
+            );
+
+            if (writeRows.length > 0) {
+                const result = await instance.bulkWrite(
+                    writeRows,
+                    'replication-master-write'
+                );
+
+                result.error.forEach(err => {
+                    if (err.status !== 409) {
+                        throw newRxError('SNH', {
+                            name: 'non conflict error',
+                            error: err as any
+                        });
+                    } else {
+                        conflicts.push(
+                            writeDocToDocState(ensureNotFalsy(err.documentInDb), hasAttachments, keepMeta)
+                        );
+                    }
+                });
+            }
+            return conflicts;
+        }
+    };
+
+    return replicationHandler;
+}
+
+
+export async function cancelRxStorageReplication(
+    replicationState: RxStorageInstanceReplicationState<any>
+) {
+    OPEN_REPLICATION_STATES.delete(replicationState);
+
+    replicationState.events.canceled.next(true);
+    replicationState.events.active.up.complete();
+    replicationState.events.active.down.complete();
+    replicationState.events.processed.up.complete();
+    replicationState.events.processed.down.complete();
+    replicationState.events.resolvedConflicts.complete();
+    replicationState.events.canceled.complete();
+    await replicationState.checkpointQueue;
+}

package/src/replication-protocol/meta-instance.ts

@@ -0,0 +1,195 @@
+import {
+    fillWithDefaultSettings,
+    getComposedPrimaryKeyOfDocumentData,
+    getLengthOfPrimaryKey
+} from '../rx-schema-helper.ts';
+import { flatCloneDocWithMeta } from '../rx-storage-helper.ts';
+import type {
+    BulkWriteRow,
+    ById,
+    RxDocumentData,
+    RxJsonSchema,
+    RxStorageInstanceReplicationState,
+    RxStorageReplicationMeta,
+    WithDeleted
+} from '../types/index.d.ts';
+import {
+    getDefaultRevision,
+    createRevision,
+    now
+} from '../plugins/utils/index.ts';
+
+
+export const META_INSTANCE_SCHEMA_TITLE = 'RxReplicationProtocolMetaData';
+
+export function getRxReplicationMetaInstanceSchema<RxDocType, CheckpointType>(
+    replicatedDocumentsSchema: RxJsonSchema<RxDocumentData<RxDocType>>,
+    encrypted: boolean
+): RxJsonSchema<RxDocumentData<RxStorageReplicationMeta<RxDocType, CheckpointType>>> {
+    const parentPrimaryKeyLength = getLengthOfPrimaryKey(replicatedDocumentsSchema);
+
+    const minItemIdLength = 4; // 'DOWN' must fit into this
+    const minIdLength = minItemIdLength + 2; // 'DOWN|1' must fit into this
+    // add +1 for the '|' and +1 for the 'isCheckpoint' flag
+    let idLength = parentPrimaryKeyLength + 2;
+    if (idLength < minIdLength) {
+        idLength = minIdLength;
+    }
+
+    const baseSchema: RxJsonSchema<RxStorageReplicationMeta<RxDocType, CheckpointType>> = {
+        title: META_INSTANCE_SCHEMA_TITLE,
+        primaryKey: {
+            key: 'id',
+            fields: [
+                'itemId',
+                'isCheckpoint'
+            ],
+            separator: '|'
+        },
+        type: 'object',
+        version: replicatedDocumentsSchema.version,
+        additionalProperties: false,
+        properties: {
+            id: {
+                type: 'string',
+                minLength: 1,
+                maxLength: idLength
+            },
+            isCheckpoint: {
+                type: 'string',
+                enum: [
+                    '0',
+                    '1'
+                ],
+                minLength: 1,
+                maxLength: 1
+            },
+            itemId: {
+                type: 'string',
+                /**
+                 * ensure that all values of RxStorageReplicationDirection ('DOWN' has 4 chars) fit into it
+                 * because checkpoints use the itemId field for that.
+                 */
+                maxLength: parentPrimaryKeyLength > minItemIdLength ? parentPrimaryKeyLength : minItemIdLength
+            },
+            checkpointData: {
+                type: 'object',
+                additionalProperties: true
+            },
+            docData: {
+                type: 'object',
+                properties: replicatedDocumentsSchema.properties
+            },
+            isResolvedConflict: {
+                type: 'string'
+            }
+        },
+        keyCompression: replicatedDocumentsSchema.keyCompression,
+        required: [
+            'id',
+            'isCheckpoint',
+            'itemId'
+        ]
+    };
+    if (encrypted) {
+        baseSchema.encrypted = ['docData'];
+    }
+    const metaInstanceSchema: RxJsonSchema<RxDocumentData<RxStorageReplicationMeta<RxDocType, CheckpointType>>> = fillWithDefaultSettings(baseSchema);
+    return metaInstanceSchema;
+}
+
+
+
+/**
+ * Returns the document states of what the fork instance
+ * assumes to be the latest state on the master instance.
+ */
+export function getAssumedMasterState<RxDocType>(
+    state: RxStorageInstanceReplicationState<RxDocType>,
+    docIds: string[]
+): Promise<ById<{
+    docData: WithDeleted<RxDocType>;
+    metaDocument: RxDocumentData<RxStorageReplicationMeta<RxDocType, any>>;
+}>> {
+    return state.input.metaInstance.findDocumentsById(
+        docIds.map(docId => {
+            const useId = getComposedPrimaryKeyOfDocumentData(
+                state.input.metaInstance.schema,
+                {
+                    itemId: docId,
+                    isCheckpoint: '0'
+                }
+            );
+            return useId;
+        }),
+        true
+    ).then(metaDocs => {
+        const ret: {
+            [docId: string]: {
+                docData: RxDocumentData<RxDocType>;
+                metaDocument: RxDocumentData<RxStorageReplicationMeta<RxDocType, any>>;
+            };
+        } = {};
+        Object
+            .values(metaDocs)
+            .forEach((metaDoc) => {
+                ret[metaDoc.itemId] = {
+                    docData: metaDoc.docData,
+                    metaDocument: metaDoc
+                };
+            });
+
+        return ret;
+    });
+}
+
+
+export async function getMetaWriteRow<RxDocType>(
+    state: RxStorageInstanceReplicationState<RxDocType>,
+    newMasterDocState: WithDeleted<RxDocType>,
+    previous?: RxDocumentData<RxStorageReplicationMeta<RxDocType, any>>,
+    isResolvedConflict?: string
+): Promise<BulkWriteRow<RxStorageReplicationMeta<RxDocType, any>>> {
+    const docId: string = (newMasterDocState as any)[state.primaryPath];
+    const newMeta: RxDocumentData<RxStorageReplicationMeta<RxDocType, any>> = previous ? flatCloneDocWithMeta(
+        previous
+    ) : {
+        id: '',
+        isCheckpoint: '0',
+        itemId: docId,
+        docData: newMasterDocState,
+        _attachments: {},
+        _deleted: false,
+        _rev: getDefaultRevision(),
+        _meta: {
+            lwt: 0
+        }
+    };
+    newMeta.docData = newMasterDocState;
+
+    /**
+     * Sending isResolvedConflict with the value undefined
+     * will throw a schema validation error because it must be either
+     * not set or have a string.
+     */
+    if (isResolvedConflict) {
+        newMeta.isResolvedConflict = isResolvedConflict;
+    }
+
+    newMeta._meta.lwt = now();
+    newMeta.id = getComposedPrimaryKeyOfDocumentData(
+        state.input.metaInstance.schema,
+        newMeta
+    );
+    newMeta._rev = createRevision(
+        await state.checkpointKey,
+        previous
+    );
+
+    const ret = {
+        previous,
+        document: newMeta
+    };
+
+    return ret;
+}