@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-microsoft-onedrive/upstream.ts

@@ -0,0 +1,282 @@
+import { RxReplicationWriteToMasterRow, WithDeletedAndAttachments } from '../../index.ts';
+import { newRxError, newRxFetchError } from '../../rx-error.ts';
+import { deepEqual, ensureNotFalsy } from '../utils/index.ts';
+import { fetchDocumentContents, getDocumentFiles, insertDocumentFiles, updateDocumentFiles } from './document-handling.ts';
+import { fillFileIfEtagMatches, getDriveBaseUrl } from './microsoft-onedrive-helper.ts';
+import type {
+    OneDriveState,
+    OneDriveItem
+} from './microsoft-onedrive-types.ts';
+import { DriveStructure } from './init.ts';
+
+export const WAL_FILE_NAME = 'rxdb-wal.json';
+
+// Batch insert limit in Graph api is often not fully restricted for files unless using batch endpoints, 
+// a reasonable size is 100-250 to avoid timeout.
+export const DRIVE_MAX_BULK_SIZE = 250;
+
+export async function fetchConflicts<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    primaryPath: keyof WithDeletedAndAttachments<RxDocType>,
+    writeRows: RxReplicationWriteToMasterRow<RxDocType>[]
+) {
+    if (writeRows.length > DRIVE_MAX_BULK_SIZE) {
+        throw newRxError('ODR18', {
+            args: {
+                DRIVE_MAX_BULK_SIZE
+            }
+        });
+    }
+
+    const ids = writeRows.map(row => (row.newDocumentState as any)[primaryPath]);
+    const filesMeta = await getDocumentFiles(
+        oneDriveState,
+        init,
+        ids as string[]
+    );
+    const fileIdByDocId = new Map<string, string>();
+    const fileIds: string[] = filesMeta.files.map((f) => {
+        const fileId = ensureNotFalsy(f.id);
+        const docId = f.name.split('.')[0];
+        fileIdByDocId.set(docId, fileId);
+        return fileId;
+    });
+    const contentsByFileId = await fetchDocumentContents<WithDeletedAndAttachments<RxDocType>>(
+        oneDriveState,
+        fileIds
+    );
+
+    const conflicts: WithDeletedAndAttachments<RxDocType>[] = [];
+    const nonConflicts: RxReplicationWriteToMasterRow<RxDocType>[] = [];
+    writeRows.forEach(row => {
+        const docId = (row.newDocumentState as any)[primaryPath];
+        let fileContent: undefined | WithDeletedAndAttachments<RxDocType>;
+        const fileId = fileIdByDocId.get(docId);
+        if (fileId) {
+            fileContent = contentsByFileId.byId[fileId];
+        }
+        if (row.assumedMasterState) {
+            if (!deepEqual(row.assumedMasterState, fileContent)) {
+                conflicts.push(ensureNotFalsy(fileContent));
+            } else {
+                nonConflicts.push(row);
+            }
+        } else if (fileContent) {
+            conflicts.push(fileContent);
+        } else {
+            nonConflicts.push(row);
+        }
+    });
+
+    if ((nonConflicts.length + conflicts.length) !== writeRows.length) {
+        throw newRxError('SNH', {
+            pushRows: writeRows,
+            args: {
+                nonConflicts,
+                conflicts,
+                contentsByFileId: contentsByFileId.byId
+            }
+        });
+    }
+
+    return {
+        conflicts,
+        nonConflicts
+    };
+}
+
+export async function writeToWal<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    writeRows?: RxReplicationWriteToMasterRow<RxDocType>[]
+) {
+    const walFileId = init.walFile.fileId;
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+
+    const metaUrl = `${baseUrl}/items/${encodeURIComponent(walFileId)}?$select=id,size,eTag`;
+    const metaRes = await fetch(metaUrl, {
+        method: "GET",
+        headers: {
+            Authorization: `Bearer ${oneDriveState.authToken}`,
+        },
+    });
+    if (!metaRes.ok) {
+        throw await newRxFetchError(metaRes);
+    }
+    const meta: OneDriveItem = await metaRes.json();
+    const sizeNum = meta.size || 0;
+
+    if (writeRows && sizeNum > 0) {
+        throw newRxError("ODR19", {
+            args: {
+                sizeNum,
+                walFileId,
+                meta,
+                writeRows: writeRows?.length
+            }
+        });
+    }
+
+    const etag = ensureNotFalsy(meta.eTag, 'etag missing');
+    const writeResult = await fillFileIfEtagMatches(
+        oneDriveState,
+        walFileId,
+        etag,
+        writeRows
+    );
+    if (writeResult.status !== 200 && writeResult.status !== 201) {
+        throw newRxError("ODR19", {
+            args: {
+                walFileId,
+                meta,
+                writeRows: writeRows?.length
+            }
+        });
+    }
+}
+
+
+export async function readWalContent<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+): Promise<{
+    etag: string;
+    rows: RxReplicationWriteToMasterRow<RxDocType>[] | undefined;
+}> {
+    const walFileId = init.walFile.fileId;
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+    const contentUrl = `${baseUrl}/items/${encodeURIComponent(walFileId)}/content`;
+
+    const res = await fetch(contentUrl, {
+        method: "GET",
+        headers: {
+            Authorization: `Bearer ${oneDriveState.authToken}`,
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res);
+    }
+
+    let etag = res.headers.get("etag") || res.headers.get("ETag");
+
+    if (!etag) {
+        const metaRes = await fetch(`${baseUrl}/items/${encodeURIComponent(walFileId)}?$select=eTag`, {
+            headers: {
+                Authorization: `Bearer ${oneDriveState.authToken}`,
+            }
+        });
+        const meta: OneDriveItem = await metaRes.json();
+        etag = meta.eTag;
+    }
+
+    const text = await res.text();
+
+    if (!text || !text.trim()) {
+        return {
+            etag: ensureNotFalsy(etag),
+            rows: undefined
+        };
+    }
+
+    return {
+        etag: ensureNotFalsy(etag),
+        rows: JSON.parse(text) as RxReplicationWriteToMasterRow<RxDocType>[]
+    };
+}
+
+
+/**
+ * Here we read the WAL file content
+ * and sort the content into the actual
+ * document files.
+ * Notice that when the JavaScript process
+ * exists at any point here, we need to have
+ * a recoverable state on the next run. So this
+ * must be idempotent.
+ */
+export async function processWalFile<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    primaryPath: keyof RxDocType
+) {
+    const content = await readWalContent<RxDocType>(
+        oneDriveState,
+        init
+    );
+    if (!content.rows) {
+        return;
+    }
+
+
+    const docIds = content.rows.map(row => row.newDocumentState[primaryPath]);
+    const docFiles = await getDocumentFiles(
+        oneDriveState,
+        init,
+        docIds as string[]
+    );
+    const fileIdByDocId: Record<string, string> = {};
+
+    docFiles.files.forEach(file => {
+        const docId = file.name.split('.')[0] as any;
+        fileIdByDocId[docId] = file.id;
+    });
+
+    const toInsert: WithDeletedAndAttachments<RxDocType>[] = [];
+    const toUpdate: WithDeletedAndAttachments<RxDocType>[] = [];
+    content.rows.filter(row => {
+        const docId = row.newDocumentState[primaryPath];
+        const fileExists = fileIdByDocId[docId as any];
+        if (!fileExists) {
+            toInsert.push(row.newDocumentState);
+        } else {
+            toUpdate.push(row.newDocumentState);
+        }
+    });
+
+    await Promise.all([
+        insertDocumentFiles(
+            oneDriveState,
+            init,
+            primaryPath,
+            toInsert
+        ),
+        updateDocumentFiles(
+            oneDriveState,
+            primaryPath,
+            toUpdate,
+            fileIdByDocId,
+        )
+    ]);
+
+
+    // overwrite wal with emptyness
+    await writeToWal(
+        oneDriveState,
+        init,
+        undefined
+    );
+
+}
+
+export async function handleUpstreamBatch<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    primaryPath: keyof WithDeletedAndAttachments<RxDocType>,
+    writeRows: RxReplicationWriteToMasterRow<RxDocType>[]
+): Promise<WithDeletedAndAttachments<RxDocType>[]> {
+    const conflictResult = await fetchConflicts(
+        oneDriveState,
+        init,
+        primaryPath,
+        writeRows
+    );
+    await writeToWal(
+        oneDriveState,
+        init,
+        conflictResult.nonConflicts
+    );
+
+    return conflictResult.conflicts;
+}

package/src/plugins/replication-mongodb/index.ts

@@ -0,0 +1,275 @@
+import {
+    ensureNotFalsy,
+    flatClone
+} from '../../plugins/utils/index.ts';
+
+import { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';
+import type {
+    RxCollection,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxReplicationWriteToMasterRow,
+    RxReplicationPullStreamItem,
+    RxDocumentData
+} from '../../types/index.d.ts';
+import {
+    RxReplicationState,
+    startReplicationOnLeaderShip
+} from '../replication/index.ts';
+import {
+    addRxPlugin,
+    WithDeleted
+} from '../../index.ts';
+
+import { Subject } from 'rxjs';
+import type {
+    MongoDbCheckpointType,
+    SyncOptionsMongoDB
+} from './mongodb-types.ts';
+
+import {
+    Db as MongoDatabase,
+    Collection as MongoCollection,
+    MongoClient,
+    ClientSession
+} from 'mongodb';
+import { MONGO_OPTIONS_DRIVER_INFO } from '../storage-mongodb/mongodb-helper.ts';
+import { iterateCheckpoint } from './mongodb-checkpoint.ts';
+import { mongodbDocToRxDB, rxdbDocToMongo, startChangeStream } from './mongodb-helper.ts';
+
+export * from './mongodb-helper.ts';
+export * from './mongodb-checkpoint.ts';
+export type * from './mongodb-types.ts';
+
+export class RxMongoDBReplicationState<RxDocType> extends RxReplicationState<RxDocType, MongoDbCheckpointType> {
+
+    constructor(
+        public readonly mongoClient: MongoClient,
+        public readonly mongoDatabase: MongoDatabase,
+        public readonly mongoCollection: MongoCollection<RxDocumentData<RxDocType> | any>,
+        public readonly options: SyncOptionsMongoDB<RxDocType>,
+        public readonly replicationIdentifier: string,
+        public readonly collection: RxCollection<RxDocType, any, any>,
+        public readonly pull?: ReplicationPullOptions<RxDocType, MongoDbCheckpointType>,
+        public readonly push?: ReplicationPushOptions<RxDocType>,
+        public readonly live: boolean = true,
+        public retryTime: number = 1000 * 5,
+        public autoStart: boolean = true
+    ) {
+
+        super(
+            replicationIdentifier,
+            collection,
+            '_deleted',
+            pull,
+            push,
+            live,
+            retryTime,
+            autoStart
+        );
+    }
+}
+
+export function replicateMongoDB<RxDocType>(options: SyncOptionsMongoDB<RxDocType>) {
+    addRxPlugin(RxDBLeaderElectionPlugin);
+    const primaryPath = options.collection.schema.primaryPath;
+    options.live = typeof options.live === 'undefined' ? true : options.live;
+    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, MongoDbCheckpointType>> = new Subject();
+
+    const mongoClient = new MongoClient(options.mongodb.connection, MONGO_OPTIONS_DRIVER_INFO);
+    const mongoDatabase = mongoClient.db(options.mongodb.databaseName);
+    const mongoCollection = mongoDatabase.collection(options.mongodb.collectionName);
+
+    let replicationPrimitivesPull: ReplicationPullOptions<RxDocType, MongoDbCheckpointType> | undefined;
+    if (options.pull) {
+        replicationPrimitivesPull = {
+            async handler(
+                lastPulledCheckpoint: MongoDbCheckpointType | undefined,
+                batchSize: number
+            ) {
+                const result = await iterateCheckpoint<RxDocType>(primaryPath, mongoCollection, batchSize, lastPulledCheckpoint);
+                return {
+                    documents: result.docs,
+                    checkpoint: result.checkpoint
+                };
+            },
+            batchSize: ensureNotFalsy(options.pull).batchSize,
+            modifier: ensureNotFalsy(options.pull).modifier,
+            stream$: pullStream$.asObservable()
+        };
+    }
+
+    let replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined;
+    if (options.push) {
+        replicationPrimitivesPush = {
+            async handler(
+                rows: RxReplicationWriteToMasterRow<RxDocType>[]
+            ) {
+                const conflicts: WithDeleted<RxDocType>[] = [];
+                const session: ClientSession = mongoClient.startSession();
+                session.startTransaction(options.mongodb.pushTransactionOptions);
+                const ids = rows.map(row => (row.newDocumentState as any)[primaryPath]);
+                const currentDocsArray = await mongoCollection.find(
+                    { [primaryPath]: { $in: ids } },
+                    { session }
+                ).toArray();
+                const currentDocsMap = new Map<any, any>();
+                currentDocsArray.forEach((doc: any) => {
+                    currentDocsMap.set(doc[primaryPath], doc);
+                });
+                let promises: Promise<any>[] = [];
+                rows.forEach(row => {
+                    const toMongoDoc = rxdbDocToMongo(row.newDocumentState as any);
+                    const docId = (row.newDocumentState as any)[primaryPath];
+                    const current = currentDocsMap.get(docId);
+                    const remoteDocState: WithDeleted<RxDocType> | undefined = current ? mongodbDocToRxDB(primaryPath, current) : undefined;
+
+                    /**
+                     * We do not want to require a deleted-flag or any RxDB specific stuff on the RxDB side.
+                     * So for deletes we have to hack around this.
+                     */
+                    let assumedMaster = row.assumedMasterState;
+                    if (row.newDocumentState._deleted) {
+                        if (remoteDocState) {
+                            if (!assumedMaster) {
+                                // remote exists but not assumed -> conflict
+                                conflicts.push(remoteDocState);
+                            } else if (assumedMaster._deleted) {
+                                // remote exists but assumed as deleted -> conflict
+                                conflicts.push(remoteDocState);
+                            } else {
+                                // remote exists and assumed to exist -> check for normal conflict or do the deletion-write
+                                if (options.collection.conflictHandler.isEqual(remoteDocState, assumedMaster, 'mongodb-pull-equal-check-deleted') === false) {
+                                    // conflict
+                                    conflicts.push(remoteDocState);
+                                } else {
+                                    promises.push(
+                                        mongoCollection.deleteOne(
+                                            {
+                                                [primaryPath]: docId
+                                            },
+                                            {
+                                                session
+                                            }
+                                        )
+                                    );
+                                }
+
+                            }
+                        } else {
+                            if (!assumedMaster) {
+                                // no remote and no assumed master -> insertion of deleted -> do nothing
+                            } else if (assumedMaster._deleted) {
+                                // no remote and assumed master also deleted -> insertion of deleted -> do nothing
+                            }
+                        }
+                    } else {
+                        /**
+                         * Non-deleted are handled normally like in every other
+                         * of the replication plugins.
+                         */
+                        if (
+                            remoteDocState &&
+                            (
+                                !row.assumedMasterState ||
+                                options.collection.conflictHandler.isEqual(remoteDocState, row.assumedMasterState, 'mongodb-pull-equal-check') === false
+                            )
+                        ) {
+                            // conflict
+                            conflicts.push(remoteDocState);
+                        } else {
+                            if (current) {
+                                if (row.newDocumentState._deleted) {
+                                    promises.push(
+                                        mongoCollection.deleteOne(
+                                            {
+                                                [primaryPath]: docId
+                                            },
+                                            {
+                                                session
+                                            }
+                                        )
+                                    );
+                                } else {
+                                    promises.push(
+                                        mongoCollection.updateOne(
+                                            { [primaryPath]: docId },
+                                            { $set: toMongoDoc },
+                                            {
+                                                upsert: true,
+                                                session
+                                            }
+                                        )
+                                    );
+                                }
+                            } else {
+                                /**
+                                 * No current but has assumed.
+                                 * This means the server state was deleted
+                                 * and we have a conflict.
+                                 */
+                                if (row.assumedMasterState) {
+                                    const conflicting = flatClone(row.assumedMasterState);
+                                    conflicting._deleted = true;
+                                    conflicts.push(conflicting);
+                                } else {
+                                    if (row.newDocumentState._deleted) {
+                                        // inserting deleted -> do nothing
+                                    } else {
+                                        promises.push(
+                                            mongoCollection.insertOne(toMongoDoc, { session })
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    }
+                });
+                await Promise.all(promises);
+                await session.commitTransaction();
+                return conflicts;
+            },
+            batchSize: options.push.batchSize,
+            modifier: options.push.modifier
+        };
+    }
+
+
+    const replicationState = new RxMongoDBReplicationState<RxDocType>(
+        mongoClient,
+        mongoDatabase,
+        mongoCollection,
+        options,
+        options.replicationIdentifier,
+        options.collection,
+        replicationPrimitivesPull,
+        replicationPrimitivesPush,
+        options.live,
+        options.retryTime,
+        options.autoStart
+    );
+
+    /**
+     * Subscribe to changes for the pull.stream$
+     */
+    if (options.live && options.pull) {
+        const startBefore = replicationState.start.bind(replicationState);
+        const cancelBefore = replicationState.cancel.bind(replicationState);
+        replicationState.start = async () => {
+            const changestream = await startChangeStream(mongoCollection, undefined, replicationState.subjects.error);
+            changestream.on('change', () => {
+                // TODO use the documents data of the change instead of emitting the RESYNC flag
+                pullStream$.next('RESYNC');
+            });
+            replicationState.cancel = async () => {
+                await changestream.close();
+                return cancelBefore();
+            };
+            return startBefore();
+        };
+    }
+
+    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);
+    return replicationState;
+}