@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-mongodb/mongodb-checkpoint.ts

@@ -0,0 +1,184 @@
+import { WithDeleted } from '../../types/rx-storage';
+import { clone, ensureNotFalsy, lastOfArray } from '../utils/index.ts';
+import { mongodbDocToRxDB } from './mongodb-helper.ts';
+import type {
+    MongoDBChangeStreamResumeToken,
+    MongoDBCheckpointIterationState,
+    MongoDbCheckpointType,
+} from './mongodb-types';
+import {
+    Collection as MongoCollection
+} from 'mongodb';
+
+
+export async function getCurrentResumeToken(
+    mongoCollection: MongoCollection
+): Promise<MongoDBChangeStreamResumeToken> {
+    const changeStream = mongoCollection.watch();
+
+    // Trigger the initial batch so postBatchResumeToken is available
+    await changeStream.tryNext().catch(() => { });
+
+    const token = changeStream.resumeToken;
+    changeStream.close();
+    return token as any;
+}
+
+export async function getDocsSinceChangestreamCheckpoint<MongoDocType>(
+    primaryPath: string,
+    mongoCollection: MongoCollection,
+    /**
+     * MongoDB has no way to start the stream from 'timestamp zero',
+     * we always need a resumeToken
+     */
+    resumeToken: MongoDBChangeStreamResumeToken,
+    limit: number
+): Promise<{ docs: WithDeleted<MongoDocType>[], nextToken: MongoDBChangeStreamResumeToken }> {
+    const resultByDocId = new Map<string, Promise<WithDeleted<MongoDocType>>>();
+    const changeStream = mongoCollection.watch(
+        [],
+        {
+            resumeAfter: resumeToken,
+            fullDocument: 'required',
+            fullDocumentBeforeChange: 'required',
+        }
+    );
+
+
+    /**
+     * We cannot use changeStream.resumeToken for the
+     * updated token because depending on the batchSize of mongoCollection.watch()
+     * it might have changes but not emitting a new token.
+     */
+    let nextToken = resumeToken;
+
+    return new Promise(async (res, rej) => {
+        changeStream.on('error', (err: any) => {
+            rej(err);
+        });
+
+        while (resultByDocId.size < limit) {
+            const change = await changeStream.tryNext();
+            if (change) {
+                nextToken = change._id as any;
+                const docId = (change as any).documentKey._id;
+
+                if (change.operationType === 'delete') {
+                    const beforeDocMongo = ensureNotFalsy(
+                        change.fullDocumentBeforeChange,
+                        'change must have pre-deletion state'
+                    );
+                    const beforeDoc = mongodbDocToRxDB(primaryPath, beforeDocMongo as any);
+                    beforeDoc._deleted = true;
+                    resultByDocId.set(docId, Promise.resolve(beforeDoc as any));
+                } else if (
+                    change.operationType === 'insert' ||
+                    change.operationType === 'update' ||
+                    change.operationType === 'replace'
+                ) {
+                    resultByDocId.set(docId, mongoCollection.findOne({ _id: docId }).then((doc: any) => {
+                        if (doc) {
+                            return mongodbDocToRxDB(primaryPath, doc);
+                        } else {
+                            const docFromChange = ensureNotFalsy(
+                                change.fullDocument as any,
+                                'change must have change.fullDocument'
+                            );
+                            const ret = mongodbDocToRxDB(primaryPath, docFromChange);
+                            ret._deleted = true;
+                            return ret;
+                        }
+                    }));
+                }
+            } else {
+                break;
+            }
+        }
+
+        changeStream.close();
+
+        const docs = await Promise.all(Array.from(resultByDocId.values()));
+        res({ docs, nextToken: nextToken as any });
+    });
+}
+
+export async function getDocsSinceDocumentCheckpoint<MongoDocType>(
+    primaryPath: string,
+    mongoCollection: MongoCollection,
+    limit: number,
+    checkpointId?: string
+): Promise<WithDeleted<MongoDocType>[]> {
+    const query = checkpointId
+        ? { [primaryPath]: { $gt: checkpointId } }
+        : {};
+
+    const docs = await mongoCollection
+        .find(query as any)
+        .sort({ [primaryPath]: 1 })
+        .limit(limit)
+        .toArray();
+
+    return docs.map((d: any) => mongodbDocToRxDB(primaryPath, d as any));
+}
+
+
+export async function iterateCheckpoint<MongoDocType>(
+    primaryPath: string,
+    mongoCollection: MongoCollection,
+    limit: number,
+    checkpoint?: MongoDbCheckpointType,
+): Promise<MongoDBCheckpointIterationState<MongoDocType>> {
+    if (!checkpoint) {
+        const token = await getCurrentResumeToken(mongoCollection);
+        checkpoint = {
+            iterate: 'docs-by-id',
+            changestreamResumeToken: token
+        }
+    } else {
+        checkpoint = clone(checkpoint);
+    }
+
+    let docs: WithDeleted<MongoDocType>[] = [];
+    if (checkpoint.iterate === 'docs-by-id') {
+        docs = await getDocsSinceDocumentCheckpoint<MongoDocType>(primaryPath, mongoCollection, limit, checkpoint.docId);
+        const last = lastOfArray(docs);
+        if (last) {
+            checkpoint.docId = (last as any)[primaryPath];
+        }
+    } else {
+        const result = await getDocsSinceChangestreamCheckpoint<MongoDocType>(primaryPath, mongoCollection, checkpoint.changestreamResumeToken, limit);
+        docs = result.docs;
+        checkpoint.changestreamResumeToken = result.nextToken;
+    }
+
+    /**
+     * If we have to toggle from docs-by-id to changestream iteration
+     * mode, the docs array might not be full while we still have some docs left.
+     */
+    if (checkpoint.iterate === 'docs-by-id' && docs.length < limit) {
+        const ids = new Set<string>();
+        docs.forEach(d => ids.add((d as any)[primaryPath]));
+        const fillUp = await getDocsSinceChangestreamCheckpoint<MongoDocType>(
+            primaryPath,
+            mongoCollection,
+            checkpoint.changestreamResumeToken,
+            limit
+        );
+
+        checkpoint.iterate = 'changestream';
+        checkpoint.changestreamResumeToken = fillUp.nextToken;
+
+        fillUp.docs.forEach(doc => {
+            const id = (doc as any)[primaryPath];
+            if (ids.has(id)) {
+                docs = docs.filter(d => (d as any)[primaryPath] !== id);
+            }
+            docs.push(doc);
+        });
+    }
+
+    return {
+        docs,
+        checkpoint
+    };
+}

package/src/plugins/replication-mongodb/mongodb-helper.ts

@@ -0,0 +1,69 @@
+import { Subject } from 'rxjs';
+import { RxError, RxTypeError, newRxError } from '../../rx-error.ts';
+import {
+    errorToPlainJson,
+    flatClone,
+    toArray
+} from '../utils/index.ts';
+import type {
+    MongoDBChangeStreamResumeToken
+} from './mongodb-types';
+import {
+    Collection as MongoCollection,
+    ChangeStream,
+    WithId
+} from 'mongodb';
+import type { RxDocumentData, WithDeleted } from '../../types/rx-storage';
+
+export async function startChangeStream(
+    mongoCollection: MongoCollection<any>,
+    resumeToken?: MongoDBChangeStreamResumeToken,
+    errorSubject?: Subject<RxError | RxTypeError>
+): Promise<ChangeStream> {
+    const changeStream = mongoCollection.watch([], resumeToken ? { resumeAfter: resumeToken } : {
+
+    });
+
+    if (errorSubject) {
+        changeStream.on('error', (err: any) => {
+            const emitError = newRxError('RC_STREAM', {
+                errors: toArray(err).map(er => errorToPlainJson(er))
+            });
+            errorSubject.next(emitError);
+        });
+    }
+    return changeStream;
+}
+
+
+export function mongodbDocToRxDB<DocType>(primaryPath: string, doc: WithId<DocType>): WithDeleted<DocType> {
+    if (primaryPath === '_id' && typeof doc._id !== 'string') {
+        throw newRxError('MG1', {
+            document: doc
+        });
+    }
+
+    const useDoc: any = flatClone(doc);
+    useDoc._deleted = false;
+
+    if (primaryPath === '_id') {
+        return useDoc;
+    } else {
+        delete useDoc._id;
+        return useDoc;
+    }
+}
+
+
+/**
+ * MongoDB operations like mongoCollection.updateOne() will mutate the input!
+ * So we have to flat-clone first here.
+ * Also we do not want to store RxDB-specific metadata in the mongodb database.
+ */
+export function rxdbDocToMongo<DocType>(doc: RxDocumentData<DocType>): DocType {
+    const ret: any = flatClone(doc);
+    delete ret._deleted;
+    delete ret._meta;
+    delete ret._attachments;
+    return ret;
+}

package/src/plugins/replication-mongodb/mongodb-types.ts

@@ -0,0 +1,50 @@
+import { TransactionOptions, WithId } from 'mongodb';
+import type {
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    WithDeleted
+} from '../../types';
+import type {
+    MongoDBConnectionString
+} from '../storage-mongodb/mongodb-types';
+
+export type MongoDBChangeStreamResumeToken = {
+    _data: string;
+};
+
+
+export type MongoDbCheckpointType = {
+    /**
+     * MongoDB has no wait to iterate over events
+     * from the beginning of time.
+     * Therefore we first iterate over the documents by their
+     * _id field and if that reached an end, we iterate from then on
+     * over the changestream resume token.
+     */
+    iterate: 'changestream' | 'docs-by-id';
+    changestreamResumeToken: MongoDBChangeStreamResumeToken;
+    docId?: string;
+}
+
+export type MongoDBCheckpointIterationState<MongoDocType> = {
+    docs: WithDeleted<MongoDocType>[];
+    checkpoint: MongoDbCheckpointType;
+};
+
+export type MongoDbConnectionConfig = {
+    connection: MongoDBConnectionString;
+    databaseName: string;
+    collectionName: string;
+    pushTransactionOptions?: TransactionOptions;
+};
+
+export type SyncOptionsMongoDB<RxDocType> = Omit<
+    ReplicationOptions<RxDocType, any>,
+    'pull' | 'push' | 'deletedField'
+> & {
+    mongodb: MongoDbConnectionConfig;
+    pull?: Omit<ReplicationPullOptions<RxDocType, MongoDbCheckpointType>, 'handler' | 'stream$'> & {
+    };
+    push?: Omit<ReplicationPushOptions<RxDocType>, 'handler'>;
+};

package/src/plugins/replication-nats/index.ts

@@ -0,0 +1,289 @@
+import {
+    ensureNotFalsy,
+    errorToPlainJson
+} from '../../plugins/utils/index.ts';
+
+
+import { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';
+import type {
+    RxCollection,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxReplicationWriteToMasterRow,
+    RxReplicationPullStreamItem
+} from '../../types/index.d.ts';
+import {
+    RxReplicationState,
+    startReplicationOnLeaderShip
+} from '../replication/index.ts';
+import {
+    addRxPlugin,
+    newRxError,
+    WithDeleted
+} from '../../index.ts';
+
+import { Subject } from 'rxjs';
+import type {
+    NatsCheckpointType,
+    NatsSyncOptions
+} from './nats-types.ts';
+import { connect, DeliverPolicy, JSONCodec, ReplayPolicy } from 'nats';
+import { getNatsServerDocumentState } from './nats-helper.ts';
+import { awaitRetry } from '../replication/replication-helper.ts';
+
+export * from './nats-types.ts';
+export * from './nats-helper.ts';
+
+
+export class RxNatsReplicationState<RxDocType> extends RxReplicationState<RxDocType, NatsCheckpointType> {
+    constructor(
+        public readonly replicationIdentifier: string,
+        public readonly collection: RxCollection<RxDocType>,
+        public readonly pull?: ReplicationPullOptions<RxDocType, NatsCheckpointType>,
+        public readonly push?: ReplicationPushOptions<RxDocType>,
+        public readonly live: boolean = true,
+        public retryTime: number = 1000 * 5,
+        public autoStart: boolean = true
+    ) {
+        super(
+            replicationIdentifier,
+            collection,
+            '_deleted',
+            pull,
+            push,
+            live,
+            retryTime,
+            autoStart
+        );
+    }
+}
+
+
+
+export function replicateNats<RxDocType>(
+    options: NatsSyncOptions<RxDocType>
+): RxNatsReplicationState<RxDocType> {
+    options.live = typeof options.live === 'undefined' ? true : options.live;
+    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+
+    const collection: RxCollection<RxDocType, any, any> = options.collection;
+    const primaryPath = collection.schema.primaryPath;
+    addRxPlugin(RxDBLeaderElectionPlugin);
+
+    const jc = JSONCodec();
+
+
+    const connectionStatePromise = (async () => {
+        const nc = await connect(options.connection);
+        const jetstreamClient = nc.jetstream();
+        const jsm = await nc.jetstreamManager();
+        await jsm.streams.add({
+            name: options.streamName, subjects: [
+                options.subjectPrefix + '.*'
+            ]
+        });
+        const natsStream = await jetstreamClient.streams.get(options.streamName);
+        return {
+            nc,
+            jetstreamClient,
+            jsm,
+            natsStream
+        };
+    })();
+    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, NatsCheckpointType>> = new Subject();
+
+    let replicationPrimitivesPull: ReplicationPullOptions<RxDocType, NatsCheckpointType> | undefined;
+    if (options.pull) {
+        replicationPrimitivesPull = {
+            async handler(
+                lastPulledCheckpoint: NatsCheckpointType | undefined,
+                batchSize: number
+            ) {
+                const cn = await connectionStatePromise;
+                const newCheckpoint: NatsCheckpointType = {
+                    sequence: lastPulledCheckpoint ? lastPulledCheckpoint.sequence : 0
+                };
+                const consumer = await cn.natsStream.getConsumer({
+                    opt_start_seq: lastPulledCheckpoint ? lastPulledCheckpoint.sequence : 0,
+                    deliver_policy: DeliverPolicy.LastPerSubject,
+                    replay_policy: ReplayPolicy.Instant
+                });
+
+                const fetchedMessages = await consumer.fetch({
+                    max_messages: batchSize
+                });
+                await (fetchedMessages as any).signal;
+                await fetchedMessages.close();
+
+                const useMessages: WithDeleted<RxDocType>[] = [];
+                for await (const m of fetchedMessages) {
+                    useMessages.push(m.json());
+                    newCheckpoint.sequence = m.seq;
+                    m.ack();
+                }
+                return {
+                    documents: useMessages,
+                    checkpoint: newCheckpoint
+                };
+            },
+            batchSize: ensureNotFalsy(options.pull).batchSize,
+            modifier: ensureNotFalsy(options.pull).modifier,
+            stream$: pullStream$.asObservable()
+        };
+    }
+
+
+    let replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined;
+    if (options.push) {
+        replicationPrimitivesPush = {
+            async handler(
+                rows: RxReplicationWriteToMasterRow<RxDocType>[]
+            ) {
+                const cn = await connectionStatePromise;
+                const conflicts: WithDeleted<RxDocType>[] = [];
+                await Promise.all(
+                    rows.map(async (writeRow) => {
+                        const docId = (writeRow.newDocumentState as any)[primaryPath];
+
+                        /**
+                         * first get the current state of the documents from the server
+                         * so that we have the sequence number for conflict detection.
+                         */
+                        let remoteDocState;
+                        try {
+                            remoteDocState = await getNatsServerDocumentState(
+                                cn.natsStream,
+                                options.subjectPrefix,
+                                docId
+                            );
+                        } catch (err: Error | any) {
+                            if (!err.message.includes('no message found')) {
+                                throw err;
+                            }
+                        }
+
+                        if (
+                            remoteDocState &&
+                            (
+                                !writeRow.assumedMasterState ||
+                                collection.conflictHandler.isEqual(remoteDocState.json(), writeRow.assumedMasterState, 'replication-nats-push') === false
+                            )
+                        ) {
+                            // conflict
+                            conflicts.push(remoteDocState.json());
+                        } else {
+                            // no conflict (yet)
+                            let pushDone = false;
+                            while (!pushDone) {
+                                try {
+                                    await cn.jetstreamClient.publish(
+                                        options.subjectPrefix + '.' + docId,
+                                        jc.encode(writeRow.newDocumentState),
+                                        {
+                                            expect: remoteDocState ? {
+                                                streamName: options.streamName,
+                                                lastSubjectSequence: remoteDocState.seq
+                                            } : undefined
+                                        }
+                                    );
+                                    pushDone = true;
+                                } catch (err: Error | any) {
+                                    if (err.message.includes('wrong last sequence')) {
+                                        // A write happened while we are doing our write -> handle conflict
+                                        const newServerState = await getNatsServerDocumentState(
+                                            cn.natsStream,
+                                            options.subjectPrefix,
+                                            docId
+                                        );
+                                        conflicts.push(ensureNotFalsy(newServerState).json());
+                                        pushDone = true;
+                                    } else {
+                                        replicationState.subjects.error.next(
+                                            newRxError('RC_STREAM', {
+                                                document: writeRow.newDocumentState,
+                                                error: errorToPlainJson(err)
+                                            })
+                                        );
+
+                                        // -> retry after wait
+                                        await awaitRetry(
+                                            collection,
+                                            replicationState.retryTime
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    })
+                );
+                return conflicts;
+            },
+            batchSize: options.push.batchSize,
+            modifier: options.push.modifier
+        };
+    }
+
+
+    const replicationState = new RxNatsReplicationState<RxDocType>(
+        options.replicationIdentifier,
+        collection,
+        replicationPrimitivesPull,
+        replicationPrimitivesPush,
+        options.live,
+        options.retryTime,
+        options.autoStart
+    );
+
+    /**
+     * Use long polling to get live changes for the pull.stream$
+     */
+    if (options.live && options.pull) {
+        const startBefore = replicationState.start.bind(replicationState);
+        const cancelBefore = replicationState.cancel.bind(replicationState);
+        replicationState.start = async () => {
+            const cn = await connectionStatePromise;
+
+            /**
+             * First get the last sequence so that we can
+             * laster only fetch 'newer' messages.
+             */
+            let lastSeq = 0;
+            try {
+                const lastDocState = await cn.natsStream.getMessage({
+                    last_by_subj: options.subjectPrefix + '.*'
+                });
+                lastSeq = lastDocState.seq;
+            } catch (err: any | Error) {
+                if (!err.message.includes('no message found')) {
+                    throw err;
+                }
+            }
+
+            const consumer = await cn.natsStream.getConsumer({
+                opt_start_seq: lastSeq
+            });
+            const newMessages = await consumer.consume();
+            (async () => {
+                for await (const m of newMessages) {
+                    const docData: WithDeleted<RxDocType> = m.json();
+                    pullStream$.next({
+                        documents: [docData],
+                        checkpoint: {
+                            sequence: m.seq
+                        }
+                    });
+                    m.ack();
+                }
+            })();
+            replicationState.cancel = () => {
+                newMessages.close();
+                return cancelBefore();
+            };
+            return startBefore();
+        };
+    }
+
+    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);
+
+    return replicationState;
+}

package/src/plugins/replication-nats/nats-helper.ts

@@ -0,0 +1,15 @@
+import type {
+    StoredMsg,
+    Stream
+} from 'nats';
+
+export async function getNatsServerDocumentState(
+    natsStream: Stream,
+    subjectPrefix: string,
+    docId: string
+): Promise<StoredMsg | undefined> {
+    const remoteDocState = await natsStream.getMessage({
+        last_by_subj: subjectPrefix + '.' + docId
+    });
+    return remoteDocState;
+}

package/src/plugins/replication-nats/nats-types.ts

@@ -0,0 +1,43 @@
+import type {
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions
+} from '../../types/index.d.ts';
+
+
+import {
+    ConnectionOptions
+} from 'nats';
+
+
+export type NatsCheckpointType = {
+    sequence: number;
+};
+
+export type NatsSyncPullOptions<RxDocType> =
+    Omit<ReplicationPullOptions<RxDocType, NatsCheckpointType>, 'handler' | 'stream$'>
+    & {
+    };
+
+export type NatsSyncPushOptions<RxDocType> = Omit<ReplicationPushOptions<RxDocType>, 'handler'>
+    & {
+};
+
+export type NatsSyncOptions<RxDocType> = Omit<
+    ReplicationOptions<RxDocType, any>,
+    'pull' | 'push'
+> & {
+
+    connection: ConnectionOptions;
+    streamName: string;
+    /**
+     * NATS subject prefix like 'foo.bar'
+     * which means a message for a document would have the subject
+     * 'foo.bar.myDoc' where the last part 'myDoc' would be the primaryKey in
+     * the RxDB document.
+     * @link https://docs.nats.io/nats-concepts/subjects
+     */
+    subjectPrefix: string;
+    pull?: NatsSyncPullOptions<RxDocType>;
+    push?: NatsSyncPushOptions<RxDocType>;
+};