@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-webrtc/index.ts

@@ -0,0 +1,307 @@
+import {
+    BehaviorSubject,
+    filter,
+    firstValueFrom,
+    map,
+    Subject,
+    Subscription
+} from 'rxjs';
+import { addRxPlugin } from '../../plugin.ts';
+import { rxStorageInstanceToReplicationHandler } from '../../replication-protocol/index.ts';
+import type {
+    RxCollection,
+    RxError,
+    RxReplicationHandler,
+    RxReplicationWriteToMasterRow,
+    RxTypeError
+} from '../../types/index.d.ts';
+import {
+    ensureNotFalsy,
+    getFromMapOrThrow,
+    randomToken
+} from '../../plugins/utils/index.ts';
+import { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';
+import { replicateRxCollection } from '../replication/index.ts';
+import {
+    isMasterInWebRTCReplication,
+    sendMessageAndAwaitAnswer
+} from './webrtc-helper.ts';
+import type {
+    PeerWithMessage,
+    PeerWithResponse,
+    WebRTCConnectionHandler,
+    WebRTCPeerState,
+    WebRTCReplicationCheckpoint,
+    WebRTCResponse,
+    RxWebRTCReplicationState,
+    SyncOptionsWebRTC
+} from './webrtc-types.ts';
+import { newRxError } from '../../rx-error.ts';
+
+
+export async function replicateWebRTC<RxDocType, PeerType>(
+    options: SyncOptionsWebRTC<RxDocType, PeerType>
+): Promise<RxWebRTCReplicationPool<RxDocType, PeerType>> {
+    const collection = options.collection;
+    addRxPlugin(RxDBLeaderElectionPlugin);
+
+    // fill defaults
+    if (options.pull) {
+        if (!options.pull.batchSize) {
+            options.pull.batchSize = 20;
+        }
+    }
+    if (options.push) {
+        if (!options.push.batchSize) {
+            options.push.batchSize = 20;
+        }
+    }
+
+    if (collection.database.multiInstance) {
+        await collection.database.waitForLeadership();
+    }
+
+    // used to easier debug stuff
+    let requestCounter = 0;
+    const requestFlag = randomToken(10);
+    function getRequestId() {
+        const count = requestCounter++;
+        return collection.database.token + '|' + requestFlag + '|' + count;
+    }
+
+    const storageToken = await collection.database.storageToken;
+    const pool = new RxWebRTCReplicationPool(
+        collection,
+        options,
+        await options.connectionHandlerCreator(options)
+    );
+
+
+    pool.subs.push(
+        pool.connectionHandler.error$.subscribe((err: RxError | RxTypeError) => pool.error$.next(err)),
+        pool.connectionHandler.disconnect$.subscribe((peer: PeerType) => pool.removePeer(peer))
+    );
+
+    /**
+     * Answer if someone requests our storage token
+     */
+    pool.subs.push(
+        pool.connectionHandler.message$.pipe(
+            filter((data: PeerWithMessage<PeerType>) => data.message.method === 'token')
+        ).subscribe((data: PeerWithMessage<PeerType>) => {
+            pool.connectionHandler.send(data.peer, {
+                id: data.message.id,
+                result: storageToken
+            });
+        })
+    );
+
+    const connectSub = pool.connectionHandler.connect$
+        .pipe(
+            filter(() => !pool.canceled)
+        )
+        .subscribe(async (peer: PeerType) => {
+            if (options.isPeerValid) {
+                const isValid = await options.isPeerValid(peer);
+                if (!isValid) {
+                    return;
+                }
+            }
+
+            let peerToken: string;
+            try {
+                const tokenResponse = await sendMessageAndAwaitAnswer(
+                    pool.connectionHandler,
+                    peer,
+                    {
+                        id: getRequestId(),
+                        method: 'token',
+                        params: []
+                    }
+                );
+                peerToken = tokenResponse.result;
+            } catch (error: any) {
+                /**
+                 * If could not get the tokenResponse,
+                 * just ignore that peer.
+                 */
+                pool.error$.next(newRxError('RC_WEBRTC_PEER', {
+                    error
+                }));
+                return;
+            }
+            const isMaster = await isMasterInWebRTCReplication(collection.database.hashFunction, storageToken, peerToken);
+
+            let replicationState: RxWebRTCReplicationState<RxDocType> | undefined;
+            if (isMaster) {
+                const masterHandler = pool.masterReplicationHandler;
+                const masterChangeStreamSub = masterHandler.masterChangeStream$.subscribe((ev: any) => {
+                    const streamResponse: WebRTCResponse = {
+                        id: 'masterChangeStream$',
+                        result: ev
+                    };
+                    pool.connectionHandler.send(peer, streamResponse);
+                });
+
+                // clean up the subscription
+                pool.subs.push(
+                    masterChangeStreamSub,
+                    pool.connectionHandler.disconnect$.pipe(
+                        filter((p: PeerType) => p === peer)
+                    ).subscribe(() => masterChangeStreamSub.unsubscribe())
+                );
+
+                const messageSub = pool.connectionHandler.message$
+                    .pipe(
+                        filter((data: PeerWithMessage<PeerType>) => data.peer === peer),
+                        filter((data: PeerWithMessage<PeerType>) => data.message.method !== 'token')
+                    )
+                    .subscribe(async (data: PeerWithMessage<PeerType>) => {
+                        const { peer: msgPeer, message } = data;
+                        /**
+                         * If it is not a function,
+                         * it means that the client requested the masterChangeStream$
+                         */
+                        const method = (masterHandler as any)[message.method].bind(masterHandler);
+                        const result = await (method as any)(...message.params);
+                        const response: WebRTCResponse = {
+                            id: message.id,
+                            result
+                        };
+                        pool.connectionHandler.send(msgPeer, response);
+                    });
+                pool.subs.push(messageSub);
+            } else {
+                replicationState = replicateRxCollection({
+                    replicationIdentifier: [collection.name, options.topic, peerToken].join('||'),
+                    collection: collection,
+                    autoStart: true,
+                    deletedField: '_deleted',
+                    live: true,
+                    retryTime: options.retryTime,
+                    waitForLeadership: false,
+                    pull: options.pull ? Object.assign({}, options.pull, {
+                        async handler(lastPulledCheckpoint: WebRTCReplicationCheckpoint | undefined) {
+                            const answer = await sendMessageAndAwaitAnswer(
+                                pool.connectionHandler,
+                                peer,
+                                {
+                                    method: 'masterChangesSince',
+                                    params: [
+                                        lastPulledCheckpoint,
+                                        ensureNotFalsy(options.pull).batchSize
+                                    ],
+                                    id: getRequestId()
+                                }
+                            );
+                            return answer.result;
+                        },
+                        stream$: pool.connectionHandler.response$.pipe(
+                            filter((m: PeerWithResponse<PeerType>) => m.response.id === 'masterChangeStream$'),
+                            map((m: PeerWithResponse<PeerType>) => m.response.result)
+                        )
+
+                    }) : undefined,
+                    push: options.push ? Object.assign({}, options.push, {
+                        async handler(docs: RxReplicationWriteToMasterRow<RxDocType>[]) {
+                            const answer = await sendMessageAndAwaitAnswer(
+                                pool.connectionHandler,
+                                peer,
+                                {
+                                    method: 'masterWrite',
+                                    params: [docs],
+                                    id: getRequestId()
+                                }
+                            );
+                            return answer.result;
+                        }
+                    }) : undefined
+                });
+            }
+            pool.addPeer(peer, replicationState);
+        });
+    pool.subs.push(connectSub);
+    return pool;
+}
+
+
+/**
+ * Because the WebRTC replication runs between many instances,
+ * we use a Pool instead of returning a single replication state.
+ */
+export class RxWebRTCReplicationPool<RxDocType, PeerType> {
+    peerStates$: BehaviorSubject<Map<PeerType, WebRTCPeerState<RxDocType, PeerType>>> = new BehaviorSubject(new Map());
+    canceled: boolean = false;
+    masterReplicationHandler: RxReplicationHandler<RxDocType, WebRTCReplicationCheckpoint>;
+    subs: Subscription[] = [];
+
+    public error$ = new Subject<RxError | RxTypeError>();
+
+    constructor(
+        public readonly collection: RxCollection<RxDocType, any, any, any>,
+        public readonly options: SyncOptionsWebRTC<RxDocType, PeerType>,
+        public readonly connectionHandler: WebRTCConnectionHandler<PeerType>
+    ) {
+        this.collection.onClose.push(() => this.cancel());
+        this.masterReplicationHandler = rxStorageInstanceToReplicationHandler(
+            collection.storageInstance,
+            collection.conflictHandler,
+            collection.database.token,
+        );
+    }
+
+    addPeer(
+        peer: PeerType,
+        // only if isMaster=false it has a replicationState
+        replicationState?: RxWebRTCReplicationState<RxDocType>
+    ) {
+        const peerState: WebRTCPeerState<RxDocType, PeerType> = {
+            peer,
+            replicationState,
+            subs: []
+        };
+        this.peerStates$.next(this.peerStates$.getValue().set(peer, peerState));
+        if (replicationState) {
+            peerState.subs.push(
+                replicationState.error$.subscribe((ev: RxError | RxTypeError) => this.error$.next(ev))
+            );
+        }
+    }
+    removePeer(peer: PeerType) {
+        const peerState: WebRTCPeerState<RxDocType, PeerType> = getFromMapOrThrow(this.peerStates$.getValue(), peer) as WebRTCPeerState<RxDocType, PeerType>;
+        this.peerStates$.getValue().delete(peer);
+        this.peerStates$.next(this.peerStates$.getValue());
+        peerState.subs.forEach((sub: Subscription) => sub.unsubscribe());
+        if (peerState.replicationState) {
+            peerState.replicationState.cancel();
+        }
+    }
+
+    // often used in unit tests
+    awaitFirstPeer() {
+        return firstValueFrom(
+            this.peerStates$.pipe(
+                filter((peerStates: Map<PeerType, WebRTCPeerState<RxDocType, PeerType>>) => peerStates.size > 0)
+            )
+        );
+    }
+
+    public async cancel() {
+        if (this.canceled) {
+            return;
+        }
+        this.canceled = true;
+        this.subs.forEach((sub: Subscription) => sub.unsubscribe());
+        (Array.from(this.peerStates$.getValue().keys()) as PeerType[]).forEach((peer: PeerType) => {
+            this.removePeer(peer);
+        });
+        await this.connectionHandler.close();
+    }
+}
+
+export * from './webrtc-helper.ts';
+export * from './signaling-server.ts';
+export * from './webrtc-types.ts';
+// export * from './connection-handler-webtorrent';
+// export * from './connection-handler-p2pcf';
+export * from './connection-handler-simple-peer.ts';

package/src/plugins/replication-webrtc/signaling-server.ts

@@ -0,0 +1,193 @@
+import {
+    getFromMapOrCreate,
+    promiseWait,
+    randomToken
+} from '../utils/index.ts';
+import {
+    SIMPLE_PEER_PING_INTERVAL,
+    type PeerMessage
+} from './connection-handler-simple-peer.ts';
+import type {
+    WebSocket,
+    ServerOptions
+} from 'ws';
+
+export const PEER_ID_LENGTH = 12;
+export type ServerPeer = {
+    id: string;
+    socket: WebSocket;
+    rooms: Set<string>;
+    lastPing: number;
+};
+
+
+/**
+ * Starts a WebRTC signaling server
+ * that can be used in tests.
+*/
+export async function startSignalingServerSimplePeer(
+    serverOptions: ServerOptions
+) {
+    const { WebSocketServer } = await import('ws');
+    const wss = new WebSocketServer(serverOptions);
+
+    const peerById = new Map<string, ServerPeer>();
+    const peersByRoom = new Map<string, Set<string>>();
+
+    let serverClosed = false;
+    wss.on('close', () => {
+        serverClosed = true
+        peerById.clear();
+        peersByRoom.clear();
+    });
+
+    /**
+     * Clients can disconnect without telling that to the
+     * server. Therefore we have to automatically disconnect clients that
+     * have not sent a ping message in the last 2 minutes.
+     */
+    (async () => {
+        while (!serverClosed) {
+            await promiseWait(1000 * 5);
+            const minTime = Date.now() - SIMPLE_PEER_PING_INTERVAL;
+            Array.from(peerById.values()).forEach(peer => {
+                if (peer.lastPing < minTime) {
+                    disconnectSocket(peer.id, 'no ping for 2 minutes');
+                }
+            });
+        }
+    })();
+
+    function disconnectSocket(peerId: string, reason: string) {
+        console.log('# disconnect peer ' + peerId + ' reason: ' + reason);
+        const peer = peerById.get(peerId);
+        if (peer) {
+            peer.socket.close && peer.socket.close(undefined, reason);
+            peer.rooms.forEach(roomId => {
+                const room = peersByRoom.get(roomId);
+                room?.delete(peerId);
+                if (room && room.size === 0) {
+                    peersByRoom.delete(roomId);
+                }
+            });
+        }
+        peerById.delete(peerId);
+    }
+
+    wss.on('connection', function (ws: WebSocket) {
+        /**
+         * PeerID is created by the server to prevent malicious
+         * actors from falsy claiming other peoples ids.
+         */
+        const peerId = randomToken(PEER_ID_LENGTH);
+        const peer: ServerPeer = {
+            id: peerId,
+            socket: ws,
+            rooms: new Set(),
+            lastPing: Date.now()
+        };
+        peerById.set(peerId, peer);
+
+        sendMessage(ws, { type: 'init', yourPeerId: peerId });
+
+
+        ws.on('error', (err: Error) => {
+            console.error('SERVER ERROR:');
+            console.dir(err);
+            disconnectSocket(peerId, 'socket errored');
+        });
+        ws.on('close', () => {
+            disconnectSocket(peerId, 'socket disconnected');
+        });
+
+        ws.on('message', (msgEvent: any) => {
+            peer.lastPing = Date.now();
+            const message = JSON.parse(msgEvent.toString());
+            const type = message.type;
+            switch (type) {
+                case 'join':
+                    const roomId = message.room;
+                    if (
+                        !validateIdString(roomId) ||
+                        !validateIdString(peerId)
+                    ) {
+                        disconnectSocket(peerId, 'invalid ids');
+                        return;
+                    }
+
+                    if (peer.rooms.has(peerId)) {
+                        return;
+                    }
+                    peer.rooms.add(roomId);
+
+
+                    const room = getFromMapOrCreate(
+                        peersByRoom,
+                        message.room,
+                        () => new Set()
+                    );
+
+                    room.add(peerId);
+
+                    // tell everyone about new room state
+                    room.forEach(otherPeerId => {
+                        const otherPeer = peerById.get(otherPeerId);
+                        if (otherPeer) {
+                            sendMessage(
+                                otherPeer.socket,
+                                {
+                                    type: 'joined',
+                                    otherPeerIds: Array.from(room)
+                                }
+                            );
+                        }
+                    });
+                    break;
+                case 'signal':
+                    if (
+                        message.senderPeerId !== peerId
+                    ) {
+                        disconnectSocket(peerId, 'spoofed sender');
+                        return;
+                    }
+                    const receiver = peerById.get(message.receiverPeerId);
+                    if (receiver) {
+                        sendMessage(
+                            receiver.socket,
+                            message
+                        );
+                    }
+                    break;
+                case 'ping':
+                    break;
+                default:
+                    disconnectSocket(peerId, 'unknown message type ' + type);
+            }
+
+        });
+    });
+
+    return {
+        port: serverOptions.port,
+        server: wss,
+        localUrl: 'ws://localhost:' + serverOptions.port
+    };
+}
+
+
+function sendMessage(ws: WebSocket, message: PeerMessage) {
+    const msgString = JSON.stringify(message);
+    ws.send(msgString);
+}
+
+function validateIdString(roomId: string): boolean {
+    if (
+        typeof roomId === 'string' &&
+        roomId.length > 5 &&
+        roomId.length < 100
+    ) {
+        return true;
+    } else {
+        return false;
+    }
+}

package/src/plugins/replication-webrtc/webrtc-helper.ts

@@ -0,0 +1,55 @@
+import type {
+    HashFunction
+} from '../../types/index.d.ts';
+import type {
+    WebRTCConnectionHandler,
+    WebRTCMessage,
+    WebRTCResponse,
+    PeerWithResponse
+} from './webrtc-types.ts';
+import { filter, firstValueFrom, map } from 'rxjs';
+
+
+
+/**
+ * To deterministically define which peer is master and
+ * which peer is fork, we compare the storage tokens.
+ * But we have to hash them before, to ensure that
+ * a storageToken like 'aaaaaa' is not always the master
+ * for all peers.
+ */
+export async function isMasterInWebRTCReplication(
+    hashFunction: HashFunction,
+    ownStorageToken: string,
+    otherStorageToken: string
+): Promise<boolean> {
+    const isMaster =
+        await hashFunction([ownStorageToken, otherStorageToken].join('|'))
+        >
+        await hashFunction([otherStorageToken, ownStorageToken].join('|'));
+    return isMaster;
+}
+
+/**
+ * Send a message to the peer and await the answer.
+ * @throws with an EmptyErrorImpl if the peer connection
+ * was closed before an answer was received.
+ */
+export function sendMessageAndAwaitAnswer<PeerType>(
+    handler: WebRTCConnectionHandler<PeerType>,
+    peer: PeerType,
+    message: WebRTCMessage
+): Promise<WebRTCResponse> {
+
+
+    const requestId = message.id;
+    const answerPromise = firstValueFrom(
+        handler.response$.pipe(
+            filter((d: PeerWithResponse<PeerType>) => d.peer === peer),
+            filter((d: PeerWithResponse<PeerType>) => d.response.id === requestId),
+            map((d: PeerWithResponse<PeerType>) => d.response)
+        )
+    );
+    handler.send(peer, message);
+    return answerPromise;
+}

package/src/plugins/replication-webrtc/webrtc-types.ts

@@ -0,0 +1,91 @@
+import { Observable, Subscription } from 'rxjs';
+import type {
+    MaybePromise,
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxError,
+    RxReplicationHandler,
+    RxStorageDefaultCheckpoint,
+    RxTypeError,
+    StringKeys
+} from '../../types/index.d.ts';
+import { RxReplicationState } from '../replication/index.ts';
+import { WebsocketMessageResponseType, WebsocketMessageType } from '../replication-websocket/index.ts';
+
+export type WebRTCReplicationCheckpoint = RxStorageDefaultCheckpoint;
+
+
+export type WebRTCMessage = Omit<WebsocketMessageType, 'method' | 'collection'> & {
+    method: StringKeys<RxReplicationHandler<any, any>> | 'token';
+};
+export type WebRTCResponse = Omit<WebsocketMessageResponseType, 'collection'>;
+export type PeerWithMessage<PeerType> = {
+    peer: PeerType;
+    message: WebRTCMessage;
+};
+export type PeerWithResponse<PeerType> = {
+    peer: PeerType;
+    response: WebRTCResponse;
+};
+
+export type WebRTCConnectionHandler<PeerType> = {
+    connect$: Observable<PeerType>;
+    disconnect$: Observable<PeerType>;
+    message$: Observable<PeerWithMessage<PeerType>>;
+    response$: Observable<PeerWithResponse<PeerType>>;
+    error$: Observable<RxError | RxTypeError>;
+    send(peer: PeerType, message: WebRTCMessage | WebRTCResponse): Promise<void>;
+    close(): Promise<void>;
+};
+
+export type WebRTCConnectionHandlerCreator<PeerType> = (
+    opts: SyncOptionsWebRTC<any, PeerType>
+) => Promise<WebRTCConnectionHandler<PeerType>>;
+
+export type WebRTCSyncPushOptions<RxDocType> = Omit<
+    ReplicationPushOptions<RxDocType>,
+    'handler'
+> & {};
+
+export type WebRTCSyncPullOptions<RxDocType> = Omit<
+    ReplicationPullOptions<RxDocType, WebRTCReplicationCheckpoint>,
+    'handler' | 'stream$'
+> & {};
+
+export type SyncOptionsWebRTC<RxDocType, PeerType> = Omit<
+    ReplicationOptions<RxDocType, WebRTCReplicationCheckpoint>,
+    'pull' |
+    'push' |
+    'replicationIdentifier' |
+    'deletedField' |
+    'live' |
+    'autostart' |
+    'waitForLeadership'
+> & {
+    /**
+     * It will only replicate with other instances
+     * that use the same topic.
+     */
+    topic: string;
+    connectionHandlerCreator: WebRTCConnectionHandlerCreator<PeerType>;
+    /**
+     * Run on new peers so that bad peers can be blocked.
+     * If returns true, the peer is valid and it will replicate.
+     * If returns false, it will drop the peer.
+     */
+    isPeerValid?: (peer: PeerType) => MaybePromise<boolean>;
+    pull?: WebRTCSyncPullOptions<RxDocType>;
+    push?: WebRTCSyncPushOptions<RxDocType>;
+};
+
+export type RxWebRTCReplicationState<RxDocType> = RxReplicationState<RxDocType, WebRTCReplicationCheckpoint>;
+
+
+export type WebRTCPeerState<RxDocType, PeerType> = {
+    peer: PeerType;
+    // only exists when the peer was picked as master and the own client was picked as fork.
+    replicationState?: RxWebRTCReplicationState<RxDocType>;
+    // clean this up when removing the peer
+    subs: Subscription[];
+};

package/src/plugins/replication-websocket/index.ts

@@ -0,0 +1,3 @@
+export * from './websocket-client.ts';
+export * from './websocket-server.ts';
+export * from './websocket-types.ts';