@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-websocket/websocket-client.ts

@@ -0,0 +1,187 @@
+import {
+    replicateRxCollection,
+    RxReplicationState
+} from '../replication/index.ts';
+import {
+    WebsocketClientOptions,
+    WebsocketMessageResponseType,
+    WebsocketMessageType
+} from './websocket-types.ts';
+
+import ReconnectingWebSocket from 'reconnecting-websocket';
+
+import { WebSocket } from 'ws';
+import {
+    errorToPlainJson,
+    randomToken,
+    toArray
+} from '../../plugins/utils/index.ts';
+import {
+    filter,
+    map,
+    Subject,
+    firstValueFrom,
+    BehaviorSubject
+} from 'rxjs';
+import type {
+    RxError,
+    RxReplicationWriteToMasterRow
+} from '../../types/index.d.ts';
+import { newRxError } from '../../rx-error.ts';
+
+export type WebsocketClient = {
+    url: string;
+    socket: any;
+    connected$: BehaviorSubject<boolean>;
+    message$: Subject<any>;
+    error$: Subject<RxError>;
+};
+
+
+export async function createWebSocketClient<RxDocType>(options: WebsocketClientOptions<RxDocType>): Promise<WebsocketClient> {
+    const wsClient = new ReconnectingWebSocket(
+        options.url,
+        [],
+        { WebSocket }
+    );
+    const connected$ = new BehaviorSubject<boolean>(false);
+    const message$ = new Subject<any>();
+    const error$ = new Subject<any>();
+    wsClient.onerror = (err: any) => {
+
+        console.log('--- WAS CLIENT GOT ERROR:');
+        console.log(err.error.message);
+
+        const emitError = newRxError('RC_STREAM', {
+            errors: toArray(err).map((er: any) => errorToPlainJson(er)),
+            direction: 'pull'
+        });
+        error$.next(emitError);
+    };
+    await new Promise<void>(res => {
+        wsClient.onopen = () => {
+
+            if (options.headers) {
+                const authMessage: WebsocketMessageType = {
+                    collection: options.collection.name,
+                    id: randomToken(10),
+                    params: [options.headers],
+                    method: 'auth'
+                };
+                wsClient.send(JSON.stringify(authMessage));
+            }
+
+            connected$.next(true);
+            res();
+        };
+    });
+    wsClient.onclose = () => {
+        connected$.next(false);
+    };
+
+    wsClient.onmessage = (messageObj: any) => {
+        const message = JSON.parse(messageObj.data);
+        message$.next(message);
+    };
+
+    return {
+        url: options.url,
+        socket: wsClient,
+        connected$,
+        message$,
+        error$
+    };
+
+}
+
+export async function replicateWithWebsocketServer<RxDocType, CheckpointType>(
+    options: WebsocketClientOptions<RxDocType>
+): Promise<RxReplicationState<RxDocType, CheckpointType>> {
+    const websocketClient = await createWebSocketClient(options);
+    const wsClient = websocketClient.socket;
+    const messages$ = websocketClient.message$;
+
+    let requestCounter = 0;
+    const requestFlag = randomToken(10);
+    function getRequestId() {
+        const count = requestCounter++;
+        return options.collection.database.token + '|' + requestFlag + '|' + count;
+    }
+    const replicationState = replicateRxCollection<RxDocType, CheckpointType>({
+        collection: options.collection,
+        replicationIdentifier: options.replicationIdentifier,
+        live: options.live,
+        pull: {
+            batchSize: options.batchSize,
+            stream$: messages$.pipe(
+                filter((msg: WebsocketMessageResponseType) => msg.id === 'stream' && msg.collection === options.collection.name),
+                map((msg: WebsocketMessageResponseType) => msg.result)
+            ),
+            async handler(lastPulledCheckpoint: CheckpointType | undefined, batchSize: number) {
+                const requestId = getRequestId();
+                const request: WebsocketMessageType = {
+                    id: requestId,
+                    collection: options.collection.name,
+                    method: 'masterChangesSince',
+                    params: [lastPulledCheckpoint, batchSize]
+                };
+                wsClient.send(JSON.stringify(request));
+                const result = await firstValueFrom(
+                    messages$.pipe(
+                        filter((msg: WebsocketMessageResponseType) => msg.id === requestId),
+                        map((msg: WebsocketMessageResponseType) => msg.result)
+                    )
+                );
+                return result;
+            }
+        },
+        push: {
+            batchSize: options.batchSize,
+            handler(docs: RxReplicationWriteToMasterRow<RxDocType>[]) {
+                const requestId = getRequestId();
+                const request: WebsocketMessageType = {
+                    id: requestId,
+                    collection: options.collection.name,
+                    method: 'masterWrite',
+                    params: [docs]
+                };
+                wsClient.send(JSON.stringify(request));
+                return firstValueFrom(
+                    messages$.pipe(
+                        filter((msg: WebsocketMessageResponseType) => msg.id === requestId),
+                        map((msg: WebsocketMessageResponseType) => msg.result)
+                    )
+                );
+            }
+        }
+    });
+
+    websocketClient.error$.subscribe((err: RxError) => replicationState.subjects.error.next(err));
+
+    websocketClient.connected$.subscribe((isConnected: boolean) => {
+        if (isConnected) {
+            /**
+             * When the client goes offline and online again,
+             * we have to send a 'RESYNC' signal because the client
+             * might have missed out events while being offline.
+             */
+            replicationState.reSync();
+
+            /**
+             * Because reconnecting creates a new websocket-instance,
+             * we have to start the changestream from the remote again
+             * each time.
+             */
+            const streamRequest: WebsocketMessageType = {
+                id: 'stream',
+                collection: options.collection.name,
+                method: 'masterChangeStream$',
+                params: []
+            };
+            wsClient.send(JSON.stringify(streamRequest));
+        }
+    });
+
+    options.collection.onClose.push(() => websocketClient.socket.close());
+    return replicationState;
+}

package/src/plugins/replication-websocket/websocket-server.ts

@@ -0,0 +1,135 @@
+import type {
+    RxCollection,
+    RxDatabase,
+    RxReplicationHandler
+} from '../../types/index.d.ts';
+
+import {
+    type WebSocket,
+    WebSocketServer,
+    type ServerOptions,
+} from 'ws';
+
+import type {
+    WebsocketMessageResponseType,
+    WebsocketMessageType,
+    WebsocketServerOptions,
+    WebsocketServerState
+} from './websocket-types.ts';
+import { rxStorageInstanceToReplicationHandler } from '../../replication-protocol/index.ts';
+import {
+    PROMISE_RESOLVE_VOID, getFromMapOrCreate
+} from '../../plugins/utils/index.ts';
+import { Subject } from 'rxjs';
+
+export function startSocketServer(options: ServerOptions): WebsocketServerState {
+    const wss = new WebSocketServer(options);
+    let closed = false;
+    function closeServer() {
+        if (closed) {
+            return PROMISE_RESOLVE_VOID;
+        }
+        closed = true;
+        onConnection$.complete();
+        return new Promise<void>((res, rej) => {
+            /**
+             * We have to close all client connections,
+             * otherwise wss.close() will never call the callback.
+             * @link https://github.com/websockets/ws/issues/1288#issuecomment-360594458
+             */
+            for (const ws of wss.clients) {
+                ws.close();
+            }
+            wss.close((err: any) => {
+                if (err) {
+                    rej(err);
+                } else {
+                    res();
+                }
+            });
+        });
+    }
+
+    const onConnection$ = new Subject<WebSocket>();
+    wss.on('connection', (ws: any) => onConnection$.next(ws));
+
+    return {
+        server: wss,
+        close: closeServer,
+        onConnection$: onConnection$.asObservable()
+    };
+}
+
+const REPLICATION_HANDLER_BY_COLLECTION: WeakMap<RxCollection, RxReplicationHandler<any, any>> = new Map();
+export function getReplicationHandlerByCollection<RxDocType>(
+    database: RxDatabase<any>,
+    collectionName: string
+): RxReplicationHandler<RxDocType, any> {
+    if (!database.collections[collectionName]) {
+        throw new Error('collection ' + collectionName + ' does not exist');
+    }
+
+    const collection = database.collections[collectionName];
+    const handler = getFromMapOrCreate<RxCollection, RxReplicationHandler<RxDocType, any>>(
+        REPLICATION_HANDLER_BY_COLLECTION,
+        collection,
+        () => {
+            return rxStorageInstanceToReplicationHandler(
+                collection.storageInstance,
+                collection.conflictHandler,
+                database.token
+            );
+        }
+    );
+    return handler;
+}
+
+export function startWebsocketServer(options: WebsocketServerOptions): WebsocketServerState {
+    const { database, ...wsOptions } = options;
+    const serverState = startSocketServer(wsOptions as any);
+
+    // auto close when the database gets closed
+    database.onClose.push(() => serverState.close());
+
+    serverState.onConnection$.subscribe((ws: WebSocket) => {
+        const onCloseHandlers: Function[] = [];
+        ws.onclose = () => {
+            onCloseHandlers.map(fn => fn());
+        };
+        ws.on('message', async (messageString: string) => {
+            const message: WebsocketMessageType = JSON.parse(messageString);
+            const handler = getReplicationHandlerByCollection(database, message.collection);
+            if (message.method === 'auth') {
+                return;
+            }
+            const method = handler[message.method];
+
+            /**
+             * If it is not a function,
+             * it means that the client requested the masterChangeStream$
+             */
+            if (typeof method !== 'function') {
+                const changeStreamSub = handler.masterChangeStream$.subscribe((ev: any) => {
+                    const streamResponse: WebsocketMessageResponseType = {
+                        id: 'stream',
+                        collection: message.collection,
+                        result: ev
+                    };
+                    ws.send(JSON.stringify(streamResponse));
+                });
+                onCloseHandlers.push(() => changeStreamSub.unsubscribe());
+                return;
+            }
+            const result = await (method as any)(...message.params);
+            const response: WebsocketMessageResponseType = {
+                id: message.id,
+                collection: message.collection,
+                result
+            };
+            ws.send(JSON.stringify(response));
+        });
+    });
+
+
+    return serverState;
+}

package/src/plugins/replication-websocket/websocket-types.ts

@@ -0,0 +1,47 @@
+import type {
+    Observable,
+} from 'rxjs';
+import type {
+    ServerOptions,
+    ClientOptions,
+    WebSocketServer,
+    WebSocket
+} from 'ws';
+import type {
+    RxCollection,
+    RxDatabase,
+    RxReplicationHandler,
+    StringKeys
+} from '../../types/index.d.ts';
+
+export type WebsocketServerOptions = {
+    database: RxDatabase<any, any, any>;
+} & ServerOptions;
+
+export type WebsocketServerState = {
+    server: WebSocketServer;
+    close: () => Promise<void>;
+    onConnection$: Observable<WebSocket>;
+};
+
+export type WebsocketClientOptions<RxDocType> = {
+    replicationIdentifier: string;
+    collection: RxCollection<RxDocType>;
+    url: string;
+    batchSize?: number;
+    live?: boolean;
+    headers?: { [k: string]: string; };
+} & ClientOptions;
+
+export type WebsocketMessageType = {
+    id: string;
+    collection: string;
+    method: StringKeys<RxReplicationHandler<any, any>> | 'auth';
+    params: any[];
+};
+
+export type WebsocketMessageResponseType = {
+    id: string;
+    collection: string;
+    result: any;
+};

package/src/plugins/state/helpers.ts

@@ -0,0 +1,86 @@
+import { overwritable } from '../../overwritable.ts';
+import { newRxError } from '../../rx-error.ts';
+import type { DeepReadonly, RxJsonSchema } from '../../types';
+import type { RxStateDocument } from './types';
+
+export const RX_STATE_SCHEMA_TITLE = 'RxStateCollection';
+export const RX_STATE_ID_LENGTH = 14;
+export const RX_STATE_COLLECTION_SCHEMA: DeepReadonly<RxJsonSchema<RxStateDocument>> = {
+    title: RX_STATE_SCHEMA_TITLE,
+    primaryKey: 'id',
+    version: 0,
+    type: 'object',
+    properties: {
+        id: {
+            type: 'string',
+            /**
+             * We store numbers in string format like '0001'
+             * with a left-pad.
+             */
+            maxLength: RX_STATE_ID_LENGTH,
+            minLength: RX_STATE_ID_LENGTH,
+            pattern: '[0-9]+'
+        },
+        sId: {
+            type: 'string',
+            maxLength: 10,
+            minLength: 10
+        },
+        ops: {
+            type: 'array',
+            minItems: 1,
+            items: {
+                type: 'object',
+                properties: {
+                    k: {
+                        type: 'string'
+                    },
+                    v: {
+                        /**
+                         * Do not define a type for the value
+                         * because anything is allowed.
+                         */
+                    }
+                },
+                required: [
+                    'k',
+                    'v'
+                ]
+            }
+        }
+    },
+    required: [
+        'id',
+        'sId',
+        'ops'
+    ]
+} as const;
+
+
+export function nextRxStateId(lastId?: string): string {
+    if (!lastId) {
+        return ''.padStart(RX_STATE_ID_LENGTH, '0');
+    }
+    const parsed = parseInt(lastId, 10);
+    const next = parsed + 1;
+    const nextString = next.toString();
+    return nextString.padStart(RX_STATE_ID_LENGTH, '0');
+}
+
+
+
+/**
+ * Only non-primitives can be used as a key in WeakMap
+ */
+export function isValidWeakMapKey(key: any) {
+    // This method is slow and must only be used in dev-mode!
+    if (!overwritable.isDevMode()) {
+        throw newRxError('SNH');
+    }
+    try {
+        new WeakMap().set(key, {});
+        return true;
+    } catch (err) {
+        return false;
+    }
+}

package/src/plugins/state/index.ts

@@ -0,0 +1,40 @@
+import type {
+    RxDatabase,
+    RxPlugin,
+    RxState
+} from '../../types/index.d.ts';
+import { getFromMapOrCreate } from '../utils/utils-map.ts';
+import { RxStateBase, createRxState } from './rx-state.ts';
+
+export * from './helpers.ts';
+
+type StateByPrefix = Map<string, Promise<RxStateBase<any>>>;
+const STATE_BY_DATABASE = new WeakMap<RxDatabase, StateByPrefix>();
+
+export async function addState<T>(
+    this: RxDatabase,
+    namespace: string = ''
+): Promise<RxState<T>> {
+    const stateCache = getFromMapOrCreate<RxDatabase, StateByPrefix>(
+        STATE_BY_DATABASE,
+        this,
+        () => new Map()
+    );
+    const state = await getFromMapOrCreate(
+        stateCache,
+        namespace,
+        () => createRxState<T>(this, namespace)
+    );
+    this.states[namespace] = state;
+    return state as any;
+}
+
+export const RxDBStatePlugin: RxPlugin = {
+    name: 'state',
+    rxdb: true,
+    prototypes: {
+        RxDatabase(proto: any) {
+            proto.addState = addState;
+        }
+    }
+};