@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-remote/storage-remote-types.ts

@@ -0,0 +1,118 @@
+import type { Observable } from 'rxjs';
+import type {
+    MaybePromise,
+    PlainJsonError,
+    RxDatabase,
+    RxStorage,
+    RxStorageInstance,
+    RxStorageInstanceCreationParams
+} from '../../types/index.d.ts';
+
+
+
+export type MessageFromRemote = {
+    connectionId: string;
+    answerTo: string; // id of the request
+    method: keyof RxStorageInstance<any, any, any> | 'create' | 'custom';
+    error?: PlainJsonError;
+    return?: any | string;
+};
+
+export type MessageToRemote = {
+    connectionId: string;
+    /**
+     * Unique ID of the request
+     */
+    requestId: string;
+    method: keyof RxStorageInstance<any, any, any> | 'create' | 'custom';
+    /**
+     * We send the RxDB version to the remote
+     * to ensure we are communicating with an RxDB instance
+     * of the same version. This is to prevent bugs
+     * when people forget to rebuild their workers.
+     */
+    version: string;
+    params:
+    RxStorageInstanceCreationParams<any, any> | // used in the create call
+    any[] | // used to call RxStorageInstance methods
+    any; // used in custom requests
+};
+
+
+/**
+ * A message channel represents a single
+ * channel that is able to communicate with the remote.
+ * For example a single websocket connection or WebWorker instance.
+ * The storage must be able to open and close MessageChannels
+ * according to the modes settings.
+ */
+export type RemoteMessageChannel = {
+    send(msg: MessageToRemote): void;
+    messages$: Observable<MessageFromRemote>;
+    close(): Promise<void>;
+};
+
+export type RxStorageRemoteSettings = {
+    identifier: string;
+    /**
+     * There are different modes
+     * that determine how many message channels are used.
+     * These modes can have different performance patterns.
+     *
+     * [default='storage']
+     */
+    mode?:
+    // create exactly one RemoteMessageChannel and reuse that everywhere.
+    | 'one'
+    // storage: create one RemoteMessageChannel per call to getRxStorage...()
+    | 'storage'
+    // database: create one RemoteMessageChannel for each database
+    | 'database'
+    // collection: create one RemoteMessageChannel for each collection
+    | 'collection';
+    messageChannelCreator: () => Promise<RemoteMessageChannel>;
+};
+
+export type RxStorageRemoteInternals = {
+    params: RxStorageInstanceCreationParams<any, any>;
+    connectionId: string;
+    messageChannel: RemoteMessageChannel;
+};
+
+export type RxStorageRemoteExposeSettingsBase = {
+    send(msg: MessageFromRemote): void;
+    messages$: Observable<MessageToRemote>;
+    customRequestHandler?: CustomRequestHandler<any, any>;
+    /**
+     * Used in tests to simulate what happens if the remote
+     * was build on a different RxDB version.
+     */
+    fakeVersion?: string;
+};
+
+export type RxStorageRemoteExposeSettingsRxDatabase = RxStorageRemoteExposeSettingsBase & {
+    /**
+     * The database which must be mapped to the remote storage server.
+     */
+    database: RxDatabase;
+};
+
+export type RxStorageRemoteExposeSettingsRxStorage = RxStorageRemoteExposeSettingsBase & {
+    /**
+     * The original storage
+     * which actually stores the data.
+     */
+    storage: RxStorage<any, any>;
+};
+
+export type RxStorageRemoteExposeSettings = RxStorageRemoteExposeSettingsRxDatabase | RxStorageRemoteExposeSettingsRxStorage;
+
+export type RxStorageRemoteExposeType = {
+    instanceByFullName: Map<string, any>;
+};
+
+/**
+ * If set, the clients can send RxDB-unrelated custom messages
+ * to the remote storage and it will  answer them.
+ */
+export type CustomRequestHandler<In, Out> = (data: In) => MaybePromise<Out>;

package/src/plugins/storage-remote-websocket/index.ts

@@ -0,0 +1,218 @@
+import { Subject } from 'rxjs';
+import type {
+    WebSocket
+} from 'ws';
+import {
+    PROMISE_RESOLVE_VOID,
+    blobToBase64String,
+    clone,
+    createBlobFromBase64
+} from '../../plugins/utils/index.ts';
+import {
+    createWebSocketClient,
+    startSocketServer
+} from '../replication-websocket/index.ts';
+import { exposeRxStorageRemote } from '../storage-remote/remote.ts';
+import { getRxStorageRemote } from '../storage-remote/rx-storage-remote.ts';
+import { createErrorAnswer } from '../storage-remote/storage-remote-helpers.ts';
+import type {
+    MessageFromRemote,
+    MessageToRemote,
+    RxStorageRemoteExposeSettings
+} from '../storage-remote/storage-remote-types.ts';
+import type {
+    RxStorageRemoteWebsocketClient,
+    RxStorageRemoteWebsocketClientOptions,
+    RxStorageRemoteWebsocketServerOptions,
+    RxStorageRemoteWebsocketServerState
+} from './types.ts';
+
+/**
+ * WebSocket transport needs Blob<->base64 conversion at the JSON boundary.
+ * These helpers handle the two message types that carry Blob data:
+ * - bulkWrite request: document._attachments[id].data (Blob)
+ * - getAttachmentData response: msg.return (Blob)
+ */
+async function serializeBlobsForWs(msg: any): Promise<string> {
+    const msgForJson: any = clone(msg);
+
+    if (msgForJson.method === 'bulkWrite' && Array.isArray(msgForJson.params)) {
+        const documentWrites = msgForJson.params[0];
+        if (Array.isArray(documentWrites)) {
+            const toConvert: { attachment: any; blob: Blob }[] = [];
+            for (const row of documentWrites) {
+                if (row.document?._attachments) {
+                    for (const attachment of Object.values(row.document._attachments)) {
+                        if ((attachment as any).data instanceof Blob) {
+                            toConvert.push({
+                                attachment,
+                                blob: (attachment as any).data as Blob,
+                            });
+                        }
+                    }
+                }
+            }
+            if (toConvert.length > 0) {
+                const base64Results = await Promise.all(
+                    toConvert.map((entry) => blobToBase64String(entry.blob)),
+                );
+                for (let i = 0; i < toConvert.length; i++) {
+                    toConvert[i].attachment.data = base64Results[i];
+                }
+            }
+        }
+    } else if (msgForJson.method === 'getAttachmentData' && msgForJson.return instanceof Blob) {
+        msgForJson.return = await blobToBase64String(msgForJson.return);
+    }
+
+    return JSON.stringify(msgForJson);
+}
+
+async function deserializeBlobsFromWs(msg: any): Promise<any> {
+    if (msg.method === 'bulkWrite' && Array.isArray(msg.params)) {
+        const documentWrites = msg.params[0];
+        if (Array.isArray(documentWrites)) {
+            for (const row of documentWrites) {
+                if (row.document?._attachments) {
+                    for (const attachment of Object.values(row.document._attachments)) {
+                        if (typeof (attachment as any).data === 'string') {
+                            (attachment as any).data = await createBlobFromBase64(
+                                (attachment as any).data,
+                                (attachment as any).type || ''
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    } else if (msg.method === 'getAttachmentData' && typeof msg.return === 'string') {
+        msg.return = await createBlobFromBase64(msg.return, '');
+    }
+    return msg;
+}
+export function startRxStorageRemoteWebsocketServer(
+    options: RxStorageRemoteWebsocketServerOptions
+): RxStorageRemoteWebsocketServerState {
+
+    (options as any).perMessageDeflate = true;
+    const serverState = startSocketServer(options as any);
+
+    const websocketByConnectionId = new Map<string, WebSocket>();
+    const messages$ = new Subject<MessageToRemote>();
+    const exposeSettings: RxStorageRemoteExposeSettings = {
+        messages$: messages$.asObservable(),
+        storage: options.storage as any,
+        database: options.database as any,
+        customRequestHandler: options.customRequestHandler,
+        send(msg) {
+            const ws = websocketByConnectionId.get(msg.connectionId);
+            if (!ws) {
+                // client disconnected, silently drop the message
+                return;
+            }
+            serializeBlobsForWs(msg)
+                .then(serialized => {
+                    try {
+                        ws.send(serialized);
+                    } catch (err) {
+                        // WebSocket might have transitioned to CLOSING or CLOSED state
+                    }
+                })
+                .catch(() => {
+                    // serialization error, silently drop
+                });
+        },
+        fakeVersion: options.fakeVersion
+    };
+    const exposeState = exposeRxStorageRemote(exposeSettings);
+
+    serverState.onConnection$.subscribe((ws: WebSocket) => {
+        const onCloseHandlers: Function[] = [];
+        const connectionIds = new Set<string>();
+        ws.onclose = () => {
+            onCloseHandlers.map(fn => fn());
+            connectionIds.forEach(id => websocketByConnectionId.delete(id));
+        };
+        ws.on('message', (messageString: string) => {
+            void (async () => {
+                let message: MessageToRemote;
+                try {
+                    message = JSON.parse(messageString);
+                    await deserializeBlobsFromWs(message);
+                } catch (err) {
+                    console.error('RxDB WebSocket server: failed to parse or deserialize message', err);
+                    return;
+                }
+                const connectionId = message.connectionId;
+                if (!websocketByConnectionId.has(connectionId)) {
+                    if (
+                        message.method !== 'create' &&
+                        message.method !== 'custom'
+                    ) {
+                        try {
+                            ws.send(
+                                JSON.stringify(
+                                    createErrorAnswer(message, new Error('First call must be a create call but is: ' + JSON.stringify(message)))
+                                )
+                            );
+                        } catch (err) {
+                            // WebSocket might be in CLOSING or CLOSED state
+                        }
+                        return;
+                    }
+                    websocketByConnectionId.set(connectionId, ws);
+                    connectionIds.add(connectionId);
+                }
+                messages$.next(message);
+            })();
+        });
+    });
+
+    return {
+        serverState,
+        exposeState
+    };
+}
+
+export function getRxStorageRemoteWebsocket(
+    options: RxStorageRemoteWebsocketClientOptions
+): RxStorageRemoteWebsocketClient {
+    const identifier = [
+        options.url,
+        'rx-remote-storage-websocket'
+    ].join('');
+    const storage = getRxStorageRemote({
+        identifier,
+        mode: options.mode,
+        async messageChannelCreator() {
+            const messages$ = new Subject<MessageFromRemote>();
+            const websocketClient = await createWebSocketClient(options as any);
+            websocketClient.message$.subscribe((msg: any) => {
+                void deserializeBlobsFromWs(msg)
+                    .then(deserialized => {
+                        messages$.next(deserialized);
+                    })
+                    .catch(err => {
+                        console.error('RxDB WebSocket client: failed to deserialize incoming message', err);
+                    });
+            });
+            return {
+                messages$,
+                async send(msg) {
+                    const serialized = await serializeBlobsForWs(msg);
+                    websocketClient.socket.send(serialized);
+                },
+                close() {
+                    websocketClient.socket.close();
+                    return PROMISE_RESOLVE_VOID;
+                }
+            };
+
+        }
+    });
+    return storage;
+}
+
+
+export * from './types.ts';
+

package/src/plugins/storage-remote-websocket/types.ts

@@ -0,0 +1,34 @@
+import type {
+    WebsocketServerState
+} from '../replication-websocket/index.ts';
+import type { ServerOptions, ClientOptions } from 'ws';
+import type { RxDatabase, RxStorage } from '../../types/index.d.ts';
+import type {
+    CustomRequestHandler,
+    RxStorageRemoteExposeType,
+    RxStorageRemoteSettings
+} from '../storage-remote/storage-remote-types.ts';
+import { RxStorageRemote } from '../storage-remote/index.ts';
+
+export type RxStorageRemoteWebsocketServerOptions = Omit<ServerOptions, 'perMessageDeflate'> & {
+    storage?: RxStorage<any, any>;
+    database?: RxDatabase<any, any, any>;
+    customRequestHandler?: CustomRequestHandler<any, any>;
+    /**
+     * Used in tests to simulate what happens if the remote
+     * was build on a different RxDB version.
+     */
+    fakeVersion?: string;
+};
+
+export type RxStorageRemoteWebsocketServerState = {
+    serverState: WebsocketServerState;
+    exposeState: RxStorageRemoteExposeType;
+};
+
+export type RxStorageRemoteWebsocketClientOptions = ClientOptions & {
+    url: string;
+    mode: RxStorageRemoteSettings['mode'];
+};
+
+export type RxStorageRemoteWebsocketClient = RxStorageRemote;

package/src/plugins/storage-sqlite/index.ts

@@ -0,0 +1,64 @@
+import {
+    ensureRxStorageInstanceParamsAreCorrect,
+    RxStorage,
+    RxStorageInstanceCreationParams
+} from '../../index.ts';
+import { RX_STORAGE_NAME_SQLITE } from './sqlite-helpers.ts';
+import {
+    createSQLiteTrialStorageInstance,
+    RxStorageInstanceSQLite
+} from './sqlite-storage-instance.ts';
+import type {
+    SQLiteInternals,
+    SQLiteInstanceCreationOptions,
+    SQLiteStorageSettings
+} from './sqlite-types.ts';
+import { RXDB_VERSION } from '../utils/utils-rxdb-version.ts';
+
+
+export * from './sqlite-helpers.ts';
+export * from './sqlite-types.ts';
+export * from './sqlite-storage-instance.ts';
+export * from './sqlite-basics-helpers.ts';
+
+export class RxStorageSQLiteTrial implements RxStorage<SQLiteInternals, SQLiteInstanceCreationOptions> {
+    public name = RX_STORAGE_NAME_SQLITE;
+    readonly rxdbVersion = RXDB_VERSION;
+    constructor(
+        public settings: SQLiteStorageSettings
+    ) { }
+
+    public createStorageInstance<RxDocType>(
+        params: RxStorageInstanceCreationParams<RxDocType, SQLiteInstanceCreationOptions>
+    ): Promise<RxStorageInstanceSQLite<RxDocType>> {
+        ensureRxStorageInstanceParamsAreCorrect(params);
+        return createSQLiteTrialStorageInstance(this, params, this.settings);
+    }
+}
+
+
+let warningShown = false;
+
+
+export function getRxStorageSQLiteTrial(
+    settings: SQLiteStorageSettings
+): RxStorageSQLiteTrial {
+    if (!warningShown) {
+        warningShown = true;
+        console.warn(
+            [
+                '-------------- RxDB SQLite Trial Version in Use -------------------------------',
+                'You are using the trial version of the SQLite RxStorage from RxDB https://rxdb.info/rx-storage-sqlite.html?console=sqlite ',
+                'While this is a great option to try out RxDB itself, notice that you should never use the trial version in production. It is way slower compared to the "real" SQLite storage',
+                'and it has several limitations like not using indexes and being limited to store a maximum of 300 documents.',
+                'For production environments, use the premium SQLite RxStorage:',
+                ' https://rxdb.info/premium/?console=sqlite ',
+                'If you already purchased premium access, ensure that you have imported the correct sqlite storage from the premium plugins.',
+                '-------------------------------------------------------------------------------'
+            ].join('\n')
+        );
+    }
+
+    const storage = new RxStorageSQLiteTrial(settings);
+    return storage;
+}