@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/storage-remote/message-channel-cache.ts

@@ -0,0 +1,88 @@
+import {
+    PROMISE_RESOLVE_VOID,
+    getFromMapOrCreate,
+    getFromMapOrThrow
+} from '../utils/index.ts';
+import {
+    RemoteMessageChannel,
+    RxStorageRemoteSettings
+} from './storage-remote-types.ts';
+
+export type RemoteMessageChannelCacheItem = {
+    identifier: string;
+    cacheKey: string;
+    messageChannel: Promise<RemoteMessageChannel>;
+    refCount: number;
+    keepAlive: boolean;
+};
+
+export const MESSAGE_CHANNEL_CACHE_BY_IDENTIFIER = new Map<string, Map<string, RemoteMessageChannelCacheItem>>();
+export const CACHE_ITEM_BY_MESSAGE_CHANNEL = new WeakMap<RemoteMessageChannel, RemoteMessageChannelCacheItem>();
+
+
+export const OPEN_REMOTE_MESSAGE_CHANNELS = new Set<RemoteMessageChannel>();
+
+function getMessageChannelCache(
+    identifier: string
+) {
+    return getFromMapOrCreate(
+        MESSAGE_CHANNEL_CACHE_BY_IDENTIFIER,
+        identifier,
+        () => new Map()
+    );
+}
+
+export function getMessageChannel(
+    settings: RxStorageRemoteSettings,
+    cacheKeys: string[],
+    keepAlive: boolean = false
+): Promise<RemoteMessageChannel> {
+    const cacheKey = getCacheKey(settings, cacheKeys);
+    const cacheItem = getFromMapOrCreate(
+        getMessageChannelCache(settings.identifier),
+        cacheKey,
+        () => {
+            const newCacheItem: RemoteMessageChannelCacheItem = {
+                identifier: settings.identifier,
+                cacheKey,
+                keepAlive,
+                refCount: 1,
+                messageChannel: settings.messageChannelCreator()
+                    .then((messageChannel) => {
+                        OPEN_REMOTE_MESSAGE_CHANNELS.add(messageChannel);
+                        CACHE_ITEM_BY_MESSAGE_CHANNEL.set(messageChannel, newCacheItem);
+                        return messageChannel;
+                    }),
+            };
+            return newCacheItem;
+        },
+        (existingCacheItem) => {
+            existingCacheItem.refCount = existingCacheItem.refCount + 1;
+        }
+    );
+    return cacheItem.messageChannel;
+}
+
+
+export function closeMessageChannel(
+    messageChannel: RemoteMessageChannel
+): Promise<void> {
+    const cacheItem = getFromMapOrThrow(CACHE_ITEM_BY_MESSAGE_CHANNEL, messageChannel);
+    cacheItem.refCount = cacheItem.refCount - 1;
+    if (cacheItem.refCount === 0 && !cacheItem.keepAlive) {
+        getMessageChannelCache(cacheItem.identifier).delete(cacheItem.cacheKey);
+        OPEN_REMOTE_MESSAGE_CHANNELS.delete(messageChannel);
+        return messageChannel.close();
+    } else {
+        return PROMISE_RESOLVE_VOID;
+    }
+}
+
+function getCacheKey(
+    settings: RxStorageRemoteSettings,
+    cacheKeys: string[]
+): string {
+    cacheKeys = cacheKeys.slice(0);
+    cacheKeys.unshift(settings.identifier);
+    return cacheKeys.join('||');
+}

package/src/plugins/storage-remote/remote.ts

@@ -0,0 +1,285 @@
+import { filter, Subscription } from 'rxjs';
+import type {
+    RxStorageInstance,
+    RxStorageInstanceCreationParams
+} from '../../types/index.d.ts';
+import {
+    deepEqual,
+    ensureNotFalsy,
+    RXDB_VERSION
+} from '../../plugins/utils/index.ts';
+import { createAnswer, createErrorAnswer } from './storage-remote-helpers.ts';
+import type {
+    MessageFromRemote,
+    MessageToRemote,
+    RxStorageRemoteExposeSettings,
+    RxStorageRemoteExposeSettingsRxDatabase,
+    RxStorageRemoteExposeSettingsRxStorage,
+    RxStorageRemoteExposeType
+} from './storage-remote-types.ts';
+import { getChangedDocumentsSince } from '../../rx-storage-helper.ts';
+import { newRxError } from '../../rx-error.ts';
+
+/**
+ * Run this on the 'remote' part,
+ * so that RxStorageMessageChannel can connect to it.
+ */
+export function exposeRxStorageRemote(settings: RxStorageRemoteExposeSettings): RxStorageRemoteExposeType {
+    type InstanceState = {
+        storageInstancePromise: Promise<RxStorageInstance<any, any, any>>;
+        connectionIds: Set<string>;
+        params: RxStorageInstanceCreationParams<any, any>;
+    };
+    const instanceByFullName: Map<string, InstanceState> = new Map();
+
+
+    settings.messages$.pipe(
+        filter((msg: MessageToRemote) => msg.method === 'custom')
+    ).subscribe(async (msg: MessageToRemote) => {
+        if (!settings.customRequestHandler) {
+            settings.send(createErrorAnswer(
+                msg,
+                new Error('Remote storage: cannot resolve custom request because settings.customRequestHandler is not set')
+            ));
+        } else {
+            try {
+                const result = await settings.customRequestHandler(msg.params);
+                settings.send(createAnswer(msg, result));
+            } catch (err: any) {
+                settings.send(createErrorAnswer(
+                    msg,
+                    err
+                ));
+            }
+        }
+    });
+
+
+    function getRxStorageInstance<RxDocType>(params: any): Promise<RxStorageInstance<RxDocType, any, any, any>> {
+        if ((settings as RxStorageRemoteExposeSettingsRxStorage).storage) {
+            return (settings as RxStorageRemoteExposeSettingsRxStorage).storage.createStorageInstance(params);
+        } else if ((settings as RxStorageRemoteExposeSettingsRxDatabase).database) {
+            const storageInstances = Array.from((settings as RxStorageRemoteExposeSettingsRxDatabase).database.storageInstances);
+            const collectionName = params.collectionName;
+            const storageInstance = storageInstances.find(instance => instance.collectionName === collectionName);
+            if (!storageInstance) {
+                console.dir(storageInstances);
+                throw new Error('storageInstance does not exist ' + JSON.stringify({
+                    collectionName
+                }));
+            }
+            const schema = params.schema;
+            if (!deepEqual(schema, storageInstance.schema)) {
+                throw new Error('Wrong schema ' + JSON.stringify({
+                    schema,
+                    existingSchema: storageInstance.schema
+                }));
+            }
+            return Promise.resolve(storageInstance);
+        } else {
+            throw new Error('no base given');
+        }
+    }
+
+    const mustBeRxDBVersion = settings.fakeVersion ? settings.fakeVersion : RXDB_VERSION;
+    settings.messages$.pipe(
+        filter((msg: MessageToRemote) => msg.method === 'create')
+    ).subscribe(async (msg: MessageToRemote) => {
+        if (msg.version !== mustBeRxDBVersion) {
+            settings.send(createErrorAnswer(msg, newRxError('RM1', {
+                args: {
+                    mainVersion: msg.version,
+                    remoteVersion: mustBeRxDBVersion
+                }
+            })));
+            return;
+        }
+
+        const connectionId = msg.connectionId;
+
+        /**
+         * Do an isArray check here
+         * for runtime check types to ensure we have
+         * instance creation params and not method input params.
+        */
+        if (Array.isArray(msg.params)) {
+            return;
+        }
+        const params = msg.params;
+        const collectionName = params.collectionName;
+
+        /**
+         * We de-duplicate the storage instances.
+         * This makes sense in many environments like
+         * electron where on main process contains the storage
+         * for multiple renderer processes. Same goes for SharedWorkers etc.
+         */
+        const fullName = [
+            params.databaseName,
+            params.collectionName,
+            params.schema.version
+        ].join('|');
+        let state = instanceByFullName.get(fullName);
+        if (!state) {
+            try {
+                state = {
+                    /**
+                     * We work with a promise here to ensure
+                     * that parallel create-calls will still end up
+                     * with exactly one instance and not more.
+                     */
+                    storageInstancePromise: getRxStorageInstance(params),
+                    connectionIds: new Set(),
+                    params
+                };
+                instanceByFullName.set(fullName, state);
+
+                /**
+                 * Must await the creation here
+                 * so that in case of an error,
+                 * it knows about the error message and can send
+                 * that back to the main process. 
+                 */
+                await state.storageInstancePromise;
+            } catch (err: any) {
+                settings.send(createErrorAnswer(msg, err));
+                return;
+            }
+        } else {
+            // if instance already existed, ensure that the schema is equal
+            if (!deepEqual(params.schema, state.params.schema)) {
+                settings.send(createErrorAnswer(msg, new Error('Remote storage: schema not equal to existing storage')));
+                return;
+            }
+        }
+        state.connectionIds.add(msg.connectionId);
+        const subs: Subscription[] = [];
+
+        const storageInstance = await state.storageInstancePromise;
+        /**
+         * Automatically subscribe to the changeStream()
+         * because we always need them.
+         */
+        subs.push(
+            storageInstance.changeStream().subscribe((changes: any) => {
+                const message: MessageFromRemote = {
+                    connectionId,
+                    answerTo: 'changestream',
+                    method: 'changeStream',
+                    return: changes
+                };
+                try {
+                    settings.send(message);
+                } catch (err) {
+                    // Connection might have been closed, ignore send errors for changeStream events
+                }
+            })
+        );
+
+        let connectionClosed = false;
+        function closeThisConnection() {
+            if (connectionClosed) {
+                return;
+            }
+            connectionClosed = true;
+            subs.forEach(sub => sub.unsubscribe());
+            ensureNotFalsy(state).connectionIds.delete(connectionId);
+            instanceByFullName.delete(fullName);
+            /**
+             * TODO how to notify the other ports on remove() ?
+             */
+        }
+
+        // also close the connection when the collection gets closed
+        if ((settings as RxStorageRemoteExposeSettingsRxDatabase).database) {
+            const database = (settings as RxStorageRemoteExposeSettingsRxDatabase).database;
+            const collection = database.collections[collectionName];
+            if (collection) {
+                collection.onClose.push(() => closeThisConnection());
+            } else {
+                database.onClose.push(() => closeThisConnection());
+            }
+        }
+
+        subs.push(
+            settings.messages$.pipe(
+                filter((subMsg: MessageToRemote) => (subMsg as MessageToRemote).connectionId === connectionId)
+            ).subscribe(async (plainMessage: MessageToRemote) => {
+                const message: MessageToRemote = plainMessage as any;
+                if (
+                    message.method === 'create' ||
+                    message.method === 'custom'
+                ) {
+                    return;
+                }
+                if (!Array.isArray(message.params)) {
+                    return;
+                }
+                let result;
+                try {
+                    if (
+                        message.method === 'close' &&
+                        (settings as RxStorageRemoteExposeSettingsRxDatabase).database
+                    ) {
+                        /**
+                         * Do not close the storageInstance if it was taken from
+                         * a running RxDatabase.
+                         * In that case we only close the instance
+                         * when the RxDatabase gets closed.
+                         */
+                        settings.send(createAnswer(message, null));
+                        return;
+                    }
+                    /**
+                     * On calls to 'close()',
+                     * we only close the main instance if there are no other
+                     * ports connected.
+                     */
+                    if (
+                        message.method === 'close' &&
+                        ensureNotFalsy(state).connectionIds.size > 1
+                    ) {
+                        settings.send(createAnswer(message, null));
+                        ensureNotFalsy(state).connectionIds.delete(connectionId);
+                        subs.forEach(sub => sub.unsubscribe());
+                        return;
+                    }
+
+                    if (message.method === 'getChangedDocumentsSince' && !storageInstance.getChangedDocumentsSince) {
+                        result = await getChangedDocumentsSince(
+                            storageInstance,
+                            message.params[0],
+                            message.params[1]
+                        );
+                    } else {
+                        result = await (storageInstance as any)[message.method](
+                            message.params[0],
+                            message.params[1],
+                            message.params[2],
+                            message.params[3]
+                        );
+                    }
+                    if (
+                        message.method === 'close' ||
+                        message.method === 'remove'
+                    ) {
+                        closeThisConnection();
+                    }
+                    settings.send(createAnswer(message, result));
+                } catch (err: any) {
+                    try {
+                        settings.send(createErrorAnswer(message, err));
+                    } catch (innerErr) {
+                        // Connection might have been closed, ignore send errors
+                    }
+                }
+            })
+        );
+
+        settings.send(createAnswer(msg, 'ok'));
+    });
+
+    return {
+        instanceByFullName
+    };
+}

package/src/plugins/storage-remote/rx-storage-remote.ts

@@ -0,0 +1,286 @@
+import {
+    firstValueFrom,
+    filter,
+    Observable,
+    Subject,
+    Subscription
+} from 'rxjs';
+import type {
+    BulkWriteRow,
+    EventBulk,
+    RxDocumentData,
+    RxJsonSchema,
+    RxStorage,
+    RxStorageBulkWriteResponse,
+    RxStorageChangeEvent,
+    RxStorageCountResult,
+    RxStorageInstance,
+    RxStorageInstanceCreationParams,
+    RxStorageQueryResult
+} from '../../types/index.d.ts';
+import {
+    RXDB_VERSION,
+    randomToken
+} from '../../plugins/utils/index.ts';
+import type {
+    MessageFromRemote,
+    MessageToRemote,
+    RemoteMessageChannel,
+    RxStorageRemoteInternals,
+    RxStorageRemoteSettings
+} from './storage-remote-types.ts';
+import { closeMessageChannel, getMessageChannel } from './message-channel-cache.ts';
+
+
+export class RxStorageRemote implements RxStorage<RxStorageRemoteInternals, any> {
+    public readonly name: string = 'remote';
+    public readonly rxdbVersion = RXDB_VERSION;
+
+    private seed: string = randomToken(10);
+    private lastRequestId: number = 0;
+    public messageChannelIfOneMode?: Promise<RemoteMessageChannel>;
+    constructor(
+        public readonly settings: RxStorageRemoteSettings
+    ) {
+        if (settings.mode === 'one') {
+            this.messageChannelIfOneMode = getMessageChannel(
+                settings,
+                [],
+                true
+            );
+        }
+    }
+
+    public getRequestId() {
+        const newId = this.lastRequestId++;
+        return this.seed + '|' + newId;
+    }
+
+    async createStorageInstance<RxDocType>(
+        params: RxStorageInstanceCreationParams<RxDocType, any>
+    ): Promise<RxStorageInstanceRemote<RxDocType>> {
+        const connectionId = 'c|' + this.getRequestId();
+
+        const cacheKeys: string[] = [
+            'mode-' + this.settings.mode
+        ];
+        switch (this.settings.mode) {
+            case 'collection':
+                cacheKeys.push('collection-' + params.collectionName);
+            // eslint-disable-next-line no-fallthrough
+            case 'database':
+                cacheKeys.push('database-' + params.databaseName);
+            // eslint-disable-next-line no-fallthrough
+            case 'storage':
+                cacheKeys.push('seed-' + this.seed);
+        }
+        const messageChannel = await (this.messageChannelIfOneMode ?
+            this.messageChannelIfOneMode :
+            getMessageChannel(
+                this.settings,
+                cacheKeys
+            )
+        );
+
+        const requestId = this.getRequestId();
+        const waitForOkPromise = firstValueFrom(messageChannel.messages$.pipe(
+            filter((msg: MessageFromRemote) => msg.answerTo === requestId)
+        ));
+        messageChannel.send({
+            connectionId,
+            method: 'create',
+            version: RXDB_VERSION,
+            requestId,
+            params
+        });
+
+        const waitForOkResult = await waitForOkPromise;
+        if (waitForOkResult.error) {
+            await closeMessageChannel(messageChannel);
+            throw new Error('could not create instance ' + JSON.stringify(waitForOkResult.error));
+        }
+
+        return new RxStorageInstanceRemote(
+            this,
+            params.databaseName,
+            params.collectionName,
+            params.schema,
+            {
+                params,
+                connectionId,
+                messageChannel
+            },
+            params.options
+        );
+    }
+
+    async customRequest<In, Out>(data: In): Promise<Out> {
+        const messageChannel = await this.settings.messageChannelCreator();
+        const requestId = this.getRequestId();
+        const connectionId = 'custom|request|' + requestId;
+        const waitForAnswerPromise = firstValueFrom(messageChannel.messages$.pipe(
+            filter((msg: MessageFromRemote) => msg.answerTo === requestId)
+        ));
+        messageChannel.send({
+            connectionId,
+            method: 'custom',
+            version: RXDB_VERSION,
+            requestId,
+            params: data
+        });
+        const response = await waitForAnswerPromise;
+        if (response.error) {
+            await messageChannel.close();
+            throw new Error('could not run customRequest(): ' + JSON.stringify({
+                data,
+                error: response.error
+            }));
+        } else {
+            await messageChannel.close();
+            return response.return;
+        }
+
+    }
+}
+
+/**
+ * Because postMessage() can be very slow on complex objects,
+ * and some RxStorage implementations do need a JSON-string internally
+ * anyway, it is allowed to transfer a string instead of an object
+ * which must then be JSON.parse()-ed before RxDB can use it.
+ * @link https://surma.dev/things/is-postmessage-slow/
+ */
+function getMessageReturn(
+    msg: MessageFromRemote
+) {
+    if (msg.method === 'getAttachmentData') {
+        return msg.return;
+    } else {
+        if (typeof msg.return === 'string') {
+            return JSON.parse(msg.return);
+        } else {
+            return msg.return;
+        }
+    }
+}
+
+export class RxStorageInstanceRemote<RxDocType> implements RxStorageInstance<RxDocType, RxStorageRemoteInternals, any, any> {
+    private changes$: Subject<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, any>> = new Subject();
+    private subs: Subscription[] = [];
+
+    private closed?: Promise<void>;
+    messages$: Observable<MessageFromRemote>;
+
+    constructor(
+        public readonly storage: RxStorageRemote,
+        public readonly databaseName: string,
+        public readonly collectionName: string,
+        public readonly schema: Readonly<RxJsonSchema<RxDocumentData<RxDocType>>>,
+        public readonly internals: RxStorageRemoteInternals,
+        public readonly options: Readonly<any>
+    ) {
+        this.messages$ = this.internals.messageChannel.messages$.pipe(
+            filter((msg: MessageFromRemote) => msg.connectionId === this.internals.connectionId)
+        );
+        this.subs.push(
+            this.messages$.subscribe((msg: MessageFromRemote) => {
+                if (msg.method === 'changeStream') {
+                    this.changes$.next(getMessageReturn(msg));
+                }
+            })
+        );
+    }
+
+    private async requestRemote(
+        methodName: keyof RxStorageInstance<any, any, any>,
+        params: any
+    ) {
+        const requestId = this.storage.getRequestId();
+        const responsePromise = firstValueFrom(
+            this.messages$.pipe(
+                filter((msg: MessageFromRemote) => msg.answerTo === requestId)
+            )
+        );
+        const message: MessageToRemote = {
+            connectionId: this.internals.connectionId,
+            requestId,
+            version: RXDB_VERSION,
+            method: methodName,
+            params
+        };
+        this.internals.messageChannel.send(message);
+        const response = await responsePromise;
+        if (response.error) {
+            throw new Error('could not requestRemote: ' + JSON.stringify({
+                methodName,
+                params,
+                error: response.error
+            }, null, 4));
+        } else {
+            return getMessageReturn(response);
+        }
+    }
+    bulkWrite(
+        documentWrites: BulkWriteRow<RxDocType>[],
+        context: string
+    ): Promise<RxStorageBulkWriteResponse<RxDocType>> {
+        return this.requestRemote('bulkWrite', [documentWrites, context]);
+    }
+    findDocumentsById(ids: string[], deleted: boolean): Promise<RxDocumentData<RxDocType>[]> {
+        return this.requestRemote('findDocumentsById', [ids, deleted]);
+    }
+    query(preparedQuery: any): Promise<RxStorageQueryResult<RxDocType>> {
+        return this.requestRemote('query', [preparedQuery]);
+    }
+    count(preparedQuery: any): Promise<RxStorageCountResult> {
+        return this.requestRemote('count', [preparedQuery]);
+    }
+    getAttachmentData(documentId: string, attachmentId: string, digest: string): Promise<Blob> {
+        return this.requestRemote('getAttachmentData', [documentId, attachmentId, digest]);
+    }
+    getChangedDocumentsSince(
+        limit: number,
+        checkpoint?: any
+    ): Promise<
+        {
+            documents: RxDocumentData<RxDocType>[];
+            checkpoint: any;
+        }> {
+        return this.requestRemote('getChangedDocumentsSince', [limit, checkpoint]);
+    }
+    changeStream(): Observable<EventBulk<RxStorageChangeEvent<RxDocumentData<RxDocType>>, any>> {
+        return this.changes$.asObservable();
+    }
+    cleanup(minDeletedTime: number): Promise<boolean> {
+        return this.requestRemote('cleanup', [minDeletedTime]);
+    }
+    async close(): Promise<void> {
+        if (this.closed) {
+            return this.closed;
+        }
+        this.closed = (async () => {
+            this.subs.forEach(sub => sub.unsubscribe());
+            this.changes$.complete();
+            await this.requestRemote('close', []);
+            await closeMessageChannel(this.internals.messageChannel);
+        })();
+        return this.closed;
+    }
+    async remove(): Promise<void> {
+        if (this.closed) {
+            throw new Error('already closed');
+        }
+        this.closed = (async () => {
+            await this.requestRemote('remove', []);
+            await closeMessageChannel(this.internals.messageChannel);
+        })();
+        return this.closed;
+    }
+}
+
+export function getRxStorageRemote(settings: RxStorageRemoteSettings): RxStorageRemote {
+    const withDefaults = Object.assign({
+        mode: 'storage'
+    }, settings);
+    return new RxStorageRemote(withDefaults);
+}

package/src/plugins/storage-remote/storage-remote-helpers.ts

@@ -0,0 +1,33 @@
+import type {
+    RxError,
+    RxTypeError
+} from '../../types/index.d.ts';
+import { errorToPlainJson } from '../../plugins/utils/index.ts';
+import type {
+    MessageFromRemote,
+    MessageToRemote
+} from './storage-remote-types.ts';
+
+export function createErrorAnswer(
+    msg: MessageToRemote,
+    error: Error | TypeError | RxError | RxTypeError
+): MessageFromRemote {
+    return {
+        connectionId: msg.connectionId,
+        answerTo: msg.requestId,
+        method: msg.method,
+        error: errorToPlainJson(error)
+    };
+}
+
+export function createAnswer(
+    msg: MessageToRemote,
+    ret: any
+): MessageFromRemote {
+    return {
+        connectionId: msg.connectionId,
+        answerTo: msg.requestId,
+        method: msg.method,
+        return: ret
+    };
+}