@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/migration-storage/index.ts

@@ -0,0 +1,255 @@
+import {
+    RxDatabase,
+    RxCollection,
+    createRevision,
+    clone,
+    BulkWriteRow,
+    RxStorageBulkWriteResponse,
+    randomToken,
+    RxStorage,
+    prepareQuery,
+    PreparedQuery,
+    FilledMangoQuery
+} from '../../index.ts';
+
+export type RxStorageOld<A, B> = RxStorage<A, B> | any;
+
+export type AfterMigrateBatchHandlerInput = {
+    databaseName: string;
+    collectionName: string;
+    oldDatabaseName: string;
+    insertToNewWriteRows: BulkWriteRow<any>[];
+    writeToNewResult: RxStorageBulkWriteResponse<any>;
+};
+export type AfterMigrateBatchHandler = (
+    input: AfterMigrateBatchHandlerInput
+) => any | Promise<any>;
+
+
+export type MigrateStorageParams = {
+    database: RxDatabase;
+    /**
+     * Using the migration plugin requires you
+     * to rename your new old database.
+     * The original name of the v11 database must be provided here.
+     */
+    oldDatabaseName: string;
+    oldStorage: RxStorageOld<any, any>;
+    batchSize?: number;
+    parallel?: boolean;
+    afterMigrateBatch?: AfterMigrateBatchHandler;
+    // to log each step, pass console.log.bind(console) here.
+    logFunction?: (message: string) => void;
+}
+
+/**
+ * Migrates collections of RxDB version A and puts them
+ * into a RxDatabase that is created with version B.
+ * This function only works from the previous major version upwards.
+ * Do not use it to migrate like rxdb v9 to v14.
+ */
+export async function migrateStorage(
+    params: MigrateStorageParams
+): Promise<void> {
+    const collections = Object.values(params.database.collections);
+    const batchSize = params.batchSize ? params.batchSize : 10;
+    if (params.parallel) {
+        await Promise.all(
+            collections.map(collection => migrateCollection(
+                collection,
+                params.oldDatabaseName,
+                params.oldStorage,
+                batchSize,
+                params.afterMigrateBatch,
+                params.logFunction
+            ))
+        );
+    } else {
+        for (const collection of collections) {
+            await migrateCollection(
+                collection,
+                params.oldDatabaseName,
+                params.oldStorage,
+                batchSize,
+                params.afterMigrateBatch,
+                params.logFunction
+            );
+        }
+    }
+}
+
+export async function migrateCollection<RxDocType>(
+    collection: RxCollection<RxDocType>,
+    oldDatabaseName: string,
+    oldStorage: RxStorageOld<any, any>,
+    batchSize: number,
+    afterMigrateBatch?: AfterMigrateBatchHandler,
+    // to log each step, pass console.log.bind(console) here.
+    logFunction?: (message: string) => void
+) {
+    function log(message: string) {
+        if (logFunction) {
+            logFunction('migrateCollection(' + collection.name + ')' + message);
+        }
+    }
+    log('start migrateCollection()');
+    let schema = collection.schema.jsonSchema;
+    const primaryPath = collection.schema.primaryPath;
+    const oldDatabaseInstanceToken = randomToken(10);
+
+    const oldStorageInstance = await oldStorage.createStorageInstance({
+        databaseName: oldDatabaseName,
+        collectionName: collection.name,
+        multiInstance: false,
+        options: {},
+        schema: schema,
+        databaseInstanceToken: oldDatabaseInstanceToken,
+        devMode: false
+    });
+
+
+    const plainQuery: FilledMangoQuery<RxDocType> = {
+        selector: {
+            _deleted: {
+                $eq: false
+            }
+        } as any,
+        limit: batchSize,
+        sort: [{ [primaryPath]: 'asc' } as any],
+        skip: 0
+    };
+
+    const preparedQuery = prepareQuery(
+        schema,
+        plainQuery
+    );
+
+    while (true) {
+        log('loop once');
+        /**
+         * Get a batch of documents
+         */
+        const queryResult = await oldStorageInstance.query(preparedQuery);
+        const docs = queryResult.documents;
+        if (docs.length === 0) {
+            /**
+             * No more documents to migrate
+             */
+            log('migration of collection done');
+            await oldStorageInstance.remove();
+            return;
+        }
+
+        const docsNonMutated = clone(docs);
+
+        /**
+         * Get attachments
+         * if defined in the schema.
+         */
+        if (schema.attachments) {
+            await Promise.all(
+                docs.map(async (doc: any) => {
+                    const docId: string = (doc as any)[primaryPath];
+                    await Promise.all(
+                        Object.entries(doc._attachments).map(async ([attachmentId, attachmentMeta]) => {
+                            const attachmentBlob = await oldStorageInstance.getAttachmentData(
+                                docId,
+                                attachmentId,
+                                (attachmentMeta as any).digest
+                            );
+                            (doc as any)._attachments[attachmentId] = {
+                                data: attachmentBlob,
+                                digest: (attachmentMeta as any).digest,
+                                length: attachmentBlob.size,
+                                type: (attachmentMeta as any).type
+                            }
+                        })
+                    );
+                })
+            );
+            log('got attachments');
+        }
+
+        /**
+         * Insert the documents to the new storage
+         */
+        const insertToNewWriteRows: BulkWriteRow<any>[] = docs.map((document: any) => {
+            return { document };
+        });
+        const writeToNewResult: RxStorageBulkWriteResponse<any> = await collection.storageInstance.bulkWrite(
+            insertToNewWriteRows,
+            'migrate-storage'
+        );
+        log('written batch to new storage');
+
+        // TODO we should throw on non-conflict errors here.
+        // if (Object.keys(writeToNewResult.error).length > 0) {
+        //     throw new Error('could not write to new storage');
+        // }
+
+        /**
+         * Remove the docs from the old storage
+         */
+        const writeToOldRows = docs.map((_doc: any, idx: number) => {
+            const previous = docsNonMutated[idx];
+            if (!previous._meta) {
+                previous._meta = {
+                    lwt: new Date().getTime()
+                };
+            }
+
+            const newDoc: typeof previous = clone(previous);
+            newDoc._deleted = true;
+            if (!newDoc._meta) {
+                newDoc._meta = {
+                    lwt: new Date().getTime()
+                };
+            }
+            newDoc._meta.lwt = new Date().getTime() + 1;
+            newDoc._rev = createRevision(
+                oldDatabaseInstanceToken,
+                previous
+            );
+
+            return {
+                previous,
+                document: newDoc,
+            }
+        });
+        try {
+            const writeToOldResult = await oldStorageInstance.bulkWrite(
+                writeToOldRows,
+                'migrate-between-rxdb-versions'
+            );
+            if (Object.keys(writeToOldResult.error).length > 0) {
+                console.dir({
+                    writeToOldRows,
+                    errors: writeToOldResult.error
+                });
+                throw new Error('got error while deleting migrated documents on the old storage');
+            }
+        } catch (err) {
+            log('could not delete on old instance');
+            console.dir(err);
+            throw err;
+        }
+        log('deleted batch on old storage');
+        await oldStorageInstance.cleanup(0);
+
+        // run the handler if provided
+        if (afterMigrateBatch) {
+            await afterMigrateBatch({
+                databaseName: collection.database.name,
+                collectionName: collection.name,
+                oldDatabaseName,
+                insertToNewWriteRows,
+                writeToNewResult
+            });
+        }
+    }
+}
+
+
+
+
+

package/src/plugins/pipeline/flagged-functions.ts

@@ -0,0 +1,131 @@
+import { ensureNotFalsy } from '../utils/index.ts';
+
+/**
+ * This is the most hacky thing we do in RxDB.
+ * When a pipeline "transaction" is running,
+ * we have to make all calls to the collection from the outside
+ * wait while still make it possible to run reads and writes
+ * from inside the transaction.
+ * 
+ * We can decide where the call came from by checking the stack `new Error().stack`
+ * for a random "flag".
+ * But creating random flagged functions requires eval which we should not use.
+ * Instead we have a list of some flagged functions here
+ * that can be used and checked for in the stacktrace.
+ * 
+ * 
+ * When doing this with eval() instead it would look like:
+ * ```ts
+ * eval(`
+ *     async function ${this.secretFunctionName}(docs){ const x = await _this.handler(docs); return x; }
+ *     o.${this.secretFunctionName} = ${this.secretFunctionName};
+ *   `);
+ * await o[this.secretFunctionName](rxDocuments);
+ * 
+ * ```
+ */
+async function rx_pipeline_fn_1_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_2_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_3_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_4_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_5_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_6_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_7_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_8_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_9_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_10_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_11_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_12_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_13_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_14_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_15_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_16_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_17_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_18_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_19_(fn: any) {
+    return await fn();
+}
+async function rx_pipeline_fn_20_(fn: any) {
+    return await fn();
+}
+
+
+
+
+
+export const FLAGGED_FUNCTIONS = {
+    rx_pipeline_fn_1_,
+    rx_pipeline_fn_2_,
+    rx_pipeline_fn_3_,
+    rx_pipeline_fn_4_,
+    rx_pipeline_fn_5_,
+    rx_pipeline_fn_6_,
+    rx_pipeline_fn_7_,
+    rx_pipeline_fn_8_,
+    rx_pipeline_fn_9_,
+    rx_pipeline_fn_10_,
+    rx_pipeline_fn_11_,
+    rx_pipeline_fn_12_,
+    rx_pipeline_fn_13_,
+    rx_pipeline_fn_14_,
+    rx_pipeline_fn_15_,
+    rx_pipeline_fn_16_,
+    rx_pipeline_fn_17_,
+    rx_pipeline_fn_18_,
+    rx_pipeline_fn_19_,
+    rx_pipeline_fn_20_,
+} as const;
+
+
+
+const ids: (keyof typeof FLAGGED_FUNCTIONS)[] = Object.keys(FLAGGED_FUNCTIONS) as any;
+
+export function blockFlaggedFunctionKey(): keyof typeof FLAGGED_FUNCTIONS {
+    /**
+     * If this happens and we have no more flagged keys left
+     * it means that more pipeline handlers are running in parallel.
+     * To fix this, add more functions.
+     */
+    const id = ensureNotFalsy(ids.pop(), 'no flagged keys left');
+    return id;
+}
+
+export function releaseFlaggedFunctionKey(key: keyof typeof FLAGGED_FUNCTIONS) {
+    ids.push(key);
+}

package/src/plugins/pipeline/index.ts

@@ -0,0 +1,18 @@
+import type {
+    RxPlugin
+} from '../../types/index.d.ts';
+import { addPipeline } from './rx-pipeline.ts';
+
+export type * from './types.ts';
+export * from './flagged-functions.ts';
+export * from './rx-pipeline.ts';
+
+export const RxDBPipelinePlugin: RxPlugin = {
+    name: 'pipeline',
+    rxdb: true,
+    prototypes: {
+        RxCollection(proto: any) {
+            proto.addPipeline = addPipeline;
+        }
+    }
+};

package/src/plugins/pipeline/rx-pipeline.ts

@@ -0,0 +1,303 @@
+import {
+    BehaviorSubject,
+    Subject,
+    Subscription,
+    filter,
+    firstValueFrom,
+    race
+} from 'rxjs';
+import type {
+    EventBulk,
+    InternalStoreDocType,
+    RxChangeEventBulk,
+    RxCollection,
+    RxDocument,
+    RxDocumentData,
+    RxStorageChangeEvent
+} from '../../types';
+import type {
+    CheckpointDocData,
+    RxPipelineHandler,
+    RxPipelineOptions
+} from './types';
+import {
+    PROMISE_RESOLVE_VOID,
+    clone,
+    createRevision,
+    ensureNotFalsy,
+    lastOfArray,
+    nameFunction,
+    now,
+    promiseWait,
+    randomToken
+} from '../utils/index.ts';
+import { getChangedDocumentsSince } from '../../rx-storage-helper.ts';
+import { mapDocumentsDataToCacheDocs } from '../../doc-cache.ts';
+import { INTERNAL_CONTEXT_PIPELINE_CHECKPOINT, getPrimaryKeyOfInternalDocument } from '../../rx-database-internal-store.ts';
+import { FLAGGED_FUNCTIONS, blockFlaggedFunctionKey, releaseFlaggedFunctionKey } from './flagged-functions.ts';
+
+export class RxPipeline<RxDocType> {
+    processQueue = PROMISE_RESOLVE_VOID;
+    subs: Subscription[] = [];
+    stopped: boolean = false;
+
+    toRun = 1;
+    checkpointId: string;
+
+    lastSourceDocTime = new BehaviorSubject(-1);
+    lastProcessedDocTime = new BehaviorSubject(0);
+    somethingChanged = new Subject();
+
+
+    secretFunctionName = 'tx_fn_' + randomToken(10)
+
+    waitBeforeWriteFn = async () => {
+        const stack = new Error().stack;
+        if (stack && (
+            stack.includes(this.secretFunctionName)
+        )) {
+        } else {
+            await this.awaitIdle();
+        }
+    }
+
+    /**
+     * The handler of the pipeline must never throw.
+     * If it did anyway, the pipeline will be stuck and always
+     * throw the previous error on all operations.
+     */
+    error: any;
+
+    constructor(
+        public readonly identifier: string,
+        public readonly source: RxCollection<RxDocType>,
+        public readonly destination: RxCollection<any>,
+        public readonly handler: RxPipelineHandler<RxDocType>,
+        public readonly batchSize = 100
+    ) {
+        this.checkpointId = 'rx-pipeline-' + identifier;
+        this.source.onClose.push(() => this.close());
+        this.destination.awaitBeforeReads.add(this.waitBeforeWriteFn);
+        this.subs.push(
+            this.source.eventBulks$.pipe(
+                filter((bulk: RxChangeEventBulk<RxDocType>) => !this.stopped && !bulk.isLocal)
+            ).subscribe((bulk: RxChangeEventBulk<RxDocType>) => {
+                this.lastSourceDocTime.next(bulk.events[0].documentData._meta.lwt);
+                this.somethingChanged.next({});
+            })
+        );
+        this.subs.push(
+            this.destination.database.internalStore
+                .changeStream()
+                .subscribe((eventBulk: EventBulk<RxStorageChangeEvent<InternalStoreDocType>, any>) => {
+                    const events = eventBulk.events;
+                    for (let index = 0; index < events.length; index++) {
+                        const event = events[index];
+                        if (
+                            event.documentData.context === INTERNAL_CONTEXT_PIPELINE_CHECKPOINT &&
+                            event.documentData.key === this.checkpointId
+                        ) {
+                            this.lastProcessedDocTime.next(event.documentData.data.lastDocTime);
+                            this.somethingChanged.next({});
+                        }
+                    }
+                })
+        );
+    }
+
+    trigger() {
+        /**
+         * Do not stack up too many
+         * so that fast writes to the source collection
+         * do not block anything too long.
+         */
+        if (this.toRun > 2) {
+            return;
+        }
+        this.toRun = this.toRun + 1;
+
+        this.processQueue = this.processQueue.then(async () => {
+            this.toRun = this.toRun - 1;
+
+            let done = false;
+            while (
+                !done &&
+                !this.stopped &&
+                !this.destination.closed &&
+                !this.source.closed &&
+                !this.error
+            ) {
+                const checkpointDoc = await getCheckpointDoc(this);
+                const checkpoint = checkpointDoc ? checkpointDoc.data.checkpoint : undefined;
+                const docsSinceResult = await getChangedDocumentsSince(
+                    this.source.storageInstance,
+                    this.batchSize,
+                    checkpoint
+                );
+
+                let lastTime = checkpointDoc ? checkpointDoc.data.lastDocTime : 0;
+                if (docsSinceResult.documents.length > 0) {
+                    const rxDocuments = mapDocumentsDataToCacheDocs(this.source._docCache, docsSinceResult.documents);
+                    const _this = this;
+
+
+
+                    // const o: any = {};
+                    // eval(`
+                    //     async function ${this.secretFunctionName}(docs){ const x = await _this.handler(docs); return x; }
+                    //     o.${this.secretFunctionName} = ${this.secretFunctionName};
+                    // `);
+                    // await o[this.secretFunctionName](rxDocuments);
+
+                    const fnKey = blockFlaggedFunctionKey();
+                    this.secretFunctionName = fnKey;
+                    try {
+                        await FLAGGED_FUNCTIONS[fnKey](() => _this.handler(rxDocuments));
+                    } catch (err: any) {
+                        this.error = err;
+                    } finally {
+                        releaseFlaggedFunctionKey(fnKey);
+                    }
+
+                    if (this.error) {
+                        return;
+                    }
+
+                    lastTime = ensureNotFalsy(lastOfArray(docsSinceResult.documents))._meta.lwt;
+                }
+                if (!this.destination.closed) {
+                    await setCheckpointDoc(this, { checkpoint: docsSinceResult.checkpoint, lastDocTime: lastTime }, checkpointDoc);
+                }
+                if (docsSinceResult.documents.length < this.batchSize) {
+                    done = true;
+                }
+            }
+        });
+    }
+
+    async awaitIdle() {
+        if (this.error) {
+            throw this.error;
+        }
+        let done = false;
+        while (!done) {
+            await this.processQueue;
+            if (this.error) {
+                throw this.error;
+            }
+            if (this.lastProcessedDocTime.getValue() >= this.lastSourceDocTime.getValue()) {
+                done = true;
+            } else {
+                await firstValueFrom(this.somethingChanged);
+            }
+        }
+    }
+
+    async close() {
+        await this.processQueue;
+        this.stopped = true;
+        this.destination.awaitBeforeReads.delete(this.waitBeforeWriteFn);
+        this.subs.forEach(s => s.unsubscribe());
+        await this.processQueue;
+    }
+
+    /**
+     * Remove the pipeline and all metadata which it has stored
+     */
+    async remove() {
+        const insternalStore = this.destination.database.internalStore;
+        const checkpointDoc = await getCheckpointDoc(this);
+        if (checkpointDoc) {
+            const newDoc: RxDocumentData<InternalStoreDocType> = clone(checkpointDoc);
+            newDoc._deleted = true;
+            const writeResult = await insternalStore.bulkWrite([{
+                previous: checkpointDoc,
+                document: newDoc,
+            }], 'rx-pipeline');
+            if (writeResult.error.length > 0) {
+                throw writeResult.error;
+            }
+        }
+        return this.close();
+    }
+}
+
+
+export async function getCheckpointDoc<RxDocType>(
+    pipeline: RxPipeline<RxDocType>
+): Promise<RxDocumentData<InternalStoreDocType<CheckpointDocData>> | undefined> {
+    const insternalStore = pipeline.destination.database.internalStore;
+    const checkpointId = getPrimaryKeyOfInternalDocument(
+        pipeline.checkpointId,
+        INTERNAL_CONTEXT_PIPELINE_CHECKPOINT
+    );
+    const results = await insternalStore.findDocumentsById([checkpointId], false);
+    const result: RxDocumentData<InternalStoreDocType> = results[0];
+    if (result) {
+        return result;
+    } else {
+        return undefined;
+    }
+}
+
+export async function setCheckpointDoc<RxDocType>(
+    pipeline: RxPipeline<RxDocType>,
+    newCheckpoint: CheckpointDocData,
+    previous?: RxDocumentData<InternalStoreDocType>
+): Promise<void> {
+    const insternalStore = pipeline.destination.database.internalStore;
+    const newDoc: RxDocumentData<InternalStoreDocType<CheckpointDocData>> = {
+        _attachments: {},
+        _deleted: false,
+        _meta: {
+            lwt: now()
+        },
+        _rev: createRevision(pipeline.destination.database.token, previous),
+        context: INTERNAL_CONTEXT_PIPELINE_CHECKPOINT,
+        data: newCheckpoint,
+        id: getPrimaryKeyOfInternalDocument(
+            pipeline.checkpointId,
+            INTERNAL_CONTEXT_PIPELINE_CHECKPOINT
+        ),
+        key: pipeline.checkpointId
+    };
+
+    const writeResult = await insternalStore.bulkWrite([{
+        previous,
+        document: newDoc,
+    }], 'rx-pipeline');
+    if (writeResult.error.length > 0) {
+        throw writeResult.error;
+    }
+}
+
+
+export async function addPipeline<RxDocType>(
+    this: RxCollection<RxDocType>,
+    options: RxPipelineOptions<RxDocType>
+): Promise<RxPipeline<RxDocType>> {
+    const pipeline = new RxPipeline<RxDocType>(
+        options.identifier,
+        this,
+        options.destination,
+        options.handler,
+        options.batchSize
+    );
+    const waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+    const startPromise = waitForLeadership ? this.database.waitForLeadership() : PROMISE_RESOLVE_VOID;
+    startPromise.then(() => {
+        pipeline.trigger();
+        pipeline.subs.push(
+            this.eventBulks$.pipe(
+                filter((bulk: RxChangeEventBulk<RxDocType>) => {
+                    if (pipeline.stopped) {
+                        return false;
+                    }
+                    return !bulk.isLocal;
+                })
+            ).subscribe(() => pipeline.trigger())
+        );
+    });
+
+    return pipeline;
+}