@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/backup/index.ts

@@ -0,0 +1,281 @@
+import * as path from 'node:path';
+import {
+    BehaviorSubject,
+    firstValueFrom,
+    Observable,
+    Subject,
+    Subscription
+} from 'rxjs';
+import {
+    filter,
+    map
+} from 'rxjs';
+import type {
+    BackupOptions,
+    RxBackupWriteEvent,
+    RxCollection,
+    RxDatabase,
+    RxDocument,
+    RxPlugin
+} from '../../types/index.d.ts';
+import {
+    getFromMapOrCreate,
+    PROMISE_RESOLVE_FALSE,
+    PROMISE_RESOLVE_TRUE,
+    PROMISE_RESOLVE_VOID
+} from '../../plugins/utils/index.ts';
+import {
+    clearFolder,
+    deleteFolder,
+    documentFolder,
+    ensureFolderExists,
+    getMeta,
+    prepareFolders,
+    setMeta,
+    writeJsonToFile,
+    writeToFile
+} from './file-util.ts';
+import { getChangedDocumentsSince } from '../../rx-storage-helper.ts';
+
+
+/**
+ * Backups a single documents,
+ * returns the paths to all written files
+ */
+export async function backupSingleDocument(
+    rxDocument: RxDocument<any, any>,
+    options: BackupOptions
+): Promise<string[]> {
+    const data = rxDocument.toJSON(true);
+    const writtenFiles: string[] = [];
+
+    const docFolder = documentFolder(options, rxDocument.primary);
+    await clearFolder(docFolder);
+
+    const fileLocation = path.join(
+        docFolder,
+        'document.json'
+    );
+    await writeJsonToFile(fileLocation, data);
+    writtenFiles.push(fileLocation);
+
+    if (options.attachments) {
+        const attachmentsFolder = path.join(
+            docFolder,
+            'attachments'
+        );
+        ensureFolderExists(attachmentsFolder);
+        const attachments = (rxDocument as RxDocument).allAttachments();
+        await Promise.all(
+            attachments
+                .map(async (attachment) => {
+                    const content = await attachment.getData();
+                    const attachmentFileLocation = path.join(
+                        attachmentsFolder,
+                        attachment.id
+                    );
+                    await writeToFile(attachmentFileLocation, content);
+                    writtenFiles.push(attachmentFileLocation);
+                })
+        );
+    }
+
+    return writtenFiles;
+}
+
+const BACKUP_STATES_BY_DB: WeakMap<RxDatabase, RxBackupState[]> = new WeakMap();
+function addToBackupStates(db: RxDatabase, state: RxBackupState) {
+    const ar = getFromMapOrCreate(
+        BACKUP_STATES_BY_DB,
+        db,
+        () => []
+    );
+    ar.push(state);
+}
+
+export class RxBackupState {
+    public isStopped: boolean = false;
+    private subs: Subscription[] = [];
+    private persistRunning: Promise<void> = PROMISE_RESOLVE_VOID;
+    private initialReplicationDone$: BehaviorSubject<boolean> = new BehaviorSubject(false as any);
+
+    private readonly internalWriteEvents$: Subject<RxBackupWriteEvent> = new Subject();
+    public readonly writeEvents$: Observable<RxBackupWriteEvent> = this.internalWriteEvents$.asObservable();
+
+    constructor(
+        public readonly database: RxDatabase,
+        public readonly options: BackupOptions
+    ) {
+        if (!this.options.batchSize) {
+            this.options.batchSize = 10;
+        }
+        addToBackupStates(database, this);
+        prepareFolders(database, options);
+    }
+
+    /**
+     * Persists all data from all collections,
+     * beginning from the oldest sequence checkpoint
+     * to the newest one.
+     * Do not call this while it is already running.
+     * Returns true if there are more documents to process
+     */
+    public persistOnce() {
+        return this.persistRunning = this.persistRunning.then(() => this._persistOnce());
+    }
+
+    public async _persistOnce() {
+        const meta = await getMeta(this.options);
+
+        await Promise.all(
+            Object
+                .entries(this.database.collections)
+                .map(async ([collectionName, collection]) => {
+                    const primaryKey = collection.schema.primaryPath;
+                    const processedDocuments: Set<string> = new Set();
+
+                    await this.database.requestIdlePromise();
+
+                    if (!meta.collectionStates[collectionName]) {
+                        meta.collectionStates[collectionName] = {};
+                    }
+                    let lastCheckpoint = meta.collectionStates[collectionName].checkpoint;
+
+                    let hasMore = true;
+                    while (hasMore && !this.isStopped) {
+                        await this.database.requestIdlePromise();
+                        const changesResult = await getChangedDocumentsSince(
+                            collection.storageInstance,
+                            this.options.batchSize ? this.options.batchSize : 0,
+                            lastCheckpoint
+                        );
+                        lastCheckpoint = changesResult.documents.length > 0 ? changesResult.checkpoint : lastCheckpoint;
+                        meta.collectionStates[collectionName].checkpoint = lastCheckpoint;
+
+                        const docIds: string[] = changesResult.documents
+                            .map(doc => doc[primaryKey])
+                            .filter(id => {
+                                if (
+                                    processedDocuments.has(id)
+                                ) {
+                                    return false;
+                                } else {
+                                    processedDocuments.add(id);
+                                    return true;
+                                }
+                            })
+                            .filter((elem, pos, arr) => arr.indexOf(elem) === pos); // unique
+                        await this.database.requestIdlePromise();
+
+                        const docs: Map<string, RxDocument> = await collection.findByIds(docIds).exec();
+                        if (docs.size === 0) {
+                            hasMore = false;
+                            continue;
+                        }
+                        await Promise.all(
+                            Array
+                                .from(docs.values())
+                                .map(async (doc) => {
+                                    const writtenFiles = await backupSingleDocument(doc, this.options);
+                                    this.internalWriteEvents$.next({
+                                        collectionName: collection.name,
+                                        documentId: doc.primary,
+                                        files: writtenFiles,
+                                        deleted: false
+                                    });
+                                })
+                        );
+                        // handle deleted documents
+                        await Promise.all(
+                            docIds
+                                .filter(docId => !docs.has(docId))
+                                .map(async (docId) => {
+                                    await deleteFolder(documentFolder(this.options, docId));
+                                    this.internalWriteEvents$.next({
+                                        collectionName: collection.name,
+                                        documentId: docId,
+                                        files: [],
+                                        deleted: true
+                                    });
+                                })
+                        );
+                    }
+                    meta.collectionStates[collectionName].checkpoint = lastCheckpoint;
+                    await setMeta(this.options, meta);
+                })
+        );
+
+        if (!this.initialReplicationDone$.getValue()) {
+            this.initialReplicationDone$.next(true);
+        }
+    }
+
+    public watchForChanges() {
+        const collections: RxCollection[] = Object.values(this.database.collections);
+        collections.forEach(collection => {
+            const changes$ = collection.storageInstance.changeStream();
+            const sub = changes$.subscribe(() => {
+                this.persistOnce();
+            });
+            this.subs.push(sub);
+        });
+    }
+
+    /**
+     * Returns a promise that resolves when the initial backup is done
+     * and the filesystem is in sync with the database state
+     */
+    public awaitInitialBackup(): Promise<boolean> {
+        return firstValueFrom(
+            this.initialReplicationDone$.pipe(
+                filter((v: boolean) => !!v),
+                map(() => true)
+            )
+        );
+    }
+
+    cancel(): Promise<boolean> {
+        if (this.isStopped) {
+            return PROMISE_RESOLVE_FALSE;
+        }
+        this.isStopped = true;
+        this.subs.forEach(sub => sub.unsubscribe());
+        return PROMISE_RESOLVE_TRUE;
+    }
+}
+
+
+export function backup(
+    this: RxDatabase,
+    options: BackupOptions
+): RxBackupState {
+    const backupState = new RxBackupState(this, options);
+    backupState.persistOnce();
+
+    if (options.live) {
+        backupState.watchForChanges();
+    }
+
+    return backupState;
+}
+
+export * from './file-util.ts';
+export const RxDBBackupPlugin: RxPlugin = {
+    name: 'backup',
+    rxdb: true,
+    prototypes: {
+        RxDatabase(proto: any) {
+            proto.backup = backup;
+        }
+    },
+    hooks: {
+        preCloseRxDatabase: {
+            after: function preCloseRxDatabase(db: RxDatabase) {
+                const states = BACKUP_STATES_BY_DB.get(db);
+                if (states) {
+                    states.forEach(state => state.cancel());
+                }
+            }
+        }
+    }
+};

package/src/plugins/cleanup/cleanup-helper.ts

@@ -0,0 +1,11 @@
+import type {
+    RxCleanupPolicy
+} from '../../types/index.d.ts';
+
+export const DEFAULT_CLEANUP_POLICY: RxCleanupPolicy = {
+    minimumDeletedTime: 1000 * 60 * 60 * 24 * 31, // one month
+    minimumCollectionAge: 1000 * 60, // 60 seconds
+    runEach: 1000 * 60 * 5, // 5 minutes
+    awaitReplicationsInSync: true,
+    waitForLeadership: true
+};

package/src/plugins/cleanup/cleanup-state.ts

@@ -0,0 +1,92 @@
+import type { RxCleanupPolicy, RxState } from '../../types/index.d.ts';
+import { PROMISE_RESOLVE_TRUE } from '../../plugins/utils/index.ts';
+import { REPLICATION_STATE_BY_COLLECTION } from '../replication/index.ts';
+import { DEFAULT_CLEANUP_POLICY } from './cleanup-helper.ts';
+import { initialCleanupWait } from './cleanup.ts';
+import { firstValueFrom } from 'rxjs';
+
+let RXSTATE_CLEANUP_QUEUE: Promise<any> = PROMISE_RESOLVE_TRUE;
+
+export async function startCleanupForRxState(state: RxState<unknown, unknown>) {
+    const rxCollection = state.collection;
+    const rxDatabase = rxCollection.database;
+    const cleanupPolicy = Object.assign(
+        {},
+        DEFAULT_CLEANUP_POLICY,
+        rxDatabase.cleanupPolicy ? rxDatabase.cleanupPolicy : {}
+    );
+
+    await initialCleanupWait(rxCollection, cleanupPolicy);
+    if (rxCollection.closed) {
+        return;
+    }
+
+    // initially cleanup the state
+    await cleanupRxState(state, cleanupPolicy);
+
+    /**
+     * Afterwards we listen to writes
+     * and only re-run the cleanup if there was a write
+     * to the state.
+     */
+    await runCleanupAfterWrite(state, cleanupPolicy);
+}
+/**
+ * Runs the cleanup for a single RxState
+ */
+export async function cleanupRxState(
+    state: RxState<unknown, unknown>,
+    cleanupPolicy: RxCleanupPolicy
+) {
+    const rxCollection = state.collection;
+    const rxDatabase = rxCollection.database;
+
+    // run cleanup() until it returns true
+    let isDone = false;
+    while (!isDone && !rxCollection.closed) {
+        if (cleanupPolicy.awaitReplicationsInSync) {
+            const replicationStates = REPLICATION_STATE_BY_COLLECTION.get(rxCollection);
+            if (replicationStates) {
+                await Promise.all(
+                    replicationStates.map(replicationState => {
+                        if (!replicationState.isStopped()) {
+                            return replicationState.awaitInSync();
+                        }
+                    })
+                );
+            }
+        }
+        if (rxCollection.closed) {
+            return;
+        }
+        RXSTATE_CLEANUP_QUEUE = RXSTATE_CLEANUP_QUEUE
+            .then(async () => {
+                if (rxCollection.closed) {
+                    return true;
+                }
+                await rxDatabase.requestIdlePromise();
+                return state._cleanup();
+            });
+        isDone = await RXSTATE_CLEANUP_QUEUE;
+    }
+}
+
+export async function runCleanupAfterWrite(
+    state: RxState<unknown, unknown>,
+    cleanupPolicy: RxCleanupPolicy
+) {
+    const rxCollection = state.collection;
+    while (!rxCollection.closed) {
+        /**
+         * We only start the timer if there was actually a write
+         * to the collection. Otherwise the cleanup would
+         * just run on intervals even if nothing has changed.
+         */
+        await firstValueFrom(rxCollection.eventBulks$).catch(() => { });
+        await rxCollection.promiseWait(cleanupPolicy.runEach);
+        if (rxCollection.closed) {
+            return;
+        }
+        await cleanupRxState(state, cleanupPolicy);
+    }
+}

package/src/plugins/cleanup/cleanup.ts

@@ -0,0 +1,140 @@
+import type { RxCleanupPolicy, RxCollection } from '../../types/index.d.ts';
+import { PROMISE_RESOLVE_TRUE, getFromMapOrCreate } from '../../plugins/utils/index.ts';
+import { REPLICATION_STATE_BY_COLLECTION } from '../replication/index.ts';
+import { DEFAULT_CLEANUP_POLICY } from './cleanup-helper.ts';
+import { runAsyncPluginHooks } from '../../hooks.ts';
+import { filter, firstValueFrom } from 'rxjs';
+
+/**
+ * Even on multiple databases,
+ * the calls to RxStorage().cleanup()
+ * must never run in parallel.
+ * The cleanup is a background task which should
+ * not affect the performance of other, more important tasks.
+ */
+let RXSTORAGE_CLEANUP_QUEUE: Promise<boolean> = PROMISE_RESOLVE_TRUE;
+
+export async function startCleanupForRxCollection(
+    rxCollection: RxCollection
+) {
+    const rxDatabase = rxCollection.database;
+    const cleanupPolicy = Object.assign(
+        {},
+        DEFAULT_CLEANUP_POLICY,
+        rxDatabase.cleanupPolicy ? rxDatabase.cleanupPolicy : {}
+    );
+
+
+    await initialCleanupWait(rxCollection, cleanupPolicy);
+    if (rxCollection.closed) {
+        return;
+    }
+
+    // initially cleanup the collection
+    await cleanupRxCollection(rxCollection, cleanupPolicy);
+
+    /**
+     * Afterwards we listen to deletes
+     * and only re-run the cleanup after
+     * minimumDeletedTime is reached.
+     */
+    await runCleanupAfterDelete(rxCollection, cleanupPolicy);
+}
+
+
+export async function initialCleanupWait(collection: RxCollection, cleanupPolicy: RxCleanupPolicy) {
+    /**
+     * Wait until minimumDatabaseInstanceAge is reached
+     * or collection is closed.
+     */
+    await collection.promiseWait(cleanupPolicy.minimumCollectionAge);
+    if (collection.closed) {
+        return;
+    }
+
+    if (collection.database.multiInstance && cleanupPolicy.waitForLeadership) {
+        await collection.database.waitForLeadership();
+    }
+}
+
+/**
+ * Runs the cleanup for a single RxCollection
+ */
+export async function cleanupRxCollection(
+    rxCollection: RxCollection,
+    cleanupPolicy: RxCleanupPolicy
+) {
+    const rxDatabase = rxCollection.database;
+    const storageInstance = rxCollection.storageInstance;
+
+    // run cleanup() until it returns true
+    let isDone = false;
+    while (!isDone && !rxCollection.closed) {
+        if (cleanupPolicy.awaitReplicationsInSync) {
+            const replicationStates = getFromMapOrCreate(
+                REPLICATION_STATE_BY_COLLECTION,
+                rxCollection,
+                () => []
+            );
+            await Promise.all(
+                replicationStates.map(replicationState => {
+                    if (!replicationState.isStopped()) {
+                        return replicationState.awaitInSync();
+                    }
+                })
+            );
+        }
+        if (rxCollection.closed) {
+            return;
+        }
+        RXSTORAGE_CLEANUP_QUEUE = RXSTORAGE_CLEANUP_QUEUE
+            .then(async () => {
+                if (rxCollection.closed) {
+                    return true;
+                }
+                await rxDatabase.requestIdlePromise();
+                const allDone: Promise<boolean>[] = [];
+                allDone.push(storageInstance.cleanup(cleanupPolicy.minimumDeletedTime));
+                const replicationStates = getFromMapOrCreate(
+                    REPLICATION_STATE_BY_COLLECTION,
+                    rxCollection,
+                    () => []
+                );
+                for (const replicationState of replicationStates) {
+                    const meta = replicationState.metaInstance;
+                    if (meta) {
+                        allDone.push(meta.cleanup(cleanupPolicy.minimumDeletedTime));
+                    }
+                }
+
+                const hasFalse = (await Promise.all(allDone)).find(v => !v);
+                return !hasFalse;
+            });
+        isDone = await RXSTORAGE_CLEANUP_QUEUE;
+    }
+    await runAsyncPluginHooks('postCleanup', {
+        collectionName: rxCollection.name,
+        databaseName: rxDatabase.name
+    });
+}
+
+export async function runCleanupAfterDelete(
+    rxCollection: RxCollection,
+    cleanupPolicy: RxCleanupPolicy
+) {
+    while (!rxCollection.closed) {
+        /**
+         * In theory we should wait here until a document is deleted.
+         * But this would mean we have to search through all events ever processed.
+         * So instead we just wait for any write event and then we anyway throttle
+         * the calls with the promiseWait() below.
+         */
+        await firstValueFrom(rxCollection.eventBulks$).catch(() => { });
+
+        await rxCollection.promiseWait(cleanupPolicy.runEach);
+        if (rxCollection.closed) {
+            return;
+        }
+        await cleanupRxCollection(rxCollection, cleanupPolicy);
+    }
+}

package/src/plugins/cleanup/index.ts

@@ -0,0 +1,45 @@
+import type {
+    RxCollection,
+    RxPlugin
+} from '../../types/index.d.ts';
+import { DEFAULT_CLEANUP_POLICY } from './cleanup-helper.ts';
+import { startCleanupForRxState } from './cleanup-state.ts';
+import { cleanupRxCollection, startCleanupForRxCollection } from './cleanup.ts';
+
+export const RxDBCleanupPlugin: RxPlugin = {
+    name: 'cleanup',
+    rxdb: true,
+    prototypes: {
+        RxCollection: (proto: any) => {
+            proto.cleanup = async function (this: RxCollection, minimumDeletedTime?: number): Promise<void> {
+                const cleanupPolicy = Object.assign(
+                    {},
+                    DEFAULT_CLEANUP_POLICY,
+                    this.database.cleanupPolicy ? this.database.cleanupPolicy : {}
+                );
+
+                if (typeof minimumDeletedTime !== 'undefined') {
+                    cleanupPolicy.minimumDeletedTime = minimumDeletedTime;
+                }
+
+                // run cleanup() until it returns true
+                await cleanupRxCollection(this, cleanupPolicy);
+            };
+        }
+    },
+    hooks: {
+        createRxCollection: {
+            after: (i) => {
+                startCleanupForRxCollection(i.collection);
+            }
+        },
+        createRxState: {
+            after: (i) => {
+                startCleanupForRxState(i.state);
+            }
+        }
+    }
+};
+
+export * from './cleanup.ts';
+export * from './cleanup-helper.ts';