@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication/replication-helper.ts

@@ -0,0 +1,122 @@
+import type {
+    RxCollection,
+    WithDeleted
+} from '../../types/index.d.ts';
+import { flatClone } from '../../plugins/utils/index.ts';
+import { getComposedPrimaryKeyOfDocumentData } from '../../rx-schema-helper.ts';
+import type { RxReplicationState } from './index.ts';
+
+// does nothing
+export const DEFAULT_MODIFIER = (d: any) => Promise.resolve(d);
+
+
+export function swapDefaultDeletedTodeletedField<RxDocType>(
+    deletedField: string,
+    doc: WithDeleted<RxDocType>
+): RxDocType {
+    if (deletedField === '_deleted') {
+        return doc;
+    } else {
+        doc = flatClone(doc);
+        const isDeleted = !!doc._deleted;
+        (doc as any)[deletedField] = isDeleted;
+        delete (doc as any)._deleted;
+        return doc;
+    }
+}
+
+/**
+ * Must be run over all plain document data
+ * that was pulled from the remote.
+ * Used to fill up fields or modify the deleted field etc.
+ */
+export function handlePulledDocuments<RxDocType>(
+    collection: RxCollection<RxDocType, unknown, unknown, unknown>,
+    deletedField: string,
+    docs: RxDocType[]
+): WithDeleted<RxDocType>[] {
+    return docs.map(doc => {
+        const useDoc: WithDeleted<RxDocType> = flatClone(doc) as any;
+
+        /**
+         * Swap out the deleted field
+         */
+        if (deletedField !== '_deleted') {
+            const isDeleted = !!(useDoc as any)[deletedField];
+            (useDoc as any)._deleted = isDeleted;
+            delete (useDoc as any)[deletedField];
+        } else {
+            // ensure we have a boolean.
+            useDoc._deleted = !!useDoc._deleted;
+        }
+
+        /**
+         * Fill up composed primary
+         */
+        const primaryPath = collection.schema.primaryPath;
+        (useDoc as any)[primaryPath] = getComposedPrimaryKeyOfDocumentData(
+            collection.schema.jsonSchema,
+            useDoc
+        );
+        return useDoc as any;
+    });
+}
+
+
+/**
+ * Like normal promiseWait()
+ * but will skip the wait time if the online-state changes.
+ */
+export function awaitRetry(
+    collection: RxCollection<any, any, any>,
+    retryTime: number
+) {
+    if (
+        typeof window === 'undefined' ||
+        typeof window !== 'object' ||
+        typeof window.addEventListener === 'undefined' ||
+        navigator.onLine
+    ) {
+        return collection.promiseWait(retryTime);
+    }
+
+    let listener: any;
+    const onlineAgain = new Promise<void>(res => {
+        listener = () => {
+            window.removeEventListener('online', listener);
+            res();
+        };
+        window.addEventListener('online', listener);
+    });
+
+    return Promise.race([
+        onlineAgain,
+        collection.promiseWait(retryTime)
+    ]).then(() => {
+        window.removeEventListener('online', listener);
+    });
+}
+
+
+/**
+ * When a replication is running and the leading tab get hibernated
+ * by the browser, the replication will be stuck.
+ * To prevent this, we fire a mouseeven each X seconds while the replication is not canceled.
+ * 
+ * If you find a better way to prevent hibernation, please make a pull request.
+ */
+export function preventHibernateBrowserTab(replicationState: RxReplicationState<any, any>) {
+    function simulateActivity() {
+        if (
+            typeof document === 'undefined' ||
+            typeof document.dispatchEvent !== 'function'
+        ) {
+            return;
+        }
+        const event = new Event('mousemove');
+        document.dispatchEvent(event);
+    }
+
+    const intervalId = setInterval(simulateActivity, 20 * 1000); // Simulate activity every 20 seconds
+    replicationState.onCancel.push(() => clearInterval(intervalId));
+}

package/src/plugins/replication-appwrite/appwrite-helpers.ts

@@ -0,0 +1,38 @@
+import type { RxDocumentData, WithDeleted, WithDeletedAndAttachments } from '../../types';
+import { flatClone } from '../utils/index.ts';
+
+export function appwriteDocToRxDB<RxDocType>(
+    appwriteDoc: any,
+    primaryKey: string,
+    deletedField: string
+
+): WithDeleted<RxDocType> {
+    const useDoc: any = {};
+    Object.keys(appwriteDoc).forEach(key => {
+        if (!key.startsWith('$')) {
+            useDoc[key] = appwriteDoc[key];
+        }
+    });
+    useDoc[primaryKey] = appwriteDoc.$id;
+    useDoc._deleted = appwriteDoc[deletedField];
+    if (deletedField !== '_deleted') {
+        delete useDoc[deletedField];
+    }
+    return useDoc;
+}
+
+
+export function rxdbDocToAppwrite<RxDocType>(
+    rxdbDoc: WithDeletedAndAttachments<RxDocType>,
+    primaryKey: string,
+    deletedField: string
+) {
+    const writeDoc: any = flatClone(rxdbDoc);
+    delete (writeDoc as WithDeletedAndAttachments<RxDocType>)._attachments;
+    delete writeDoc[primaryKey];
+    writeDoc[deletedField] = writeDoc._deleted;
+    if (deletedField !== '_deleted') {
+        delete writeDoc._deleted;
+    }
+    return writeDoc;
+}

package/src/plugins/replication-appwrite/appwrite-types.ts

@@ -0,0 +1,31 @@
+import type {
+    ById,
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions
+} from '../../types/index.d.ts';
+
+import type { Client, Account } from 'appwrite';
+
+export type AppwriteCheckpointType = {
+    updatedAt: string;
+    id: string;
+};
+
+export type FetchMethodType = typeof fetch;
+export type SyncOptionsAppwrite<RxDocType> = Omit<
+    ReplicationOptions<RxDocType, any>,
+    'pull' | 'push' | 'deletedField'
+> & {
+    databaseId: string;
+    collectionId: string;
+    client: Client;
+    deletedField: string;
+
+    pull?: Omit<ReplicationPullOptions<RxDocType, AppwriteCheckpointType>, 'handler' | 'stream$'> & {
+    };
+    push?: Omit<ReplicationPushOptions<RxDocType>, 'handler'>;
+};
+
+
+export type URLQueryParams = ById<string | number | undefined | boolean>;

package/src/plugins/replication-appwrite/index.ts

@@ -0,0 +1,269 @@
+import type {
+    SyncOptionsAppwrite,
+    AppwriteCheckpointType
+} from './appwrite-types';
+import {
+    RxReplicationState,
+    startReplicationOnLeaderShip
+} from '../replication/index.ts';
+import type {
+    ById,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxCollection,
+    RxReplicationPullStreamItem,
+    RxReplicationWriteToMasterRow,
+    WithDeleted
+} from '../../types';
+import { addRxPlugin } from '../../plugin.ts';
+import { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';
+import {
+    Databases,
+    Query,
+    Models
+} from 'appwrite';
+import { lastOfArray } from '../utils/utils-array.ts';
+import { appwriteDocToRxDB, rxdbDocToAppwrite } from './appwrite-helpers.ts';
+import { flatClone } from '../utils/utils-object.ts';
+import { Subject } from 'rxjs';
+import { getFromMapOrThrow } from '../utils/index.ts';
+
+export class RxAppwriteReplicationState<RxDocType> extends RxReplicationState<RxDocType, AppwriteCheckpointType> {
+    constructor(
+        public readonly replicationIdentifierHash: string,
+        public readonly collection: RxCollection<RxDocType>,
+        public readonly pull?: ReplicationPullOptions<RxDocType, AppwriteCheckpointType>,
+        public readonly push?: ReplicationPushOptions<RxDocType>,
+        public readonly live: boolean = true,
+        public retryTime: number = 1000 * 5,
+        public autoStart: boolean = true
+    ) {
+        super(
+            replicationIdentifierHash,
+            collection,
+            '_deleted',
+            pull,
+            push,
+            live,
+            retryTime,
+            autoStart
+        );
+    }
+}
+
+export function replicateAppwrite<RxDocType>(
+    options: SyncOptionsAppwrite<RxDocType>
+): RxAppwriteReplicationState<RxDocType> {
+    const collection: RxCollection<RxDocType, any, any> = options.collection;
+    const primaryKey = collection.schema.primaryPath;
+    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, AppwriteCheckpointType>> = new Subject();
+
+    addRxPlugin(RxDBLeaderElectionPlugin);
+    options.live = typeof options.live === 'undefined' ? true : options.live;
+    options.deletedField = options.deletedField ? options.deletedField : '_deleted';
+    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+
+    const databases = new Databases(options.client);
+
+    const replicationPrimitivesPull: ReplicationPullOptions<RxDocType, AppwriteCheckpointType> | undefined = options.pull ? {
+        batchSize: options.pull.batchSize,
+        modifier: options.pull.modifier,
+        stream$: pullStream$.asObservable(),
+        initialCheckpoint: options.pull.initialCheckpoint,
+        handler: async (
+            lastPulledCheckpoint: AppwriteCheckpointType | undefined,
+            batchSize: number
+        ) => {
+            const queries: string[] = [];
+            if (lastPulledCheckpoint) {
+                queries.push(
+                    Query.or([
+                        Query.greaterThan('$updatedAt', lastPulledCheckpoint.updatedAt),
+                        Query.and([
+                            Query.equal('$updatedAt', lastPulledCheckpoint.updatedAt),
+                            Query.greaterThan('$id', lastPulledCheckpoint.id)
+                        ])
+                    ])
+                );
+            }
+            queries.push(Query.orderAsc('$updatedAt'));
+            queries.push(Query.orderAsc('$id'));
+            queries.push(Query.limit(batchSize));
+
+            const result = await databases.listDocuments(
+                options.databaseId,
+                options.collectionId,
+                queries
+            );
+            const lastDoc: any = lastOfArray(result.documents);
+            const newCheckpoint: AppwriteCheckpointType | undefined = lastDoc ? {
+                id: lastDoc.$id,
+                updatedAt: lastDoc.$updatedAt
+            } : undefined;
+            const resultDocs: WithDeleted<RxDocType>[] = result.documents.map((doc: any) => {
+                return appwriteDocToRxDB<RxDocType>(
+                    doc,
+                    primaryKey,
+                    options.deletedField
+                );
+            });
+
+            return {
+                checkpoint: newCheckpoint,
+                documents: resultDocs
+            };
+        }
+    } : undefined;
+
+    const replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined = options.push ? {
+        async handler(
+            rows: RxReplicationWriteToMasterRow<RxDocType>[]
+        ) {
+            let query: string;
+
+            // inserts will conflict on write
+            const nonInsertRows = rows.filter(row => row.assumedMasterState);
+            const updateDocsInDbById = new Map<string, RxDocType>();
+            if (nonInsertRows.length > 0) {
+                if (nonInsertRows.length > 1) {
+                    query = Query.or(
+                        nonInsertRows.map(row => {
+                            const id: string = (row.newDocumentState as any)[primaryKey];
+                            return Query.equal('$id', id);
+                        })
+                    );
+                } else {
+                    const id: string = (nonInsertRows[0].newDocumentState as any)[primaryKey];
+                    query = Query.equal('$id', id);
+                }
+                const updateDocsOnServer = await databases.listDocuments(
+                    options.databaseId,
+                    options.collectionId,
+                    [query]
+                );
+                updateDocsOnServer.documents.forEach((doc: any) => {
+                    const docDataInDb = appwriteDocToRxDB<RxDocType>(doc, primaryKey, options.deletedField);
+                    const docId: string = doc.$id;
+                    (docDataInDb as any)[primaryKey] = docId;
+                    updateDocsInDbById.set(docId, docDataInDb);
+                });
+            }
+
+            const conflicts: WithDeleted<RxDocType>[] = [];
+            await Promise.all(
+                rows.map(async (writeRow) => {
+                    const docId = (writeRow.newDocumentState as any)[primaryKey];
+
+                    if (!writeRow.assumedMasterState) {
+                        // INSERT
+                        const insertDoc = rxdbDocToAppwrite<RxDocType>(
+                            writeRow.newDocumentState,
+                            primaryKey,
+                            options.deletedField
+                        );
+                        try {
+                            await databases.createDocument(
+                                options.databaseId,
+                                options.collectionId,
+                                docId,
+                                insertDoc,
+                                // ["read("any")"] // permissions (optional)
+                            );
+                        } catch (err: any) {
+                            if (err.code == 409 && err.name === 'AppwriteException') {
+                                // document already exists -> conflict
+                                const docOnServer = await databases.getDocument(
+                                    options.databaseId,
+                                    options.collectionId,
+                                    docId
+                                );
+                                const docOnServerData = appwriteDocToRxDB<RxDocType>(docOnServer, primaryKey, options.deletedField);
+                                conflicts.push(docOnServerData);
+                            } else {
+                                throw err;
+                            }
+                        }
+                    } else {
+                        // UPDATE
+                        /**
+                         * TODO appwrite does not have a update-if-equals-X method,
+                         * so we pre-fetch the documents and compare them locally.
+                         * This might cause problems when multiple users update the
+                         * same documents very fast.
+                         */
+                        const docInDb: RxDocType = getFromMapOrThrow(updateDocsInDbById, docId);
+                        if (
+                            !writeRow.assumedMasterState ||
+                            collection.conflictHandler.isEqual(docInDb as any, writeRow.assumedMasterState, 'replication-appwrite-push') === false
+                        ) {
+                            // conflict
+                            conflicts.push(docInDb as any);
+                        } else {
+                            // no conflict
+                            const writeDoc: any = flatClone(writeRow.newDocumentState);
+                            delete writeDoc[primaryKey];
+                            writeDoc[options.deletedField] = writeDoc._deleted;
+                            if (options.deletedField !== '_deleted') {
+                                delete writeDoc._deleted;
+                            }
+
+                            await databases.updateDocument(
+                                options.databaseId,
+                                options.collectionId,
+                                docId,
+                                writeDoc,
+                                // ["read("any")"] // permissions (optional)
+                            );
+                        }
+
+                    }
+                })
+            );
+
+            return conflicts;
+        }
+    } : undefined;
+
+    const replicationState = new RxAppwriteReplicationState<RxDocType>(
+        options.replicationIdentifier,
+        collection,
+        replicationPrimitivesPull,
+        replicationPrimitivesPush,
+        options.live,
+        options.retryTime,
+        options.autoStart
+    );
+
+    /**
+     * Subscribe to changes for the pull.stream$
+     */
+    if (options.live && options.pull) {
+        const startBefore = replicationState.start.bind(replicationState);
+        const cancelBefore = replicationState.cancel.bind(replicationState);
+        replicationState.start = () => {
+            const channel = 'databases.' + options.databaseId + '.collections.' + options.collectionId + '.documents';
+            const unsubscribe = options.client.subscribe(
+                channel,
+                (response: any) => {
+                    const docData = appwriteDocToRxDB<RxDocType>(response.payload, primaryKey, options.deletedField);
+                    pullStream$.next({
+                        checkpoint: {
+                            id: (docData as any)[primaryKey],
+                            updatedAt: (response.payload as any).$updatedAt
+                        },
+                        documents: [docData]
+                    });
+
+                }
+            );
+            replicationState.cancel = () => {
+                unsubscribe();
+                return cancelBefore();
+            };
+            return startBefore();
+        };
+    }
+
+    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);
+    return replicationState;
+}

package/src/plugins/replication-couchdb/couchdb-helper.ts

@@ -0,0 +1,100 @@
+import type {
+    RxDocumentData,
+    StringKeys,
+    WithDeleted
+} from '../../types/index.d.ts';
+import { b64EncodeUnicode, flatClone } from '../../plugins/utils/index.ts';
+import { URLQueryParams } from './couchdb-types.ts';
+
+
+export const COUCHDB_NEW_REPLICATION_PLUGIN_IDENTITY_PREFIX = 'couchdb';
+
+
+export function mergeUrlQueryParams(
+    params: URLQueryParams
+): string {
+    return Object.entries(params)
+        .filter(([_k, value]) => typeof value !== 'undefined')
+        .map(([key, value]) => key + '=' + value)
+        .join('&');
+}
+
+export function couchDBDocToRxDocData<RxDocType>(
+    primaryPath: string,
+    couchDocData: any
+): WithDeleted<RxDocType> {
+    const doc = couchSwapIdToPrimary(primaryPath as any, couchDocData);
+
+    // ensure deleted flag is set.
+    doc._deleted = !!doc._deleted;
+
+    delete doc._rev;
+
+    return doc;
+}
+
+
+export function couchSwapIdToPrimary<T>(
+    primaryKey: StringKeys<RxDocumentData<T>>,
+    docData: any
+): any {
+    if (primaryKey === '_id' || docData[primaryKey]) {
+        return flatClone(docData);
+    }
+    docData = flatClone(docData);
+    docData[primaryKey] = docData._id;
+    delete docData._id;
+
+    return docData;
+}
+
+/**
+ * Swaps the primaryKey of the document
+ * to the _id property.
+ */
+export function couchSwapPrimaryToId<RxDocType>(
+    primaryKey: StringKeys<RxDocumentData<RxDocType>>,
+    docData: any
+): RxDocType & { _id: string; } {
+    // optimisation shortcut
+    if (primaryKey === '_id') {
+        return docData;
+    }
+
+    const idValue = docData[primaryKey];
+    const ret = flatClone(docData);
+    delete ret[primaryKey];
+    ret._id = idValue;
+    return ret;
+}
+
+
+export function getDefaultFetch() {
+    if (
+        typeof window === 'object' &&
+        (window as any)['fetch']
+    ) {
+        /**
+         * @link https://stackoverflow.com/a/47180009/3443137
+         */
+        return window.fetch.bind(window);
+    } else {
+        return fetch;
+    }
+}
+
+/**
+ * Returns a fetch handler that contains the username and password
+ * in the Authorization header
+ */
+export function getFetchWithCouchDBAuthorization(username: string, password: string): typeof fetch {
+    const ret: typeof fetch = (url, options) => {
+        options = Object.assign({}, options);
+        if (!options.headers) {
+            options.headers = {};
+        }
+        (options as any).headers['Authorization'] = 'Basic ' + b64EncodeUnicode(username + ':' + password);
+        return fetch(url as any, options);
+    };
+    return ret;
+}

package/src/plugins/replication-couchdb/couchdb-types.ts

@@ -0,0 +1,34 @@
+import type {
+    ById,
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions
+} from '../../types/index.d.ts';
+
+export type CouchDBCheckpointType = {
+    sequence: number;
+};
+
+export type FetchMethodType = typeof fetch;
+export type SyncOptionsCouchDB<RxDocType> = Omit<
+    ReplicationOptions<RxDocType, any>,
+    'pull' | 'push'
+> & {
+    url: string;
+    /**
+     * Here you can set a custom fetch method
+     * to use http headers or credentials when doing requests.
+     */
+    fetch?: FetchMethodType;
+    pull?: Omit<ReplicationPullOptions<RxDocType, CouchDBCheckpointType>, 'handler' | 'stream$'> & {
+        /**
+         * Heartbeat time in milliseconds
+         * for the long polling of the changestream.
+         */
+        heartbeat?: number;
+    };
+    push?: Omit<ReplicationPushOptions<RxDocType>, 'handler'>;
+};
+
+
+export type URLQueryParams = ById<string | number | undefined | boolean>;