@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-firestore/index.ts

@@ -0,0 +1,387 @@
+import {
+    asyncFilter,
+    ensureNotFalsy,
+    errorToPlainJson,
+    flatClone,
+    lastOfArray,
+    toArray
+} from '../../plugins/utils/index.ts';
+
+import {
+    doc,
+    query,
+    where,
+    orderBy,
+    limit,
+    getDocs,
+    getDoc,
+    onSnapshot,
+    runTransaction,
+    writeBatch,
+    serverTimestamp,
+    QueryDocumentSnapshot,
+    waitForPendingWrites,
+    documentId,
+    FirestoreError,
+    QueryConstraint
+} from 'firebase/firestore';
+
+import { RxDBLeaderElectionPlugin } from '../leader-election/index.ts';
+import type {
+    RxCollection,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxReplicationWriteToMasterRow,
+    RxReplicationPullStreamItem
+} from '../../types/index.d.ts';
+import {
+    RxReplicationState,
+    startReplicationOnLeaderShip
+} from '../replication/index.ts';
+import {
+    addRxPlugin,
+    ById,
+    getSchemaByObjectPath,
+    newRxError,
+    WithDeleted
+} from '../../index.ts';
+
+import type {
+    FirestoreCheckpointType,
+    FirestoreOptions,
+    SyncOptionsFirestore
+} from './firestore-types.ts';
+import { Subject } from 'rxjs';
+import {
+    firestoreRowToDocData,
+    getContentByIds,
+    isoStringToServerTimestamp,
+    serverTimestampToIsoString,
+    stripPrimaryKey,
+    stripServerTimestampField
+} from './firestore-helper.ts';
+
+export * from './firestore-helper.ts';
+export * from './firestore-types.ts';
+
+export class RxFirestoreReplicationState<RxDocType> extends RxReplicationState<RxDocType, FirestoreCheckpointType> {
+    constructor(
+        public readonly firestore: FirestoreOptions<RxDocType>,
+        public readonly replicationIdentifierHash: string,
+        public readonly collection: RxCollection<RxDocType>,
+        public readonly pull?: ReplicationPullOptions<RxDocType, FirestoreCheckpointType>,
+        public readonly push?: ReplicationPushOptions<RxDocType>,
+        public readonly live: boolean = true,
+        public retryTime: number = 1000 * 5,
+        public autoStart: boolean = true
+    ) {
+        super(
+            replicationIdentifierHash,
+            collection,
+            '_deleted',
+            pull,
+            push,
+            live,
+            retryTime,
+            autoStart
+        );
+    }
+}
+
+export function replicateFirestore<RxDocType>(
+    options: SyncOptionsFirestore<RxDocType>
+): RxFirestoreReplicationState<RxDocType> {
+    const collection: RxCollection<RxDocType, any, any> = options.collection;
+    addRxPlugin(RxDBLeaderElectionPlugin);
+    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, FirestoreCheckpointType>> = new Subject();
+    let replicationPrimitivesPull: ReplicationPullOptions<RxDocType, FirestoreCheckpointType> | undefined;
+    options.live = typeof options.live === 'undefined' ? true : options.live;
+    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+    const serverTimestampField = typeof options.serverTimestampField === 'undefined' ? 'serverTimestamp' : options.serverTimestampField;
+    options.serverTimestampField = serverTimestampField;
+    const primaryPath = collection.schema.primaryPath;
+
+    /**
+     * The serverTimestampField MUST NOT be part of the collections RxJsonSchema.
+     */
+    const schemaPart = getSchemaByObjectPath(collection.schema.jsonSchema, serverTimestampField);
+    if (
+        schemaPart ||
+        // also must not be nested.
+        serverTimestampField.includes('.')
+    ) {
+        throw newRxError('RC6', {
+            field: serverTimestampField,
+            schema: collection.schema.jsonSchema
+        });
+    }
+
+    const pullFilters: QueryConstraint[] = options.pull?.filter !== undefined
+        ? toArray(options.pull.filter) as QueryConstraint[]
+        : [];
+
+    const pullQuery = query(options.firestore.collection, ...pullFilters);
+
+    if (options.pull) {
+        replicationPrimitivesPull = {
+            async handler(
+                lastPulledCheckpoint: FirestoreCheckpointType | undefined,
+                batchSize: number
+            ) {
+                let newerQuery: ReturnType<typeof query>;
+                let sameTimeQuery: ReturnType<typeof query> | undefined;
+
+                if (lastPulledCheckpoint) {
+                    const lastServerTimestamp = isoStringToServerTimestamp(lastPulledCheckpoint.serverTimestamp);
+                    newerQuery = query(pullQuery,
+                        where(serverTimestampField, '>', lastServerTimestamp),
+                        orderBy(serverTimestampField, 'asc'),
+                        limit(batchSize)
+                    );
+                    sameTimeQuery = query(pullQuery,
+                        where(serverTimestampField, '==', lastServerTimestamp),
+                        where(documentId(), '>', lastPulledCheckpoint.id),
+                        orderBy(documentId(), 'asc'),
+                        limit(batchSize)
+                    );
+                } else {
+                    newerQuery = query(pullQuery,
+                        orderBy(serverTimestampField, 'asc'),
+                        limit(batchSize)
+                    );
+                }
+
+                let mustsReRun = true;
+                let useDocs: QueryDocumentSnapshot<RxDocType>[] = [];
+                while (mustsReRun) {
+                    /**
+                     * Local writes that have not been persisted to the server
+                     * are in pending state and do not have a correct serverTimestamp set.
+                     * We have to ensure we only use document states that are in sync with the server.
+                     * @link https://medium.com/firebase-developers/the-secrets-of-firestore-fieldvalue-servertimestamp-revealed-29dd7a38a82b
+                     */
+                    await waitForPendingWrites(options.firestore.database);
+                    await runTransaction(options.firestore.database, async (_tx: any) => {
+                        useDocs = [];
+                        const [
+                            newerQueryResult,
+                            sameTimeQueryResult
+                        ] = await Promise.all([
+                            getDocs(newerQuery),
+                            sameTimeQuery ? getDocs(sameTimeQuery) : undefined
+                        ]);
+
+                        if (
+                            newerQueryResult.metadata.hasPendingWrites ||
+                            (sameTimeQuery && ensureNotFalsy(sameTimeQueryResult).metadata.hasPendingWrites)
+                        ) {
+                            return;
+                        } else {
+                            mustsReRun = false;
+
+                            if (sameTimeQuery) {
+                                useDocs = ensureNotFalsy(sameTimeQueryResult).docs as any;
+                            }
+                            const missingAmount = batchSize - useDocs.length;
+                            if (missingAmount > 0) {
+                                const additionalDocs = newerQueryResult.docs.slice(0, missingAmount).filter((x: any) => !!x);
+                                useDocs = useDocs.concat(additionalDocs as any);
+                            }
+                        }
+                    });
+                }
+
+                if (useDocs.length === 0) {
+                    return {
+                        checkpoint: lastPulledCheckpoint ?? undefined,
+                        documents: []
+                    };
+                }
+                const lastDoc = ensureNotFalsy(lastOfArray(useDocs));
+                const documents: WithDeleted<RxDocType>[] = useDocs
+                    .map(row => firestoreRowToDocData(
+                        serverTimestampField,
+                        primaryPath,
+                        row
+                    ));
+                const newCheckpoint: FirestoreCheckpointType = {
+                    id: lastDoc.id,
+                    serverTimestamp: serverTimestampToIsoString(serverTimestampField, lastDoc.data())
+                };
+                const ret = {
+                    documents: documents,
+                    checkpoint: newCheckpoint
+                };
+                return ret;
+            },
+            batchSize: ensureNotFalsy(options.pull).batchSize,
+            modifier: ensureNotFalsy(options.pull).modifier,
+            stream$: pullStream$.asObservable(),
+            initialCheckpoint: options.pull.initialCheckpoint
+        };
+    }
+
+    let replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined;
+    if (options.push) {
+        const pushFilter = options.push?.filter;
+        replicationPrimitivesPush = {
+            async handler(
+                rows: RxReplicationWriteToMasterRow<RxDocType>[]
+            ) {
+                if (pushFilter !== undefined) {
+                    rows = await asyncFilter(rows, (row) => pushFilter(row.newDocumentState));
+                }
+
+                const writeRowsById: ById<RxReplicationWriteToMasterRow<RxDocType>> = {};
+                const docIds: string[] = rows.map(row => {
+                    const docId = (row.newDocumentState as any)[primaryPath];
+                    writeRowsById[docId] = row;
+                    return docId;
+                });
+                await waitForPendingWrites(options.firestore.database);
+                let conflicts: WithDeleted<RxDocType>[] = [];
+
+                /**
+                 * Everything must run INSIDE of the transaction
+                 * because on tx-errors, firebase will re-run the transaction on some cases.
+                 * @link https://firebase.google.com/docs/firestore/manage-data/transactions#transaction_failure
+                 * @link https://firebase.google.com/docs/firestore/manage-data/transactions
+                 */
+                await runTransaction(options.firestore.database, async (_tx: any) => {
+                    conflicts = []; // reset in case the tx has re-run.
+                    /**
+                     * @link https://stackoverflow.com/a/48423626/3443137
+                     */
+
+                    const getQuery = (ids: string[]) => {
+                        return getDocs(
+                            query(
+                                options.firestore.collection,
+                                where(documentId(), 'in', ids)
+                            )
+                        )
+                        .then((result: any) => result.docs)
+                        .catch((error: any) => {
+                            if (error?.code && (error as FirestoreError).code === 'permission-denied') {
+                                // Query may fail due to rules using 'resource' with non existing ids
+                                // So try to get the docs one by one
+                                return Promise.all(
+                                    ids.map(
+                                        id => getDoc(doc(options.firestore.collection, id))
+                                    )
+                                )
+                                .then((docs: any[]) => docs.filter((doc: any) => doc.exists()));
+                            }
+                            throw error;
+                        });
+                    };
+
+                    const docsInDbResult = await getContentByIds<RxDocType>(docIds, getQuery);
+
+                    const docsInDbById: ById<RxDocType> = {};
+                    docsInDbResult.forEach(row => {
+                        const docDataInDb = stripServerTimestampField(serverTimestampField, row.data());
+                        const docId = row.id;
+                        (docDataInDb as any)[primaryPath] = docId;
+                        docsInDbById[docId] = docDataInDb;
+                    });
+
+                    /**
+                     * @link https://firebase.google.com/docs/firestore/manage-data/transactions#batched-writes
+                     */
+                    const batch = writeBatch(options.firestore.database);
+                    let hasWrite = false;
+                    await Promise.all(
+                        Object.entries(writeRowsById).map(async ([docId, writeRow]) => {
+                            const docInDb: RxDocType | undefined = docsInDbById[docId];
+
+                            if (
+                                docInDb &&
+                                (
+                                    !writeRow.assumedMasterState ||
+                                    collection.conflictHandler.isEqual(docInDb as any, writeRow.assumedMasterState, 'replication-firestore-push') === false
+                                )
+                            ) {
+                                // conflict
+                                conflicts.push(docInDb as any);
+                            } else {
+                                // no conflict
+                                hasWrite = true;
+                                const docRef = doc(options.firestore.collection, docId);
+                                const writeDocData = flatClone(writeRow.newDocumentState);
+                                (writeDocData as any)[serverTimestampField] = serverTimestamp();
+                                if (!docInDb) {
+                                    // insert
+                                    batch.set(docRef, stripPrimaryKey(primaryPath, writeDocData));
+                                } else {
+                                    // update
+                                    batch.update(docRef, stripPrimaryKey(primaryPath, writeDocData));
+                                }
+                            }
+                        })
+                    );
+
+                    if (hasWrite) {
+                        await batch.commit();
+                    }
+                });
+                await waitForPendingWrites(options.firestore.database);
+                return conflicts;
+            },
+            batchSize: options.push.batchSize,
+            modifier: options.push.modifier
+        };
+    }
+
+
+    const replicationState = new RxFirestoreReplicationState<RxDocType>(
+        options.firestore,
+        options.replicationIdentifier,
+        collection,
+        replicationPrimitivesPull,
+        replicationPrimitivesPush,
+        options.live,
+        options.retryTime,
+        options.autoStart
+    );
+
+    /**
+     * Use long polling to get live changes for the pull.stream$
+     */
+    if (options.live && options.pull) {
+        const startBefore = replicationState.start.bind(replicationState);
+        const cancelBefore = replicationState.cancel.bind(replicationState);
+        replicationState.start = () => {
+            const lastChangeQuery = query(
+                pullQuery,
+                orderBy(serverTimestampField, 'desc'),
+                limit(1)
+            );
+            const unsubscribe = onSnapshot(
+                lastChangeQuery,
+                (_querySnapshot: any) => {
+                    /**
+                     * There is no good way to observe the event stream in firestore.
+                     * So instead we listen to any write to the collection
+                     * and then emit a 'RESYNC' flag.
+                     */
+                    replicationState.reSync();
+                },
+                (error: FirestoreError) => {
+                    replicationState.subjects.error.next(
+                        newRxError('RC_STREAM', { error: errorToPlainJson(error) })
+                    );
+                }
+            );
+            replicationState.cancel = () => {
+                unsubscribe();
+                return cancelBefore();
+            };
+            return startBefore();
+        };
+    }
+
+    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);
+
+    return replicationState;
+}

package/src/plugins/replication-google-drive/AGENTS.md

@@ -0,0 +1,5 @@
+# RxDB replication google drive plugin
+
+## Testing
+
+Whenever you change something in this folder, run `npm run test:replication-google-drive` to verify that the changes are correct.

package/src/plugins/replication-google-drive/document-handling.ts

@@ -0,0 +1,203 @@
+import { newRxError, newRxFetchError } from '../../rx-error.ts';
+import { ById } from '../../types/util';
+import { ensureNotFalsy } from '../utils/index.ts';
+import { insertMultipartFile } from './google-drive-helper.ts';
+import type {
+    DriveFileListResponse,
+    GoogleDriveOptionsWithDefaults
+} from './google-drive-types.ts';
+import { DriveStructure } from './init.ts';
+
+const MAX_DRIVE_PAGE_SIZE = 1000;
+
+
+
+export async function getDocumentFiles(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    init: DriveStructure,
+    docIds: string[]
+): Promise<DriveFileListResponse> {
+    const fileNames = docIds.map(id => id + '.json');
+    let q = fileNames
+        .map(name => `name = '${name.replace("'", "\\'")}'`)
+        .join(' or ');
+    q = '(' + q + ') and trashed = false';
+    q += ' and \'' + init.docsFolderId + '\' in parents';
+    if (docIds.length >= MAX_DRIVE_PAGE_SIZE) {
+        throw newRxError('SNH');
+    }
+    const params = new URLSearchParams({
+        q,
+        pageSize: MAX_DRIVE_PAGE_SIZE + '',
+        fields: "nextPageToken, files(id,name,mimeType,parents,modifiedTime,size)",
+        // Shared drives support (safe to include always)
+        includeItemsFromAllDrives: "true",
+        supportsAllDrives: "true",
+    });
+    const url = googleDriveOptions.apiEndpoint + '/drive/v3/files?' + params.toString();
+    const res = await fetch(url, {
+        method: "GET",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res, {
+            ids: docIds,
+        });
+    }
+
+    const data = await res.json();
+    return data;
+}
+
+export async function batchGetFilesMetadata(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileIds: string[]
+) {
+    const boundary = "batch_" + Math.random().toString(16).slice(2);
+
+    const parts = fileIds.map((id, i) => {
+        return (
+            `--${boundary}\r\n` +
+            `Content-Type: application/http\r\n` +
+            `Content-ID: <item-${i}>\r\n\r\n` +
+            `GET /drive/v3/files/${encodeURIComponent(id)}?` +
+            `fields=id,name,mimeType,parents,modifiedTime,size&supportsAllDrives=true HTTP/1.1\r\n\r\n`
+        );
+    });
+
+    const batchBody = parts.join("") + `--${boundary}--`;
+
+    const res = await fetch(googleDriveOptions.apiEndpoint + "/batch/drive/v3", {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+            "Content-Type": `multipart/mixed; boundary=${boundary}`,
+        },
+        body: batchBody,
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res);
+    }
+
+    // multipart/mixed; you can parse it, or just return raw.
+    return await res.text();
+}
+
+export async function insertDocumentFiles<RxDocType>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    init: DriveStructure,
+    primaryPath: keyof RxDocType,
+    docs: RxDocType[]
+) {
+    // Run uploads in parallel
+    await Promise.all(docs.map(async (doc) => {
+        const id = (doc as any)[primaryPath];
+        await insertMultipartFile(
+            googleDriveOptions,
+            init.docsFolderId,
+            id + '.json',
+            doc
+        );
+    }));
+}
+
+export async function updateDocumentFiles<DocType>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    primaryPath: keyof DocType,
+    docs: DocType[],
+    fileIdByDocId: Record<string, string>,
+    concurrency = 5
+) {
+    const queue = docs.slice(0);
+    const results: Record<string, { id: string }> = {};
+
+    async function worker() {
+        while (queue.length) {
+            const doc = queue.shift()!;
+            const docId = (doc as any)[primaryPath] as string;
+            const fileId = ensureNotFalsy(fileIdByDocId[docId]);
+
+            const url =
+                googleDriveOptions.apiEndpoint +
+                `/upload/drive/v3/files/${encodeURIComponent(fileId)}` +
+                `?uploadType=media&supportsAllDrives=true&fields=id`;
+
+            const res = await fetch(url, {
+                method: "PATCH",
+                headers: {
+                    Authorization: `Bearer ${googleDriveOptions.authToken}`,
+                    "Content-Type": "application/json; charset=UTF-8",
+                },
+                body: JSON.stringify(doc),
+            });
+
+            if (!res.ok) {
+                throw await newRxFetchError(res, {
+                    args: {
+                        docId,
+                        fileId
+                    }
+                });
+            }
+
+            results[docId] = await res.json(); // { id }
+        }
+    }
+
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+    return results;
+}
+
+
+export async function fetchDocumentContents<DocType>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileIds: string[],
+    concurrency = 5
+): Promise<{ byId: ById<DocType>; ordered: (DocType)[] }> {
+    const byId: ById<DocType> = {};
+    const ordered = new Array<DocType>(fileIds.length);
+    let nextIndex = 0;
+
+    const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+    async function fetchOne(fileId: string, attempt = 0): Promise<DocType | undefined> {
+        const url =
+            googleDriveOptions.apiEndpoint +
+            `/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&supportsAllDrives=true`;
+
+        const res = await fetch(url, {
+            headers: { Authorization: `Bearer ${googleDriveOptions.authToken}` },
+        });
+
+        if ([429, 500, 502, 503, 504].includes(res.status) && attempt < 4) {
+            const backoffMs = 250 * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
+            await sleep(backoffMs);
+            return fetchOne(fileId, attempt + 1);
+        }
+
+        if (!res.ok) throw await newRxFetchError(res, { args: { fileId } });
+
+        const text = await res.text();
+        return text ? (JSON.parse(text) as DocType) : undefined;
+    }
+
+    async function worker() {
+        while (true) {
+            const i = nextIndex++;
+            if (i >= fileIds.length) return;
+
+            const fileId = fileIds[i];
+            const doc = await fetchOne(fileId);
+
+            ordered[i] = ensureNotFalsy(doc);
+            byId[fileId] = doc as any; // if ById allows undefined, remove `as any`
+        }
+    }
+
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+    return { byId, ordered };
+}

package/src/plugins/replication-google-drive/download-specs.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+
+# This script downloads the Google Drive API v2 and v3 specs.
+# So your AI/LLM can read them and understand the API.
+
+# Directory where the script is located
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SPECS_DIR="$SCRIPT_DIR/specs"
+
+# Create specs directory if it doesn't exist
+mkdir -p "$SPECS_DIR"
+
+# URL 1: Google Drive API v2 JSON
+URL1="https://raw.githubusercontent.com/rakyll/drivefuse/refs/heads/master/third_party/code.google.com/p/google-api-go-client/drive/v2/drive-api.json"
+FILE1="$SPECS_DIR/drive-api.json"
+
+if [ ! -f "$FILE1" ]; then
+    echo "Downloading drive-api.json..."
+    curl -L "$URL1" -o "$FILE1" -s
+    echo "Downloaded drive-api.json"
+else
+    echo "drive-api.json already exists, skipping."
+fi
+
+# URL 2: Google Drive API v3 OpenAPI YAML
+URL2="https://raw.githubusercontent.com/APIs-guru/openapi-directory/refs/heads/main/APIs/googleapis.com/drive/v3/openapi.yaml"
+FILE2="$SPECS_DIR/openapi.yaml"
+
+if [ ! -f "$FILE2" ]; then
+    echo "Downloading openapi.yaml..."
+    curl -L "$URL2" -o "$FILE2" -s
+    echo "Downloaded openapi.yaml"
+else
+    echo "openapi.yaml already exists, skipping."
+fi
+
+
+# URL 3: Google Drive API v2 Specs
+URL3="https://www.googleapis.com/discovery/v1/apis/drive/v2/rest"
+FILE3="$SPECS_DIR/google-drive-api-v2.json"
+
+if [ ! -f "$FILE3" ]; then
+    echo "Downloading google-drive-api-v2.json..."
+    curl -L "$URL3" -o "$FILE3" -s
+    echo "Downloaded google-drive-api-v2.json"
+else
+    echo "google-drive-api-v2.json already exists, skipping."
+fi