@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-microsoft-onedrive/document-handling.ts

@@ -0,0 +1,216 @@
+import { newRxError, newRxFetchError } from '../../rx-error.ts';
+import { ById } from '../../types/util';
+import { ensureNotFalsy } from '../utils/index.ts';
+import type {
+    OneDriveState,
+    OneDriveItem,
+    OneDriveResponse
+} from './microsoft-onedrive-types.ts';
+import { DriveStructure } from './init.ts';
+import { getDriveBaseUrl } from './microsoft-onedrive-helper.ts';
+
+const MAX_DRIVE_PAGE_SIZE = 999;
+
+export async function getDocumentFiles(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    docIds: string[]
+): Promise<{ files: OneDriveItem[] }> {
+
+    if (docIds.length >= MAX_DRIVE_PAGE_SIZE) {
+        throw newRxError('SNH');
+    }
+
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+    const files: OneDriveItem[] = [];
+
+    // Microsoft Graph doesn't support complex "OR" in search easily, but we can do a `$filter` 
+    // or just fetch by folder children and then filter locally if amount is small, 
+    // or use batch requests. For simplicity we use search where name in (x, y, z) is not supported,
+    // so we can use a $batch request to get all files by id/name from the docs folder.
+
+    // An alternative is `$batch` requests to look up by path. 
+    // Graph batch has a limit of 20 requests per batch.
+
+    const BATCH_SIZE = 20;
+
+    for (let i = 0; i < docIds.length; i += BATCH_SIZE) {
+        const chunk = docIds.slice(i, i + BATCH_SIZE);
+        const driveBaseUrl = getDriveBaseUrl(oneDriveState);
+        const apiEndpoint = oneDriveState.apiEndpoint || 'https://graph.microsoft.com/v1.0';
+        let pathPrefix = driveBaseUrl.substring(apiEndpoint.length);
+        if (!pathPrefix.startsWith('/')) {
+            pathPrefix = '/' + pathPrefix;
+        }
+
+        const batchRequests = chunk.map((id, index) => {
+            return {
+                id: index.toString(),
+                method: "GET",
+                url: `${pathPrefix}/items/${init.docsFolderId}:/${encodeURIComponent(id + '.json')}?$select=id,name,eTag,createdDateTime,lastModifiedDateTime,size,file`
+            };
+        });
+
+        // The URL for batch is at the domain root, not the drive root.
+        // E.g. https://graph.microsoft.com/v1.0/$batch
+        const batchUrl = (oneDriveState.apiEndpoint || 'https://graph.microsoft.com/v1.0') + '/$batch';
+
+        const batchBody = JSON.stringify({ requests: batchRequests });
+
+        const res = await fetch(batchUrl, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${oneDriveState.authToken}`,
+                'Content-Type': 'application/json'
+            },
+            body: batchBody
+        });
+
+        if (!res.ok) {
+            throw await newRxFetchError(res, {
+                ids: docIds,
+            });
+        }
+
+        const data = await res.json();
+        const responses = data.responses || [];
+        for (const response of responses) {
+            if (response.status === 200) {
+                files.push(response.body);
+            } else if (response.status !== 404) {
+                // If it's a 404, the file doesn't exist, which is fine for getDocumentFiles.
+                // It just returns the files that ARE found.
+                // If it's another error, we could throw, but skipping is safer to match Google Drive behavior if one fails.
+            }
+        }
+    }
+
+    return { files };
+}
+
+
+export async function insertDocumentFiles<RxDocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    primaryPath: keyof RxDocType,
+    docs: RxDocType[]
+) {
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+
+    // Run uploads in parallel
+    await Promise.all(docs.map(async (doc) => {
+        const id = (doc as any)[primaryPath];
+        const fileName = id + '.json';
+        const url = `${baseUrl}/items/${init.docsFolderId}:/${encodeURIComponent(fileName)}:/content`;
+
+        const res = await fetch(url, {
+            method: 'PUT',
+            headers: {
+                Authorization: `Bearer ${oneDriveState.authToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(doc)
+        });
+
+        if (!res.ok) {
+            throw await newRxFetchError(res);
+        }
+    }));
+}
+
+export async function updateDocumentFiles<DocType>(
+    oneDriveState: OneDriveState,
+    primaryPath: keyof DocType,
+    docs: DocType[],
+    fileIdByDocId: Record<string, string>,
+    concurrency = 5
+) {
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+    const queue = docs.slice(0);
+    const results: Record<string, { id: string }> = {};
+
+    async function worker() {
+        let doc = queue.shift();
+        while (doc) {
+            const docId = (doc as any)[primaryPath] as string;
+            const fileId = ensureNotFalsy(fileIdByDocId[docId]);
+
+            const url = `${baseUrl}/items/${encodeURIComponent(fileId)}/content`;
+
+            const res = await fetch(url, {
+                method: "PUT",
+                headers: {
+                    Authorization: `Bearer ${oneDriveState.authToken}`,
+                    "Content-Type": "application/json; charset=UTF-8",
+                },
+                body: JSON.stringify(doc),
+            });
+
+            if (!res.ok) {
+                throw await newRxFetchError(res, {
+                    args: {
+                        docId,
+                        fileId
+                    }
+                });
+            }
+
+            const resData: OneDriveItem = await res.json();
+            results[docId] = { id: resData.id };
+
+            doc = queue.shift();
+        }
+    }
+
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+    return results;
+}
+
+
+export async function fetchDocumentContents<DocType>(
+    oneDriveState: OneDriveState,
+    fileIds: string[],
+    concurrency = 5
+): Promise<{ byId: ById<DocType>; ordered: (DocType)[] }> {
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+    const byId: ById<DocType> = {};
+    const ordered = new Array<DocType>(fileIds.length);
+    let nextIndex = 0;
+
+    const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+    async function fetchOne(fileId: string, attempt = 0): Promise<DocType | undefined> {
+        const url = `${baseUrl}/items/${encodeURIComponent(fileId)}/content`;
+
+        const res = await fetch(url, {
+            headers: { Authorization: `Bearer ${oneDriveState.authToken}` },
+        });
+
+        if ([429, 500, 502, 503, 504].includes(res.status) && attempt < 4) {
+            const backoffMs = 250 * Math.pow(2, attempt) + Math.floor(Math.random() * 200);
+            await sleep(backoffMs);
+            return fetchOne(fileId, attempt + 1);
+        }
+
+        if (!res.ok) throw await newRxFetchError(res, { args: { fileId } });
+
+        const text = await res.text();
+        return text ? (JSON.parse(text) as DocType) : undefined;
+    }
+
+    async function worker() {
+        while (true) {
+            const i = nextIndex++;
+            if (i >= fileIds.length) return;
+
+            const fileId = fileIds[i];
+            const doc = await fetchOne(fileId);
+
+            ordered[i] = ensureNotFalsy(doc);
+            byId[fileId] = doc as any;
+        }
+    }
+
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+    return { byId, ordered };
+}

package/src/plugins/replication-microsoft-onedrive/downstream.ts

@@ -0,0 +1,121 @@
+import { newRxFetchError } from '../../rx-error.ts';
+import { ensureNotFalsy, lastOfArray } from '../utils/index.ts';
+import { fetchDocumentContents } from './document-handling.ts';
+import type {
+    OneDriveItem,
+    OneDriveCheckpointType,
+    OneDriveState
+} from './microsoft-onedrive-types.ts';
+import { DriveStructure } from './init.ts';
+import { getDriveBaseUrl } from './microsoft-onedrive-helper.ts';
+
+export async function fetchChanges<DocType>(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    checkpoint?: OneDriveCheckpointType,
+    batchSize: number = 10
+): Promise<{
+    checkpoint: OneDriveCheckpointType | undefined,
+    documents: DocType[]
+}> {
+    const filesResult = await fetchChangesFiles(
+        oneDriveState,
+        init,
+        checkpoint,
+        batchSize
+    );
+
+    const contents = await fetchDocumentContents<DocType>(
+        oneDriveState,
+        filesResult.files.map(file => file.id)
+    );
+
+    return {
+        checkpoint: filesResult.checkpoint,
+        documents: contents.ordered
+    };
+}
+
+
+export async function fetchChangesFiles(
+    oneDriveState: OneDriveState,
+    init: DriveStructure,
+    checkpoint?: OneDriveCheckpointType,
+    batchSize: number = 10
+): Promise<{
+    checkpoint?: OneDriveCheckpointType,
+    files: OneDriveItem[]
+}> {
+
+    const baseUrl = getDriveBaseUrl(oneDriveState);
+
+    // Intentionally overfetch in case multiple docs have the same lastModifiedDateTime.
+    const OVERFETCH_AMOUNT = 6;
+    let url = `${baseUrl}/items/${init.docsFolderId}/children?`;
+
+    const params = new URLSearchParams();
+    params.set('$top', (batchSize + OVERFETCH_AMOUNT).toString());
+    params.set('$select', 'id,name,eTag,lastModifiedDateTime,size,file');
+    params.set('$orderby', 'lastModifiedDateTime asc,name asc');
+
+    if (checkpoint) {
+        // Warning: MS Graph $filter with string timestamps 
+        params.set('$filter', `lastModifiedDateTime ge '${checkpoint.lastModifiedDateTime}'`);
+    }
+
+    url += params.toString();
+
+    const res = await fetch(url, {
+        headers: {
+            Authorization: `Bearer ${oneDriveState.authToken}`,
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res);
+    }
+
+    const data: { value: OneDriveItem[] } = await res.json();
+
+    let files = data.value || [];
+    if (checkpoint) {
+        files = files.filter(file => {
+            if (file.lastModifiedDateTime === checkpoint.lastModifiedDateTime) {
+                // If it's the exact same time, check if we already processed it
+                const docId = file.name.split('.')[0];
+                return !(checkpoint as any).docIdsWithSameModifiedTime?.includes(docId);
+            }
+            return true;
+        });
+    }
+
+    files = files.slice(0, batchSize);
+    const first = files[0];
+
+    let newCheckpoint = checkpoint;
+    const last = lastOfArray(files);
+
+    if (last) {
+        const lastModified = ensureNotFalsy(last.lastModifiedDateTime);
+        let docIdsWithSameModifiedTime = files
+            .filter(file => file.lastModifiedDateTime === lastModified)
+            .map(file => file.name.split('.')[0]);
+
+        if (checkpoint && first && first.lastModifiedDateTime === checkpoint.lastModifiedDateTime) {
+            const oldIds = (checkpoint as any).docIdsWithSameModifiedTime || [];
+            docIdsWithSameModifiedTime = docIdsWithSameModifiedTime.concat(oldIds);
+        }
+
+        newCheckpoint = {
+            docIdsWithSameModifiedTime,
+            lastModifiedDateTime: lastModified
+        } as any;
+        // using 'any' to dynamically assign docIdsWithSameModifiedTime, 
+        // as doing it on the type strictly might conflict with standard Checkpoint usage in RxDB.
+    }
+
+    return {
+        checkpoint: newCheckpoint,
+        files
+    };
+}

package/src/plugins/replication-microsoft-onedrive/index.ts

@@ -0,0 +1,220 @@
+import {
+    RxDBLeaderElectionPlugin
+} from '../leader-election/index.ts';
+import type {
+    RxCollection,
+    ReplicationPullOptions,
+    ReplicationPushOptions,
+    RxReplicationWriteToMasterRow,
+    RxReplicationPullStreamItem
+} from '../../types/index.d.ts';
+import {
+    RxReplicationState,
+    startReplicationOnLeaderShip
+} from '../replication/index.ts';
+import {
+    addRxPlugin
+} from '../../index.ts';
+
+import type {
+    OneDriveCheckpointType,
+    OneDriveState,
+    SyncOptionsOneDrive
+} from './microsoft-onedrive-types.ts';
+import { Subject } from 'rxjs';
+import { DriveStructure, initDriveStructure } from './init.ts';
+import { handleUpstreamBatch } from './upstream.ts';
+import { fetchChanges } from './downstream.ts';
+import { commitTransaction, runInTransaction, startTransaction } from './transaction.ts';
+import {
+    ensureProcessNextTickIsSet
+} from '../replication-webrtc/connection-handler-simple-peer.ts';
+import { SignalingOptions, SignalingState } from './signaling.ts';
+
+export * from './microsoft-onedrive-types.ts';
+export * from './microsoft-onedrive-helper.ts';
+export * from './transaction.ts';
+export * from './init.ts';
+export * from './document-handling.ts';
+export * from './downstream.ts';
+export * from './upstream.ts';
+export * from './signaling.ts';
+
+export const DEFAULT_TRANSACTION_TIMEOUT = 60 * 1000;
+
+export class RxOneDriveReplicationState<RxDocType> extends RxReplicationState<RxDocType, OneDriveCheckpointType> {
+
+    /**
+     * Only exists on live replication
+     */
+    public signalingState?: SignalingState;
+
+    constructor(
+        public readonly oneDrive: OneDriveState,
+        public readonly driveStructure: DriveStructure,
+        public readonly replicationIdentifierHash: string,
+        public readonly collection: RxCollection<RxDocType, any>,
+        public readonly pull?: ReplicationPullOptions<RxDocType, OneDriveCheckpointType>,
+        public readonly push?: ReplicationPushOptions<RxDocType>,
+        public readonly signalingOptions?: SignalingOptions,
+        public readonly live: boolean = true,
+        public retryTime: number = 1000 * 5,
+        public autoStart: boolean = true
+    ) {
+        super(
+            replicationIdentifierHash,
+            collection,
+            '_deleted',
+            pull,
+            push,
+            live,
+            retryTime,
+            autoStart
+        );
+    }
+
+
+    /**
+     * Notify other peers that something
+     * has or might have changed so that
+     * they can pull from their checkpoints.
+     */
+    async notifyPeers() {
+        if (this.signalingState) {
+            await this.signalingState.pingPeers('RESYNC');
+        }
+    }
+
+}
+
+
+
+export async function replicateMicrosoftOneDrive<RxDocType>(
+    options: SyncOptionsOneDrive<RxDocType>
+): Promise<RxOneDriveReplicationState<RxDocType>> {
+    const collection: RxCollection<RxDocType, any, any> = options.collection;
+    addRxPlugin(RxDBLeaderElectionPlugin);
+
+    const oneDriveState: OneDriveState = Object.assign(
+        {
+            apiEndpoint: 'https://graph.microsoft.com/v1.0',
+            driveId: 'me/drive',
+            transactionTimeout: DEFAULT_TRANSACTION_TIMEOUT
+        },
+        options.oneDrive
+    );
+    const driveStructure = await initDriveStructure(oneDriveState);
+
+
+    let replicationState: RxOneDriveReplicationState<RxDocType>;
+
+    const pullStream$: Subject<RxReplicationPullStreamItem<RxDocType, OneDriveCheckpointType>> = new Subject();
+    let replicationPrimitivesPull: ReplicationPullOptions<RxDocType, OneDriveCheckpointType> | undefined;
+
+    options.live = typeof options.live === 'undefined' ? true : options.live;
+    options.waitForLeadership = typeof options.waitForLeadership === 'undefined' ? true : options.waitForLeadership;
+
+    if (options.pull) {
+        replicationPrimitivesPull = {
+            async handler(
+                lastPulledCheckpoint: OneDriveCheckpointType | undefined,
+                batchSize: number
+            ) {
+                return runInTransaction(
+                    oneDriveState,
+                    driveStructure,
+                    collection.schema.primaryPath,
+                    async () => {
+                        const changes = await fetchChanges<RxDocType>(
+                            oneDriveState,
+                            driveStructure,
+                            lastPulledCheckpoint,
+                            batchSize
+                        );
+                        return changes as any;
+                    }
+                );
+            },
+            batchSize: options.pull.batchSize,
+            modifier: options.pull.modifier,
+            stream$: pullStream$.asObservable(),
+            initialCheckpoint: options.pull.initialCheckpoint
+        };
+    }
+
+    let replicationPrimitivesPush: ReplicationPushOptions<RxDocType> | undefined;
+    if (options.push) {
+        replicationPrimitivesPush = {
+            async handler(
+                rows: RxReplicationWriteToMasterRow<RxDocType>[]
+            ) {
+                return runInTransaction(
+                    oneDriveState,
+                    driveStructure,
+                    collection.schema.primaryPath,
+                    async () => {
+                        const conflicts = await handleUpstreamBatch(
+                            oneDriveState,
+                            driveStructure,
+                            options.collection.schema.primaryPath as any,
+                            rows
+                        );
+                        return conflicts;
+                    },
+                    () => replicationState.notifyPeers().catch(() => { })
+                );
+            },
+            batchSize: options.push.batchSize,
+            modifier: options.push.modifier
+        };
+    }
+
+
+
+    replicationState = new RxOneDriveReplicationState<RxDocType>(
+        oneDriveState,
+        driveStructure,
+        options.replicationIdentifier,
+        collection,
+        replicationPrimitivesPull,
+        replicationPrimitivesPush,
+        options.signalingOptions,
+        options.live,
+        options.retryTime,
+        options.autoStart
+    );
+
+    /**
+     * OneDrive has no websocket or server-send-events
+     * to observe file changes reliably enough for real-time web. 
+     * Therefore we use WebRTC to connect clients which then can ping each other on changes.
+     * Instead of a signaling server, we use OneDrive itself
+     * to exchange signaling data.
+     */
+    if (options.live && options.pull) {
+        ensureProcessNextTickIsSet();
+        const startBefore = replicationState.start.bind(replicationState);
+        const cancelBefore = replicationState.cancel.bind(replicationState);
+        replicationState.start = () => {
+            replicationState.signalingState = new SignalingState(
+                replicationState.oneDrive,
+                replicationState.driveStructure,
+                options.signalingOptions ? options.signalingOptions : {}
+            );
+
+            const sub = replicationState.signalingState.resync$.subscribe(() => {
+                replicationState.reSync();
+            });
+
+            replicationState.cancel = () => {
+                sub.unsubscribe();
+                replicationState.signalingState?.close();
+                return cancelBefore();
+            };
+            return startBefore();
+        };
+    }
+
+    startReplicationOnLeaderShip(options.waitForLeadership, replicationState);
+    return replicationState;
+}

package/src/plugins/replication-microsoft-onedrive/init.ts

@@ -0,0 +1,107 @@
+import type { OneDriveState } from './microsoft-onedrive-types.ts';
+import {
+    ensureFolderExists,
+    listFilesInFolder,
+    createEmptyFile,
+    fillFileIfEtagMatches,
+    readJsonFileContent
+} from './microsoft-onedrive-helper.ts';
+import { newRxError } from '../../rx-error.ts';
+import { randomToken } from '../utils/utils-string.ts';
+import { TRANSACTION_FILE_NAME } from './transaction.ts';
+import { ensureNotFalsy } from '../utils/index.ts';
+import { WAL_FILE_NAME } from './upstream.ts';
+
+const NON_ALLOWED_ROOT_FOLDERS = [
+    '/',
+    '',
+    null,
+    false,
+    undefined,
+    'root'
+];
+
+export async function initDriveStructure(
+    oneDriveState: OneDriveState
+) {
+
+    NON_ALLOWED_ROOT_FOLDERS.forEach(nonAllowed => {
+        if (oneDriveState.folderPath === nonAllowed) {
+            throw newRxError('ODR1', {
+                folderPath: oneDriveState.folderPath
+            });
+        }
+    });
+
+    // root folder
+    const rootFolderId = await ensureFolderExists(oneDriveState, oneDriveState.folderPath);
+
+    const rootFolderContent = await listFilesInFolder(oneDriveState, rootFolderId);
+
+    /**
+     * Folder must either be empty
+     * or already used as a RxDB onedrive sync target.
+     */
+    const hasRxDBJson = rootFolderContent.find((file: any) => file.name === 'rxdb.json');
+    const hasOther = rootFolderContent.find((file: any) => file.name !== 'rxdb.json');
+    if (hasOther && !hasRxDBJson) {
+        throw newRxError('ODR9', {
+            folderPath: oneDriveState.folderPath
+        });
+    }
+
+    /**
+     * Create rxdb.json file.
+     * This must always be the first step.
+     */
+    const rxdbJson = await createEmptyFile(oneDriveState, rootFolderId, 'rxdb.json');
+    let replicationIdentifier: string;
+    if (rxdbJson.size === 0) {
+        const rxdbJsonData = await fillFileIfEtagMatches<{ replicationIdentifier: string }>(
+            oneDriveState,
+            rxdbJson.fileId,
+            rxdbJson.etag,
+            { replicationIdentifier: randomToken(10) }
+        );
+        replicationIdentifier = ensureNotFalsy(rxdbJsonData.content).replicationIdentifier;
+    } else {
+        const rxdbJsonData = await readJsonFileContent<{ replicationIdentifier: string }>(
+            oneDriveState,
+            rxdbJson.fileId
+        );
+        replicationIdentifier = ensureNotFalsy(rxdbJsonData.content).replicationIdentifier;
+    }
+
+
+    // docs folder
+    const docsFolderId = await ensureFolderExists(oneDriveState, oneDriveState.folderPath + '/docs');
+
+    // signaling folder
+    const signalingFolderId = await ensureFolderExists(oneDriveState, oneDriveState.folderPath + '/signaling');
+
+    // transaction file
+    const transactionFile = await createEmptyFile(
+        oneDriveState,
+        rootFolderId,
+        TRANSACTION_FILE_NAME
+    );
+
+    // WAL file
+    const walFile = await createEmptyFile(
+        oneDriveState,
+        rootFolderId,
+        WAL_FILE_NAME
+    );
+
+    return {
+        rootFolderId,
+        docsFolderId,
+        signalingFolderId,
+        replicationIdentifier,
+        rxdbJson,
+        transactionFile,
+        walFile
+    };
+}
+
+export type DriveStructure = Awaited<ReturnType<typeof initDriveStructure>>;