@connecttomahdi/rxdb 17.0.0-beta.17

package/src/plugins/replication-google-drive/downstream.ts

@@ -0,0 +1,118 @@
+import { newRxFetchError } from '../../rx-error.ts';
+import { ensureNotFalsy, lastOfArray } from '../utils/index.ts';
+import { fetchDocumentContents } from './document-handling.ts';
+import type {
+    DriveFileMetadata,
+    GoogleDriveCheckpointType,
+    GoogleDriveOptionsWithDefaults
+} from './google-drive-types';
+import { DriveStructure } from './init.ts';
+
+export async function fetchChanges<DocType>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    init: DriveStructure,
+    checkpoint?: GoogleDriveCheckpointType,
+    batchSize: number = 10
+): Promise<{
+    checkpoint: GoogleDriveCheckpointType | undefined,
+    documents: DocType[]
+}> {
+    const filesResult = await fetchChangesFiles(
+        googleDriveOptions,
+        init,
+        checkpoint,
+        batchSize
+    );
+
+    const contents = await fetchDocumentContents<DocType>(
+        googleDriveOptions,
+        filesResult.files.map(file => file.id)
+    );
+
+    return {
+        checkpoint: filesResult.checkpoint,
+        documents: contents.ordered
+    };
+}
+export async function fetchChangesFiles(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    init: DriveStructure,
+    checkpoint?: GoogleDriveCheckpointType,
+    batchSize: number = 10
+): Promise<{
+    checkpoint?: GoogleDriveCheckpointType,
+    files: DriveFileMetadata[]
+}> {
+
+    const queryParts = [
+        `'${init.docsFolderId}' in parents`,
+        `and trashed = false`
+    ];
+
+    if (checkpoint) {
+        queryParts.push(`and modifiedTime >= '${checkpoint.modifiedTime}'`);
+    }
+
+    /**
+     * Intentionally overfetch in case
+     * multiple docs have the same modifiedTime.
+     * We later have to strip the additional ones.
+     */
+    const OVERFETCH_AMOUNT = 6;
+    const params = new URLSearchParams({
+        q: queryParts.join(' '),
+        pageSize: (batchSize + OVERFETCH_AMOUNT) + '',
+        orderBy: "modifiedTime asc,name asc",
+        fields: "files(id,name,mimeType,parents,modifiedTime,size)",
+        supportsAllDrives: "true",
+        includeItemsFromAllDrives: "true",
+    });
+
+    const url =
+        googleDriveOptions.apiEndpoint +
+        "/drive/v3/files?" +
+        params.toString();
+
+    const res = await fetch(url, {
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+        },
+    });
+    if (!res.ok) {
+        throw await newRxFetchError(res);
+    }
+
+    const data: { files: DriveFileMetadata[] } = await res.json();
+
+    let files = data.files;
+    if (checkpoint) {
+        files = files
+            .filter(file => !(file.modifiedTime === checkpoint.modifiedTime && checkpoint.docIdsWithSameModifiedTime.includes(file.name)));
+    }
+    files = files.slice(0, batchSize);
+    const first = files[0];
+
+
+    let newCheckpoint = checkpoint;
+    const last = lastOfArray(files);
+    if (last) {
+        const lastModified = ensureNotFalsy(last.modifiedTime);
+        let docIdsWithSameModifiedTime = files
+            .filter(file => file.modifiedTime === lastModified)
+            .map(file => file.name);
+
+        if (checkpoint && first.modifiedTime === checkpoint.modifiedTime) {
+            docIdsWithSameModifiedTime = docIdsWithSameModifiedTime.concat(checkpoint.docIdsWithSameModifiedTime);
+        }
+
+        newCheckpoint = {
+            docIdsWithSameModifiedTime,
+            modifiedTime: lastModified
+        }
+    }
+
+    return {
+        checkpoint: newCheckpoint,
+        files
+    };
+}

package/src/plugins/replication-google-drive/google-drive-helper.ts

@@ -0,0 +1,467 @@
+import { newRxError, newRxFetchError } from '../../rx-error.ts';
+import { ensureNotFalsy } from '../utils/index.ts';
+import type {
+    GoogleDriveOptionsWithDefaults,
+    DriveFileMetadata
+} from './google-drive-types.ts';
+import { DriveStructure } from './init.ts';
+
+export const DRIVE_API_VERSION = 'v3';
+export const DRIVE_MAX_PAGE_SIZE = 1000;
+export const DRIVE_MAX_BULK_SIZE = DRIVE_MAX_PAGE_SIZE / 4;
+export const FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder';
+
+export async function createFolder(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    parentId: string = 'root',
+    folderName: string
+): Promise<string> {
+    const url = googleDriveOptions.apiEndpoint + '/drive/v3/files?fields=id,name,mimeType,trashed';
+    const body = {
+        name: folderName,
+        mimeType: FOLDER_MIME_TYPE,
+        parents: [parentId]
+    };
+
+    const response = await fetch(url, {
+        method: 'POST',
+        headers: {
+            Authorization: 'Bearer ' + googleDriveOptions.authToken,
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(body)
+    });
+
+    if (!response.ok) {
+        const errorText = await response.text();
+
+        if (response.status == 409) {
+            // someone else created the same folder, return that one instead.
+            const found = await findFolder(googleDriveOptions, parentId, folderName);
+            return ensureNotFalsy(found);
+        }
+
+        throw await newRxFetchError(response, {
+            folderName,
+            parentId
+        });
+    }
+
+    await response.json();
+
+    /**
+     * To make the function idempotent, we do not use the id from the creation-response.
+     * Instead after creating the folder, we search for it again so that in case
+     * some other instance created the same folder, we use the oldest one always.
+     */
+    const foundFolder = await findFolder(
+        googleDriveOptions,
+        parentId,
+        folderName
+    );
+
+    return ensureNotFalsy(foundFolder);
+}
+
+
+export async function findFolder(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    parentId: string = 'root',
+    folderName: string
+): Promise<string | undefined> {
+    const query = "name = '" + folderName + "' and '" + parentId + "' in parents and trashed = false and mimeType = '" + FOLDER_MIME_TYPE + "'";
+    /**
+     * We sort by createdTime ASC
+     * so in case the same folder was created multiple times, we always pick the same
+     * one which is the oldest one.
+     */
+    const searchUrl = googleDriveOptions.apiEndpoint + '/drive/v3/files?fields=files(id,mimeType)&orderBy=createdTime asc&q=' + encodeURIComponent(query);
+    const searchResponse = await fetch(searchUrl, {
+        method: 'GET',
+        headers: {
+            Authorization: 'Bearer ' + googleDriveOptions.authToken
+        }
+    });
+    const searchData = await searchResponse.json();
+
+    if (searchData.files && searchData.files.length > 0) {
+        const file = searchData.files[0];
+        if (file.mimeType !== FOLDER_MIME_TYPE) {
+            throw newRxError('GDR3', {
+                folderName,
+                args: {
+                    file,
+                    FOLDER_MIME_TYPE
+                }
+            });
+        }
+        return file.id;
+    } else {
+        return undefined;
+    }
+}
+
+export async function ensureFolderExists(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    folderPath: string
+): Promise<string> {
+    const parts = folderPath.split('/').filter(p => p.length > 0);
+    let parentId = 'root';
+    for (const part of parts) {
+        const newParentId = await findFolder(googleDriveOptions, parentId, part);
+        if (newParentId) {
+            parentId = newParentId
+        } else {
+            parentId = await createFolder(googleDriveOptions, parentId, part);
+        }
+    }
+    return parentId;
+}
+
+
+export async function createEmptyFile(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    parentId: string,
+    fileName: string
+) {
+    const url = googleDriveOptions.apiEndpoint + '/drive/v3/files?fields=id';
+    const body = {
+        name: fileName,
+        parents: [parentId],
+        mimeType: 'application/json'
+    };
+
+    const response = await fetch(url, {
+        method: 'POST',
+        headers: {
+            Authorization: 'Bearer ' + googleDriveOptions.authToken,
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(body)
+    });
+
+    /**
+     * Do not throw on duplicates,
+     * if the file is there already, find its id
+     * and return that one.
+     */
+    if (!response.ok && response.status !== 409) {
+        throw await newRxFetchError(response, {
+            folderName: fileName
+        });
+    }
+
+    /**
+     * For idempotent runs, fetch the file again
+     * after creating it.
+     */
+    const query = [
+        `name = '${fileName}'`,
+        `'${parentId}' in parents`,
+        `trashed = false`,
+    ].join(' and ');
+    const url2 =
+        googleDriveOptions.apiEndpoint + '/drive/v3/files' +
+        '?fields=files(id,etag,size,createdTime)' +
+        '&orderBy=createdTime asc' +
+        '&q=' + encodeURIComponent(query);
+    const res = await fetch(url2, {
+        headers: {
+            Authorization: 'Bearer ' + googleDriveOptions.authToken,
+        },
+    });
+    const data = await res.json();
+    const file = ensureNotFalsy(data.files[0]);
+    return {
+        status: response.status,
+        etag: ensureNotFalsy(file.etag),
+        createdTime: ensureNotFalsy(file.createdTime),
+        fileId: ensureNotFalsy(file.id),
+        size: parseInt(file.size, 10)
+    }
+}
+
+
+export async function fillFileIfEtagMatches<T = any>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileId: string,
+    etag: string,
+    jsonContent?: any
+): Promise<{
+    status: number;
+    etag: string;
+    content: T | undefined;
+    serverTime: number;
+}> {
+    const url =
+        `${googleDriveOptions.apiEndpoint}` +
+        `/upload/drive/v2/files/${encodeURIComponent(fileId)}` +
+        `?uploadType=media`;
+
+    const writeContent = typeof jsonContent !== 'undefined' ? JSON.stringify(jsonContent) : '';
+    const res = await fetch(url, {
+        method: "PUT",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+            "Content-Type": "application/json; charset=utf-8",
+            "If-Match": etag,
+        },
+        body: writeContent,
+    });
+    if (res.status !== 412 && res.status !== 200) {
+        throw await newRxFetchError(res);
+    }
+
+    return readJsonFileContent<T>(
+        googleDriveOptions,
+        fileId
+    ).then(r => {
+        return {
+            content: r.content,
+            etag: r.etag,
+            status: res.status,
+            serverTime: r.serverTime
+        };
+    });
+}
+
+export async function deleteIfEtagMatches(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileId: string,
+    etag: string
+): Promise<void> {
+    const url =
+        `${googleDriveOptions.apiEndpoint}` +
+        `/drive/v2/files/${encodeURIComponent(fileId)}`;
+
+    const res = await fetch(url, {
+        method: "DELETE",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+            "If-Match": etag,
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res, {
+            args: {
+                etag,
+                fileId
+            }
+        });
+    }
+
+    if (res.ok) {
+        // Drive v2 returns 204 No Content on successful delete
+        return;
+    }
+}
+
+export async function deleteFile(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileId: string
+): Promise<void> {
+    const url =
+        `${googleDriveOptions.apiEndpoint}` +
+        `/drive/v2/files/${encodeURIComponent(fileId)}`;
+
+    const res = await fetch(url, {
+        method: "DELETE",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res, {
+            args: {
+                fileId
+            }
+        });
+    }
+
+    if (res.ok) {
+        // Drive v2 returns 204 No Content on successful delete
+        return;
+    }
+}
+
+export async function readJsonFileContent<T>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    fileId: string
+): Promise<{
+    etag: string;
+    content: T | undefined;
+    serverTime: number;
+}> {
+    const url =
+        `${googleDriveOptions.apiEndpoint}` +
+        `/drive/v2/files/${encodeURIComponent(fileId)}?alt=media`;
+
+    const res = await fetch(url, {
+        method: "GET",
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+            Accept: "application/json",
+        },
+    });
+
+    if (!res.ok) {
+        throw await newRxFetchError(res, {
+            args: {
+                fileId
+            }
+        });
+    }
+
+    const dateHeader = res.headers.get('date');
+    const unixMs = Date.parse(ensureNotFalsy(dateHeader));
+
+    const contentType = res.headers.get("content-type") || "";
+    if (!contentType.includes("application/json")) {
+        const err = new Error("NOT_A_JSON_FILE but " + contentType);
+        (err as any).code = "NOT_A_JSON_FILE";
+        (err as any).contentType = contentType;
+        throw err;
+    }
+
+    const contentText = await res.text();
+    const content = contentText.length > 0 ? JSON.parse(contentText) : undefined;
+    const etag = ensureNotFalsy(res.headers.get('etag'));
+    return {
+        etag,
+        content: content as T,
+        serverTime: unixMs
+    };
+}
+
+
+export async function readFolder(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    folderPath: string
+): Promise<DriveFileMetadata[]> {
+    let parentId = 'root';
+    const parts = folderPath.split('/').filter(p => p.length > 0);
+
+    // Resolve folder path
+    for (const part of parts) {
+        const query = "name = '" + part + "' and '" + parentId + "' in parents and trashed = false and mimeType = '" + FOLDER_MIME_TYPE + "'";
+        const searchUrl = googleDriveOptions.apiEndpoint + '/drive/v3/files?fields=files(id)&q=' + encodeURIComponent(query);
+        const searchResponse = await fetch(searchUrl, {
+            method: 'GET',
+            headers: {
+                Authorization: 'Bearer ' + googleDriveOptions.authToken
+            }
+        });
+        const searchData = await searchResponse.json();
+        if (searchData.files && searchData.files.length > 0) {
+            parentId = searchData.files[0].id;
+        } else {
+            throw newRxError('SNH', { folderPath });
+        }
+    }
+
+    // List children
+    const query = "'" + parentId + "' in parents and trashed = false";
+    const listUrl = googleDriveOptions.apiEndpoint + '/drive/v3/files?fields=files(id,name,mimeType,trashed,parents)&q=' + encodeURIComponent(query);
+    const listResponse = await fetch(listUrl, {
+        method: 'GET',
+        headers: {
+            Authorization: 'Bearer ' + googleDriveOptions.authToken
+        }
+    });
+
+    if (!listResponse.ok) {
+        throw await newRxFetchError(listResponse, {
+            folderName: folderPath
+        });
+    }
+
+    const listData = await listResponse.json();
+    return listData.files || [];
+}
+
+export async function insertMultipartFile<T>(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    folderId: string,
+    filename: string,
+    jsonData: T
+) {
+    const content = JSON.stringify(jsonData);
+
+    const metadata = {
+        name: filename,
+        mimeType: 'application/json',
+        parents: folderId,
+    };
+
+    const postData = createMultipartBody(
+        metadata,
+        content
+    );
+
+    const res = await fetch(googleDriveOptions.apiEndpoint + "/upload/drive/v3/files?uploadType=multipart", {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+            'Content-Type': 'multipart/related; boundary="' + postData.boundary + '"'
+        },
+        body: postData.body
+    });
+    if (!res.ok) {
+        throw await newRxFetchError(res);
+    }
+}
+
+export function createMultipartBody(
+    metadata: Record<string, unknown>,
+    content: string
+) {
+    const multipartBoundary = '-------1337-use-RxDB-7355608-' + Math.random().toString(16).slice(2);
+    const delimiter = '\r\n--' + multipartBoundary + '\r\n';
+    const closeDelim = '\r\n--' + multipartBoundary + '--';
+    const body = delimiter +
+        'Content-Type: application/json\r\n\r\n' +
+        JSON.stringify(metadata) +
+        delimiter +
+        'Content-Type: application/json\r\n\r\n' +
+        content +
+        closeDelim;
+    return { body, boundary: multipartBoundary };
+};
+
+
+export async function listFilesInFolder(
+    googleDriveOptions: GoogleDriveOptionsWithDefaults,
+    folderId: string
+): Promise<DriveFileMetadata[]> {
+
+    const q = `'${folderId}' in parents and trashed = false`;
+
+    const params = new URLSearchParams({
+        q,
+        pageSize: "1000", // max allowed
+        fields: "files(id,name,mimeType,parents,modifiedTime,size)",
+        supportsAllDrives: "true",
+        includeItemsFromAllDrives: "true",
+    });
+
+    const url =
+        googleDriveOptions.apiEndpoint +
+        "/drive/v3/files?" +
+        params.toString();
+
+    const res = await fetch(url, {
+        headers: {
+            Authorization: `Bearer ${googleDriveOptions.authToken}`,
+        },
+    });
+
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        throw new Error(`files.list failed: ${res.status} ${text}`);
+    }
+
+    const data = await res.json();
+    return data.files ?? [];
+}

package/src/plugins/replication-google-drive/google-drive-types.ts

@@ -0,0 +1,84 @@
+import type {
+    ReplicationOptions,
+    ReplicationPullOptions,
+    ReplicationPushOptions
+} from '../../types/index.d.ts';
+import { SignalingOptions } from './signaling.ts';
+
+export type GoogleDriveCheckpointType = {
+    modifiedTime: string;
+    /**
+     * There is no way to do a comparison (> or >=)
+     * on fileNames or ids in google drive.
+     * Also modifiedTime can be sometimes the same
+     * on multiple files.
+     * Therefore we might overfetch documents and
+     * after the request removed the ones that have been known before.
+     */
+    docIdsWithSameModifiedTime: string[];
+};
+
+export interface DriveFileMetadata {
+    id: string;
+    name: string;
+    mimeType: string;
+    parents?: string[];
+    modifiedTime?: string;
+    size?: string; // Drive returns size as string
+    fileSize?: string; // v2
+}
+
+export type GoogleDriveOptions = {
+    oauthClientId: string;
+    authToken: string;
+    /**
+     * like "https://www.googleapis.com"
+     * No trailing slash!
+     */
+    apiEndpoint?: string;
+    /**
+     * Path to a folder in Google Drive where all data is stored.
+     * Example: 'rxdb-replication/my-app'
+     */
+    folderPath: string;
+
+    /**
+     * Time in milliseconds.
+     */
+    transactionTimeout?: number;
+};
+
+export type GoogleDriveOptionsWithDefaults = {
+    oauthClientId: string;
+    authToken: string;
+    /**
+     * like "https://www.googleapis.com"
+     * No trailing slash!
+     */
+    apiEndpoint: string;
+    folderPath: string;
+    /**
+     * Time in milliseconds.
+     */
+    transactionTimeout: number;
+};
+
+export interface DriveFileListResponse {
+    nextPageToken?: string;
+    files: DriveFileMetadata[];
+}
+
+
+export type GoogleDriveSyncPullOptions<RxDocType> = Omit<ReplicationPullOptions<RxDocType, GoogleDriveCheckpointType>, 'handler' | 'stream$'>;
+
+export type GoogleDriveSyncPushOptions<RxDocType> = Omit<ReplicationPushOptions<RxDocType>, 'handler'>;
+
+export type SyncOptionsGoogleDrive<RxDocType> = Omit<
+    ReplicationOptions<RxDocType, any>,
+    'pull' | 'push'
+> & {
+    googleDrive: GoogleDriveOptions;
+    signalingOptions?: SignalingOptions;
+    pull?: GoogleDriveSyncPullOptions<RxDocType>;
+    push?: GoogleDriveSyncPushOptions<RxDocType>;
+};