document-drive 1.0.0-experimental.8 → 1.0.0-websockets

package/src/storage/prisma.ts

@@ -1,32 +1,53 @@
-import { PrismaClient, Prisma } from '@prisma/client';
+import { Prisma, PrismaClient } from '@prisma/client';
 import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library';
-import { backOff, IBackOffOptions } from "exponential-backoff";
 import {
     DocumentDriveAction,
     DocumentDriveLocalState,
     DocumentDriveState
 } from 'document-model-libs/document-drive';
 import type {
+    Action,
+    AttachmentInput,
     BaseAction,
+    Document,
     DocumentHeader,
+    DocumentOperations,
     ExtendedState,
+    FileRegistry,
     Operation,
     OperationScope,
     State
 } from 'document-model/document';
+import { IBackOffOptions, backOff } from 'exponential-backoff';
 import { ConflictOperationError } from '../server/error';
+import type { SynchronizationUnitQuery } from '../server/types';
 import { logger } from '../utils/logger';
-import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
-
-type Transaction = Omit<
-    PrismaClient<Prisma.PrismaClientOptions, never>,
-    '$connect' | '$disconnect' | '$on' | '$transaction' | '$use' | '$extends'
-> | ExtendedPrismaClient;
+import {
+    DocumentDriveStorage,
+    DocumentStorage,
+    IDriveStorage,
+    IStorageDelegate
+} from './types';
+
+type Transaction =
+    | Omit<
+          PrismaClient<Prisma.PrismaClientOptions, never>,
+          | '$connect'
+          | '$disconnect'
+          | '$on'
+          | '$transaction'
+          | '$use'
+          | '$extends'
+      >
+    | ExtendedPrismaClient;
 
 function storageToOperation(
-    op: Prisma.$OperationPayload['scalars']
+    op: Prisma.$OperationPayload['scalars'] & {
+        attachments?: AttachmentInput[];
+    }
 ): Operation {
     const operation: Operation = {
+        id: op.opId || undefined,
         skip: op.skip,
         hash: op.hash,
         index: op.index,
@@ -34,7 +55,10 @@ function storageToOperation(
         input: JSON.parse(op.input),
         type: op.type,
         scope: op.scope as OperationScope,
-        // attachments: fileRegistry
+        resultingState: op.resultingState
+            ? op.resultingState.toString()
+            : undefined,
+        attachments: op.attachments
     };
     if (op.context) {
         operation.context = op.context as Prisma.JsonObject;
@@ -44,38 +68,47 @@ function storageToOperation(
 
 export type PrismaStorageOptions = {
     transactionRetryBackoff?: IBackOffOptions;
-}
+};
 
-function getRetryTransactionsClient<T extends PrismaClient>(prisma: T, backOffOptions?: Partial<IBackOffOptions>) {
+function getRetryTransactionsClient<T extends PrismaClient>(
+    prisma: T,
+    backOffOptions?: Partial<IBackOffOptions>
+) {
     return prisma.$extends({
         client: {
-            $transaction: (...args: Parameters<T["$transaction"]>) => {
+            $transaction: (...args: Parameters<T['$transaction']>) => {
                 // eslint-disable-next-line prefer-spread
                 return backOff(() => prisma.$transaction.apply(prisma, args), {
-                    retry: (e) => {
+                    retry: e => {
                         // Retry the transaction only if the error was due to a write conflict or deadlock
                         // See: https://www.prisma.io/docs/reference/api-reference/error-reference#p2034
-                        return (e as { code: string }).code === "P2034";
+                        return (e as { code: string }).code === 'P2034';
                     },
-                    ...backOffOptions,
+                    ...backOffOptions
                 });
             }
         }
     });
 }
 
-type ExtendedPrismaClient = ReturnType<typeof getRetryTransactionsClient<PrismaClient>>;
+type ExtendedPrismaClient = ReturnType<
+    typeof getRetryTransactionsClient<PrismaClient>
+>;
 
 export class PrismaStorage implements IDriveStorage {
     private db: ExtendedPrismaClient;
+    private delegate: IStorageDelegate | undefined;
 
     constructor(db: PrismaClient, options?: PrismaStorageOptions) {
         const backOffOptions = options?.transactionRetryBackoff;
         this.db = getRetryTransactionsClient(db, {
             ...backOffOptions,
-            jitter: backOffOptions?.jitter ?? "full"
+            jitter: backOffOptions?.jitter ?? 'full'
         });
+    }
 
+    setStorageDelegate(delegate: IStorageDelegate): void {
+        this.delegate = delegate;
     }
 
     async createDrive(id: string, drive: DocumentDriveStorage): Promise<void> {
@@ -133,11 +166,10 @@ export class PrismaStorage implements IDriveStorage {
                 name: document.name,
                 documentType: document.documentType,
                 driveId: drive,
-                initialState: document.initialState as Prisma.InputJsonObject,
+                initialState: JSON.stringify(document.initialState),
                 lastModified: document.lastModified,
-                revision: document.revision,
-                id,
-                state: document.initialState
+                revision: JSON.stringify(document.revision),
+                id
             }
         });
     }
@@ -147,14 +179,8 @@ export class PrismaStorage implements IDriveStorage {
         drive: string,
         id: string,
         operations: Operation[],
-        header: DocumentHeader,
-        newState: State<any, any> | undefined = undefined
+        header: DocumentHeader
     ): Promise<void> {
-        const document = await this.getDocument(drive, id, tx);
-        if (!document) {
-            throw new Error(`Document with id ${id} not found`);
-        }
-
         try {
             await tx.operation.createMany({
                 data: operations.map(op => ({
@@ -168,7 +194,10 @@ export class PrismaStorage implements IDriveStorage {
                     scope: op.scope,
                     branch: 'main',
                     skip: op.skip,
-                    context: op.context
+                    context: op.context,
+                    resultingState: op.resultingState
+                        ? Buffer.from(JSON.stringify(op.resultingState))
+                        : undefined
                 }))
             });
 
@@ -179,10 +208,34 @@ export class PrismaStorage implements IDriveStorage {
                 },
                 data: {
                     lastModified: header.lastModified,
-                    revision: header.revision,
-                    state: newState
+                    revision: JSON.stringify(header.revision)
                 }
             });
+
+            await Promise.all(
+                operations
+                    .filter(o => o.attachments?.length)
+                    .map(op => {
+                        return tx.operation.update({
+                            where: {
+                                unique_operation: {
+                                    driveId: drive,
+                                    documentId: id,
+                                    index: op.index,
+                                    scope: op.scope,
+                                    branch: 'main'
+                                }
+                            },
+                            data: {
+                                attachments: {
+                                    createMany: {
+                                        data: op.attachments ?? []
+                                    }
+                                }
+                            }
+                        });
+                    })
+            );
         } catch (e) {
             // P2002: Unique constraint failed
             // Operation with existing index
@@ -209,6 +262,7 @@ export class PrismaStorage implements IDriveStorage {
                 );
 
                 if (!existingOperation || !conflictOp) {
+                    console.error(e);
                     throw e;
                 } else {
                     throw new ConflictOperationError(
@@ -228,7 +282,7 @@ export class PrismaStorage implements IDriveStorage {
         callback: (document: DocumentStorage) => Promise<{
             operations: Operation[];
             header: DocumentHeader;
-            newState?: State<any, any> | undefined
+            newState?: State<any, any> | undefined;
         }>
     ) {
         let result: {
@@ -237,24 +291,25 @@ export class PrismaStorage implements IDriveStorage {
             newState?: State<any, any> | undefined;
         } | null = null;
 
-        await this.db.$transaction(async tx => {
-            const document = await this.getDocument(drive, id, tx);
-            if (!document) {
-                throw new Error(`Document with id ${id} not found`);
-            }
-            result = await callback(document);
-
-            const { operations, header, newState } = result;
-            return this._addDocumentOperations(
-                tx,
-                drive,
-                id,
-                operations,
-                header,
-                newState
-            );
-        }, { isolationLevel: "Serializable" });
+        await this.db.$transaction(
+            async tx => {
+                const document = await this.getDocument(drive, id, tx);
+                if (!document) {
+                    throw new Error(`Document with id ${id} not found`);
+                }
+                result = await callback(document);
 
+                const { operations, header, newState } = result;
+                return this._addDocumentOperations(
+                    tx,
+                    drive,
+                    id,
+                    operations,
+                    header
+                );
+            },
+            { isolationLevel: 'Serializable' }
+        );
 
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
         if (!result) {
@@ -268,19 +323,22 @@ export class PrismaStorage implements IDriveStorage {
         drive: string,
         id: string,
         operations: Operation[],
-        header: DocumentHeader,
+        header: DocumentHeader
     ): Promise<void> {
         return this._addDocumentOperations(
             this.db,
             drive,
             id,
             operations,
-            header,
+            header
         );
     }
 
     async getDocuments(drive: string) {
         const docs = await this.db.document.findMany({
+            select: {
+                id: true
+            },
             where: {
                 AND: {
                     driveId: drive,
@@ -299,25 +357,18 @@ export class PrismaStorage implements IDriveStorage {
             where: {
                 id: id,
                 driveId: driveId
-            },
+            }
         });
         return count > 0;
     }
 
     async getDocument(driveId: string, id: string, tx?: Transaction) {
-        const result = await (tx ?? this.db).document.findFirst({
+        const prisma = tx ?? this.db;
+        const result = await prisma.document.findUnique({
             where: {
-                id: id,
-                driveId: driveId
-            },
-            include: {
-                operations: {
-                    orderBy: {
-                        index: 'asc'
-                    },
-                    include: {
-                        attachments: true
-                    }
+                id_driveId: {
+                    driveId,
+                    id
                 }
             }
         });
@@ -326,31 +377,119 @@ export class PrismaStorage implements IDriveStorage {
             throw new Error(`Document with id ${id} not found`);
         }
 
+        const cachedOperations = (await this.delegate?.getCachedOperations(
+            driveId,
+            id
+        )) ?? {
+            global: [],
+            local: []
+        };
+        const scopeIndex = Object.keys(cachedOperations).reduceRight<
+            Record<OperationScope, number>
+        >(
+            (acc, value) => {
+                const scope = value as OperationScope;
+                const lastIndex = cachedOperations[scope]?.at(-1)?.index ?? -1;
+                acc[scope] = lastIndex;
+                return acc;
+            },
+            { global: -1, local: -1 }
+        );
+
+        const conditions = Object.entries(scopeIndex).map(
+            ([scope, index]) => `("scope" = '${scope}' AND "index" > ${index})`
+        );
+        conditions.push(
+            `("scope" NOT IN (${Object.keys(cachedOperations)
+                .map(s => `'${s}'`)
+                .join(', ')}))`
+        );
+
+        // retrieves operations with resulting state
+        // for the last operation of each scope
+        // TODO prevent SQL injection
+        const queryOperations = await prisma.$queryRawUnsafe<
+            Prisma.$OperationPayload['scalars'][]
+        >(
+            `WITH ranked_operations AS (
+            SELECT
+                *,
+                ROW_NUMBER() OVER (PARTITION BY scope ORDER BY index DESC) AS rn
+            FROM "Operation"
+            )
+            SELECT
+            "id",
+            "opId",
+            "scope",
+            "branch",
+            "index",
+            "skip",
+            "hash",
+            "timestamp",
+            "input",
+            "type",
+            "context",
+            CASE
+                WHEN rn = 1 THEN "resultingState"
+                ELSE NULL
+            END AS "resultingState"
+            FROM ranked_operations
+            WHERE "driveId" = $1 AND "documentId" = $2
+            AND (${conditions.join(' OR ')})
+            ORDER BY scope, index;
+        `,
+            driveId,
+            id
+        );
+        const operationIds = queryOperations.map(o => o.id);
+        const attachments = await prisma.attachment.findMany({
+            where: {
+                operationId: {
+                    in: operationIds
+                }
+            }
+        });
+
+        // TODO add attachments from cached operations
+        const fileRegistry: FileRegistry = {};
+
+        const operationsByScope = queryOperations.reduce<
+            DocumentOperations<Action>
+        >((acc, operation) => {
+            const scope = operation.scope as OperationScope;
+            if (!acc[scope]) {
+                acc[scope] = [];
+            }
+            const result = storageToOperation(operation);
+            result.attachments = attachments.filter(
+                a => a.operationId === operation.id
+            );
+            result.attachments.forEach(({ hash, ...file }) => {
+                fileRegistry[hash] = file;
+            });
+            acc[scope].push(result);
+            return acc;
+        }, cachedOperations);
+
         const dbDoc = result;
-        const doc = {
+        const doc: Document = {
             created: dbDoc.created.toISOString(),
             name: dbDoc.name ? dbDoc.name : '',
             documentType: dbDoc.documentType,
-            initialState: dbDoc.initialState as ExtendedState<
+            initialState: JSON.parse(dbDoc.initialState) as ExtendedState<
                 DocumentDriveState,
                 DocumentDriveLocalState
             >,
-            state: dbDoc.state as State<unknown, unknown>,
+            state: undefined,
             lastModified: new Date(dbDoc.lastModified).toISOString(),
-            operations: {
-                global: dbDoc.operations
-                    .filter(op => op.scope === 'global' && !op.clipboard)
-                    .map(storageToOperation),
-                local: dbDoc.operations
-                    .filter(op => op.scope === 'local' && !op.clipboard)
-                    .map(storageToOperation)
-            },
-            clipboard: dbDoc.operations
-                .filter(op => op.clipboard)
-                .map(storageToOperation),
-            revision: dbDoc.revision as Record<OperationScope, number>
+            operations: operationsByScope,
+            clipboard: [],
+            revision: JSON.parse(dbDoc.revision) as Record<
+                OperationScope,
+                number
+            >,
+            attachments: {}
         };
-
         return doc;
     }
 
@@ -401,11 +540,107 @@ export class PrismaStorage implements IDriveStorage {
     }
 
     async deleteDrive(id: string) {
+        // delete drive documents and operations
         await this.db.document.deleteMany({
             where: {
                 driveId: id
             }
         });
+
+        // delete drive and associated slug
+        await this.db.drive.deleteMany({
+            where: {
+                id
+            }
+        });
+
+        // delete drive itself
         await this.deleteDocument('drives', id);
     }
+
+    async getOperationResultingState(
+        driveId: string,
+        documentId: string,
+        index: number,
+        scope: string,
+        branch: string
+    ): Promise<unknown> {
+        const operation = await this.db.operation.findUnique({
+            where: {
+                unique_operation: {
+                    driveId,
+                    documentId,
+                    index,
+                    scope,
+                    branch
+                }
+            }
+        });
+        return operation?.resultingState?.toString();
+    }
+
+    getDriveOperationResultingState(
+        drive: string,
+        index: number,
+        scope: string,
+        branch: string
+    ): Promise<unknown> {
+        return this.getOperationResultingState(
+            'drives',
+            drive,
+            index,
+            scope,
+            branch
+        );
+    }
+
+    async getSynchronizationUnitsRevision(
+        units: SynchronizationUnitQuery[]
+    ): Promise<
+        {
+            driveId: string;
+            documentId: string;
+            scope: string;
+            branch: string;
+            lastUpdated: string;
+            revision: number;
+        }[]
+    > {
+        // TODO add branch condition
+        const whereClauses = units
+            .map((_, index) => {
+                return `("driveId" = $${index * 3 + 1} AND "documentId" = $${index * 3 + 2} AND "scope" = $${index * 3 + 3})`;
+            })
+            .join(' OR ');
+
+        const query = `
+            SELECT "driveId", "documentId", "scope", "branch", MAX("timestamp") as "lastUpdated", MAX("index") as revision FROM "Operation"
+            WHERE ${whereClauses}
+            GROUP BY "driveId", "documentId", "scope", "branch"
+        `;
+
+        const params = units
+            .map(unit => [
+                unit.documentId ? unit.driveId : 'drives',
+                unit.documentId || unit.driveId,
+                unit.scope
+            ])
+            .flat();
+        const results = await this.db.$queryRawUnsafe<
+            {
+                driveId: string;
+                documentId: string;
+                lastUpdated: string;
+                scope: OperationScope;
+                branch: string;
+                revision: number;
+            }[]
+        >(query, ...params);
+        return results.map(row => ({
+            ...row,
+            driveId: row.driveId === 'drives' ? row.documentId : row.driveId,
+            documentId: row.driveId === 'drives' ? '' : row.documentId,
+            lastUpdated: new Date(row.lastUpdated).toISOString()
+        }));
+    }
 }

package/src/storage/sequelize.ts

@@ -11,6 +11,7 @@ import {
 } from 'document-model/document';
 import { DataTypes, Options, Sequelize } from 'sequelize';
 import { DocumentDriveStorage, DocumentStorage, IDriveStorage } from './types';
+import type { SynchronizationUnitQuery } from '../server/types';
 
 export class SequelizeStorage implements IDriveStorage {
     private db: Sequelize;
@@ -25,8 +26,8 @@ export class SequelizeStorage implements IDriveStorage {
                 type: DataTypes.STRING,
                 primaryKey: true
             },
-            id: DataTypes.STRING,
-        })
+            id: DataTypes.STRING
+        });
         const Document = this.db.define('document', {
             id: {
                 type: DataTypes.STRING,
@@ -155,7 +156,7 @@ export class SequelizeStorage implements IDriveStorage {
         drive: string,
         id: string,
         operations: Operation[],
-        header: DocumentHeader,
+        header: DocumentHeader
     ): Promise<void> {
         const document = await this.getDocument(drive, id);
         if (!document) {
@@ -177,7 +178,8 @@ export class SequelizeStorage implements IDriveStorage {
                 timestamp: op.timestamp,
                 type: op.type,
                 scope: op.scope,
-                branch: 'main'
+                branch: 'main',
+                opId: op.id
             }))
         );
 
@@ -284,8 +286,8 @@ export class SequelizeStorage implements IDriveStorage {
             where: {
                 id: id,
                 driveId: driveId
-            },
-        })
+            }
+        });
 
         return count > 0;
     }
@@ -323,6 +325,7 @@ export class SequelizeStorage implements IDriveStorage {
                     input: JSON;
                     type: string;
                     scope: string;
+                    opId?: string;
                 }
             ];
             revision: Required<Record<OperationScope, number>>;
@@ -348,13 +351,15 @@ export class SequelizeStorage implements IDriveStorage {
                 input: JSON;
                 type: string;
                 scope: string;
+                opId?: string;
             }) => ({
                 hash: op.hash,
                 index: op.index,
                 timestamp: new Date(op.timestamp).toISOString(),
                 input: op.input,
                 type: op.type,
-                scope: op.scope as OperationScope
+                scope: op.scope as OperationScope,
+                id: op.opId
                 // attachments: fileRegistry
             })
         );
@@ -444,4 +449,61 @@ export class SequelizeStorage implements IDriveStorage {
             });
         }
     }
+
+    async getSynchronizationUnitsRevision(
+        units: SynchronizationUnitQuery[]
+    ): Promise<
+        {
+            driveId: string;
+            documentId: string;
+            scope: string;
+            branch: string;
+            lastUpdated: string;
+            revision: number;
+        }[]
+    > {
+        const results = await Promise.allSettled(
+            units.map(async unit => {
+                try {
+                    const document = await (unit.documentId
+                        ? this.getDocument(unit.driveId, unit.documentId)
+                        : this.getDrive(unit.driveId));
+                    if (!document) {
+                        return undefined;
+                    }
+                    const operation =
+                        document.operations[unit.scope as OperationScope]?.at(
+                            -1
+                        );
+                    if (operation) {
+                        return {
+                            driveId: unit.driveId,
+                            documentId: unit.documentId,
+                            scope: unit.scope,
+                            branch: unit.branch,
+                            lastUpdated: operation.timestamp,
+                            revision: operation.index
+                        };
+                    }
+                } catch {
+                    return undefined;
+                }
+            })
+        );
+        return results.reduce<
+            {
+                driveId: string;
+                documentId: string;
+                scope: string;
+                branch: string;
+                lastUpdated: string;
+                revision: number;
+            }[]
+        >((acc, curr) => {
+            if (curr.status === 'fulfilled' && curr.value !== undefined) {
+                acc.push(curr.value);
+            }
+            return acc;
+        }, []);
+    }
 }