document-drive 1.0.0-alpha.80 → 1.0.0-alpha.81

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.80",
+    "version": "1.0.0-alpha.81",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",

package/src/server/index.ts

@@ -194,7 +194,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         const drive = await this.getDrive(driveId);
         let driveTriggers = this.triggerMap.get(driveId);
 
-        const syncUnits = await this.getSynchronizationUnitsIds(driveId);
+        const syncUnits = await this.getSynchronizationUnitsIds(
+            driveId,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            drive
+        );
 
         for (const trigger of drive.state.local.triggers) {
             if (driveTriggers?.get(trigger.id)) {
@@ -374,9 +381,10 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         documentId?: string[],
         scope?: string[],
         branch?: string[],
-        documentType?: string[]
+        documentType?: string[],
+        loadedDrive?: DocumentDriveDocument
     ) {
-        const drive = await this.getDrive(driveId);
+        const drive = loadedDrive || (await this.getDrive(driveId));
 
         const synchronizationUnitsQuery = await this.getSynchronizationUnitsIds(
             driveId,
@@ -490,9 +498,10 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
     public async getSynchronizationUnitIdInfo(
         driveId: string,
-        syncId: string
+        syncId: string,
+        loadedDrive?: DocumentDriveDocument
     ): Promise<SynchronizationUnitQuery | undefined> {
-        const drive = await this.getDrive(driveId);
+        const drive = loadedDrive || (await this.getDrive(driveId));
         const node = drive.state.global.nodes.find(
             node =>
                 isFileNode(node) &&
@@ -523,11 +532,13 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
     public async getSynchronizationUnit(
         driveId: string,
-        syncId: string
+        syncId: string,
+        loadedDrive?: DocumentDriveDocument
     ): Promise<SynchronizationUnit | undefined> {
         const syncUnit = await this.getSynchronizationUnitIdInfo(
             driveId,
-            syncId
+            syncId,
+            loadedDrive
         );
 
         if (!syncUnit) {
@@ -559,12 +570,17 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         filter: {
             since?: string | undefined;
             fromRevision?: number | undefined;
-        }
+        },
+        loadedDrive?: DocumentDriveDocument
     ): Promise<OperationUpdate[]> {
         const syncUnit =
             syncId === '0'
                 ? { documentId: '', scope: 'global' }
-                : await this.getSynchronizationUnitIdInfo(driveId, syncId);
+                : await this.getSynchronizationUnitIdInfo(
+                      driveId,
+                      syncId,
+                      loadedDrive
+                  );
 
         if (!syncUnit) {
             throw new Error(`Invalid Sync Id ${syncId} in drive ${driveId}`);
@@ -572,7 +588,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
         const document =
             syncId === '0'
-                ? await this.getDrive(driveId)
+                ? loadedDrive || (await this.getDrive(driveId))
                 : await this.getDocument(driveId, syncUnit.documentId); // TODO replace with getDocumentOperations
 
         const operations =
@@ -830,19 +846,24 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
     async _processOperations<T extends Document, A extends Action>(
         drive: string,
         documentId: string | undefined,
-        storageDocument: DocumentStorage<T>,
+        documentStorage: DocumentStorage<T>,
         operations: Operation<A | BaseAction>[]
     ) {
         const operationsApplied: Operation<A | BaseAction>[] = [];
         const signals: SignalResult[] = [];
-        let document: T = this._buildDocument(storageDocument);
+        const documentStorageWithState = await this._addDocumentResultingStage(
+            documentStorage,
+            drive,
+            documentId
+        );
 
+        let document: T = this._buildDocument(documentStorageWithState);
         let error: OperationError | undefined; // TODO: replace with an array of errors/consistency issues
         const operationsByScope = groupOperationsByScope(operations);
 
         for (const scope of Object.keys(operationsByScope)) {
             const storageDocumentOperations =
-                storageDocument.operations[scope as OperationScope];
+                documentStorage.operations[scope as OperationScope];
 
             // TODO two equal operations done by two clients will be considered the same, ie: { type: "INCREMENT" }
             const branch = removeExistingOperations(
@@ -922,6 +943,57 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         } as const;
     }
 
+    private async _addDocumentResultingStage<T extends Document>(
+        document: DocumentStorage<T>,
+        drive: string,
+        documentId?: string,
+        options?: GetDocumentOptions
+    ): Promise<DocumentStorage<T>> {
+        // apply skip header operations to all scopes
+        const operations =
+            options?.revisions !== undefined
+                ? filterOperationsByRevision(
+                      document.operations,
+                      options.revisions
+                  )
+                : document.operations;
+        const documentOperations =
+            DocumentUtils.documentHelpers.garbageCollectDocumentOperations(
+                operations
+            );
+
+        for (const scope of Object.keys(documentOperations)) {
+            const lastRemainingOperation =
+                documentOperations[scope as OperationScope].at(-1);
+            // if the latest operation doesn't have a resulting state then tries
+            // to retrieve it from the db to avoid rerunning all the operations
+            if (
+                lastRemainingOperation &&
+                !lastRemainingOperation.resultingState
+            ) {
+                lastRemainingOperation.resultingState = await (documentId
+                    ? this.storage.getOperationResultingState?.(
+                          drive,
+                          documentId,
+                          lastRemainingOperation.index,
+                          lastRemainingOperation.scope,
+                          'main'
+                      )
+                    : this.storage.getDriveOperationResultingState?.(
+                          drive,
+                          lastRemainingOperation.index,
+                          lastRemainingOperation.scope,
+                          'main'
+                      ));
+            }
+        }
+
+        return {
+            ...document,
+            operations: documentOperations
+        };
+    }
+
     private _buildDocument<T extends Document>(
         documentStorage: DocumentStorage<T>,
         options?: GetDocumentOptions
@@ -1614,14 +1686,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         }
 
         const prevSyncUnits = await this.getSynchronizationUnitsIds(drive);
-
         try {
             await this._addDriveOperations(drive, async documentStorage => {
                 const result = await this._processOperations<
                     DocumentDriveDocument,
                     DocumentDriveAction
                 >(drive, undefined, documentStorage, operations.slice());
-
                 document = result.document;
                 operationsApplied.push(...result.operationsApplied);
                 signals.push(...result.signals);
@@ -1654,7 +1724,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 }
             }
 
-            const syncUnits = await this.getSynchronizationUnitsIds(drive);
+            const syncUnits = await this.getSynchronizationUnitsIds(
+                drive,
+                undefined,
+                undefined,
+                undefined,
+                undefined,
+                document
+            );
 
             const prevSyncUnitsIds = prevSyncUnits.map(unit => unit.syncId);
             const syncUnitsIds = syncUnits.map(unit => unit.syncId);

package/src/server/listener/manager.ts

@@ -239,43 +239,47 @@ export class ListenerManager extends BaseListenerManager {
                 );
 
                 const strandUpdates: StrandUpdate[] = [];
-                for (const syncUnit of syncUnits) {
-                    const unitState = listener.syncUnits.get(syncUnit.syncId);
-
-                    if (
-                        unitState &&
-                        unitState.listenerRev >= syncUnit.revision
-                    ) {
-                        continue;
-                    }
-
-                    const opData: OperationUpdate[] = [];
-                    try {
-                        const data = await this.drive.getOperationData(
-                            // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
-                            driveId,
-                            syncUnit.syncId,
-                            {
-                                fromRevision: unitState?.listenerRev
-                            }
+                await Promise.all(
+                    syncUnits.map(async syncUnit => {
+                        const unitState = listener.syncUnits.get(
+                            syncUnit.syncId
                         );
-                        opData.push(...data);
-                    } catch (e) {
-                        logger.error(e);
-                    }
 
-                    if (!opData.length) {
-                        continue;
-                    }
+                        if (
+                            unitState &&
+                            unitState.listenerRev >= syncUnit.revision
+                        ) {
+                            return;
+                        }
 
-                    strandUpdates.push({
-                        driveId,
-                        documentId: syncUnit.documentId,
-                        branch: syncUnit.branch,
-                        operations: opData,
-                        scope: syncUnit.scope as OperationScope
-                    });
-                }
+                        const opData: OperationUpdate[] = [];
+                        try {
+                            const data = await this.drive.getOperationData(
+                                // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
+                                driveId,
+                                syncUnit.syncId,
+                                {
+                                    fromRevision: unitState?.listenerRev
+                                }
+                            );
+                            opData.push(...data);
+                        } catch (e) {
+                            logger.error(e);
+                        }
+
+                        if (!opData.length) {
+                            return;
+                        }
+
+                        strandUpdates.push({
+                            driveId,
+                            documentId: syncUnit.documentId,
+                            branch: syncUnit.branch,
+                            operations: opData,
+                            scope: syncUnit.scope as OperationScope
+                        });
+                    })
+                );
 
                 if (strandUpdates.length == 0) {
                     continue;
@@ -369,7 +373,11 @@ export class ListenerManager extends BaseListenerManager {
         return false;
     }
 
-    getListenerSyncUnits(driveId: string, listenerId: string) {
+    getListenerSyncUnits(
+        driveId: string,
+        listenerId: string,
+        loadedDrive?: DocumentDriveDocument
+    ) {
         const listener = this.listenerState.get(driveId)?.get(listenerId);
         if (!listener) {
             return [];
@@ -380,7 +388,8 @@ export class ListenerManager extends BaseListenerManager {
             filter.documentId ?? ['*'],
             filter.scope ?? ['*'],
             filter.branch ?? ['*'],
-            filter.documentType ?? ['*']
+            filter.documentType ?? ['*'],
+            loadedDrive
         );
     }
 
@@ -453,45 +462,53 @@ export class ListenerManager extends BaseListenerManager {
         // fetch operations from drive  and prepare strands
         const strands: StrandUpdate[] = [];
 
-        const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
+        const drive = await this.drive.getDrive(driveId);
+        const syncUnits = await this.getListenerSyncUnits(
+            driveId,
+            listenerId,
+            drive
+        );
 
-        for (const syncUnit of syncUnits) {
-            if (syncUnit.revision < 0) {
-                continue;
-            }
-            const entry = listener.syncUnits.get(syncUnit.syncId);
-            if (entry && entry.listenerRev >= syncUnit.revision) {
-                continue;
-            }
+        await Promise.all(
+            syncUnits.map(async syncUnit => {
+                if (syncUnit.revision < 0) {
+                    return;
+                }
+                const entry = listener.syncUnits.get(syncUnit.syncId);
+                if (entry && entry.listenerRev >= syncUnit.revision) {
+                    return;
+                }
 
-            const { documentId, driveId, scope, branch } = syncUnit;
-            try {
-                const operations = await this.drive.getOperationData(
-                    // DEAL WITH INVALID SYNC ID ERROR
-                    driveId,
-                    syncUnit.syncId,
-                    {
-                        since,
-                        fromRevision: entry?.listenerRev
+                const { documentId, driveId, scope, branch } = syncUnit;
+                try {
+                    const operations = await this.drive.getOperationData(
+                        // DEAL WITH INVALID SYNC ID ERROR
+                        driveId,
+                        syncUnit.syncId,
+                        {
+                            since,
+                            fromRevision: entry?.listenerRev
+                        },
+                        drive
+                    );
+
+                    if (!operations.length) {
+                        return;
                     }
-                );
 
-                if (!operations.length) {
-                    continue;
+                    strands.push({
+                        driveId,
+                        documentId,
+                        scope: scope as OperationScope,
+                        branch,
+                        operations
+                    });
+                } catch (error) {
+                    logger.error(error);
+                    return;
                 }
-
-                strands.push({
-                    driveId,
-                    documentId,
-                    scope: scope as OperationScope,
-                    branch,
-                    operations
-                });
-            } catch (error) {
-                logger.error(error);
-                continue;
-            }
-        }
+            })
+        );
 
         return strands;
     }

package/src/server/types.ts

@@ -289,12 +289,14 @@ export abstract class BaseDocumentDriveServer {
         documentId?: string[],
         scope?: string[],
         branch?: string[],
-        documentType?: string[]
+        documentType?: string[],
+        loadedDrive?: DocumentDriveDocument
     ): Promise<SynchronizationUnit[]>;
 
     abstract getSynchronizationUnit(
         driveId: string,
-        syncId: string
+        syncId: string,
+        loadedDrive?: DocumentDriveDocument
     ): Promise<SynchronizationUnit | undefined>;
 
     abstract getSynchronizationUnitsIds(
@@ -311,7 +313,8 @@ export abstract class BaseDocumentDriveServer {
         filter: {
             since?: string;
             fromRevision?: number;
-        }
+        },
+        loadedDrive?: DocumentDriveDocument
     ): Promise<OperationUpdate[]>;
 
     /** Internal methods **/

package/src/storage/prisma.ts

@@ -309,7 +309,7 @@ export class PrismaStorage implements IDriveStorage {
                     header
                 );
             },
-            { isolationLevel: 'Serializable' }
+            { isolationLevel: 'Serializable', maxWait: 10000, timeout: 20000 }
         );
 
         // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition