document-drive 1.0.0-alpha.26 → 1.0.0-alpha.28

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-alpha.26",
+    "version": "1.0.0-alpha.28",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",
@@ -31,7 +31,7 @@
     "peerDependencies": {
         "@prisma/client": "5.11.0",
         "document-model": "^1.0.35",
-        "document-model-libs": "^1.17.1",
+        "document-model-libs": "^1.18.1",
         "localforage": "^1.10.0",
         "sequelize": "^6.35.2",
         "sqlite3": "^5.1.7"
@@ -53,9 +53,9 @@
         "@types/node": "^20.11.16",
         "@typescript-eslint/eslint-plugin": "^6.18.1",
         "@typescript-eslint/parser": "^6.18.1",
-        "@vitest/coverage-v8": "^0.34.6",
+        "@vitest/coverage-v8": "^1.4.0",
         "document-model": "^1.0.34",
-        "document-model-libs": "^1.17.1",
+        "document-model-libs": "^1.18.1",
         "eslint": "^8.56.0",
         "eslint-config-prettier": "^9.1.0",
         "fake-indexeddb": "^5.0.1",
@@ -68,6 +68,6 @@
         "sequelize": "^6.35.2",
         "sqlite3": "^5.1.7",
         "typescript": "^5.3.2",
-        "vitest": "^1.2.2"
+        "vitest": "^1.4.0"
     }
 }

package/src/server/index.ts

@@ -121,12 +121,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         const result = await (!strand.documentId
             ? this.addDriveOperations(
                   strand.driveId,
-                  operations as Operation<DocumentDriveAction | BaseAction>[]
+                  operations as Operation<DocumentDriveAction | BaseAction>[],
+                  false
               )
             : this.addOperations(
                   strand.driveId,
                   strand.documentId,
-                  operations
+                  operations,
+                  false
               ));
 
         if (result.status === 'ERROR') {
@@ -229,7 +231,8 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         driveId: string,
         documentId?: string[],
         scope?: string[],
-        branch?: string[]
+        branch?: string[],
+        documentType?: string[]
     ) {
         const drive = await this.getDrive(driveId);
 
@@ -238,21 +241,23 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 isFileNode(node) &&
                 (!documentId?.length ||
                     documentId.includes(node.id) ||
-                    documentId.includes('*'))
+                    documentId.includes('*')) &&
+                (!documentType?.length ||
+                    documentType.includes(node.documentType) ||
+                    documentType.includes('*'))
         ) as Pick<
             FileNode,
             'id' | 'documentType' | 'scopes' | 'synchronizationUnits'
         >[];
 
-        if (documentId && !nodes.length) {
-            throw new Error('File node not found');
-        }
-
         // checks if document drive synchronization unit should be added
         if (
-            !documentId ||
-            documentId.includes('*') ||
-            documentId.includes('')
+            (!documentId ||
+                documentId.includes('*') ||
+                documentId.includes('')) &&
+            (!documentType?.length ||
+                documentType.includes('powerhouse/document-drive') ||
+                documentType.includes('*'))
         ) {
             nodes.unshift({
                 id: '',
@@ -275,8 +280,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 scope?.length || branch?.length
                     ? node.synchronizationUnits.filter(
                           unit =>
-                              (!scope?.length || scope.includes(unit.scope)) &&
-                              (!branch?.length || branch.includes(unit.branch))
+                              (!scope?.length ||
+                                  scope.includes(unit.scope) ||
+                                  scope.includes('*')) &&
+                              (!branch?.length ||
+                                  branch.includes(unit.branch) ||
+                                  branch.includes('*'))
                       )
                     : node.synchronizationUnits;
             if (!nodeUnits.length) {
@@ -506,31 +515,15 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
         await this.storage.createDocument(driveId, input.id, document);
 
-        await this.listenerStateManager.addSyncUnits(
-            input.synchronizationUnits.map(({ syncId, scope, branch }) => {
-                const lastOperation = document.operations[scope].slice().pop();
-                return {
-                    syncId,
-                    scope,
-                    branch,
-                    driveId,
-                    documentId: input.id,
-                    documentType: document.documentType,
-                    lastUpdated:
-                        lastOperation?.timestamp ?? document.lastModified,
-                    revision: lastOperation?.index ?? 0
-                };
-            })
-        );
         return document;
     }
 
     async deleteDocument(driveId: string, id: string) {
         try {
             const syncUnits = await this.getSynchronizationUnits(driveId, [id]);
-            this.listenerStateManager.removeSyncUnits(syncUnits);
-        } catch {
-            /* empty */
+            await this.listenerStateManager.removeSyncUnits(driveId, syncUnits);
+        } catch (error) {
+            console.warn('Error deleting document', error);
         }
         return this.storage.deleteDocument(driveId, id);
     }
@@ -788,7 +781,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         }
     }
 
-    async addOperations(drive: string, id: string, operations: Operation[]) {
+    async addOperations(
+        drive: string,
+        id: string,
+        operations: Operation[],
+        forceSync = true
+    ) {
         let document: Document | undefined;
         const operationsApplied: Operation[] = [];
         const updatedOperations: Operation[] = [];
@@ -841,29 +839,26 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 branches
             );
             // update listener cache
-            for (const syncUnit of syncUnits) {
-                this.listenerStateManager
-                    .updateSynchronizationRevision(
-                        drive,
-                        syncUnit.syncId,
-                        syncUnit.revision,
-                        syncUnit.lastUpdated,
-                        () => this.updateSyncStatus(drive, 'SYNCING'),
-                        this.handleListenerError.bind(this)
-                    )
-                    .then(
-                        updates =>
-                            updates.length &&
-                            this.updateSyncStatus(drive, 'SUCCESS')
-                    )
-                    .catch(error => {
-                        console.error(
-                            'Non handled error updating sync revision',
-                            error
-                        );
-                        this.updateSyncStatus(drive, 'ERROR', error as Error);
-                    });
-            }
+            this.listenerStateManager
+                .updateSynchronizationRevisions(
+                    drive,
+                    syncUnits,
+                    () => this.updateSyncStatus(drive, 'SYNCING'),
+                    this.handleListenerError.bind(this),
+                    forceSync
+                )
+                .then(
+                    updates =>
+                        updates.length &&
+                        this.updateSyncStatus(drive, 'SUCCESS')
+                )
+                .catch(error => {
+                    console.error(
+                        'Non handled error updating sync revision',
+                        error
+                    );
+                    this.updateSyncStatus(drive, 'ERROR', error as Error);
+                });
 
             // after applying all the valid operations,throws
             // an error if there was an invalid operation
@@ -943,7 +938,8 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
     async addDriveOperations(
         drive: string,
-        operations: Operation<DocumentDriveAction | BaseAction>[]
+        operations: Operation<DocumentDriveAction | BaseAction>[],
+        forceSync = true
     ) {
         let document: DocumentDriveDocument | undefined;
         const operationsApplied: Operation<DocumentDriveAction | BaseAction>[] =
@@ -994,13 +990,23 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 .pop();
             if (lastOperation) {
                 this.listenerStateManager
-                    .updateSynchronizationRevision(
+                    .updateSynchronizationRevisions(
                         drive,
-                        '0',
-                        lastOperation.index,
-                        lastOperation.timestamp,
+                        [
+                            {
+                                syncId: '0',
+                                driveId: drive,
+                                documentId: '',
+                                scope: 'global',
+                                branch: 'main',
+                                documentType: 'powerhouse/document-drive',
+                                lastUpdated: lastOperation.timestamp,
+                                revision: lastOperation.index
+                            }
+                        ],
                         () => this.updateSyncStatus(drive, 'SYNCING'),
-                        this.handleListenerError.bind(this)
+                        this.handleListenerError.bind(this),
+                        forceSync
                     )
                     .then(
                         updates =>

package/src/server/listener/manager.ts

@@ -19,7 +19,35 @@ import { InternalTransmitter } from './transmitter/internal';
 import { SwitchboardPushTransmitter } from './transmitter/switchboard-push';
 import { ITransmitter } from './transmitter/types';
 
+function debounce<T extends unknown[], R>(
+    func: (...args: T) => Promise<R>,
+    delay = 250
+) {
+    let timer: number;
+    return (immediate: boolean, ...args: T) => {
+        if (timer) {
+            clearTimeout(timer);
+        }
+        return new Promise<R>((resolve, reject) => {
+            if (immediate) {
+                func(...args)
+                    .then(resolve)
+                    .catch(reject);
+            } else {
+                timer = setTimeout(() => {
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+                    func(...args)
+                        .then(resolve)
+                        .catch(reject);
+                }, delay) as unknown as number;
+            }
+        });
+    };
+}
+
 export class ListenerManager extends BaseListenerManager {
+    static LISTENER_UPDATE_DELAY = 250;
+
     async getTransmitter(
         driveId: string,
         listenerId: string
@@ -30,14 +58,6 @@ export class ListenerManager extends BaseListenerManager {
     async addListener(listener: Listener) {
         const drive = listener.driveId;
 
-        const syncUnits = await this.drive.getSynchronizationUnits(drive);
-        const filteredSyncUnits = [];
-        for (const syncUnit of syncUnits) {
-            if (this._checkFilter(listener.filter, syncUnit)) {
-                filteredSyncUnits.push(syncUnit);
-            }
-        }
-
         if (!this.listenerState.has(drive)) {
             this.listenerState.set(drive, new Map());
         }
@@ -50,11 +70,7 @@ export class ListenerManager extends BaseListenerManager {
             pendingTimeout: '0',
             listener,
             listenerStatus: 'CREATED',
-            syncUnits: filteredSyncUnits.map(e => ({
-                ...e,
-                listenerRev: -1,
-                syncRev: e.revision
-            }))
+            syncUnits: new Map()
         });
 
         let transmitter: ITransmitter | undefined;
@@ -101,17 +117,28 @@ export class ListenerManager extends BaseListenerManager {
         return driveMap.delete(listenerId);
     }
 
-    async updateSynchronizationRevision(
+    async removeSyncUnits(driveId: string, syncUnits: SynchronizationUnit[]) {
+        const listeners = this.listenerState.get(driveId);
+        if (!listeners) {
+            return;
+        }
+        for (const [, listener] of listeners) {
+            for (const syncUnit of syncUnits) {
+                listener.syncUnits.delete(syncUnit.syncId);
+            }
+        }
+    }
+
+    async updateSynchronizationRevisions(
         driveId: string,
-        syncId: string,
-        syncRev: number,
-        lastUpdated: string,
+        syncUnits: SynchronizationUnit[],
         willUpdate?: (listeners: Listener[]) => void,
         onError?: (
             error: Error,
             driveId: string,
             listener: ListenerState
-        ) => void
+        ) => void,
+        forceSync = false
     ) {
         const drive = this.listenerState.get(driveId);
         if (!drive) {
@@ -120,100 +147,37 @@ export class ListenerManager extends BaseListenerManager {
 
         const outdatedListeners: Listener[] = [];
         for (const [, listener] of drive) {
-            const syncUnits = listener.syncUnits.filter(
-                e => e.syncId === syncId
-            );
-            if (listener.driveId !== driveId) {
+            if (
+                outdatedListeners.find(
+                    l => l.listenerId === listener.listener.listenerId
+                )
+            ) {
                 continue;
             }
-
             for (const syncUnit of syncUnits) {
-                if (syncUnit.syncId !== syncId) {
+                if (!this._checkFilter(listener.listener.filter, syncUnit)) {
                     continue;
                 }
 
-                syncUnit.syncRev = syncRev;
-                syncUnit.lastUpdated = lastUpdated;
+                const listenerRev = listener.syncUnits.get(syncUnit.syncId);
+
                 if (
-                    !outdatedListeners.find(
-                        l => l.listenerId === listener.listener.listenerId
-                    )
+                    !listenerRev ||
+                    listenerRev.listenerRev < syncUnit.revision
                 ) {
                     outdatedListeners.push(listener.listener);
+                    break;
                 }
             }
         }
 
         if (outdatedListeners.length) {
             willUpdate?.(outdatedListeners);
-            return this.triggerUpdate(onError);
+            return this.triggerUpdate(forceSync, onError);
         }
         return [];
     }
 
-    async addSyncUnits(syncUnits: SynchronizationUnit[]) {
-        for (const [driveId, drive] of this.listenerState) {
-            for (const [id, listenerState] of drive) {
-                const transmitter = await this.getTransmitter(driveId, id);
-                if (!transmitter) {
-                    continue;
-                }
-                const filteredSyncUnits = [];
-                const { listener } = listenerState;
-                for (const syncUnit of syncUnits) {
-                    if (!this._checkFilter(listener.filter, syncUnit)) {
-                        continue;
-                    }
-                    const existingSyncUnit = listenerState.syncUnits.find(
-                        unit => unit.syncId === syncUnit.syncId
-                    );
-                    if (existingSyncUnit) {
-                        existingSyncUnit.syncRev = syncUnit.revision;
-                        existingSyncUnit.lastUpdated = syncUnit.lastUpdated;
-                    } else {
-                        filteredSyncUnits.push(syncUnit);
-                    }
-                }
-
-                // TODO is this possible?
-                if (!this.listenerState.has(driveId)) {
-                    this.listenerState.set(driveId, new Map());
-                }
-
-                const driveMap = this.listenerState.get(driveId)!;
-
-                // TODO reuse existing state
-                driveMap.set(listener.listenerId, {
-                    block: listener.block,
-                    driveId: listener.driveId,
-                    pendingTimeout: '0',
-                    listener,
-                    listenerStatus: 'CREATED',
-                    syncUnits: listenerState.syncUnits.concat(
-                        filteredSyncUnits.map(e => ({
-                            ...e,
-                            listenerRev: -1,
-                            syncRev: e.revision
-                        }))
-                    )
-                });
-            }
-        }
-    }
-
-    removeSyncUnits(syncUnits: SynchronizationUnit[]) {
-        for (const [driveId, drive] of this.listenerState) {
-            const syncIds = syncUnits
-                .filter(s => s.driveId === driveId)
-                .map(s => s.syncId);
-            for (const [, listenerState] of drive) {
-                listenerState.syncUnits = listenerState.syncUnits.filter(
-                    s => !syncIds.includes(s.syncId)
-                );
-            }
-        }
-    }
-
     async updateListenerRevision(
         listenerId: string,
         driveId: string,
@@ -230,14 +194,22 @@ export class ListenerManager extends BaseListenerManager {
             return;
         }
 
-        const entry = listener.syncUnits.find(s => s.syncId === syncId);
+        const lastUpdated = new Date().toISOString();
+        const entry = listener.syncUnits.get(syncId);
         if (entry) {
             entry.listenerRev = listenerRev;
-            entry.lastUpdated = new Date().toISOString();
+            entry.lastUpdated = lastUpdated;
+        } else {
+            listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
         }
     }
 
-    async triggerUpdate(
+    triggerUpdate = debounce(
+        this._triggerUpdate.bind(this),
+        ListenerManager.LISTENER_UPDATE_DELAY
+    );
+
+    private async _triggerUpdate(
         onError?: (
             error: Error,
             driveId: string,
@@ -252,18 +224,19 @@ export class ListenerManager extends BaseListenerManager {
                     continue;
                 }
 
+                const syncUnits = await this.getListenerSyncUnits(
+                    driveId,
+                    listener.listener.listenerId
+                );
+
                 const strandUpdates: StrandUpdate[] = [];
-                for (const unit of listener.syncUnits) {
-                    const {
-                        syncRev,
-                        syncId,
-                        listenerRev,
-                        driveId,
-                        documentId,
-                        scope,
-                        branch
-                    } = unit;
-                    if (listenerRev >= syncRev) {
+                for (const syncUnit of syncUnits) {
+                    const unitState = listener.syncUnits.get(syncUnit.syncId);
+
+                    if (
+                        unitState &&
+                        unitState.listenerRev >= syncUnit.revision
+                    ) {
                         continue;
                     }
 
@@ -271,9 +244,9 @@ export class ListenerManager extends BaseListenerManager {
                     try {
                         const data = await this.drive.getOperationData(
                             driveId,
-                            syncId,
+                            syncUnit.syncId,
                             {
-                                fromRevision: listenerRev
+                                fromRevision: unitState?.listenerRev
                             }
                         );
                         opData.push(...data);
@@ -287,10 +260,10 @@ export class ListenerManager extends BaseListenerManager {
 
                     strandUpdates.push({
                         driveId,
-                        documentId,
-                        branch,
+                        documentId: syncUnit.documentId,
+                        branch: syncUnit.branch,
                         operations: opData,
-                        scope: scope as OperationScope
+                        scope: syncUnit.scope as OperationScope
                     });
                 }
 
@@ -311,15 +284,25 @@ export class ListenerManager extends BaseListenerManager {
                     listener.pendingTimeout = '0';
                     listener.listenerStatus = 'PENDING';
 
-                    for (const unit of listener.syncUnits) {
-                        const revision = listenerRevisions.find(
-                            e =>
-                                e.documentId === unit.documentId &&
-                                e.scope === unit.scope &&
-                                e.branch === unit.branch
+                    const lastUpdated = new Date().toISOString();
+
+                    for (const revision of listenerRevisions) {
+                        const syncUnit = syncUnits.find(
+                            unit =>
+                                revision.documentId === unit.documentId &&
+                                revision.scope === unit.scope &&
+                                revision.branch === unit.branch
                         );
-                        if (revision) {
-                            unit.listenerRev = revision.revision;
+                        if (syncUnit) {
+                            listener.syncUnits.set(syncUnit.syncId, {
+                                lastUpdated,
+                                listenerRev: revision.revision
+                            });
+                        } else {
+                            console.warn(
+                                `Received revision for untracked unit for listener ${listener.listener.listenerId}`,
+                                revision
+                            );
                         }
                     }
                     const revisionError = listenerRevisions.find(
@@ -328,7 +311,9 @@ export class ListenerManager extends BaseListenerManager {
                     if (revisionError) {
                         throw new OperationError(
                             revisionError.status as ErrorStatus,
-                            undefined
+                            undefined,
+                            revisionError.error,
+                            revisionError.error
                         );
                     }
                     listener.listenerStatus = 'SUCCESS';
@@ -372,6 +357,21 @@ export class ListenerManager extends BaseListenerManager {
         return false;
     }
 
+    getListenerSyncUnits(driveId: string, listenerId: string) {
+        const listener = this.listenerState.get(driveId)?.get(listenerId);
+        if (!listener) {
+            return [];
+        }
+        const filter = listener.listener.filter;
+        return this.drive.getSynchronizationUnits(
+            driveId,
+            filter.documentId ?? ['*'],
+            filter.scope ?? ['*'],
+            filter.branch ?? ['*'],
+            filter.documentType ?? ['*']
+        );
+    }
+
     async initDrive(drive: DocumentDriveDocument) {
         const {
             state: {
@@ -411,23 +411,26 @@ export class ListenerManager extends BaseListenerManager {
         since?: string
     ): Promise<StrandUpdate[]> {
         // fetch listenerState from listenerManager
-        const entries = await this.getListener(driveId, listenerId);
+        const listener = await this.getListener(driveId, listenerId);
 
         // fetch operations from drive  and prepare strands
         const strands: StrandUpdate[] = [];
 
-        for (const entry of entries.syncUnits) {
-            if (entry.listenerRev >= entry.syncRev) {
+        const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
+
+        for (const syncUnit of syncUnits) {
+            const entry = listener.syncUnits.get(syncUnit.syncId);
+            if (entry && entry.listenerRev >= syncUnit.revision) {
                 continue;
             }
 
-            const { documentId, driveId, scope, branch } = entry;
+            const { documentId, driveId, scope, branch } = syncUnit;
             const operations = await this.drive.getOperationData(
-                entry.driveId,
-                entry.syncId,
+                driveId,
+                syncUnit.syncId,
                 {
                     since,
-                    fromRevision: entry.listenerRev
+                    fromRevision: entry?.listenerRev
                 }
             );
 

package/src/server/listener/transmitter/pull-responder.ts

@@ -71,11 +71,13 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
         listenerId: string,
         revisions: ListenerRevision[]
     ): Promise<boolean> {
-        const listener = await this.manager.getListener(driveId, listenerId);
-
+        const syncUnits = await this.manager.getListenerSyncUnits(
+            driveId,
+            listenerId
+        );
         let success = true;
         for (const revision of revisions) {
-            const syncUnit = listener.syncUnits.find(
+            const syncUnit = syncUnits.find(
                 s =>
                     s.scope === revision.scope &&
                     s.branch === revision.branch &&
@@ -83,7 +85,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
                     s.documentId == revision.documentId
             );
             if (!syncUnit) {
-                console.log('Sync unit not found', revision);
+                console.warn('Unknown sync unit was acknowledged', revision);
                 success = false;
                 continue;
             }
@@ -254,11 +256,6 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
                         : 'SUCCESS',
                     error
                 });
-
-                // TODO: Should try to parse remaining strands?
-                // if (error) {
-                //     break;
-                // }
             }
 
             onRevisions?.(listenerRevisions);

package/src/server/listener/transmitter/switchboard-push.ts

@@ -35,6 +35,7 @@ export class SwitchboardPushTransmitter implements ITransmitter {
                             branch
                             status
                             revision
+                            error
                         }
                     }
                 `,

package/src/server/types.ts

@@ -92,9 +92,12 @@ export type ListenerRevision = {
     branch: string;
     status: UpdateStatus;
     revision: number;
+    error?: string;
 };
 
-export type ListenerRevisionWithError = ListenerRevision & { error?: Error };
+export type ListenerRevisionWithError = Omit<ListenerRevision, 'error'> & {
+    error?: Error;
+};
 
 export type ListenerUpdate = {
     listenerId: string;
@@ -163,12 +166,14 @@ export abstract class BaseDocumentDriveServer {
     abstract addOperation(
         drive: string,
         id: string,
-        operation: Operation
+        operation: Operation,
+        forceSync?: boolean
     ): Promise<IOperationResult>;
     abstract addOperations(
         drive: string,
         id: string,
-        operations: Operation[]
+        operations: Operation[],
+        forceSync?: boolean
     ): Promise<IOperationResult>;
 
     abstract addDriveOperation(
@@ -207,7 +212,8 @@ export abstract class BaseDocumentDriveServer {
         driveId: string,
         documentId?: string[],
         scope?: string[],
-        branch?: string[]
+        branch?: string[],
+        documentType?: string[]
     ): Promise<SynchronizationUnit[]>;
 
     abstract getSynchronizationUnit(
@@ -289,11 +295,9 @@ export abstract class BaseListenerManager {
         since?: string
     ): Promise<StrandUpdate[]>;
 
-    abstract updateSynchronizationRevision(
+    abstract updateSynchronizationRevisions(
         driveId: string,
-        syncId: string,
-        syncRev: number,
-        lastUpdated: string,
+        syncUnits: SynchronizationUnit[],
         willUpdate?: (listeners: Listener[]) => void,
         onError?: (
             error: Error,
@@ -328,11 +332,11 @@ export interface ListenerState {
     block: boolean;
     pendingTimeout: string;
     listener: Listener;
-    syncUnits: SyncronizationUnitState[];
+    syncUnits: Map<SynchronizationUnit['syncId'], SyncronizationUnitState>;
     listenerStatus: ListenerStatus;
 }
 
-export interface SyncronizationUnitState extends SynchronizationUnit {
+export interface SyncronizationUnitState {
     listenerRev: number;
-    syncRev: number;
+    lastUpdated: string;
 }