document-drive 0.0.30 → 1.0.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "0.0.30",
+    "version": "1.0.0-alpha.2",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",
@@ -24,6 +24,7 @@
         "lint": "eslint src --ext .js,.jsx,.ts,.tsx && yarn check-types",
         "lint:fix": "eslint src --ext .js,.jsx,.ts,.tsx --fix",
         "format": "prettier . --write",
+        "release": "semantic-release",
         "test": "vitest run --coverage",
         "test:watch": "vitest watch"
     },
@@ -43,14 +44,18 @@
         "sanitize-filename": "^1.6.3"
     },
     "devDependencies": {
+        "@commitlint/cli": "^18.6.1",
+        "@commitlint/config-conventional": "^18.6.2",
         "@prisma/client": "5.8.1",
+        "@semantic-release/changelog": "^6.0.3",
+        "@semantic-release/git": "^10.0.1",
         "@total-typescript/ts-reset": "^0.5.1",
         "@types/node": "^20.11.16",
         "@typescript-eslint/eslint-plugin": "^6.18.1",
         "@typescript-eslint/parser": "^6.18.1",
         "@vitest/coverage-v8": "^0.34.6",
         "document-model": "^1.0.29",
-        "document-model-libs": "^1.1.44",
+        "document-model-libs": "^1.1.48",
         "eslint": "^8.56.0",
         "eslint-config-prettier": "^9.1.0",
         "fake-indexeddb": "^5.0.1",
@@ -58,6 +63,7 @@
         "msw": "^2.1.2",
         "prettier": "^3.1.1",
         "prettier-plugin-organize-imports": "^3.2.4",
+        "semantic-release": "^23.0.2",
         "sequelize": "^6.35.2",
         "sqlite3": "^5.1.7",
         "typescript": "^5.3.2",

package/src/server/index.ts

@@ -23,8 +23,11 @@ import { generateUUID, isDocumentDrive, isNoopUpdate } from '../utils';
 import { requestPublicDrive } from '../utils/graphql';
 import { OperationError } from './error';
 import { ListenerManager } from './listener/manager';
-import { PullResponderTransmitter } from './listener/transmitter';
-import type { ITransmitter } from './listener/transmitter/types';
+import {
+    CancelPullLoop,
+    ITransmitter,
+    PullResponderTransmitter
+} from './listener/transmitter';
 import {
     BaseDocumentDriveServer,
     DriveEvents,
@@ -52,7 +55,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
     private listenerStateManager: ListenerManager;
     private triggerMap = new Map<
         DocumentDriveState['id'],
-        Map<Trigger['id'], number>
+        Map<Trigger['id'], CancelPullLoop>
     >();
     private syncStatus = new Map<DocumentDriveState['id'], SyncStatus>();
 
@@ -100,7 +103,9 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                   operations
               ));
 
-        this.updateSyncStatus(strand.driveId, result.status);
+        if (result.status === 'ERROR') {
+            this.updateSyncStatus(strand.driveId, result.status, result.error);
+        }
         this.emit('strandUpdate', strand);
         return result;
     }
@@ -138,11 +143,11 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
             if (!driveTriggers) {
                 driveTriggers = new Map();
-                this.updateSyncStatus(driveId, 'SYNCING');
             }
 
+            this.updateSyncStatus(driveId, 'SYNCING');
             if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
-                const intervalId = PullResponderTransmitter.setupPull(
+                const cancelPullLoop = PullResponderTransmitter.setupPull(
                     driveId,
                     trigger,
                     this.saveStrand.bind(this),
@@ -151,12 +156,20 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                             driveId,
                             error instanceof OperationError
                                 ? error.status
-                                : 'ERROR'
+                                : 'ERROR',
+                            error
                         );
                     },
-                    acknowledgeSuccess => {}
+                    revisions => {
+                        const errorRevision = revisions.find(
+                            r => r.status !== 'SUCCESS'
+                        );
+                        if (!errorRevision) {
+                            this.updateSyncStatus(driveId, 'SUCCESS');
+                        }
+                    }
                 );
-                driveTriggers.set(trigger.id, intervalId);
+                driveTriggers.set(trigger.id, cancelPullLoop);
                 this.triggerMap.set(driveId, driveTriggers);
             }
         }
@@ -164,7 +177,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
     private async stopSyncRemoteDrive(driveId: string) {
         const triggers = this.triggerMap.get(driveId);
-        triggers?.forEach(clearInterval);
+        triggers?.forEach(cancel => cancel());
         return this.triggerMap.delete(driveId);
     }
 
@@ -607,9 +620,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 );
                 continue;
             } else if (op.index < nextIndex) {
-                const existingOperation = scopeOperations.find(
-                    existingOperation => existingOperation.index === op.index
-                );
+                const existingOperation = scopeOperations
+                    .concat(pastOperations)
+                    .find(
+                        existingOperation =>
+                            existingOperation.index === op.index
+                    );
                 if (existingOperation && existingOperation.hash !== op.hash) {
                     error = new OperationError(
                         'CONFLICT',
@@ -786,8 +802,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                         syncUnit.syncId,
                         syncUnit.revision,
                         syncUnit.lastUpdated,
+                        () => this.updateSyncStatus(drive, 'SYNCING'),
                         this.handleListenerError.bind(this)
                     )
+                    .then(
+                        updates =>
+                            updates.length &&
+                            this.updateSyncStatus(drive, 'SUCCESS')
+                    )
                     .catch(error => {
                         console.error(
                             'Non handled error updating sync revision',
@@ -917,8 +939,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                         '0',
                         lastOperation.index,
                         lastOperation.timestamp,
+                        () => this.updateSyncStatus(drive, 'SYNCING'),
                         this.handleListenerError.bind(this)
                     )
+                    .then(
+                        updates =>
+                            updates.length &&
+                            this.updateSyncStatus(drive, 'SUCCESS')
+                    )
                     .catch(error => {
                         console.error(
                             'Non handled error updating sync revision',
@@ -974,6 +1002,13 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         return this.listenerStateManager.getTransmitter(driveId, listenerId);
     }
 
+    getListener(
+        driveId: string,
+        listenerId: string
+    ): Promise<ListenerState | undefined> {
+        return this.listenerStateManager.getListener(driveId, listenerId);
+    }
+
     getSyncStatus(drive: string): SyncStatus {
         const status = this.syncStatus.get(drive);
         if (!status) {

package/src/server/listener/decorator.ts

@@ -0,0 +1,28 @@
+import { DocumentDriveServer } from '..';
+
+function ListenerManagerDecorator(constructor: new () => DocumentDriveServer) {
+    return class extends constructor {
+        // Define extra methods here
+        extraMethod(): void {
+            // Access private variables of the original class
+            console.log('Accessing private variable:', this.getLi);
+        }
+    };
+}
+
+// Define your original class
+class OriginalClass {
+    private privateVariable: string;
+
+    constructor(privateVariable: string) {
+        this.privateVariable = privateVariable;
+    }
+
+    // Define other methods and properties here
+}
+
+// Use the decorator to augment the original class with extra methods
+const AugmentedClass = ExtraMethodsDecorator(OriginalClass);
+
+// Create an instance of the augmented class
+const instance = new AugmentedClass('private data');

package/src/server/listener/manager.ts

@@ -9,6 +9,7 @@ import {
     ErrorStatus,
     Listener,
     ListenerState,
+    ListenerUpdate,
     OperationUpdate,
     StrandUpdate,
     SynchronizationUnit
@@ -122,6 +123,7 @@ export class ListenerManager extends BaseListenerManager {
         syncId: string,
         syncRev: number,
         lastUpdated: string,
+        willUpdate?: (listeners: Listener[]) => void,
         onError?: (
             error: Error,
             driveId: string,
@@ -130,10 +132,10 @@ export class ListenerManager extends BaseListenerManager {
     ) {
         const drive = this.listenerState.get(driveId);
         if (!drive) {
-            return;
+            return [];
         }
 
-        let newRevision = false;
+        const outdatedListeners: Listener[] = [];
         for (const [, listener] of drive) {
             const syncUnits = listener.syncUnits.filter(
                 e => e.syncId === syncId
@@ -149,13 +151,21 @@ export class ListenerManager extends BaseListenerManager {
 
                 syncUnit.syncRev = syncRev;
                 syncUnit.lastUpdated = lastUpdated;
-                newRevision = true;
+                if (
+                    !outdatedListeners.find(
+                        l => l.listenerId === listener.listener.listenerId
+                    )
+                ) {
+                    outdatedListeners.push(listener.listener);
+                }
             }
         }
 
-        if (newRevision) {
+        if (outdatedListeners.length) {
+            willUpdate?.(outdatedListeners);
             return this.triggerUpdate(onError);
         }
+        return [];
     }
 
     async addSyncUnits(syncUnits: SynchronizationUnit[]) {
@@ -251,6 +261,7 @@ export class ListenerManager extends BaseListenerManager {
             listener: ListenerState
         ) => void
     ) {
+        const listenerUpdates: ListenerUpdate[] = [];
         for (const [driveId, drive] of this.listenerState) {
             for (const [id, listener] of drive) {
                 const transmitter = await this.getTransmitter(driveId, id);
@@ -338,6 +349,10 @@ export class ListenerManager extends BaseListenerManager {
                         );
                     }
                     listener.listenerStatus = 'SUCCESS';
+                    listenerUpdates.push({
+                        listenerId: listener.listener.listenerId,
+                        listenerRevisions
+                    });
                 } catch (e) {
                     // TODO: Handle error based on listener params (blocking, retry, etc)
                     onError?.(e as Error, driveId, listener);
@@ -346,6 +361,7 @@ export class ListenerManager extends BaseListenerManager {
                 }
             }
         }
+        return listenerUpdates;
     }
 
     private _checkFilter(
@@ -403,11 +419,53 @@ export class ListenerManager extends BaseListenerManager {
         }
     }
 
-    getListener(driveId: string, listenerId: string): ListenerState {
+    getListener(driveId: string, listenerId: string): Promise<ListenerState> {
         const drive = this.listenerState.get(driveId);
         if (!drive) throw new Error('Drive not found');
         const listener = drive.get(listenerId);
         if (!listener) throw new Error('Listener not found');
-        return listener;
+        return Promise.resolve(listener);
+    }
+
+    async getStrands(
+        driveId: string,
+        listenerId: string,
+        since?: string
+    ): Promise<StrandUpdate[]> {
+        // fetch listenerState from listenerManager
+        const entries = await this.getListener(driveId, listenerId);
+
+        // fetch operations from drive  and prepare strands
+        const strands: StrandUpdate[] = [];
+
+        for (const entry of entries.syncUnits) {
+            if (entry.listenerRev >= entry.syncRev) {
+                continue;
+            }
+
+            const { documentId, driveId, scope, branch } = entry;
+            const operations = await this.drive.getOperationData(
+                entry.driveId,
+                entry.syncId,
+                {
+                    since,
+                    fromRevision: entry.listenerRev
+                }
+            );
+
+            if (!operations.length) {
+                continue;
+            }
+
+            strands.push({
+                driveId,
+                documentId,
+                scope: scope as OperationScope,
+                branch,
+                operations
+            });
+        }
+
+        return strands;
     }
 }

package/src/server/listener/transmitter/pull-responder.ts

@@ -8,6 +8,7 @@ import {
     IOperationResult,
     Listener,
     ListenerRevision,
+    ListenerRevisionWithError,
     OperationUpdate,
     StrandUpdate
 } from '../../types';
@@ -26,11 +27,17 @@ export type PullStrandsGraphQL = {
     };
 };
 
+export type CancelPullLoop = () => void;
+
 export type StrandUpdateGraphQL = Omit<StrandUpdate, 'operations'> & {
     operations: OperationUpdateGraphQL[];
 };
 
-export class PullResponderTransmitter implements ITransmitter {
+export interface IPullResponderTransmitter extends ITransmitter {
+    getStrands(since?: string): Promise<StrandUpdate[]>;
+}
+
+export class PullResponderTransmitter implements IPullResponderTransmitter {
     private drive: BaseDocumentDriveServer;
     private listener: Listener;
     private manager: ListenerManager;
@@ -49,48 +56,12 @@ export class PullResponderTransmitter implements ITransmitter {
         return [];
     }
 
-    async getStrands(
-        listenerId: string,
-        since?: string
-    ): Promise<StrandUpdate[]> {
-        // fetch listenerState from listenerManager
-        const entries = this.manager.getListener(
+    getStrands(since?: string | undefined): Promise<StrandUpdate[]> {
+        return this.manager.getStrands(
             this.listener.driveId,
-            listenerId
+            this.listener.listenerId,
+            since
         );
-
-        // fetch operations from drive  and prepare strands
-        const strands: StrandUpdate[] = [];
-
-        for (const entry of entries.syncUnits) {
-            if (entry.listenerRev >= entry.syncRev) {
-                continue;
-            }
-
-            const { documentId, driveId, scope, branch } = entry;
-            const operations = await this.drive.getOperationData(
-                entry.driveId,
-                entry.syncId,
-                {
-                    since,
-                    fromRevision: entry.listenerRev
-                }
-            );
-
-            if (!operations.length) {
-                continue;
-            }
-
-            strands.push({
-                driveId,
-                documentId,
-                scope: scope as OperationScope,
-                branch,
-                operations
-            });
-        }
-
-        return strands;
     }
 
     async processAcknowledge(
@@ -98,7 +69,7 @@ export class PullResponderTransmitter implements ITransmitter {
         listenerId: string,
         revisions: ListenerRevision[]
     ): Promise<boolean> {
-        const listener = this.manager.getListener(driveId, listenerId);
+        const listener = await this.manager.getListener(driveId, listenerId);
 
         let success = true;
         for (const revision of revisions) {
@@ -218,14 +189,102 @@ export class PullResponderTransmitter implements ITransmitter {
         return result.acknowledge;
     }
 
+    private static async executePull(
+        driveId: string,
+        trigger: PullResponderTrigger,
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
+        onAcknowledge?: (success: boolean) => void
+    ) {
+        try {
+            const { url, listenerId } = trigger.data;
+            const strands = await PullResponderTransmitter.pullStrands(
+                driveId,
+                url,
+                listenerId
+                // since ?
+            );
+
+            // if there are no new strands then do nothing
+            if (!strands.length) {
+                return;
+            }
+
+            const listenerRevisions: ListenerRevisionWithError[] = [];
+
+            for (const strand of strands) {
+                const operations: Operation[] = strand.operations.map(
+                    ({ index, type, hash, input, skip, timestamp }) => ({
+                        index,
+                        type,
+                        hash,
+                        input,
+                        skip,
+                        timestamp,
+                        scope: strand.scope,
+                        branch: strand.branch
+                    })
+                );
+
+                let error: Error | undefined = undefined;
+                try {
+                    const result = await onStrandUpdate(strand);
+                    if (result.error) {
+                        throw result.error;
+                    }
+                } catch (e) {
+                    error = e as Error;
+                    onError(error);
+                }
+
+                listenerRevisions.push({
+                    branch: strand.branch,
+                    documentId: strand.documentId || '',
+                    driveId: strand.driveId,
+                    revision: operations.pop()?.index ?? -1,
+                    scope: strand.scope as OperationScope,
+                    status: error
+                        ? error instanceof OperationError
+                            ? error.status
+                            : 'ERROR'
+                        : 'SUCCESS',
+                    error
+                });
+
+                // TODO: Should try to parse remaining strands?
+                // if (error) {
+                //     break;
+                // }
+            }
+
+            onRevisions?.(listenerRevisions);
+
+            await PullResponderTransmitter.acknowledgeStrands(
+                driveId,
+                url,
+                listenerId,
+                listenerRevisions.map(revision => {
+                    const { error, ...rest } = revision;
+                    return rest;
+                })
+            )
+                .then(result => onAcknowledge?.(result))
+                .catch(error => console.error('ACK error', error));
+        } catch (error) {
+            onError(error as Error);
+        }
+    }
+
     static setupPull(
         driveId: string,
         trigger: PullResponderTrigger,
         onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
         onError: (error: Error) => void,
+        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
         onAcknowledge?: (success: boolean) => void
-    ): number {
-        const { url, listenerId, interval } = trigger.data;
+    ): CancelPullLoop {
+        const { interval } = trigger.data;
         let loopInterval = PULL_DRIVE_INTERVAL;
         if (interval) {
             try {
@@ -238,80 +297,36 @@ export class PullResponderTransmitter implements ITransmitter {
             }
         }
 
-        const timeout = setInterval(async () => {
-            try {
-                const strands = await PullResponderTransmitter.pullStrands(
+        let isCancelled = false;
+        let timeout: number | undefined;
+
+        const executeLoop = async () => {
+            while (!isCancelled) {
+                await this.executePull(
                     driveId,
-                    url,
-                    listenerId
-                    // since ?
+                    trigger,
+                    onStrandUpdate,
+                    onError,
+                    onRevisions,
+                    onAcknowledge
                 );
+                await new Promise(resolve => {
+                    timeout = setTimeout(
+                        resolve,
+                        loopInterval
+                    ) as unknown as number;
+                });
+            }
+        };
 
-                // if there are no new strands then do nothing
-                if (!strands.length) {
-                    return;
-                }
-
-                const listenerRevisions: ListenerRevision[] = [];
-
-                for (const strand of strands) {
-                    const operations: Operation[] = strand.operations.map(
-                        ({ index, type, hash, input, skip, timestamp }) => ({
-                            index,
-                            type,
-                            hash,
-                            input,
-                            skip,
-                            timestamp,
-                            scope: strand.scope,
-                            branch: strand.branch
-                        })
-                    );
-
-                    let error: Error | undefined = undefined;
-
-                    try {
-                        const result = await onStrandUpdate(strand);
-                        if (result.error) {
-                            throw result.error;
-                        }
-                    } catch (e) {
-                        error = e as Error;
-                        onError?.(error);
-                    }
-
-                    listenerRevisions.push({
-                        branch: strand.branch,
-                        documentId: strand.documentId ?? '',
-                        driveId: strand.driveId,
-                        revision: operations.pop()?.index ?? -1,
-                        scope: strand.scope as OperationScope,
-                        status: error
-                            ? error instanceof OperationError
-                                ? error.status
-                                : 'ERROR'
-                            : 'SUCCESS'
-                    });
+        executeLoop().catch(console.error);
 
-                    // TODO: Should try to parse remaining strands?
-                    if (error) {
-                        break;
-                    }
-                }
-
-                await PullResponderTransmitter.acknowledgeStrands(
-                    driveId,
-                    url,
-                    listenerId,
-                    listenerRevisions
-                )
-                    .then(result => onAcknowledge?.(result))
-                    .catch(error => console.error('ACK error', error));
-            } catch (error) {
-                onError(error as Error);
+        return () => {
+            isCancelled = true;
+            if (timeout !== undefined) {
+                clearTimeout(timeout);
             }
-        }, loopInterval);
-        return timeout as unknown as number;
+        };
     }
 
     static isPullResponderTrigger(

package/src/server/types.ts

@@ -93,6 +93,13 @@ export type ListenerRevision = {
     revision: number;
 };
 
+export type ListenerRevisionWithError = ListenerRevision & { error?: Error };
+
+export type ListenerUpdate = {
+    listenerId: string;
+    listenerRevisions: ListenerRevision[];
+};
+
 export type UpdateStatus = 'SUCCESS' | 'CONFLICT' | 'MISSING' | 'ERROR';
 export type ErrorStatus = Exclude<UpdateStatus, 'SUCCESS'>;
 
@@ -185,11 +192,6 @@ export abstract class BaseDocumentDriveServer {
     ): Promise<Document>;
     protected abstract deleteDocument(drive: string, id: string): Promise<void>;
 
-    abstract getTransmitter(
-        driveId: string,
-        listenerId: string
-    ): Promise<ITransmitter | undefined>;
-
     /** Event methods **/
     protected abstract emit<K extends keyof DriveEvents>(
         this: this,
@@ -201,6 +203,11 @@ export abstract class BaseDocumentDriveServer {
         event: K,
         cb: DriveEvents[K]
     ): Unsubscribe;
+
+    abstract getTransmitter(
+        driveId: string,
+        listenerId: string
+    ): Promise<ITransmitter | undefined>;
 }
 
 export abstract class BaseListenerManager {
@@ -220,21 +227,39 @@ export abstract class BaseListenerManager {
     }
 
     abstract init(): Promise<void>;
+
     abstract addListener(listener: Listener): Promise<ITransmitter>;
     abstract removeListener(
         driveId: string,
         listenerId: string
     ): Promise<boolean>;
+    abstract getListener(
+        driveId: string,
+        listenerId: string
+    ): Promise<ListenerState | undefined>;
+
     abstract getTransmitter(
         driveId: string,
         listenerId: string
     ): Promise<ITransmitter | undefined>;
+
+    abstract getStrands(
+        listenerId: string,
+        since?: string
+    ): Promise<StrandUpdate[]>;
+
     abstract updateSynchronizationRevision(
         driveId: string,
         syncId: string,
         syncRev: number,
-        lastUpdated: string
-    ): Promise<void>;
+        lastUpdated: string,
+        willUpdate?: (listeners: Listener[]) => void,
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void
+    ): Promise<ListenerUpdate[]>;
 
     abstract updateListenerRevision(
         listenerId: string,

package/src/storage/prisma.ts

@@ -119,28 +119,6 @@ export class PrismaStorage implements IDriveStorage {
         } catch (e) {
             console.log(e);
         }
-
-        await this.db.document.upsert({
-            where: {
-                id_driveId: {
-                    id: 'drives',
-                    driveId: id
-                }
-            },
-            create: {
-                id: 'drives',
-                driveId: id,
-                documentType: header.documentType,
-                initialState: document.initialState,
-                lastModified: header.lastModified,
-                revision: header.revision,
-                created: header.created
-            },
-            update: {
-                lastModified: header.lastModified,
-                revision: header.revision
-            }
-        });
     }
 
     async getDocuments(drive: string) {