document-drive 0.0.29 → 1.0.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "0.0.29",
+    "version": "1.0.0-alpha.1",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",
@@ -24,12 +24,13 @@
         "lint": "eslint src --ext .js,.jsx,.ts,.tsx && yarn check-types",
         "lint:fix": "eslint src --ext .js,.jsx,.ts,.tsx --fix",
         "format": "prettier . --write",
+        "release": "semantic-release",
         "test": "vitest run --coverage",
         "test:watch": "vitest watch"
     },
     "peerDependencies": {
         "@prisma/client": "5.8.1",
-        "document-model": "^1.0.28",
+        "document-model": "^1.0.29",
         "document-model-libs": "^1.1.44",
         "localforage": "^1.10.0",
         "sequelize": "^6.35.2",
@@ -39,16 +40,21 @@
         "graphql": "^16.8.1",
         "graphql-request": "^6.1.0",
         "json-stringify-deterministic": "^1.0.12",
+        "nanoevents": "^9.0.0",
         "sanitize-filename": "^1.6.3"
     },
     "devDependencies": {
+        "@commitlint/cli": "^18.6.1",
+        "@commitlint/config-conventional": "^18.6.2",
         "@prisma/client": "5.8.1",
+        "@semantic-release/changelog": "^6.0.3",
+        "@semantic-release/git": "^10.0.1",
         "@total-typescript/ts-reset": "^0.5.1",
         "@types/node": "^20.11.16",
         "@typescript-eslint/eslint-plugin": "^6.18.1",
         "@typescript-eslint/parser": "^6.18.1",
         "@vitest/coverage-v8": "^0.34.6",
-        "document-model": "^1.0.28",
+        "document-model": "^1.0.29",
         "document-model-libs": "^1.1.44",
         "eslint": "^8.56.0",
         "eslint-config-prettier": "^9.1.0",
@@ -57,6 +63,7 @@
         "msw": "^2.1.2",
         "prettier": "^3.1.1",
         "prettier-plugin-organize-imports": "^3.2.4",
+        "semantic-release": "^23.0.2",
         "sequelize": "^6.35.2",
         "sqlite3": "^5.1.7",
         "typescript": "^5.3.2",

package/src/server/index.ts

@@ -3,19 +3,20 @@ import {
     DocumentDriveDocument,
     DocumentDriveState,
     FileNode,
-    Trigger,
     isFileNode,
+    Trigger,
     utils
 } from 'document-model-libs/document-drive';
 import {
     Action,
     BaseAction,
+    utils as baseUtils,
     Document,
     DocumentModel,
     Operation,
-    OperationScope,
-    utils as baseUtils
+    OperationScope
 } from 'document-model/document';
+import { createNanoEvents, Unsubscribe } from 'nanoevents';
 import { MemoryStorage } from '../storage/memory';
 import type { DocumentStorage, IDriveStorage } from '../storage/types';
 import { generateUUID, isDocumentDrive, isNoopUpdate } from '../utils';
@@ -26,7 +27,9 @@ import { PullResponderTransmitter } from './listener/transmitter';
 import type { ITransmitter } from './listener/transmitter/types';
 import {
     BaseDocumentDriveServer,
+    DriveEvents,
     IOperationResult,
+    ListenerState,
     RemoteDriveOptions,
     StrandUpdate,
     SyncStatus,
@@ -43,6 +46,7 @@ export type * from './types';
 export const PULL_DRIVE_INTERVAL = 5000;
 
 export class DocumentDriveServer extends BaseDocumentDriveServer {
+    private emitter = createNanoEvents<DriveEvents>();
     private documentModels: DocumentModel[];
     private storage: IDriveStorage;
     private listenerStateManager: ListenerManager;
@@ -62,6 +66,15 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         this.storage = storage;
     }
 
+    private updateSyncStatus(
+        driveId: string,
+        status: SyncStatus,
+        error?: Error
+    ) {
+        this.syncStatus.set(driveId, status);
+        this.emit('syncStatus', driveId, status, error);
+    }
+
     private async saveStrand(strand: StrandUpdate) {
         const operations: Operation[] = strand.operations.map(
             ({ index, type, hash, input, skip, timestamp }) => ({
@@ -86,10 +99,27 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                   strand.documentId,
                   operations
               ));
-        this.syncStatus.set(strand.driveId, result.status);
+
+        this.updateSyncStatus(strand.driveId, result.status);
+        this.emit('strandUpdate', strand);
         return result;
     }
 
+    private handleListenerError(
+        error: Error,
+        driveId: string,
+        listener: ListenerState
+    ) {
+        console.error(
+            `Listener ${listener.listener.label ?? listener.listener.listenerId} error: ${error.message}`
+        );
+        this.updateSyncStatus(
+            driveId,
+            error instanceof OperationError ? error.status : 'ERROR',
+            error
+        );
+    }
+
     private shouldSyncRemoteDrive(drive: DocumentDriveDocument) {
         return (
             drive.state.local.availableOffline &&
@@ -108,16 +138,16 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
             if (!driveTriggers) {
                 driveTriggers = new Map();
-                this.syncStatus.set(driveId, 'SYNCING');
             }
 
+            this.updateSyncStatus(driveId, 'SYNCING');
             if (PullResponderTransmitter.isPullResponderTrigger(trigger)) {
                 const intervalId = PullResponderTransmitter.setupPull(
                     driveId,
                     trigger,
                     this.saveStrand.bind(this),
                     error => {
-                        this.syncStatus.set(
+                        this.updateSyncStatus(
                             driveId,
                             error instanceof OperationError
                                 ? error.status
@@ -265,7 +295,7 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 (filter.since !== undefined &&
                     filter.since <= operation.timestamp) ||
                 (filter.fromRevision !== undefined &&
-                    operation.index >= filter.fromRevision)
+                    operation.index > filter.fromRevision)
         );
 
         return filteredOperations.map(operation => ({
@@ -456,6 +486,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
     }
 
     async deleteDocument(driveId: string, id: string) {
+        try {
+            const syncUnits = await this.getSynchronizationUnits(driveId, [id]);
+            this.listenerStateManager.removeSyncUnits(syncUnits);
+        } catch {
+            /* empty */
+        }
         return this.storage.deleteDocument(driveId, id);
     }
 
@@ -664,10 +700,10 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             );
         }
 
-        const results = await Promise.all(
-            operationSignals.map(handler => handler())
-        );
-        signalResults.push(...results);
+        for (const signalHandler of operationSignals) {
+            const result = await signalHandler();
+            signalResults.push(result);
+        }
 
         return {
             document: newDocument,
@@ -749,10 +785,21 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                         drive,
                         syncUnit.syncId,
                         syncUnit.revision,
-                        syncUnit.lastUpdated
+                        syncUnit.lastUpdated,
+                        () => this.updateSyncStatus(drive, 'SYNCING'),
+                        this.handleListenerError.bind(this)
+                    )
+                    .then(
+                        updates =>
+                            updates.length &&
+                            this.updateSyncStatus(drive, 'SUCCESS')
                     )
                     .catch(error => {
-                        console.error('Error updating sync revision', error);
+                        console.error(
+                            'Non handled error updating sync revision',
+                            error
+                        );
+                        this.updateSyncStatus(drive, 'ERROR', error as Error);
                     });
             }
 
@@ -875,10 +922,21 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                         drive,
                         '0',
                         lastOperation.index,
-                        lastOperation.timestamp
+                        lastOperation.timestamp,
+                        () => this.updateSyncStatus(drive, 'SYNCING'),
+                        this.handleListenerError.bind(this)
+                    )
+                    .then(
+                        updates =>
+                            updates.length &&
+                            this.updateSyncStatus(drive, 'SUCCESS')
                     )
                     .catch(error => {
-                        console.error('Error updating sync revision', error);
+                        console.error(
+                            'Non handled error updating sync revision',
+                            error
+                        );
+                        this.updateSyncStatus(drive, 'ERROR', error as Error);
                     });
             }
 
@@ -935,4 +993,15 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         }
         return status;
     }
+
+    on<K extends keyof DriveEvents>(event: K, cb: DriveEvents[K]): Unsubscribe {
+        return this.emitter.on(event, cb);
+    }
+
+    protected emit<K extends keyof DriveEvents>(
+        event: K,
+        ...args: Parameters<DriveEvents[K]>
+    ): void {
+        return this.emitter.emit(event, ...args);
+    }
 }

package/src/server/listener/manager.ts

@@ -9,6 +9,8 @@ import {
     ErrorStatus,
     Listener,
     ListenerState,
+    ListenerUpdate,
+    OperationUpdate,
     StrandUpdate,
     SynchronizationUnit
 } from '../types';
@@ -120,14 +122,20 @@ export class ListenerManager extends BaseListenerManager {
         driveId: string,
         syncId: string,
         syncRev: number,
-        lastUpdated: string
+        lastUpdated: string,
+        willUpdate?: (listeners: Listener[]) => void,
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void
     ) {
         const drive = this.listenerState.get(driveId);
         if (!drive) {
-            return;
+            return [];
         }
 
-        let newRevision = false;
+        const outdatedListeners: Listener[] = [];
         for (const [, listener] of drive) {
             const syncUnits = listener.syncUnits.filter(
                 e => e.syncId === syncId
@@ -143,13 +151,21 @@ export class ListenerManager extends BaseListenerManager {
 
                 syncUnit.syncRev = syncRev;
                 syncUnit.lastUpdated = lastUpdated;
-                newRevision = true;
+                if (
+                    !outdatedListeners.find(
+                        l => l.listenerId === listener.listener.listenerId
+                    )
+                ) {
+                    outdatedListeners.push(listener.listener);
+                }
             }
         }
 
-        if (newRevision) {
-            return this.triggerUpdate();
+        if (outdatedListeners.length) {
+            willUpdate?.(outdatedListeners);
+            return this.triggerUpdate(onError);
         }
+        return [];
     }
 
     async addSyncUnits(syncUnits: SynchronizationUnit[]) {
@@ -202,6 +218,19 @@ export class ListenerManager extends BaseListenerManager {
         }
     }
 
+    removeSyncUnits(syncUnits: SynchronizationUnit[]) {
+        for (const [driveId, drive] of this.listenerState) {
+            const syncIds = syncUnits
+                .filter(s => s.driveId === driveId)
+                .map(s => s.syncId);
+            for (const [, listenerState] of drive) {
+                listenerState.syncUnits = listenerState.syncUnits.filter(
+                    s => !syncIds.includes(s.syncId)
+                );
+            }
+        }
+    }
+
     async updateListenerRevision(
         listenerId: string,
         driveId: string,
@@ -225,7 +254,14 @@ export class ListenerManager extends BaseListenerManager {
         }
     }
 
-    async triggerUpdate() {
+    async triggerUpdate(
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void
+    ) {
+        const listenerUpdates: ListenerUpdate[] = [];
         for (const [driveId, drive] of this.listenerState) {
             for (const [id, listener] of drive) {
                 const transmitter = await this.getTransmitter(driveId, id);
@@ -248,13 +284,19 @@ export class ListenerManager extends BaseListenerManager {
                         continue;
                     }
 
-                    const opData = await this.drive.getOperationData(
-                        driveId,
-                        syncId,
-                        {
-                            fromRevision: listenerRev
-                        }
-                    );
+                    const opData: OperationUpdate[] = [];
+                    try {
+                        const data = await this.drive.getOperationData(
+                            driveId,
+                            syncId,
+                            {
+                                fromRevision: listenerRev
+                            }
+                        );
+                        opData.push(...data);
+                    } catch (e) {
+                        console.error(e);
+                    }
 
                     if (!opData.length) {
                         continue;
@@ -307,14 +349,19 @@ export class ListenerManager extends BaseListenerManager {
                         );
                     }
                     listener.listenerStatus = 'SUCCESS';
+                    listenerUpdates.push({
+                        listenerId: listener.listener.listenerId,
+                        listenerRevisions
+                    });
                 } catch (e) {
                     // TODO: Handle error based on listener params (blocking, retry, etc)
+                    onError?.(e as Error, driveId, listener);
                     listener.listenerStatus =
                         e instanceof OperationError ? e.status : 'ERROR';
-                    throw e;
                 }
             }
         }
+        return listenerUpdates;
     }
 
     private _checkFilter(

package/src/server/listener/transmitter/pull-responder.ts

@@ -218,6 +218,87 @@ export class PullResponderTransmitter implements ITransmitter {
         return result.acknowledge;
     }
 
+    private static async executePull(
+        driveId: string,
+        trigger: PullResponderTrigger,
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onAcknowledge?: (success: boolean) => void
+    ) {
+        try {
+            const { url, listenerId } = trigger.data;
+            const strands = await PullResponderTransmitter.pullStrands(
+                driveId,
+                url,
+                listenerId
+                // since ?
+            );
+
+            // if there are no new strands then do nothing
+            if (!strands.length) {
+                return;
+            }
+
+            const listenerRevisions: ListenerRevision[] = [];
+
+            for (const strand of strands) {
+                const operations: Operation[] = strand.operations.map(
+                    ({ index, type, hash, input, skip, timestamp }) => ({
+                        index,
+                        type,
+                        hash,
+                        input,
+                        skip,
+                        timestamp,
+                        scope: strand.scope,
+                        branch: strand.branch
+                    })
+                );
+
+                let error: Error | undefined = undefined;
+
+                try {
+                    const result = await onStrandUpdate(strand);
+                    if (result.error) {
+                        throw result.error;
+                    }
+                } catch (e) {
+                    error = e as Error;
+                    onError(error);
+                }
+
+                listenerRevisions.push({
+                    branch: strand.branch,
+                    documentId: strand.documentId || '',
+                    driveId: strand.driveId,
+                    revision: operations.pop()?.index ?? -1,
+                    scope: strand.scope as OperationScope,
+                    status: error
+                        ? error instanceof OperationError
+                            ? error.status
+                            : 'ERROR'
+                        : 'SUCCESS'
+                });
+
+                // TODO: Should try to parse remaining strands?
+                if (error) {
+                    break;
+                }
+            }
+
+            await PullResponderTransmitter.acknowledgeStrands(
+                driveId,
+                url,
+                listenerId,
+                listenerRevisions
+            )
+                .then(result => onAcknowledge?.(result))
+                .catch(error => console.error('ACK error', error));
+        } catch (error) {
+            onError(error as Error);
+        }
+    }
+
     static setupPull(
         driveId: string,
         trigger: PullResponderTrigger,
@@ -225,7 +306,7 @@ export class PullResponderTransmitter implements ITransmitter {
         onError: (error: Error) => void,
         onAcknowledge?: (success: boolean) => void
     ): number {
-        const { url, listenerId, interval } = trigger.data;
+        const { interval } = trigger.data;
         let loopInterval = PULL_DRIVE_INTERVAL;
         if (interval) {
             try {
@@ -238,79 +319,25 @@ export class PullResponderTransmitter implements ITransmitter {
             }
         }
 
-        const timeout = setInterval(async () => {
-            try {
-                const strands = await PullResponderTransmitter.pullStrands(
-                    driveId,
-                    url,
-                    listenerId
-                    // since ?
-                );
-
-                // if there are no new strands then do nothing
-                if (!strands.length) {
-                    return;
-                }
-
-                const listenerRevisions: ListenerRevision[] = [];
-
-                for (const strand of strands) {
-                    const operations: Operation[] = strand.operations.map(
-                        ({ index, type, hash, input, skip, timestamp }) => ({
-                            index,
-                            type,
-                            hash,
-                            input,
-                            skip,
-                            timestamp,
-                            scope: strand.scope,
-                            branch: strand.branch
-                        })
-                    );
-
-                    let error: Error | undefined = undefined;
-
-                    try {
-                        const result = await onStrandUpdate(strand);
-                        if (result.error) {
-                            throw result.error;
-                        }
-                    } catch (e) {
-                        error = e as Error;
-                        onError?.(error);
-                    }
-
-                    listenerRevisions.push({
-                        branch: strand.branch,
-                        documentId: strand.documentId ?? '',
-                        driveId: strand.driveId,
-                        revision: operations.pop()?.index ?? -1,
-                        scope: strand.scope as OperationScope,
-                        status: error
-                            ? error instanceof OperationError
-                                ? error.status
-                                : 'ERROR'
-                            : 'SUCCESS'
-                    });
-
-                    // TODO: Should try to parse remaining strands?
-                    if (error) {
-                        break;
-                    }
-                }
+        this.executePull(
+            driveId,
+            trigger,
+            onStrandUpdate,
+            onError,
+            onAcknowledge
+        );
 
-                await PullResponderTransmitter.acknowledgeStrands(
+        const timeout = setInterval(
+            async () =>
+                this.executePull(
                     driveId,
-                    url,
-                    listenerId,
-                    listenerRevisions
-                )
-                    .then(result => onAcknowledge?.(result))
-                    .catch(error => console.error('ACK error', error));
-            } catch (error) {
-                onError(error as Error);
-            }
-        }, loopInterval);
+                    trigger,
+                    onStrandUpdate,
+                    onError,
+                    onAcknowledge
+                ),
+            loopInterval
+        );
         return timeout as unknown as number;
     }
 

package/src/server/types.ts

@@ -15,6 +15,7 @@ import type {
     Signal,
     State
 } from 'document-model/document';
+import { Unsubscribe } from 'nanoevents';
 import { OperationError } from './error';
 import { ITransmitter } from './listener/transmitter/types';
 
@@ -92,6 +93,11 @@ export type ListenerRevision = {
     revision: number;
 };
 
+export type ListenerUpdate = {
+    listenerId: string;
+    listenerRevisions: ListenerRevision[];
+};
+
 export type UpdateStatus = 'SUCCESS' | 'CONFLICT' | 'MISSING' | 'ERROR';
 export type ErrorStatus = Exclude<UpdateStatus, 'SUCCESS'>;
 
@@ -114,6 +120,11 @@ export type StrandUpdate = {
 
 export type SyncStatus = 'SYNCING' | UpdateStatus;
 
+export interface DriveEvents {
+    syncStatus: (driveId: string, status: SyncStatus, error?: Error) => void;
+    strandUpdate: (update: StrandUpdate) => void;
+}
+
 export abstract class BaseDocumentDriveServer {
     /** Public methods **/
     abstract getDrives(): Promise<string[]>;
@@ -132,12 +143,12 @@ export abstract class BaseDocumentDriveServer {
         drive: string,
         id: string,
         operation: Operation
-    ): Promise<IOperationResult<Document>>;
+    ): Promise<IOperationResult>;
     abstract addOperations(
         drive: string,
         id: string,
         operations: Operation[]
-    ): Promise<IOperationResult<Document>>;
+    ): Promise<IOperationResult>;
 
     abstract addDriveOperation(
         drive: string,
@@ -183,12 +194,23 @@ export abstract class BaseDocumentDriveServer {
         driveId: string,
         listenerId: string
     ): Promise<ITransmitter | undefined>;
+
+    /** Event methods **/
+    protected abstract emit<K extends keyof DriveEvents>(
+        this: this,
+        event: K,
+        ...args: Parameters<DriveEvents[K]>
+    ): void;
+    abstract on<K extends keyof DriveEvents>(
+        this: this,
+        event: K,
+        cb: DriveEvents[K]
+    ): Unsubscribe;
 }
 
 export abstract class BaseListenerManager {
     protected drive: BaseDocumentDriveServer;
-    protected listenerState: Map<string, Map<string, ListenerState>> =
-        new Map();
+    protected listenerState = new Map<string, Map<string, ListenerState>>();
     protected transmitters: Record<
         DocumentDriveState['id'],
         Record<Listener['listenerId'], ITransmitter>
@@ -196,7 +218,7 @@ export abstract class BaseListenerManager {
 
     constructor(
         drive: BaseDocumentDriveServer,
-        listenerState: Map<string, Map<string, ListenerState>> = new Map()
+        listenerState = new Map<string, Map<string, ListenerState>>()
     ) {
         this.drive = drive;
         this.listenerState = listenerState;
@@ -216,8 +238,14 @@ export abstract class BaseListenerManager {
         driveId: string,
         syncId: string,
         syncRev: number,
-        lastUpdated: string
-    ): Promise<void>;
+        lastUpdated: string,
+        willUpdate?: (listeners: Listener[]) => void,
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void
+    ): Promise<ListenerUpdate[]>;
 
     abstract updateListenerRevision(
         listenerId: string,

package/src/storage/prisma.ts

@@ -119,28 +119,6 @@ export class PrismaStorage implements IDriveStorage {
         } catch (e) {
             console.log(e);
         }
-
-        await this.db.document.upsert({
-            where: {
-                id_driveId: {
-                    id: 'drives',
-                    driveId: id
-                }
-            },
-            create: {
-                id: 'drives',
-                driveId: id,
-                documentType: header.documentType,
-                initialState: document.initialState,
-                lastModified: header.lastModified,
-                revision: header.revision,
-                created: header.created
-            },
-            update: {
-                lastModified: header.lastModified,
-                revision: header.revision
-            }
-        });
     }
 
     async getDocuments(drive: string) {