document-drive 1.0.0-experimental.8 → 1.0.0-websockets

package/src/server/listener/manager.ts

@@ -14,7 +14,7 @@ import {
     StrandUpdate,
     SynchronizationUnit
 } from '../types';
-import { PullResponderTransmitter } from './transmitter';
+import { PullResponderTransmitter, SubscriptionTransmitter } from './transmitter';
 import { InternalTransmitter } from './transmitter/internal';
 import { SwitchboardPushTransmitter } from './transmitter/switchboard-push';
 import { ITransmitter } from './transmitter/types';
@@ -53,7 +53,7 @@ export class ListenerManager extends BaseListenerManager {
         driveId: string,
         listenerId: string
     ): Promise<ITransmitter | undefined> {
-        return this.transmitters[driveId]?.[listenerId];
+        return Promise.resolve(this.transmitters[driveId]?.[listenerId]);
     }
 
     async addListener(listener: Listener) {
@@ -93,6 +93,10 @@ export class ListenerManager extends BaseListenerManager {
                 );
                 break;
             }
+            case "Subscription": {
+                transmitter = new SubscriptionTransmitter(listener, this);
+                break;
+            }
             case 'Internal': {
                 transmitter = new InternalTransmitter(listener, this.drive);
                 break;
@@ -106,7 +110,7 @@ export class ListenerManager extends BaseListenerManager {
         const driveTransmitters = this.transmitters[drive] || {};
         driveTransmitters[listener.listenerId] = transmitter;
         this.transmitters[drive] = driveTransmitters;
-        return transmitter;
+        return Promise.resolve(transmitter);
     }
 
     async removeListener(driveId: string, listenerId: string) {
@@ -115,10 +119,10 @@ export class ListenerManager extends BaseListenerManager {
             return false;
         }
 
-        return driveMap.delete(listenerId);
+        return Promise.resolve(driveMap.delete(listenerId));
     }
 
-    async removeSyncUnits(driveId: string, syncUnits: SynchronizationUnit[]) {
+    async removeSyncUnits(driveId: string, syncUnits: Pick<SynchronizationUnit, "syncId">[]) {
         const listeners = this.listenerState.get(driveId);
         if (!listeners) {
             return;
@@ -128,6 +132,7 @@ export class ListenerManager extends BaseListenerManager {
                 listener.syncUnits.delete(syncUnit.syncId);
             }
         }
+        return Promise.resolve();
     }
 
     async updateSynchronizationRevisions(
@@ -203,6 +208,8 @@ export class ListenerManager extends BaseListenerManager {
         } else {
             listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
         }
+
+        return Promise.resolve();
     }
 
     triggerUpdate = debounce(
@@ -243,7 +250,7 @@ export class ListenerManager extends BaseListenerManager {
 
                     const opData: OperationUpdate[] = [];
                     try {
-                        const data = await this.drive.getOperationData(
+                        const data = await this.drive.getOperationData( // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
                             driveId,
                             syncUnit.syncId,
                             {
@@ -373,6 +380,21 @@ export class ListenerManager extends BaseListenerManager {
         );
     }
 
+    getListenerSyncUnitIds(driveId: string, listenerId: string) {
+        const listener = this.listenerState.get(driveId)?.get(listenerId);
+        if (!listener) {
+            return [];
+        }
+        const filter = listener.listener.filter;
+        return this.drive.getSynchronizationUnitsIds(
+            driveId,
+            filter.documentId ?? ['*'],
+            filter.scope ?? ['*'],
+            filter.branch ?? ['*'],
+            filter.documentType ?? ['*']
+        );
+    }
+
     async initDrive(drive: DocumentDriveDocument) {
         const {
             state: {
@@ -398,6 +420,14 @@ export class ListenerManager extends BaseListenerManager {
         }
     }
 
+    async removeDrive(driveId: string): Promise<void> {
+        this.listenerState.delete(driveId);
+        const transmitters = this.transmitters[driveId];
+        if (transmitters) {
+            await Promise.all(Object.values(transmitters).map(t => t.disconnect?.()));
+        }
+    }
+
     getListener(driveId: string, listenerId: string): Promise<ListenerState> {
         const drive = this.listenerState.get(driveId);
         if (!drive) throw new Error('Drive not found');
@@ -420,32 +450,40 @@ export class ListenerManager extends BaseListenerManager {
         const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
 
         for (const syncUnit of syncUnits) {
+            if (syncUnit.revision < 0) {
+                continue;
+            }
             const entry = listener.syncUnits.get(syncUnit.syncId);
             if (entry && entry.listenerRev >= syncUnit.revision) {
                 continue;
             }
 
             const { documentId, driveId, scope, branch } = syncUnit;
-            const operations = await this.drive.getOperationData(
-                driveId,
-                syncUnit.syncId,
-                {
-                    since,
-                    fromRevision: entry?.listenerRev
+            try {
+                const operations = await this.drive.getOperationData( // DEAL WITH INVALID SYNC ID ERROR
+                    driveId,
+                    syncUnit.syncId,
+                    {
+                        since,
+                        fromRevision: entry?.listenerRev
+                    }
+                );
+
+                if (!operations.length) {
+                    continue;
                 }
-            );
 
-            if (!operations.length) {
+                strands.push({
+                    driveId,
+                    documentId,
+                    scope: scope as OperationScope,
+                    branch,
+                    operations
+                });
+            } catch (error) {
+                logger.error(error);
                 continue;
             }
-
-            strands.push({
-                driveId,
-                documentId,
-                scope: scope as OperationScope,
-                branch,
-                operations
-            });
         }
 
         return strands;

package/src/server/listener/transmitter/index.ts

@@ -2,3 +2,4 @@ export * from './pull-responder';
 export * from './switchboard-push';
 export * from './internal';
 export * from './types';
+export * from './subscription';

package/src/server/listener/transmitter/internal.ts

@@ -12,6 +12,7 @@ import { logger } from '../../../utils/logger';
 
 export interface IReceiver {
     transmit: (strands: InternalTransmitterUpdate[]) => Promise<void>;
+    disconnect: () => Promise<void>;
 }
 
 export type InternalTransmitterUpdate<
@@ -88,4 +89,8 @@ export class InternalTransmitter implements ITransmitter {
     setReceiver(receiver: IReceiver) {
         this.receiver = receiver;
     }
+
+    async disconnect(): Promise<void> {
+        await this.receiver?.disconnect();
+    }
 }

package/src/server/listener/transmitter/pull-responder.ts

@@ -1,5 +1,5 @@
-import { ListenerFilter, Trigger, z } from 'document-model-libs/document-drive';
-import { Operation, OperationScope } from 'document-model/document';
+import { ListenerFilter, Trigger } from 'document-model-libs/document-drive';
+import { OperationScope } from 'document-model/document';
 import { PULL_DRIVE_INTERVAL } from '../..';
 import { generateUUID } from '../../../utils';
 import { gql, requestGraphql } from '../../../utils/graphql';
@@ -16,7 +16,7 @@ import {
     StrandUpdate
 } from '../../types';
 import { ListenerManager } from '../manager';
-import { ITransmitter, PullResponderTrigger } from './types';
+import { ITriggerTransmitter, PullResponderTrigger } from './types';
 
 export type OperationUpdateGraphQL = Omit<OperationUpdate, 'input'> & {
     input: string;
@@ -36,7 +36,7 @@ export type StrandUpdateGraphQL = Omit<StrandUpdate, 'operations'> & {
     operations: OperationUpdateGraphQL[];
 };
 
-export interface IPullResponderTransmitter extends ITransmitter {
+export interface IPullResponderTransmitter extends ITriggerTransmitter {
     getStrands(since?: string): Promise<StrandUpdate[]>;
 }
 
@@ -72,7 +72,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
         listenerId: string,
         revisions: ListenerRevision[]
     ): Promise<boolean> {
-        const syncUnits = await this.manager.getListenerSyncUnits(
+        const syncUnits = await this.manager.getListenerSyncUnitIds(
             driveId,
             listenerId
         );
@@ -153,6 +153,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
                                 scope
                                 branch
                                 operations {
+                                    id
                                     timestamp
                                     skip
                                     type
@@ -237,17 +238,10 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
             const listenerRevisions: ListenerRevisionWithError[] = [];
 
             for (const strand of strands) {
-                const operations: Operation[] = strand.operations.map(
-                    (op) => ({
-                        ...op,
-                        scope: strand.scope,
-                        branch: strand.branch
-                    })
-                );
-
+                let result: IOperationResult | undefined = undefined;
                 let error: Error | undefined = undefined;
                 try {
-                    const result = await onStrandUpdate(strand);
+                    result = await onStrandUpdate(strand);
                     if (result.error) {
                         throw result.error;
                     }
@@ -260,7 +254,9 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
                     branch: strand.branch,
                     documentId: strand.documentId || '',
                     driveId: strand.driveId,
-                    revision: operations.pop()?.index ?? -1,
+                    revision:
+                        result?.document?.operations[strand.scope]?.at(-1)
+                            ?.index ?? -1,
                     scope: strand.scope as OperationScope,
                     status: error
                         ? error instanceof OperationError

package/src/server/listener/transmitter/subscription.ts

@@ -0,0 +1,364 @@
+import { Client, createClient } from 'graphql-ws';
+import WebSocket from 'isomorphic-ws';
+import { logger } from '../../../utils/logger';
+import {
+    IOperationResult,
+    Listener,
+    ListenerRevision,
+    ListenerRevisionWithError,
+    RemoteDriveOptions,
+    StrandUpdate
+} from '../../types';
+import { ListenerManager } from '../manager';
+import { ITriggerTransmitter, SubscriptionTrigger } from './types';
+import { ListenerFilter, Trigger, z } from 'document-model-libs/document-drive';
+import { gql, requestGraphql } from '../../../utils/graphql';
+import { generateUUID } from '../../../utils';
+import { OperationScope } from 'document-model/document';
+import { OperationError } from '../../error';
+import { StrandUpdateGraphQL } from './pull-responder';
+
+
+export class SubscriptionTransmitter implements ITriggerTransmitter {
+    private _strands: StrandUpdate[] = [];
+    private listener: Listener;
+    private manager: ListenerManager;
+    private _init: Promise<StrandUpdate[]> | null = null;
+    private handler: ((strands: StrandUpdate[]) => void) | null = null;
+
+    constructor(
+        listener: Listener,
+        manager: ListenerManager,
+    ) {
+        this.listener = listener;
+        this.manager = manager;
+    }
+
+    async init() {
+        if (this._init) {
+            return this._init;
+        }
+        this._init = this.#refreshStrands();
+        return this._init;
+    }
+
+    #updateStrands(strands: StrandUpdate[]): void {
+        this._strands = strands;
+        this.handler?.(strands);
+    }
+
+    async #refreshStrands() {
+        const strands = await this.manager.getStrands(
+            this.listener.driveId,
+            this.listener.listenerId
+        );
+        this.#updateStrands(strands);
+        return Promise.resolve(strands);
+    }
+
+    async * strandsGenerator(since?: number): AsyncGenerator<StrandUpdate[]> {
+        await this.init();
+        let waitHandler = null;
+        let firstTime = true;
+        // eslint-disable-next-line
+        while (true) {
+            if (waitHandler) {
+                await waitHandler;
+            }
+            waitHandler = new Promise<StrandUpdate[]>(resolve => {
+                this.handler = resolve;
+            });
+
+            // only return empty array on first call
+            if (this._strands.length || firstTime) {
+                firstTime = false;
+                // TODO add support for 'since' parameter
+                yield this._strands.slice();
+            }
+        }
+    }
+
+    async transmit(strands: StrandUpdate[]): Promise<ListenerRevision[]> {
+        // if subscription has not been initiated by
+        // the client then ignores new strands
+        if (!this._init) {
+            return [];
+        }
+        console.log("TRANSMIT", this.listener.listenerId, strands.map(s => s.operations.length))
+        this.#updateStrands([...this._strands, ...strands]);
+        return Promise.resolve([]);
+    }
+
+    async processAcknowledge(
+        driveId: string,
+        listenerId: string,
+        revisions: ListenerRevision[]
+    ): Promise<boolean> {
+        const syncUnits = await this.manager.getListenerSyncUnits(
+            driveId,
+            listenerId
+        );
+        let success = true;
+        let acknowledged = false;
+        for (const revision of revisions) {
+            const syncUnit = syncUnits.find(
+                s =>
+                    s.scope === revision.scope &&
+                    s.branch === revision.branch &&
+                    s.driveId === revision.driveId &&
+                    s.documentId == revision.documentId
+            );
+            if (!syncUnit) {
+                logger.warn(
+                    'Unknown sync unit was acknowledged',
+                    revision
+                );
+                success = false;
+                continue;
+            }
+
+            await this.manager.updateListenerRevision(
+                listenerId,
+                driveId,
+                syncUnit.syncId,
+                revision.revision
+            );
+            acknowledged = true;
+        }
+        if (acknowledged) {
+            await this.#refreshStrands();
+        }
+
+        return success;
+    }
+
+    static async registerSubscription(
+        driveId: string,
+        url: string,
+        filter: ListenerFilter
+    ): Promise<Listener['listenerId']> {
+        // graphql request to switchboard
+        const { registerListener } = await requestGraphql<{
+            registerListener: {
+                listenerId: Listener['listenerId'];
+            };
+        }>(
+            url,
+            gql`
+                mutation registerListener(
+                    $filter: InputListenerFilter!
+                    $type: TransmitterType!
+                ) {
+                    registerListener(filter: $filter, type: $type) {
+                        listenerId
+                    }
+                }
+            `,
+            { filter, type: 'Subscription' }
+        );
+        return registerListener.listenerId;
+    }
+
+    static async createTrigger(
+        driveId: string,
+        url: string,
+        options: Pick<RemoteDriveOptions, 'pullFilter'>
+    ): Promise<Trigger> {
+        const { pullFilter } = options;
+        const listenerId = await SubscriptionTransmitter.registerSubscription(
+            driveId,
+            url,
+            pullFilter ?? {
+                documentId: ['*'],
+                documentType: ['*'],
+                branch: ['*'],
+                scope: ['*']
+            }
+        );
+
+        const trigger: Trigger = {
+            id: generateUUID(),
+            type: 'Subscription',
+            data: {
+                url,
+                listenerId,
+            }
+        };
+        return trigger;
+    }
+
+    static isTrigger(
+        trigger: Trigger
+    ): trigger is SubscriptionTrigger {
+        return (
+            trigger.type === 'Subscription' &&
+            z.SubscriptionTriggerDataSchema().safeParse(trigger.data).success
+        );
+    }
+
+    static setup(driveId: string,
+        trigger: SubscriptionTrigger,
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
+        onAcknowledge?: (success: boolean) => void) {
+
+        const { url } = trigger.data;
+        let subscriptionUrl = url.replace("http", "ws")
+        subscriptionUrl += subscriptionUrl.endsWith("/") ? "ws" : "/ws";
+
+        const client = createClient({
+            url: subscriptionUrl,
+            webSocketImpl: WebSocket,
+        });
+        try {
+            SubscriptionTransmitter.subscribeStrands(client, trigger, onStrandUpdate, onError, onRevisions, onAcknowledge).catch(onError);
+        } catch (error) {
+            onError(error as Error);
+        }
+        return () => { return client.dispose(); };
+    }
+
+    private static async subscribeStrands(client: Client, trigger: SubscriptionTrigger,
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
+        onAcknowledge?: (success: boolean) => void) {
+        const { listenerId } = trigger.data;
+        const subscription = client.iterate<{ subscribeStrands: StrandUpdateGraphQL[] }>({
+            query: `
+            subscription($listenerId: ID) {
+                subscribeStrands(listenerId: $listenerId) {
+                branch
+                documentId
+                driveId
+                operations {
+                    timestamp
+                    skip
+                    type
+                    input
+                    hash
+                    index
+                    context {
+                        signer {
+                            user {
+                                address
+                                networkId
+                                chainId
+                            }
+                            app {
+                                name
+                                key
+                            }
+                            signature
+                        }
+                    }
+                }
+                scope
+                }
+            }`,
+            variables: {
+                listenerId,
+            }
+        });
+        for await (const { errors, data } of subscription) {
+            const error = errors?.at(0);
+            if (error) {
+                onError(error);
+            } else {
+                const strands = data?.subscribeStrands ?? [];
+                console.log(listenerId, "Save strands", strands);
+                SubscriptionTransmitter.saveStrands(trigger, client, strands, onStrandUpdate, onError, onRevisions, onAcknowledge).catch(onError);
+            }
+        }
+    }
+
+    static async saveStrands(
+        trigger: SubscriptionTrigger,
+        client: Client,
+        strandsQL: StrandUpdateGraphQL[],
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
+        onAcknowledge?: (success: boolean) => void) {
+        const strands = strandsQL.map(s => ({
+            ...s,
+            operations: s.operations.map(o => ({
+                ...o,
+                scope: s.scope,
+                branch: s.branch,
+                input: JSON.parse(o.input) as object
+            }))
+        }));
+        // if there are no new strands then do nothing
+        if (!strands.length) {
+            onRevisions?.([]);
+            return;
+        }
+
+        const listenerRevisions: ListenerRevisionWithError[] = [];
+
+
+        for (const strand of strands) {
+            let error: Error | undefined = undefined;
+            let result: IOperationResult | undefined = undefined;
+            try {
+                result = await onStrandUpdate(strand);
+                if (result.error) {
+                    throw result.error;
+                }
+            } catch (e) {
+                error = e as Error;
+                onError(error);
+            }
+
+            listenerRevisions.push({
+                branch: strand.branch,
+                documentId: strand.documentId || '',
+                driveId: strand.driveId,
+                revision: result?.document?.operations[strand.scope]?.at(-1)?.index ?? -1,
+                scope: strand.scope as OperationScope,
+                status: error
+                    ? error instanceof OperationError
+                        ? error.status
+                        : 'ERROR'
+                    : 'SUCCESS',
+                error
+            });
+        }
+
+        onRevisions?.(listenerRevisions);
+
+        await SubscriptionTransmitter.acknowledgeStrands(
+            client,
+            trigger.data.listenerId,
+            listenerRevisions.map(revision => {
+                const { error, ...rest } = revision;
+                return rest;
+            })
+        )
+            .then(result => onAcknowledge?.(result))
+            .catch(error => logger.error('ACK error', error));
+    }
+
+    static async acknowledgeStrands(
+        client: Client,
+        listenerId: string,
+        revisions: ListenerRevision[]
+    ): Promise<boolean> {
+        const subscription = client.iterate<{ acknowledge: boolean }>({
+            query: `
+            mutation acknowledge(
+                $listenerId: String!
+                $revisions: [ListenerRevisionInput]
+            ) {
+                acknowledge(listenerId: $listenerId, revisions: $revisions)
+            }
+        `,
+            variables: { listenerId, revisions }
+        });
+        const result = await subscription.next();
+
+        return (result.value as { acknowledge: boolean }).acknowledge as boolean;
+    }
+}

package/src/server/listener/transmitter/types.ts

@@ -1,11 +1,21 @@
 import {
     PullResponderTriggerData,
+    SubscriptionTriggerData,
     Trigger
 } from 'document-model-libs/document-drive';
 import { ListenerRevision, StrandUpdate } from '../..';
 
 export interface ITransmitter {
     transmit(strands: StrandUpdate[]): Promise<ListenerRevision[]>;
+    disconnect?(): Promise<void>;
+}
+
+export interface ITriggerTransmitter extends ITransmitter {
+    processAcknowledge(
+        driveId: string,
+        listenerId: string,
+        revisions: ListenerRevision[]
+    ): Promise<boolean>
 }
 
 export interface InternalTransmitterService extends ITransmitter {
@@ -16,3 +26,8 @@ export type PullResponderTrigger = Omit<Trigger, 'data' | 'type'> & {
     data: PullResponderTriggerData;
     type: 'PullResponder';
 };
+
+export type SubscriptionTrigger = Omit<Trigger, 'data' | 'type'> & {
+    data: SubscriptionTriggerData;
+    type: 'Subscription';
+};