document-drive 0.0.26 → 0.0.28

package/src/server/listener/index.ts

@@ -0,0 +1,2 @@
+export * from './manager';
+export * from './transmitter';

package/src/server/listener/manager.ts

@@ -0,0 +1,382 @@
+import {
+    ListenerCallInfo,
+    ListenerFilter
+} from 'document-model-libs/document-drive';
+import { OperationScope } from 'document-model/document';
+import { OperationError } from '../error';
+import {
+    BaseListenerManager,
+    ErrorStatus,
+    Listener,
+    ListenerState,
+    StrandUpdate,
+    SynchronizationUnit
+} from '../types';
+import { PullResponderTransmitter } from './transmitter';
+import { SwitchboardPushTransmitter } from './transmitter/switchboard-push';
+import { ITransmitter } from './transmitter/types';
+
+export class ListenerManager extends BaseListenerManager {
+    async getTransmitter(
+        driveId: string,
+        listenerId: string
+    ): Promise<ITransmitter | undefined> {
+        return this.transmitters[driveId]?.[listenerId];
+    }
+
+    async addListener(listener: Listener) {
+        const drive = listener.driveId;
+
+        const syncUnits = await this.drive.getSynchronizationUnits(drive);
+        const filteredSyncUnits = [];
+        for (const syncUnit of syncUnits) {
+            if (this._checkFilter(listener.filter, syncUnit)) {
+                filteredSyncUnits.push(syncUnit);
+            }
+        }
+
+        if (!this.listenerState.has(drive)) {
+            this.listenerState.set(drive, new Map());
+        }
+
+        const driveMap = this.listenerState.get(drive)!;
+
+        const driveDocument = await this.drive.getDrive(drive);
+
+        const lastDriveOperation = driveDocument.operations.global
+            .slice()
+            .pop();
+
+        driveMap.set(listener.listenerId, {
+            block: listener.block,
+            driveId: listener.driveId,
+            pendingTimeout: '0',
+            listener,
+            listenerStatus: 'CREATED',
+            syncUnits: [
+                {
+                    syncId: '0',
+                    driveId: listener.driveId,
+                    documentId: '',
+                    documentType: driveDocument.documentType,
+                    scope: 'global',
+                    branch: 'main',
+                    lastUpdated:
+                        lastDriveOperation?.timestamp ??
+                        driveDocument.lastModified,
+                    revision: lastDriveOperation?.index ?? 0,
+                    listenerRev: -1,
+                    syncRev: lastDriveOperation?.index ?? 0
+                }
+            ].concat(
+                filteredSyncUnits.map(e => ({
+                    ...e,
+                    listenerRev: -1,
+                    syncRev: e.revision
+                }))
+            )
+        });
+
+        let transmitter: ITransmitter | undefined;
+
+        switch (listener.callInfo?.transmitterType) {
+            case 'SwitchboardPush': {
+                transmitter = new SwitchboardPushTransmitter(
+                    listener,
+                    this.drive
+                );
+                break;
+            }
+
+            case 'PullResponder': {
+                transmitter = new PullResponderTransmitter(
+                    listener,
+                    this.drive,
+                    this
+                );
+            }
+        }
+
+        if (!transmitter) {
+            throw new Error('Transmitter not found');
+        }
+
+        const driveTransmitters = this.transmitters[drive] || {};
+        driveTransmitters[listener.listenerId] = transmitter;
+        this.transmitters[drive] = driveTransmitters;
+        return transmitter;
+    }
+
+    async removeListener(driveId: string, listenerId: string) {
+        const driveMap = this.listenerState.get(driveId);
+        if (!driveMap) {
+            return false;
+        }
+
+        return driveMap.delete(listenerId);
+    }
+
+    async updateSynchronizationRevision(
+        driveId: string,
+        syncId: string,
+        syncRev: number,
+        lastUpdated: string
+    ) {
+        const drive = this.listenerState.get(driveId);
+        if (!drive) {
+            return;
+        }
+
+        let newRevision = false;
+        for (const [, listener] of drive) {
+            const syncUnits = listener.syncUnits.filter(
+                e => e.syncId === syncId
+            );
+            if (listener.driveId !== driveId) {
+                continue;
+            }
+
+            for (const syncUnit of syncUnits) {
+                if (syncUnit.syncId !== syncId) {
+                    continue;
+                }
+
+                syncUnit.syncRev = syncRev;
+                syncUnit.lastUpdated = lastUpdated;
+                newRevision = true;
+            }
+        }
+
+        if (newRevision) {
+            return this.triggerUpdate();
+        }
+    }
+
+    async addSyncUnits(syncUnits: SynchronizationUnit[]) {
+        for (const [driveId, drive] of this.listenerState) {
+            for (const [id, listenerState] of drive) {
+                const transmitter = await this.getTransmitter(driveId, id);
+                if (!transmitter) {
+                    continue;
+                }
+                const filteredSyncUnits = [];
+                const { listener } = listenerState;
+                for (const syncUnit of syncUnits) {
+                    if (!this._checkFilter(listener.filter, syncUnit)) {
+                        continue;
+                    }
+                    const existingSyncUnit = listenerState.syncUnits.find(
+                        unit => unit.syncId === syncUnit.syncId
+                    );
+                    if (existingSyncUnit) {
+                        existingSyncUnit.syncRev = syncUnit.revision;
+                        existingSyncUnit.lastUpdated = syncUnit.lastUpdated;
+                    } else {
+                        filteredSyncUnits.push(syncUnit);
+                    }
+                }
+
+                // TODO is this possible?
+                if (!this.listenerState.has(driveId)) {
+                    this.listenerState.set(driveId, new Map());
+                }
+
+                const driveMap = this.listenerState.get(driveId)!;
+
+                // TODO reuse existing state
+                driveMap.set(listener.listenerId, {
+                    block: listener.block,
+                    driveId: listener.driveId,
+                    pendingTimeout: '0',
+                    listener,
+                    listenerStatus: 'CREATED',
+                    syncUnits: listenerState.syncUnits.concat(
+                        filteredSyncUnits.map(e => ({
+                            ...e,
+                            listenerRev: -1,
+                            syncRev: e.revision
+                        }))
+                    )
+                });
+            }
+        }
+    }
+
+    async updateListenerRevision(
+        listenerId: string,
+        driveId: string,
+        syncId: string,
+        listenerRev: number
+    ): Promise<void> {
+        const drive = this.listenerState.get(driveId);
+        if (!drive) {
+            return;
+        }
+
+        const listener = drive.get(listenerId);
+        if (!listener) {
+            return;
+        }
+
+        const entry = listener.syncUnits.find(s => s.syncId === syncId);
+        if (entry) {
+            entry.listenerRev = listenerRev;
+            entry.lastUpdated = new Date().toISOString();
+        }
+    }
+
+    async triggerUpdate() {
+        for (const [driveId, drive] of this.listenerState) {
+            for (const [id, listener] of drive) {
+                const transmitter = await this.getTransmitter(driveId, id);
+                if (!transmitter) {
+                    continue;
+                }
+
+                const strandUpdates: StrandUpdate[] = [];
+                for (const unit of listener.syncUnits) {
+                    const {
+                        syncRev,
+                        syncId,
+                        listenerRev,
+                        driveId,
+                        documentId,
+                        scope,
+                        branch
+                    } = unit;
+                    if (listenerRev >= syncRev) {
+                        continue;
+                    }
+
+                    const opData = await this.drive.getOperationData(
+                        driveId,
+                        syncId,
+                        {
+                            fromRevision: listenerRev
+                        }
+                    );
+
+                    if (!opData.length) {
+                        continue;
+                    }
+
+                    strandUpdates.push({
+                        driveId,
+                        documentId,
+                        branch,
+                        operations: opData,
+                        scope: scope as OperationScope
+                    });
+                }
+
+                if (strandUpdates.length == 0) {
+                    continue;
+                }
+
+                listener.pendingTimeout = new Date(
+                    new Date().getTime() / 1000 + 300
+                ).toISOString();
+                listener.listenerStatus = 'PENDING';
+
+                // TODO update listeners in parallel, blocking for listeners with block=true
+                try {
+                    const listenerRevisions =
+                        await transmitter?.transmit(strandUpdates);
+
+                    listener.pendingTimeout = '0';
+                    listener.listenerStatus = 'PENDING';
+
+                    for (const unit of listener.syncUnits) {
+                        const revision = listenerRevisions.find(
+                            e =>
+                                e.documentId === unit.documentId &&
+                                e.scope === unit.scope &&
+                                e.branch === unit.branch
+                        );
+                        if (revision) {
+                            unit.listenerRev = revision.revision;
+                        }
+                    }
+                    const revisionError = listenerRevisions.find(
+                        l => l.status !== 'SUCCESS'
+                    );
+                    if (revisionError) {
+                        throw new OperationError(
+                            revisionError.status as ErrorStatus,
+                            undefined
+                        );
+                    }
+                    listener.listenerStatus = 'SUCCESS';
+                } catch (e) {
+                    // TODO: Handle error based on listener params (blocking, retry, etc)
+                    listener.listenerStatus =
+                        e instanceof OperationError ? e.status : 'ERROR';
+                    throw e;
+                }
+            }
+        }
+    }
+
+    private _checkFilter(
+        filter: ListenerFilter,
+        syncUnit: SynchronizationUnit
+    ) {
+        const { branch, documentId, scope, documentType } = syncUnit;
+        // TODO: Needs to be optimized
+        if (
+            (!filter.branch ||
+                filter.branch.includes(branch) ||
+                filter.branch.includes('*')) &&
+            (!filter.documentId ||
+                filter.documentId.includes(documentId) ||
+                filter.documentId.includes('*')) &&
+            (!filter.scope ||
+                filter.scope.includes(scope) ||
+                filter.scope.includes('*')) &&
+            (!filter.documentType ||
+                filter.documentType.includes(documentType) ||
+                filter.documentType.includes('*'))
+        ) {
+            return true;
+        }
+        return false;
+    }
+
+    async init() {
+        const drives = await this.drive.getDrives();
+        for (const driveId of drives) {
+            const drive = await this.drive.getDrive(driveId);
+            const {
+                state: {
+                    local: { listeners }
+                }
+            } = drive;
+
+            for (const listener of listeners) {
+                this.addListener({
+                    block: listener.block,
+                    driveId,
+                    filter: {
+                        branch: listener.filter.branch ?? [],
+                        documentId: listener.filter.documentId ?? [],
+                        documentType: listener.filter.documentType,
+                        scope: listener.filter.scope ?? []
+                    },
+                    listenerId: listener.listenerId,
+                    system: listener.system,
+                    callInfo:
+                        (listener.callInfo as ListenerCallInfo) ?? undefined,
+                    label: listener.label ?? ''
+                });
+            }
+        }
+    }
+
+    getListener(driveId: string, listenerId: string): ListenerState {
+        const drive = this.listenerState.get(driveId);
+        if (!drive) throw new Error('Drive not found');
+        const listener = drive.get(listenerId);
+        if (!listener) throw new Error('Listener not found');
+        return listener;
+    }
+}

package/src/server/listener/transmitter/index.ts

@@ -0,0 +1,3 @@
+export * from './pull-responder';
+export * from './switchboard-push';
+export * from './types';

package/src/server/listener/transmitter/pull-responder.ts

@@ -0,0 +1,308 @@
+import { ListenerFilter, Trigger, z } from 'document-model-libs/document-drive';
+import { Operation, OperationScope } from 'document-model/document';
+import { PULL_DRIVE_INTERVAL } from '../..';
+import { gql, requestGraphql } from '../../../utils/graphql';
+import { OperationError } from '../../error';
+import {
+    BaseDocumentDriveServer,
+    IOperationResult,
+    Listener,
+    ListenerRevision,
+    OperationUpdate,
+    StrandUpdate
+} from '../../types';
+import { ListenerManager } from '../manager';
+import { ITransmitter, PullResponderTrigger } from './types';
+
+export type OperationUpdateGraphQL = Omit<OperationUpdate, 'input'> & {
+    input: string;
+};
+
+export type PullStrandsGraphQL = {
+    system: {
+        sync: {
+            strands: StrandUpdateGraphQL[];
+        };
+    };
+};
+
+export type StrandUpdateGraphQL = Omit<StrandUpdate, 'operations'> & {
+    operations: OperationUpdateGraphQL[];
+};
+
+export class PullResponderTransmitter implements ITransmitter {
+    private drive: BaseDocumentDriveServer;
+    private listener: Listener;
+    private manager: ListenerManager;
+
+    constructor(
+        listener: Listener,
+        drive: BaseDocumentDriveServer,
+        manager: ListenerManager
+    ) {
+        this.listener = listener;
+        this.drive = drive;
+        this.manager = manager;
+    }
+
+    async transmit(): Promise<ListenerRevision[]> {
+        return [];
+    }
+
+    async getStrands(
+        listenerId: string,
+        since?: string
+    ): Promise<StrandUpdate[]> {
+        // fetch listenerState from listenerManager
+        const entries = this.manager.getListener(
+            this.listener.driveId,
+            listenerId
+        );
+
+        // fetch operations from drive  and prepare strands
+        const strands: StrandUpdate[] = [];
+
+        for (const entry of entries.syncUnits) {
+            if (entry.listenerRev >= entry.syncRev) {
+                continue;
+            }
+
+            const { documentId, driveId, scope, branch } = entry;
+            const operations = await this.drive.getOperationData(
+                entry.driveId,
+                entry.syncId,
+                {
+                    since,
+                    fromRevision: entry.listenerRev
+                }
+            );
+            strands.push({
+                driveId,
+                documentId,
+                scope: scope as OperationScope,
+                branch,
+                operations
+            });
+        }
+
+        return strands;
+    }
+
+    async processAcknowledge(
+        driveId: string,
+        listenerId: string,
+        revisions: ListenerRevision[]
+    ): Promise<boolean> {
+        const listener = this.manager.getListener(driveId, listenerId);
+        let success = true;
+        for (const revision of revisions) {
+            const syncId = listener.syncUnits.find(
+                s => s.scope === revision.scope && s.branch === revision.branch && s.documentId === revision.documentId && s.driveId === driveId
+            )?.syncId;
+            if (!syncId) {
+                success = false;
+                continue;
+            }
+
+            await this.manager.updateListenerRevision(
+                listenerId,
+                driveId,
+                syncId,
+                revision.revision
+            );
+        }
+
+        return success;
+    }
+
+    static async registerPullResponder(
+        driveId: string,
+        url: string,
+        filter: ListenerFilter
+    ): Promise<Listener['listenerId']> {
+        // graphql request to switchboard
+        const { registerPullResponderListener } = await requestGraphql<{
+            registerPullResponderListener: {
+                listenerId: Listener['listenerId'];
+            };
+        }>(
+            url,
+            gql`
+                mutation registerPullResponderListener(
+                    $filter: InputListenerFilter!
+                ) {
+                    registerPullResponderListener(filter: $filter) {
+                        listenerId
+                    }
+                }
+            `,
+            { filter }
+        );
+        return registerPullResponderListener.listenerId;
+    }
+
+    static async pullStrands(
+        driveId: string,
+        url: string,
+        listenerId: string,
+        since?: string // TODO add support for since
+    ): Promise<StrandUpdate[]> {
+        const {
+            system: {
+                sync: { strands }
+            }
+        } = await requestGraphql<PullStrandsGraphQL>(
+            url,
+            gql`
+                query strands($listenerId: ID!) {
+                    system {
+                        sync {
+                            strands(listenerId: $listenerId) {
+                                driveId
+                                documentId
+                                scope
+                                branch
+                                operations {
+                                    timestamp
+                                    skip
+                                    type
+                                    input
+                                    hash
+                                    index
+                                }
+                            }
+                        }
+                    }
+                }
+            `,
+            { listenerId }
+        );
+        return strands.map(s => ({
+            ...s,
+            operations: s.operations.map(o => ({
+                ...o,
+                input: JSON.parse(o.input) as object
+            }))
+        }));
+    }
+
+    static async acknowledgeStrands(
+        driveId: string,
+        url: string,
+        listenerId: string,
+        revisions: ListenerRevision[]
+    ): Promise<boolean> {
+        const result = await requestGraphql<{ acknowledge: boolean }>(
+            url,
+            gql`
+                mutation acknowledge(
+                    $listenerId: String!
+                    $revisions: [ListenerRevisionInput]
+                ) {
+                    acknowledge(listenerId: $listenerId, revisions: $revisions)
+                }
+            `,
+            { listenerId, revisions }
+        );
+        return result.acknowledge;
+    }
+
+    static setupPull(
+        driveId: string,
+        trigger: PullResponderTrigger,
+        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
+        onError: (error: Error) => void,
+        onAcknowledge?: (success: boolean) => void
+    ): number {
+        const { url, listenerId, interval } = trigger.data;
+        let loopInterval = PULL_DRIVE_INTERVAL;
+        if (interval) {
+            try {
+                const intervalNumber = parseInt(interval);
+                if (intervalNumber) {
+                    loopInterval = intervalNumber;
+                }
+            } catch {
+                // ignore invalid interval
+            }
+        }
+
+        const timeout = setInterval(async () => {
+            try {
+                const strands = await PullResponderTransmitter.pullStrands(
+                    driveId,
+                    url,
+                    listenerId
+                    // since ?
+                );
+                const listenerRevisions: ListenerRevision[] = [];
+
+                for (const strand of strands) {
+                    const operations: Operation[] = strand.operations.map(
+                        ({ index, type, hash, input, skip, timestamp }) => ({
+                            index,
+                            type,
+                            hash,
+                            input,
+                            skip,
+                            timestamp,
+                            scope: strand.scope,
+                            branch: strand.branch
+                        })
+                    );
+
+                    let error: Error | undefined = undefined;
+
+                    try {
+                        const result = await onStrandUpdate(strand);
+                        if (result.error) {
+                            throw result.error;
+                        }
+                    } catch (e) {
+                        error = e as Error;
+                        onError?.(error);
+                    }
+
+                    listenerRevisions.push({
+                        branch: strand.branch,
+                        documentId: strand.documentId ?? '',
+                        driveId: strand.driveId,
+                        revision: operations.pop()?.index ?? -1,
+                        scope: strand.scope as OperationScope,
+                        status: error
+                            ? error instanceof OperationError
+                                ? error.status
+                                : 'ERROR'
+                            : 'SUCCESS'
+                    });
+
+                    // TODO: Should try to parse remaining strands?
+                    if (error) {
+                        break;
+                    }
+                }
+
+                await PullResponderTransmitter.acknowledgeStrands(
+                    driveId,
+                    url,
+                    listenerId,
+                    listenerRevisions
+                )
+                    .then(result => onAcknowledge?.(result))
+                    .catch(error => console.error('ACK error', error));
+            } catch (error) {
+                onError(error as Error);
+            }
+        }, loopInterval);
+        return timeout as unknown as number;
+    }
+
+    static isPullResponderTrigger(
+        trigger: Trigger
+    ): trigger is PullResponderTrigger {
+        return (
+            trigger.type === 'PullResponder' &&
+            z.PullResponderTriggerDataSchema().safeParse(trigger.data).success
+        );
+    }
+}