document-drive 1.0.0-experimental.79 → 1.0.0-experimental.80

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [1.0.0-alpha.79](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.78...v1.0.0-alpha.79) (2024-06-24)
+
+
+### Bug Fixes
+
+* do not store resulting state in cache ([f5ca275](https://github.com/powerhouse-inc/document-drive/commit/f5ca27549a1ef36cbb7600b47ae27e3d53a94c33))
+
 # [1.0.0-alpha.78](https://github.com/powerhouse-inc/document-drive/compare/v1.0.0-alpha.77...v1.0.0-alpha.78) (2024-06-20)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "document-drive",
-    "version": "1.0.0-experimental.79",
+    "version": "1.0.0-experimental.80",
     "license": "AGPL-3.0-only",
     "type": "module",
     "module": "./src/index.ts",

package/src/server/index.ts

@@ -66,6 +66,7 @@ import {
 import {
     AddOperationOptions,
     BaseDocumentDriveServer,
+    BaseListenerManager,
     DriveEvents,
     GetDocumentOptions,
     IOperationResult,
@@ -81,6 +82,8 @@ import {
     type SynchronizationUnit
 } from './types';
 import { filterOperationsByRevision } from './utils';
+import { RedisListenerManager } from './listener/redis-manager';
+import { RedisClientType } from 'redis';
 
 export * from './listener';
 export type * from './types';
@@ -105,10 +108,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         documentModels: DocumentModel[],
         storage: IDriveStorage = new MemoryStorage(),
         cache: ICache = new InMemoryCache(),
-        queueManager: IQueueManager = new BaseQueueManager()
+        queueManager: IQueueManager = new BaseQueueManager(),
+        redisClient: RedisClientType | undefined = undefined
     ) {
         super();
-        this.listenerStateManager = new ListenerManager(this);
+        this.listenerStateManager = redisClient ? new RedisListenerManager(this, redisClient) : new ListenerManager(this);
+
         this.documentModels = documentModels;
         this.storage = storage;
         this.cache = cache;
@@ -149,16 +154,16 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
 
         const result = await (!strand.documentId
             ? this.queueDriveOperations(
-                  strand.driveId,
-                  operations as Operation<DocumentDriveAction | BaseAction>[],
-                  { source }
-              )
+                strand.driveId,
+                operations as Operation<DocumentDriveAction | BaseAction>[],
+                { source }
+            )
             : this.queueOperations(
-                  strand.driveId,
-                  strand.documentId,
-                  operations,
-                  { source }
-              ));
+                strand.driveId,
+                strand.documentId,
+                operations,
+                { source }
+            ));
 
         if (result.status === 'ERROR') {
             this.updateSyncStatus(strand.driveId, result.status, result.error);
@@ -290,12 +295,12 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             return documentId
                 ? this.addOperations(driveId, documentId, operations, options)
                 : this.addDriveOperations(
-                      driveId,
-                      operations as Operation<
-                          DocumentDriveAction | BaseAction
-                      >[],
-                      options
-                  );
+                    driveId,
+                    operations as Operation<
+                        DocumentDriveAction | BaseAction
+                    >[],
+                    options
+                );
         },
         processActionJob: async ({
             driveId,
@@ -306,10 +311,10 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             return documentId
                 ? this.addActions(driveId, documentId, actions, options)
                 : this.addDriveActions(
-                      driveId,
-                      actions as Operation<DocumentDriveAction | BaseAction>[],
-                      options
-                  );
+                    driveId,
+                    actions as Operation<DocumentDriveAction | BaseAction>[],
+                    options
+                );
         },
         processJob: async (job: Job) => {
             if (isOperationJob(job)) {
@@ -462,14 +467,14 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
             const nodeUnits =
                 scope?.length || branch?.length
                     ? node.synchronizationUnits.filter(
-                          unit =>
-                              (!scope?.length ||
-                                  scope.includes(unit.scope) ||
-                                  scope.includes('*')) &&
-                              (!branch?.length ||
-                                  branch.includes(unit.branch) ||
-                                  branch.includes('*'))
-                      )
+                        unit =>
+                            (!scope?.length ||
+                                scope.includes(unit.scope) ||
+                                scope.includes('*')) &&
+                            (!branch?.length ||
+                                branch.includes(unit.branch) ||
+                                branch.includes('*'))
+                    )
                     : node.synchronizationUnits;
             if (!nodeUnits.length) {
                 continue;
@@ -902,11 +907,11 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                         e instanceof OperationError
                             ? e
                             : new OperationError(
-                                  'ERROR',
-                                  nextOperation,
-                                  (e as Error).message,
-                                  (e as Error).cause
-                              );
+                                'ERROR',
+                                nextOperation,
+                                (e as Error).message,
+                                (e as Error).cause
+                            );
 
                     // TODO: don't break on errors...
                     break;
@@ -940,9 +945,9 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         const revisionOperations =
             options?.revisions !== undefined
                 ? filterOperationsByRevision(
-                      documentStorage.operations,
-                      options.revisions
-                  )
+                    documentStorage.operations,
+                    options.revisions
+                )
                 : documentStorage.operations;
         const operations =
             baseUtils.documentHelpers.garbageCollectDocumentOperations(
@@ -992,18 +997,18 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
         if (lastRemainingOperation && !lastRemainingOperation.resultingState) {
             lastRemainingOperation.resultingState = await (id
                 ? this.storage.getOperationResultingState?.(
-                      drive,
-                      id,
-                      lastRemainingOperation.index,
-                      lastRemainingOperation.scope,
-                      'main'
-                  )
+                    drive,
+                    id,
+                    lastRemainingOperation.index,
+                    lastRemainingOperation.scope,
+                    'main'
+                )
                 : this.storage.getDriveOperationResultingState?.(
-                      drive,
-                      lastRemainingOperation.index,
-                      lastRemainingOperation.scope,
-                      'main'
-                  ));
+                    drive,
+                    lastRemainingOperation.index,
+                    lastRemainingOperation.scope,
+                    'main'
+                ));
         }
 
         const operationSignals: (() => Promise<SignalResult>)[] = [];
@@ -1446,11 +1451,11 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 error instanceof OperationError
                     ? error
                     : new OperationError(
-                          'ERROR',
-                          undefined,
-                          (error as Error).message,
-                          (error as Error).cause
-                      );
+                        'ERROR',
+                        undefined,
+                        (error as Error).message,
+                        (error as Error).cause
+                    );
 
             return {
                 status: operationError.status,
@@ -1778,11 +1783,11 @@ export class DocumentDriveServer extends BaseDocumentDriveServer {
                 error instanceof OperationError
                     ? error
                     : new OperationError(
-                          'ERROR',
-                          undefined,
-                          (error as Error).message,
-                          (error as Error).cause
-                      );
+                        'ERROR',
+                        undefined,
+                        (error as Error).message,
+                        (error as Error).cause
+                    );
 
             return {
                 status: operationError.status,

package/src/server/listener/redis-manager.ts

@@ -0,0 +1,473 @@
+import {
+    DocumentDriveDocument,
+    ListenerFilter
+} from 'document-model-libs/document-drive';
+import { OperationScope } from 'document-model/document';
+import { logger } from '../../utils/logger';
+import { OperationError } from '../error';
+import {
+    BaseDocumentDriveServer,
+    BaseListenerManager,
+    ErrorStatus,
+    Listener,
+    ListenerState,
+    ListenerUpdate,
+    OperationUpdate,
+    StrandUpdate,
+    SynchronizationUnit
+} from '../types';
+import { PullResponderTransmitter } from './transmitter';
+import { InternalTransmitter } from './transmitter/internal';
+import { SwitchboardPushTransmitter } from './transmitter/switchboard-push';
+import { ITransmitter, StrandUpdateSource } from './transmitter/types';
+import { RedisClientType } from "redis";
+import { ListenerManager } from './manager';
+
+function debounce<T extends unknown[], R>(
+    func: (...args: T) => Promise<R>,
+    delay = 250
+) {
+    let timer: number;
+    return (immediate: boolean, ...args: T) => {
+        if (timer) {
+            clearTimeout(timer);
+        }
+        return new Promise<R>((resolve, reject) => {
+            if (immediate) {
+                func(...args)
+                    .then(resolve)
+                    .catch(reject);
+            } else {
+                timer = setTimeout(() => {
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+                    func(...args)
+                        .then(resolve)
+                        .catch(reject);
+                }, delay) as unknown as number;
+            }
+        });
+    };
+}
+
+export class RedisListenerManager extends ListenerManager {
+    static LISTENER_UPDATE_DELAY = 250;
+
+    private redisClient: RedisClientType
+
+    constructor(
+        drive: BaseDocumentDriveServer,
+        redisClient: RedisClientType
+    ) {
+        super(drive);
+        this.redisClient = redisClient;
+    }
+
+    async getTransmitter(
+        driveId: string,
+        listenerId: string
+    ): Promise<ITransmitter | undefined> {
+        const { listener } = await this.getListener(driveId, listenerId);
+        let transmitter: ITransmitter | undefined;
+
+        switch (listener.callInfo?.transmitterType) {
+            case 'SwitchboardPush': {
+                transmitter = new SwitchboardPushTransmitter(
+                    listener,
+                    this.drive
+                );
+                break;
+            }
+
+            case 'PullResponder': {
+                transmitter = new PullResponderTransmitter(
+                    listener,
+                    this.drive,
+                    this
+                );
+                break;
+            }
+            case 'Internal': {
+                transmitter = new InternalTransmitter(listener, this.drive);
+                break;
+            }
+        }
+
+        if (!transmitter) {
+            throw new Error('Transmitter not found');
+        }
+
+        return transmitter;
+    }
+
+    async addListener(listener: Listener) {
+        const { driveId, listenerId } = listener;
+        await this.redisClient.hSet(driveId, listenerId, JSON.stringify({
+            block: listener.block,
+            driveId: listener.driveId,
+            pendingTimeout: '0',
+            listener,
+            listenerStatus: 'CREATED',
+            syncUnits: new Map()
+        }));
+
+        const transmitter = await this.getTransmitter(driveId, listenerId);
+        if (!transmitter) {
+            throw new Error('Transmitter not found');
+        }
+        return transmitter;
+    }
+
+    async updateListener(listenerState: ListenerState) {
+        const { driveId, listenerId, block } = listenerState.listener;
+        await this.redisClient.hSet(driveId, listenerId, JSON.stringify({
+            block: block,
+            driveId: driveId,
+            pendingTimeout: '0',
+            listener: listenerState.listener,
+            listenerStatus: 'CREATED',
+            syncUnits: listenerState.syncUnits
+        }));
+    }
+
+    async removeListener(driveId: string, listenerId: string) {
+        return (await this.redisClient.hDel(driveId, listenerId)) > 0;
+    }
+
+    async removeSyncUnits(
+        driveId: string,
+        syncUnits: Pick<SynchronizationUnit, 'syncId'>[]
+    ) {
+        const listeners = await this.getListeners(driveId)
+        await Promise.all(listeners.map((listener) => {
+            for (const syncUnit of syncUnits) {
+                listener.syncUnits.delete(syncUnit.syncId);
+            }
+
+            return this.updateListener(listener);
+
+        }))
+
+        return;
+    }
+
+    async updateSynchronizationRevisions(
+        driveId: string,
+        syncUnits: SynchronizationUnit[],
+        source: StrandUpdateSource,
+        willUpdate?: (listeners: Listener[]) => void,
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void,
+        forceSync = false
+    ) {
+        const listeners = await this.getListeners(driveId);
+        const outdatedListeners: Listener[] = [];
+        for (const listener of listeners) {
+            if (
+                outdatedListeners.find(
+                    l => l.listenerId === listener.listener.listenerId
+                )
+            ) {
+                continue;
+            }
+            for (const syncUnit of syncUnits) {
+                if (!this._checkFilter(listener.listener.filter, syncUnit)) {
+                    continue;
+                }
+
+                const listenerRev = listener.syncUnits.get(syncUnit.syncId);
+
+                if (
+                    !listenerRev ||
+                    listenerRev.listenerRev < syncUnit.revision
+                ) {
+                    outdatedListeners.push(listener.listener);
+                    break;
+                }
+            }
+        }
+
+        if (outdatedListeners.length) {
+            willUpdate?.(outdatedListeners);
+            return this.triggerUpdate(forceSync, source, onError);
+        }
+        return [];
+    }
+
+    async updateListenerRevision(
+        listenerId: string,
+        driveId: string,
+        syncId: string,
+        listenerRev: number
+    ): Promise<void> {
+        const listener = await this.getListener(driveId, listenerId);
+        const lastUpdated = new Date().toISOString();
+        const entry = listener.syncUnits.get(syncId);
+        if (entry) {
+            entry.listenerRev = listenerRev;
+            entry.lastUpdated = lastUpdated;
+        } else {
+            listener.syncUnits.set(syncId, { listenerRev, lastUpdated });
+        }
+
+        await this.updateListener(listener);
+
+        return Promise.resolve();
+    }
+
+    triggerUpdate = debounce(
+        this.__triggerUpdate.bind(this),
+        ListenerManager.LISTENER_UPDATE_DELAY
+    );
+
+    private async __triggerUpdate(
+        source: StrandUpdateSource,
+        onError?: (
+            error: Error,
+            driveId: string,
+            listener: ListenerState
+        ) => void
+    ) {
+        const listenerUpdates: ListenerUpdate[] = [];
+        for (const [driveId, drive] of this.listenerState) {
+            for (const [id, listener] of drive) {
+                const transmitter = await this.getTransmitter(driveId, id);
+                if (!transmitter) {
+                    continue;
+                }
+
+                const syncUnits = await this.getListenerSyncUnits(
+                    driveId,
+                    listener.listener.listenerId
+                );
+
+                const strandUpdates: StrandUpdate[] = [];
+                for (const syncUnit of syncUnits) {
+                    const unitState = listener.syncUnits.get(syncUnit.syncId);
+
+
+
+                    if (
+                        unitState &&
+                        unitState.listenerRev >= syncUnit.revision
+                    ) {
+                        continue;
+                    }
+
+                    const opData: OperationUpdate[] = [];
+                    try {
+                        const data = await this.drive.getOperationData(
+                            // TODO - join queries, DEAL WITH INVALID SYNC ID ERROR
+                            driveId,
+                            syncUnit.syncId,
+                            {
+                                fromRevision: unitState?.listenerRev
+                            }
+                        );
+                        opData.push(...data);
+                    } catch (e) {
+                        logger.error(e);
+                    }
+
+                    if (!opData.length) {
+                        continue;
+                    }
+
+                    strandUpdates.push({
+                        driveId,
+                        documentId: syncUnit.documentId,
+                        branch: syncUnit.branch,
+                        operations: opData,
+                        scope: syncUnit.scope as OperationScope
+                    });
+                }
+
+                if (strandUpdates.length == 0) {
+                    continue;
+                }
+
+                listener.pendingTimeout = new Date(
+                    new Date().getTime() / 1000 + 300
+                ).toISOString();
+                listener.listenerStatus = 'PENDING';
+
+                // TODO update listeners in parallel, blocking for listeners with block=true
+                try {
+                    const listenerRevisions = await transmitter?.transmit(
+                        strandUpdates,
+                        source
+                    );
+
+                    listener.pendingTimeout = '0';
+                    listener.listenerStatus = 'PENDING';
+
+                    const lastUpdated = new Date().toISOString();
+
+                    for (const revision of listenerRevisions) {
+                        const syncUnit = syncUnits.find(
+                            unit =>
+                                revision.documentId === unit.documentId &&
+                                revision.scope === unit.scope &&
+                                revision.branch === unit.branch
+                        );
+                        if (syncUnit) {
+                            listener.syncUnits.set(syncUnit.syncId, {
+                                lastUpdated,
+                                listenerRev: revision.revision
+                            });
+                        } else {
+                            logger.warn(
+                                `Received revision for untracked unit for listener ${listener.listener.listenerId}`,
+                                revision
+                            );
+                        }
+                    }
+                    const revisionError = listenerRevisions.find(
+                        l => l.status !== 'SUCCESS'
+                    );
+                    if (revisionError) {
+                        throw new OperationError(
+                            revisionError.status as ErrorStatus,
+                            undefined,
+                            revisionError.error,
+                            revisionError.error
+                        );
+                    }
+                    listener.listenerStatus = 'SUCCESS';
+                    listenerUpdates.push({
+                        listenerId: listener.listener.listenerId,
+                        listenerRevisions
+                    });
+                } catch (e) {
+                    // TODO: Handle error based on listener params (blocking, retry, etc)
+                    onError?.(e as Error, driveId, listener);
+                    listener.listenerStatus =
+                        e instanceof OperationError ? e.status : 'ERROR';
+                }
+            }
+        }
+        return listenerUpdates;
+    }
+
+    async getListenerSyncUnits(driveId: string, listenerId: string) {
+        const listener = await this.getListener(driveId, listenerId)
+        const { filter } = listener.listener;
+        return this.drive.getSynchronizationUnits(
+            driveId,
+            filter.documentId ?? ['*'],
+            filter.scope ?? ['*'],
+            filter.branch ?? ['*'],
+            filter.documentType ?? ['*']
+        );
+    }
+
+    async getListenerSyncUnitIds(driveId: string, listenerId: string) {
+        const listener = await this.getListener(driveId, listenerId)
+        const { filter } = listener.listener;
+        return this.drive.getSynchronizationUnitsIds(
+            driveId,
+            filter.documentId ?? ['*'],
+            filter.scope ?? ['*'],
+            filter.branch ?? ['*'],
+            filter.documentType ?? ['*']
+        );
+    }
+
+    async initDrive(drive: DocumentDriveDocument) {
+        const {
+            state: {
+                local: { listeners }
+            }
+        } = drive;
+
+        for (const listener of listeners) {
+            await this.addListener({
+                block: listener.block,
+                driveId: drive.state.global.id,
+                filter: {
+                    branch: listener.filter.branch ?? [],
+                    documentId: listener.filter.documentId ?? [],
+                    documentType: listener.filter.documentType,
+                    scope: listener.filter.scope ?? []
+                },
+                listenerId: listener.listenerId,
+                system: listener.system,
+                callInfo: listener.callInfo ?? undefined,
+                label: listener.label ?? ''
+            });
+        }
+    }
+
+    async removeDrive(driveId: string): Promise<void> {
+        await this.redisClient.del(driveId);
+    }
+
+    async getListener(driveId: string, listenerId: string): Promise<ListenerState> {
+        const result = await this.redisClient.hGet(driveId, listenerId);
+        if (!result) throw new Error('Listener not found');
+        return JSON.parse(result);
+    }
+
+    async getListeners(driveId: string): Promise<ListenerState[]> {
+        const result = await this.redisClient.hGetAll(driveId);
+        if (!result) throw new Error('Listener not found');
+        const listenerIds = Object.keys(result);
+        return listenerIds.map(id => JSON.parse(result[id]!));
+    }
+
+    async getStrands(
+        driveId: string,
+        listenerId: string,
+        since?: string
+    ): Promise<StrandUpdate[]> {
+        // fetch listenerState from listenerManager
+        const listener = await this.getListener(driveId, listenerId);
+
+        // fetch operations from drive  and prepare strands
+        const strands: StrandUpdate[] = [];
+
+        const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
+
+        for (const syncUnit of syncUnits) {
+            if (syncUnit.revision < 0) {
+                continue;
+            }
+            const entry = listener.syncUnits.get(syncUnit.syncId);
+            if (entry && entry.listenerRev >= syncUnit.revision) {
+                continue;
+            }
+
+            const { documentId, driveId, scope, branch } = syncUnit;
+            try {
+                const operations = await this.drive.getOperationData(
+                    // DEAL WITH INVALID SYNC ID ERROR
+                    driveId,
+                    syncUnit.syncId,
+                    {
+                        since,
+                        fromRevision: entry?.listenerRev
+                    }
+                );
+
+                if (!operations.length) {
+                    continue;
+                }
+
+                strands.push({
+                    driveId,
+                    documentId,
+                    scope: scope as OperationScope,
+                    branch,
+                    operations
+                });
+            } catch (error) {
+                logger.error(error);
+                continue;
+            }
+        }
+
+        return strands;
+    }
+}

package/src/server/listener/transmitter/pull-responder.ts

@@ -7,6 +7,7 @@ import { logger as defaultLogger } from '../../../utils/logger';
 import { OperationError } from '../../error';
 import {
     BaseDocumentDriveServer,
+    BaseListenerManager,
     IOperationResult,
     Listener,
     ListenerRevision,
@@ -47,12 +48,12 @@ export interface IPullResponderTransmitter extends ITransmitter {
 export class PullResponderTransmitter implements IPullResponderTransmitter {
     private drive: BaseDocumentDriveServer;
     private listener: Listener;
-    private manager: ListenerManager;
+    private manager: BaseListenerManager;
 
     constructor(
         listener: Listener,
         drive: BaseDocumentDriveServer,
-        manager: ListenerManager
+        manager: BaseListenerManager
     ) {
         this.listener = listener;
         this.drive = drive;

package/src/server/types.ts

@@ -341,6 +341,12 @@ export abstract class BaseDocumentDriveServer {
     abstract clearStorage(): Promise<void>;
 }
 
+export abstract class IListenerState {
+    abstract setListener(listener: ListenerState): Promise<void>;
+    abstract getListener(driveId: string, listenerId: string): Promise<ListenerState>;
+    abstract removeListener(driveId: string, listenerId: string): Promise<void>;
+}
+
 export abstract class BaseListenerManager {
     protected drive: BaseDocumentDriveServer;
     protected listenerState = new Map<string, Map<string, ListenerState>>();

package/src/server/listener/transmitter/subscription-push.ts

@@ -1,364 +0,0 @@
-import { Client, createClient } from 'graphql-ws';
-import WebSocket from 'isomorphic-ws';
-import { logger } from '../../../utils/logger';
-import {
-    IOperationResult,
-    Listener,
-    ListenerRevision,
-    ListenerRevisionWithError,
-    RemoteDriveOptions,
-    StrandUpdate
-} from '../../types';
-import { ListenerManager } from '../manager';
-import { ITriggerTransmitter, SubscriptionTrigger } from './types';
-import { ListenerFilter, Trigger, z } from 'document-model-libs/document-drive';
-import { gql, requestGraphql } from '../../../utils/graphql';
-import { generateUUID } from '../../../utils';
-import { OperationScope } from 'document-model/document';
-import { OperationError } from '../../error';
-import { StrandUpdateGraphQL } from './pull-responder';
-
-
-export class SubscriptionPushTransmitter implements ITriggerTransmitter {
-    private _strands: StrandUpdate[] = [];
-    private listener: Listener;
-    private manager: ListenerManager;
-    private _init: Promise<StrandUpdate[]> | null = null;
-    private handler: ((strands: StrandUpdate[]) => void) | null = null;
-
-    constructor(
-        listener: Listener,
-        manager: ListenerManager,
-    ) {
-        this.listener = listener;
-        this.manager = manager;
-    }
-
-    async init() {
-        if (this._init) {
-            return this._init;
-        }
-        this._init = this.#refreshStrands();
-        return this._init;
-    }
-
-    #updateStrands(strands: StrandUpdate[]): void {
-        this._strands = strands;
-        this.handler?.(strands);
-    }
-
-    async #refreshStrands() {
-        const strands = await this.manager.getStrands(
-            this.listener.driveId,
-            this.listener.listenerId
-        );
-        this.#updateStrands(strands);
-        return Promise.resolve(strands);
-    }
-
-    async * strandsGenerator(since?: number): AsyncGenerator<StrandUpdate[]> {
-        await this.init();
-        let waitHandler = null;
-        let firstTime = true;
-        // eslint-disable-next-line
-        while (true) {
-            if (waitHandler) {
-                await waitHandler;
-            }
-            waitHandler = new Promise<StrandUpdate[]>(resolve => {
-                this.handler = resolve;
-            });
-
-            // only return empty array on first call
-            if (this._strands.length || firstTime) {
-                firstTime = false;
-                // TODO add support for 'since' parameter
-                yield this._strands.slice();
-            }
-        }
-    }
-
-    async transmit(strands: StrandUpdate[]): Promise<ListenerRevision[]> {
-        // if subscription has not been initiated by
-        // the client then ignores new strands
-        if (!this._init) {
-            return [];
-        }
-        console.log("TRANSMIT", this.listener.listenerId, strands.map(s => s.operations.length))
-        this.#updateStrands([...this._strands, ...strands]);
-        return Promise.resolve([]);
-    }
-
-    async processAcknowledge(
-        driveId: string,
-        listenerId: string,
-        revisions: ListenerRevision[]
-    ): Promise<boolean> {
-        const syncUnits = await this.manager.getListenerSyncUnits(
-            driveId,
-            listenerId
-        );
-        let success = true;
-        let acknowledged = false;
-        for (const revision of revisions) {
-            const syncUnit = syncUnits.find(
-                s =>
-                    s.scope === revision.scope &&
-                    s.branch === revision.branch &&
-                    s.driveId === revision.driveId &&
-                    s.documentId == revision.documentId
-            );
-            if (!syncUnit) {
-                logger.warn(
-                    'Unknown sync unit was acknowledged',
-                    revision
-                );
-                success = false;
-                continue;
-            }
-
-            await this.manager.updateListenerRevision(
-                listenerId,
-                driveId,
-                syncUnit.syncId,
-                revision.revision
-            );
-            acknowledged = true;
-        }
-        if (acknowledged) {
-            await this.#refreshStrands();
-        }
-
-        return success;
-    }
-
-    static async registerSubscription(
-        driveId: string,
-        url: string,
-        filter: ListenerFilter
-    ): Promise<Listener['listenerId']> {
-        // graphql request to switchboard
-        const { registerListener } = await requestGraphql<{
-            registerListener: {
-                listenerId: Listener['listenerId'];
-            };
-        }>(
-            url,
-            gql`
-                mutation registerListener(
-                    $filter: InputListenerFilter!
-                    $type: TransmitterType!
-                ) {
-                    registerListener(filter: $filter, type: $type) {
-                        listenerId
-                    }
-                }
-            `,
-            { filter, type: 'Subscription' }
-        );
-        return registerListener.listenerId;
-    }
-
-    static async createTrigger(
-        driveId: string,
-        url: string,
-        options: Pick<RemoteDriveOptions, 'pullFilter'>
-    ): Promise<Trigger> {
-        const { pullFilter } = options;
-        const listenerId = await SubscriptionTransmitter.registerSubscription(
-            driveId,
-            url,
-            pullFilter ?? {
-                documentId: ['*'],
-                documentType: ['*'],
-                branch: ['*'],
-                scope: ['*']
-            }
-        );
-
-        const trigger: Trigger = {
-            id: generateUUID(),
-            type: 'Subscription',
-            data: {
-                url,
-                listenerId,
-            }
-        };
-        return trigger;
-    }
-
-    static isTrigger(
-        trigger: Trigger
-    ): trigger is SubscriptionTrigger {
-        return (
-            trigger.type === 'Subscription' &&
-            z.SubscriptionTriggerDataSchema().safeParse(trigger.data).success
-        );
-    }
-
-    static setup(driveId: string,
-        trigger: SubscriptionTrigger,
-        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
-        onError: (error: Error) => void,
-        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
-        onAcknowledge?: (success: boolean) => void) {
-
-        const { url } = trigger.data;
-        let subscriptionUrl = url.replace("http", "ws")
-        subscriptionUrl += subscriptionUrl.endsWith("/") ? "ws" : "/ws";
-
-        const client = createClient({
-            url: subscriptionUrl,
-            webSocketImpl: WebSocket,
-        });
-        try {
-            SubscriptionTransmitter.subscribeStrands(client, trigger, onStrandUpdate, onError, onRevisions, onAcknowledge).catch(onError);
-        } catch (error) {
-            onError(error as Error);
-        }
-        return () => { return client.dispose(); };
-    }
-
-    private static async subscribeStrands(client: Client, trigger: SubscriptionTrigger,
-        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
-        onError: (error: Error) => void,
-        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
-        onAcknowledge?: (success: boolean) => void) {
-        const { listenerId } = trigger.data;
-        const subscription = client.iterate<{ subscribeStrands: StrandUpdateGraphQL[] }>({
-            query: `
-            subscription($listenerId: ID) {
-                subscribeStrands(listenerId: $listenerId) {
-                branch
-                documentId
-                driveId
-                operations {
-                    timestamp
-                    skip
-                    type
-                    input
-                    hash
-                    index
-                    context {
-                        signer {
-                            user {
-                                address
-                                networkId
-                                chainId
-                            }
-                            app {
-                                name
-                                key
-                            }
-                            signature
-                        }
-                    }
-                }
-                scope
-                }
-            }`,
-            variables: {
-                listenerId,
-            }
-        });
-        for await (const { errors, data } of subscription) {
-            const error = errors?.at(0);
-            if (error) {
-                onError(error);
-            } else {
-                const strands = data?.subscribeStrands ?? [];
-                console.log(listenerId, "Save strands", strands);
-                SubscriptionTransmitter.saveStrands(trigger, client, strands, onStrandUpdate, onError, onRevisions, onAcknowledge).catch(onError);
-            }
-        }
-    }
-
-    static async saveStrands(
-        trigger: SubscriptionTrigger,
-        client: Client,
-        strandsQL: StrandUpdateGraphQL[],
-        onStrandUpdate: (strand: StrandUpdate) => Promise<IOperationResult>,
-        onError: (error: Error) => void,
-        onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
-        onAcknowledge?: (success: boolean) => void) {
-        const strands = strandsQL.map(s => ({
-            ...s,
-            operations: s.operations.map(o => ({
-                ...o,
-                scope: s.scope,
-                branch: s.branch,
-                input: JSON.parse(o.input) as object
-            }))
-        }));
-        // if there are no new strands then do nothing
-        if (!strands.length) {
-            onRevisions?.([]);
-            return;
-        }
-
-        const listenerRevisions: ListenerRevisionWithError[] = [];
-
-
-        for (const strand of strands) {
-            let error: Error | undefined = undefined;
-            let result: IOperationResult | undefined = undefined;
-            try {
-                result = await onStrandUpdate(strand);
-                if (result.error) {
-                    throw result.error;
-                }
-            } catch (e) {
-                error = e as Error;
-                onError(error);
-            }
-
-            listenerRevisions.push({
-                branch: strand.branch,
-                documentId: strand.documentId || '',
-                driveId: strand.driveId,
-                revision: result?.document?.operations[strand.scope]?.at(-1)?.index ?? -1,
-                scope: strand.scope as OperationScope,
-                status: error
-                    ? error instanceof OperationError
-                        ? error.status
-                        : 'ERROR'
-                    : 'SUCCESS',
-                error
-            });
-        }
-
-        onRevisions?.(listenerRevisions);
-
-        await SubscriptionTransmitter.acknowledgeStrands(
-            client,
-            trigger.data.listenerId,
-            listenerRevisions.map(revision => {
-                const { error, ...rest } = revision;
-                return rest;
-            })
-        )
-            .then(result => onAcknowledge?.(result))
-            .catch(error => logger.error('ACK error', error));
-    }
-
-    static async acknowledgeStrands(
-        client: Client,
-        listenerId: string,
-        revisions: ListenerRevision[]
-    ): Promise<boolean> {
-        const subscription = client.iterate<{ acknowledge: boolean }>({
-            query: `
-            mutation acknowledge(
-                $listenerId: String!
-                $revisions: [ListenerRevisionInput]
-            ) {
-                acknowledge(listenerId: $listenerId, revisions: $revisions)
-            }
-        `,
-            variables: { listenerId, revisions }
-        });
-        const result = await subscription.next();
-
-        return (result.value as { acknowledge: boolean }).acknowledge as boolean;
-    }
-}