document-drive 1.17.2 → 1.19.0

package/src/server/listener/transmitter/pull-responder.ts

@@ -1,5 +1,5 @@
 import { ListenerFilter, Trigger } from "document-model-libs/document-drive";
-import { Operation, OperationScope } from "document-model/document";
+import { Operation } from "document-model/document";
 import { PULL_DRIVE_INTERVAL } from "../..";
 import { generateUUID } from "../../../utils";
 import { gql, requestGraphql } from "../../../utils/graphql";
@@ -7,7 +7,7 @@ import { logger as defaultLogger } from "../../../utils/logger";
 import { OperationError } from "../../error";
 import {
   GetStrandsOptions,
-  IBaseDocumentDriveServer,
+  IListenerManager,
   IOperationResult,
   Listener,
   ListenerRevision,
@@ -16,13 +16,14 @@ import {
   RemoteDriveOptions,
   StrandUpdate,
 } from "../../types";
-import { ListenerManager } from "../manager";
 import {
   ITransmitter,
   PullResponderTrigger,
   StrandUpdateSource,
 } from "./types";
 
+const ENABLE_SYNC_DEBUG = false;
+
 export type OperationUpdateGraphQL = Omit<OperationUpdate, "input"> & {
   input: string;
 };
@@ -45,22 +46,52 @@ export interface IPullResponderTransmitter extends ITransmitter {
   getStrands(options?: GetStrandsOptions): Promise<StrandUpdate[]>;
 }
 
+const STATIC_DEBUG_ID = `[PRT #static]`;
+
+function staticDebugLog(...data: any[]) {
+  if (!ENABLE_SYNC_DEBUG) {
+    return;
+  }
+
+  if (data.length > 0 && typeof data[0] === "string") {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    console.log(`${STATIC_DEBUG_ID} ${data[0]}`, ...data.slice(1));
+  } else {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    console.log(STATIC_DEBUG_ID, ...data);
+  }
+}
+
 export class PullResponderTransmitter implements IPullResponderTransmitter {
-  private drive: IBaseDocumentDriveServer;
+  private debugID = `[PRT #${Math.floor(Math.random() * 999)}]`;
   private listener: Listener;
-  private manager: ListenerManager;
+  private manager: IListenerManager;
 
-  constructor(
-    listener: Listener,
-    drive: IBaseDocumentDriveServer,
-    manager: ListenerManager,
-  ) {
+  constructor(listener: Listener, manager: IListenerManager) {
     this.listener = listener;
-    this.drive = drive;
     this.manager = manager;
+    this.debugLog(`constructor(listener: ${listener.listenerId})`);
+  }
+
+  private debugLog(...data: any[]) {
+    if (!ENABLE_SYNC_DEBUG) {
+      return;
+    }
+
+    if (data.length > 0 && typeof data[0] === "string") {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(`${this.debugID} ${data[0]}`, ...data.slice(1));
+    } else {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(this.debugID, ...data);
+    }
   }
 
   getStrands(options?: GetStrandsOptions): Promise<StrandUpdate[]> {
+    this.debugLog(
+      `getStrands(drive: ${this.listener.driveId}, listener: ${this.listener.listenerId})`,
+    );
+
     return this.manager.getStrands(
       this.listener.driveId,
       this.listener.listenerId,
@@ -78,6 +109,11 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     revisions: ListenerRevision[],
   ): Promise<boolean> {
+    this.debugLog(
+      `processAcknowledge(drive: ${driveId}, listener: ${listenerId})`,
+      revisions,
+    );
+
     const syncUnits = await this.manager.getListenerSyncUnitIds(
       driveId,
       listenerId,
@@ -113,6 +149,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     url: string,
     filter: ListenerFilter,
   ): Promise<Listener["listenerId"]> {
+    staticDebugLog(`registerPullResponder(url: ${url})`, filter);
     // graphql request to switchboard
     const result = await requestGraphql<{
       registerPullResponderListener: {
@@ -148,6 +185,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     options?: GetStrandsOptions, // TODO add support for since
   ): Promise<StrandUpdate[]> {
+    staticDebugLog(`pullStrands(url: ${url}, listener: ${listenerId})`);
     const result = await requestGraphql<PullStrandsGraphQL>(
       url,
       gql`
@@ -214,6 +252,11 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     revisions: ListenerRevision[],
   ): Promise<boolean> {
+    staticDebugLog(
+      `acknowledgeStrands(url: ${url}, listener: ${listenerId})`,
+      revisions,
+    );
+
     const result = await requestGraphql<{ acknowledge: boolean }>(
       url,
       gql`
@@ -248,6 +291,8 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
     onAcknowledge?: (success: boolean) => void,
   ) {
+    staticDebugLog(`executePull(driveId: ${driveId}), trigger:`, trigger);
+
     try {
       const { url, listenerId } = trigger.data;
       const strands = await PullResponderTransmitter.pullStrands(
@@ -330,6 +375,8 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
     onAcknowledge?: (success: boolean) => void,
   ): CancelPullLoop {
+    staticDebugLog(`setupPull(drive: ${driveId}), trigger:`, trigger);
+
     const { interval } = trigger.data;
     let loopInterval = PULL_DRIVE_INTERVAL;
     if (interval) {
@@ -348,6 +395,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
 
     const executeLoop = async () => {
       while (!isCancelled) {
+        staticDebugLog("Execute loop...");
         await this.executePull(
           driveId,
           trigger,
@@ -357,6 +405,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
           onAcknowledge,
         );
         await new Promise((resolve) => {
+          staticDebugLog(`Scheduling next pull in ${loopInterval} ms`);
           timeout = setTimeout(resolve, loopInterval) as unknown as number;
         });
       }
@@ -377,6 +426,10 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     url: string,
     options: Pick<RemoteDriveOptions, "pullInterval" | "pullFilter">,
   ): Promise<PullResponderTrigger> {
+    staticDebugLog(
+      `createPullResponderTrigger(drive: ${driveId}, url: ${url})`,
+    );
+
     const { pullFilter, pullInterval } = options;
     const listenerId = await PullResponderTransmitter.registerPullResponder(
       driveId,

package/src/server/listener/transmitter/switchboard-push.ts

@@ -1,23 +1,32 @@
 import stringify from "json-stringify-deterministic";
 import { gql, requestGraphql } from "../../../utils/graphql";
 import { logger } from "../../../utils/logger";
-import {
-  IBaseDocumentDriveServer,
-  Listener,
-  ListenerRevision,
-  StrandUpdate,
-} from "../../types";
+import { ListenerRevision, StrandUpdate } from "../../types";
 import { ITransmitter, StrandUpdateSource } from "./types";
 
+const ENABLE_SYNC_DEBUG = false;
+const SYNC_OPS_BATCH_LIMIT = 10;
+
 export class SwitchboardPushTransmitter implements ITransmitter {
-  private drive: IBaseDocumentDriveServer;
-  private listener: Listener;
   private targetURL: string;
+  private debugID = `[SPT #${Math.floor(Math.random() * 999)}]`;
+
+  constructor(targetURL: string) {
+    this.targetURL = targetURL;
+  }
+
+  private debugLog(...data: any[]) {
+    if (!ENABLE_SYNC_DEBUG) {
+      return false;
+    }
 
-  constructor(listener: Listener, drive: IBaseDocumentDriveServer) {
-    this.listener = listener;
-    this.drive = drive;
-    this.targetURL = listener.callInfo!.data!;
+    if (data.length > 0 && typeof data[0] === "string") {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(`${this.debugID} ${data[0]}`, ...data.slice(1));
+    } else {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(this.debugID, ...data);
+    }
   }
 
   async transmit(
@@ -28,6 +37,7 @@ export class SwitchboardPushTransmitter implements ITransmitter {
       source.type === "trigger" &&
       source.trigger.data?.url === this.targetURL
     ) {
+      this.debugLog(`Cutting trigger loop from ${this.targetURL}.`);
       return strands.map((strand) => ({
         driveId: strand.driveId,
         documentId: strand.documentId,
@@ -38,6 +48,39 @@ export class SwitchboardPushTransmitter implements ITransmitter {
       }));
     }
 
+    const culledStrands: StrandUpdate[] = [];
+    let opsCounter = 0;
+
+    for (
+      let s = 0;
+      opsCounter <= SYNC_OPS_BATCH_LIMIT && s < strands.length;
+      s++
+    ) {
+      const currentStrand = strands.at(s);
+      if (!currentStrand) {
+        break;
+      }
+      const newOps = Math.min(
+        SYNC_OPS_BATCH_LIMIT - opsCounter,
+        currentStrand.operations.length,
+      );
+
+      culledStrands.push({
+        ...currentStrand,
+        operations: currentStrand.operations.slice(0, newOps),
+      });
+
+      opsCounter += newOps;
+    }
+
+    this.debugLog(
+      ` Total update: [${strands.map((s) => s.operations.length).join(", ")}] operations`,
+    );
+
+    this.debugLog(
+      `Culled update: [${culledStrands.map((s) => s.operations.length).join(", ")}] operations`,
+    );
+
     // Send Graphql mutation to switchboard
     try {
       const { pushUpdates } = await requestGraphql<{
@@ -58,7 +101,7 @@ export class SwitchboardPushTransmitter implements ITransmitter {
           }
         `,
         {
-          strands: strands.map((strand) => ({
+          strands: culledStrands.map((strand) => ({
             ...strand,
             operations: strand.operations.map((op) => ({
               ...op,

package/src/server/types.ts

@@ -22,15 +22,15 @@ import type {
 } from "document-model/document";
 import { Unsubscribe } from "nanoevents";
 import { BaseDocumentDriveServer } from ".";
-import {
-  IReceiver as IInternalListener,
-  IInternalTransmitter,
-} from "./listener/transmitter/internal";
 import { IReadModeDriveServer } from "../read-mode/types";
 import { RunAsap } from "../utils";
 import { IDefaultDrivesManager } from "../utils/default-drives-manager";
 import { DriveInfo } from "../utils/graphql";
 import { OperationError, SynchronizationUnitNotFoundError } from "./error";
+import {
+  IReceiver as IInternalListener,
+  IInternalTransmitter,
+} from "./listener/transmitter/internal";
 import {
   ITransmitter,
   PullResponderTrigger,
@@ -105,6 +105,7 @@ export type Listener = {
   system: boolean;
   filter: ListenerFilter;
   callInfo?: ListenerCallInfo;
+  transmitter?: ITransmitter;
 };
 
 export type CreateListenerInput = {
@@ -317,137 +318,150 @@ export type GetStrandsOptions = {
   fromRevision?: number;
 };
 
-export abstract class AbstractDocumentDriveServer {
-  /** Public methods **/
-  abstract initialize(): Promise<Error[] | null>;
-  abstract setDocumentModels(models: DocumentModel[]): void;
-  abstract getDrives(): Promise<string[]>;
-  abstract addDrive(drive: DriveInput): Promise<DocumentDriveDocument>;
-  abstract addRemoteDrive(
+export type ListenerManagerOptions = {
+  sequentialUpdates?: boolean;
+};
+
+export const DefaultListenerManagerOptions = {
+  sequentialUpdates: true,
+};
+
+type PublicKeys<T> = {
+  [K in keyof T]: T extends { [P in K]: T[K] } ? K : never;
+}[keyof T];
+
+type PublicPart<T> = Pick<T, PublicKeys<T>>;
+
+export interface IBaseDocumentDriveServer {
+  initialize(): Promise<Error[] | null>;
+  setDocumentModels(models: DocumentModel[]): void;
+  getDrives(): Promise<string[]>;
+  addDrive(input: DriveInput): Promise<DocumentDriveDocument>;
+  addRemoteDrive(
     url: string,
     options: RemoteDriveOptions,
   ): Promise<DocumentDriveDocument>;
-  abstract deleteDrive(id: string): Promise<void>;
-  abstract getDrive(
-    id: string,
+  deleteDrive(driveId: string): Promise<void>;
+  getDrive(
+    driveId: string,
     options?: GetDocumentOptions,
   ): Promise<DocumentDriveDocument>;
 
-  abstract getDriveBySlug(slug: string): Promise<DocumentDriveDocument>;
+  getDriveBySlug(slug: string): Promise<DocumentDriveDocument>;
 
-  abstract getDocuments(drive: string): Promise<string[]>;
-  abstract getDocument(
-    drive: string,
-    id: string,
+  getDocuments(driveId: string): Promise<string[]>;
+  getDocument(
+    driveId: string,
+    documentId: string,
     options?: GetDocumentOptions,
   ): Promise<Document>;
 
-  abstract addOperation(
-    drive: string,
-    id: string,
+  addOperation(
+    driveId: string,
+    documentId: string,
     operation: Operation,
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract addOperations(
-    drive: string,
-    id: string,
+  addOperations(
+    driveId: string,
+    documentId: string,
     operations: Operation[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract queueOperation(
-    drive: string,
-    id: string,
+  queueOperation(
+    driveId: string,
+    documentId: string,
     operation: Operation,
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract queueOperations(
-    drive: string,
-    id: string,
+  queueOperations(
+    driveId: string,
+    documentId: string,
     operations: Operation[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract queueAction(
-    drive: string,
-    id: string,
+  queueAction(
+    driveId: string,
+    documentId: string,
     action: Action,
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract queueActions(
-    drive: string,
-    id: string,
+  queueActions(
+    driveId: string,
+    documentId: string,
     actions: Action[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract addDriveOperation(
-    drive: string,
+  addDriveOperation(
+    driveId: string,
     operation: Operation<DocumentDriveAction | BaseAction>,
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
-  abstract addDriveOperations(
-    drive: string,
+  addDriveOperations(
+    driveId: string,
     operations: Operation<DocumentDriveAction | BaseAction>[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract queueDriveOperation(
-    drive: string,
+  queueDriveOperation(
+    driveId: string,
     operation: Operation<DocumentDriveAction | BaseAction>,
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract queueDriveOperations(
-    drive: string,
+  queueDriveOperations(
+    driveId: string,
     operations: Operation<DocumentDriveAction | BaseAction>[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract queueDriveAction(
-    drive: string,
+  queueDriveAction(
+    driveId: string,
     action: DocumentDriveAction | BaseAction,
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract queueDriveActions(
-    drive: string,
+  queueDriveActions(
+    driveId: string,
     actions: Array<DocumentDriveAction | BaseAction>,
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract addAction(
-    drive: string,
-    id: string,
+  addAction(
+    driveId: string,
+    documentId: string,
     action: Action,
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
-  abstract addActions(
-    drive: string,
-    id: string,
+  addActions(
+    driveId: string,
+    documentId: string,
     actions: Action[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult>;
 
-  abstract addDriveAction(
-    drive: string,
+  addDriveAction(
+    driveId: string,
     action: DocumentDriveAction | BaseAction,
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
-  abstract addDriveActions(
-    drive: string,
+  addDriveActions(
+    driveId: string,
     actions: (DocumentDriveAction | BaseAction)[],
     options?: AddOperationOptions,
   ): Promise<IOperationResult<DocumentDriveDocument>>;
 
-  abstract getSyncStatus(
+  getSyncStatus(
     syncUnitId: string,
   ): SyncStatus | SynchronizationUnitNotFoundError;
 
-  abstract addInternalListener(
+  addInternalListener(
     driveId: string,
     receiver: IInternalListener,
     options: {
@@ -459,7 +473,7 @@ export abstract class AbstractDocumentDriveServer {
   ): Promise<IInternalTransmitter>;
 
   /** Synchronization methods */
-  abstract getSynchronizationUnits(
+  getSynchronizationUnits(
     driveId: string,
     documentId?: string[],
     scope?: string[],
@@ -468,13 +482,13 @@ export abstract class AbstractDocumentDriveServer {
     loadedDrive?: DocumentDriveDocument,
   ): Promise<SynchronizationUnit[]>;
 
-  abstract getSynchronizationUnit(
+  getSynchronizationUnit(
     driveId: string,
     syncId: string,
     loadedDrive?: DocumentDriveDocument,
   ): Promise<SynchronizationUnit | undefined>;
 
-  abstract getSynchronizationUnitsIds(
+  getSynchronizationUnitsIds(
     driveId: string,
     documentId?: string[],
     scope?: string[],
@@ -482,7 +496,7 @@ export abstract class AbstractDocumentDriveServer {
     documentType?: string[],
   ): Promise<SynchronizationUnitQuery[]>;
 
-  abstract getOperationData(
+  getOperationData(
     driveId: string,
     syncId: string,
     filter: GetStrandsOptions,
@@ -490,117 +504,73 @@ export abstract class AbstractDocumentDriveServer {
   ): Promise<OperationUpdate[]>;
 
   /** Internal methods **/
-  protected abstract createDocument(
-    drive: string,
-    document: CreateDocumentInput,
-  ): Promise<Document>;
-  protected abstract deleteDocument(drive: string, id: string): Promise<void>;
-
-  protected abstract getDocumentModel(documentType: string): DocumentModel;
-  abstract getDocumentModels(): DocumentModel[];
-
-  /** Event methods **/
-  protected abstract emit<K extends keyof DriveEvents>(
-    event: K,
-    ...args: Parameters<DriveEvents[K]>
-  ): void;
-  abstract on<K extends keyof DriveEvents>(
-    event: K,
-    cb: DriveEvents[K],
-  ): Unsubscribe;
-
-  abstract getTransmitter(
+  getDocumentModels(): DocumentModel[];
+
+  getTransmitter(
     driveId: string,
     listenerId: string,
   ): Promise<ITransmitter | undefined>;
 
-  abstract clearStorage(): Promise<void>;
+  clearStorage(): Promise<void>;
 
-  abstract registerPullResponderTrigger(
+  registerPullResponderTrigger(
     id: string,
     url: string,
     options: Pick<RemoteDriveOptions, "pullFilter" | "pullInterval">,
   ): Promise<PullResponderTrigger>;
-}
-
-export type ListenerManagerOptions = {
-  sequentialUpdates?: boolean;
-};
-
-export const DefaultListenerManagerOptions = {
-  sequentialUpdates: true,
-};
-
-type PublicKeys<T> = {
-  [K in keyof T]: T extends { [P in K]: T[K] } ? K : never;
-}[keyof T];
-
-type PublicPart<T> = Pick<T, PublicKeys<T>>;
 
-export type IBaseDocumentDriveServer = PublicPart<AbstractDocumentDriveServer>;
+  on<K extends keyof DriveEvents>(event: K, cb: DriveEvents[K]): Unsubscribe;
+}
 
 export type IDocumentDriveServer = IBaseDocumentDriveServer &
   IDefaultDrivesManager &
   IReadModeDriveServer;
 
-export abstract class BaseListenerManager {
-  protected drive: IBaseDocumentDriveServer;
-  protected listenerState = new Map<string, Map<string, ListenerState>>();
-  protected options: ListenerManagerOptions;
-  protected transmitters: Record<
-    DocumentDriveState["id"],
-    Record<Listener["listenerId"], ITransmitter>
-  > = {};
-
-  constructor(
-    drive: IBaseDocumentDriveServer,
-    listenerState = new Map<string, Map<string, ListenerState>>(),
-    options: ListenerManagerOptions = DefaultListenerManagerOptions,
-  ) {
-    this.drive = drive;
-    this.listenerState = listenerState;
-    this.options = { ...DefaultListenerManagerOptions, ...options };
-  }
-
-  abstract initDrive(drive: DocumentDriveDocument): Promise<void>;
-  abstract removeDrive(driveId: DocumentDriveState["id"]): Promise<void>;
-
-  abstract driveHasListeners(driveId: string): boolean;
-  abstract addListener(listener: Listener): Promise<ITransmitter>;
-  abstract removeListener(
-    driveId: string,
-    listenerId: string,
-  ): Promise<boolean>;
-  abstract getListener(
-    driveId: string,
-    listenerId: string,
-  ): Promise<ListenerState | undefined>;
+export type DriveUpdateErrorHandler = (
+  error: Error,
+  driveId: string,
+  listener: ListenerState,
+) => void;
 
-  abstract getTransmitter(
-    driveId: string,
-    listenerId: string,
-  ): Promise<ITransmitter | undefined>;
+export interface IListenerManager {
+  initialize(handler: DriveUpdateErrorHandler): Promise<void>;
 
-  abstract getStrands(
+  removeDrive(driveId: DocumentDriveState["id"]): Promise<void>;
+  driveHasListeners(driveId: string): boolean;
+
+  setListener(driveId: string, listener: Listener): Promise<void>;
+  removeListener(driveId: string, listenerId: string): Promise<boolean>;
+  getListenerState(driveId: string, listenerId: string): ListenerState;
+
+  getStrands(
     driveId: string,
     listenerId: string,
     options?: GetStrandsOptions,
   ): Promise<StrandUpdate[]>;
-
-  abstract updateSynchronizationRevisions(
+  updateSynchronizationRevisions(
     driveId: string,
     syncUnits: SynchronizationUnit[],
     source: StrandUpdateSource,
     willUpdate?: (listeners: Listener[]) => void,
     onError?: (error: Error, driveId: string, listener: ListenerState) => void,
+    forceSync?: boolean,
   ): Promise<ListenerUpdate[]>;
-
-  abstract updateListenerRevision(
+  updateListenerRevision(
     listenerId: string,
     driveId: string,
     syncId: string,
     listenerRev: number,
   ): Promise<void>;
+
+  // todo: re-evaluate
+  getListenerSyncUnitIds(
+    driveId: string,
+    listenerId: string,
+  ): Promise<SynchronizationUnitQuery[]>;
+  removeSyncUnits(
+    driveId: string,
+    syncUnits: Pick<SynchronizationUnit, "syncId">[],
+  ): Promise<void>;
 }
 
 export type ListenerStatus =