document-drive 1.18.0 → 1.19.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "document-drive",
-  "version": "1.18.0",
+  "version": "1.19.1",
   "license": "AGPL-3.0-only",
   "type": "module",
   "module": "./src/index.ts",
@@ -33,9 +33,9 @@
     "nanoevents": "^9.0.0",
     "prisma": "^5.18.0",
     "sanitize-filename": "^1.6.3",
-    "@powerhousedao/scalars": "1.23.0",
+    "@powerhousedao/scalars": "1.24.0",
     "document-model": "2.20.0",
-    "document-model-libs": "1.132.0"
+    "document-model-libs": "1.133.0"
   },
   "optionalDependencies": {
     "@prisma/client": "^5.18.0",
@@ -44,7 +44,7 @@
     "redis": "^4.6.15",
     "sequelize": "^6.37.3",
     "sqlite3": "^5.1.7",
-    "@powerhousedao/scalars": "1.23.0"
+    "@powerhousedao/scalars": "1.24.0"
   },
   "devDependencies": {
     "@prisma/client": "5.17.0",
@@ -66,7 +66,7 @@
     "vitest-fetch-mock": "^0.3.0",
     "webdriverio": "^9.0.9",
     "document-model": "2.20.0",
-    "document-model-libs": "1.132.0"
+    "document-model-libs": "1.133.0"
   },
   "scripts": {
     "check-types": "tsc --build",

package/src/server/index.ts

@@ -662,7 +662,15 @@ export class BaseDocumentDriveServer implements IBaseDocumentDriveServer {
     for (const zodListener of drive.state.local.listeners) {
       const transmitter = this.transmitterFactory.instance(
         zodListener.callInfo?.transmitterType ?? "",
-        zodListener as any,
+        {
+          driveId,
+          listenerId: zodListener.listenerId,
+          block: zodListener.block,
+          filter: zodListener.filter,
+          system: zodListener.system,
+          label: zodListener.label || undefined,
+          callInfo: zodListener.callInfo || undefined,
+        },
         this,
       );
 
@@ -2102,7 +2110,15 @@ export class BaseDocumentDriveServer implements IBaseDocumentDriveServer {
             // create the transmitter
             const transmitter = this.transmitterFactory.instance(
               zodListener.callInfo?.transmitterType ?? "",
-              zodListener as any,
+              {
+                driveId,
+                listenerId: zodListener.listenerId,
+                block: zodListener.block,
+                filter: zodListener.filter,
+                system: zodListener.system,
+                label: zodListener.label || undefined,
+                callInfo: zodListener.callInfo || undefined,
+              },
               this,
             );
 

package/src/server/listener/manager.ts

@@ -23,6 +23,8 @@ import {
 } from "../types";
 import { StrandUpdateSource } from "./transmitter/types";
 
+const ENABLE_SYNC_DEBUG = false;
+
 function debounce<T extends unknown[], R>(
   func: (...args: T) => Promise<R>,
   delay = 250,
@@ -50,26 +52,43 @@ function debounce<T extends unknown[], R>(
 
 export class ListenerManager implements IListenerManager {
   static LISTENER_UPDATE_DELAY = 250;
-
+  private debugID = `[LM  #${Math.floor(Math.random() * 999)}]`;
   protected driveServer: IBaseDocumentDriveServer;
+  protected options: ListenerManagerOptions;
+
   // driveId -> listenerId -> listenerState
   protected listenerStateByDriveId = new Map<
     string,
     Map<string, ListenerState>
   >();
-  protected options: ListenerManagerOptions;
 
   constructor(
     drive: IBaseDocumentDriveServer,
     listenerState = new Map<string, Map<string, ListenerState>>(),
     options: ListenerManagerOptions = DefaultListenerManagerOptions,
   ) {
+    this.debugLog(`constructor(...)`);
     this.driveServer = drive;
     this.listenerStateByDriveId = listenerState;
     this.options = { ...DefaultListenerManagerOptions, ...options };
   }
 
+  private debugLog(...data: any[]) {
+    if (!ENABLE_SYNC_DEBUG) {
+      return;
+    }
+
+    if (data.length > 0 && typeof data[0] === "string") {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(`${this.debugID} ${data[0]}`, ...data.slice(1));
+    } else {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(this.debugID, ...data);
+    }
+  }
+
   async initialize(handler: DriveUpdateErrorHandler) {
+    this.debugLog("initialize(...)");
     // if network connect comes back online
     // then triggers the listeners update
     if (typeof window !== "undefined") {
@@ -86,6 +105,10 @@ export class ListenerManager implements IListenerManager {
   }
 
   async setListener(driveId: string, listener: Listener) {
+    this.debugLog(
+      `setListener(drive: ${driveId}, listener: ${listener.listenerId})`,
+    );
+
     // slight code smell -- drive id may not need to be on listener or not passed in
     if (driveId !== listener.driveId) {
       throw new Error("Drive ID mismatch");
@@ -113,6 +136,8 @@ export class ListenerManager implements IListenerManager {
   }
 
   async removeListener(driveId: string, listenerId: string) {
+    this.debugLog("setListener()");
+
     const driveMap = this.listenerStateByDriveId.get(driveId);
     if (!driveMap) {
       return false;
@@ -222,27 +247,46 @@ export class ListenerManager implements IListenerManager {
   private async _triggerUpdate(
     source: StrandUpdateSource,
     onError?: (error: Error, driveId: string, listener: ListenerState) => void,
+    maxContinues = 500,
   ) {
+    this.debugLog(
+      `_triggerUpdate(source: ${source.type}, maxContinues: ${maxContinues})`,
+      this.listenerStateByDriveId,
+    );
+
+    if (maxContinues < 0) {
+      throw new Error("Maximum retries exhausted.");
+    }
+
     const listenerUpdates: ListenerUpdate[] = [];
+
     for (const [driveId, drive] of this.listenerStateByDriveId) {
-      for (const [_, listenerState] of drive) {
+      for (const [listenerId, listenerState] of drive) {
         const transmitter = listenerState.listener.transmitter;
+
         if (!transmitter?.transmit) {
+          this.debugLog(`Transmitter not set on listener: ${listenerId}`);
           continue;
         }
 
-        const syncUnits = await this.getListenerSyncUnits(
-          driveId,
-          listenerState.listener.listenerId,
-        );
-
+        const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
         const strandUpdates: StrandUpdate[] = [];
+
+        this.debugLog("syncUnits", syncUnits);
+
         // TODO change to push one after the other, reusing operation data
         const tasks = syncUnits.map((syncUnit) => async () => {
           const unitState = listenerState.syncUnits.get(syncUnit.syncId);
 
           if (unitState && unitState.listenerRev >= syncUnit.revision) {
+            this.debugLog(
+              `Abandoning push for sync unit ${syncUnit.syncId}: already up-to-date (${unitState.listenerRev} >= ${syncUnit.revision})`,
+            );
             return;
+          } else {
+            this.debugLog(
+              `Listener out-of-date for sync unit ${syncUnit.syncId}: ${unitState?.listenerRev} < ${syncUnit.revision}`,
+            );
           }
 
           const opData: OperationUpdate[] = [];
@@ -261,6 +305,9 @@ export class ListenerManager implements IListenerManager {
           }
 
           if (!opData.length) {
+            this.debugLog(
+              `Abandoning push for ${syncUnit.syncId}: no operations found`,
+            );
             return;
           }
 
@@ -272,34 +319,53 @@ export class ListenerManager implements IListenerManager {
             scope: syncUnit.scope as OperationScope,
           });
         });
+
         if (this.options.sequentialUpdates) {
+          this.debugLog(
+            `Collecting ${tasks.length} syncUnit strandUpdates in sequence`,
+          );
           for (const task of tasks) {
             await task();
           }
         } else {
+          this.debugLog(
+            `Collecting ${tasks.length} syncUnit strandUpdates in parallel`,
+          );
           await Promise.all(tasks.map((task) => task()));
         }
 
         if (strandUpdates.length == 0) {
+          this.debugLog(`No strandUpdates needed for listener ${listenerId}`);
           continue;
         }
 
         listenerState.pendingTimeout = new Date(
           new Date().getTime() / 1000 + 300,
         ).toISOString();
+
         listenerState.listenerStatus = "PENDING";
 
         // TODO update listeners in parallel, blocking for listeners with block=true
         try {
+          this.debugLog(
+            `_triggerUpdate(source: ${source.type}) > transmitter.transmit`,
+          );
+
           const listenerRevisions = await transmitter.transmit(
             strandUpdates,
             source,
           );
 
+          this.debugLog(
+            `_triggerUpdate(source: ${source.type}) > transmission succeeded`,
+            listenerRevisions,
+          );
+
           listenerState.pendingTimeout = "0";
           listenerState.listenerStatus = "PENDING";
 
           const lastUpdated = new Date().toISOString();
+          let continuationNeeded = false;
 
           for (const revision of listenerRevisions) {
             const syncUnit = syncUnits.find(
@@ -308,11 +374,42 @@ export class ListenerManager implements IListenerManager {
                 revision.scope === unit.scope &&
                 revision.branch === unit.branch,
             );
+
             if (syncUnit) {
               listenerState.syncUnits.set(syncUnit.syncId, {
                 lastUpdated,
                 listenerRev: revision.revision,
               });
+
+              // Check for revision status vv
+              const su = strandUpdates.find(
+                (su) =>
+                  su.driveId === revision.driveId &&
+                  su.documentId === revision.documentId &&
+                  su.scope === revision.scope &&
+                  su.branch === revision.branch,
+              );
+
+              if (su && su.operations.length > 0) {
+                const suIndex = su.operations.at(
+                  su.operations.length - 1,
+                )?.index;
+                if (suIndex !== revision.revision) {
+                  this.debugLog(
+                    `Revision still out-of-date for ${su.documentId}:${su.scope}:${su.branch} ${suIndex} <> ${revision.revision}`,
+                  );
+                  continuationNeeded = true;
+                } else {
+                  this.debugLog(
+                    `Revision match for ${su.documentId}:${su.scope}:${su.branch} ${suIndex}`,
+                  );
+                }
+              } else {
+                this.debugLog(
+                  `Cannot find strand update for (${revision.documentId}:${revision.scope}:${revision.branch} in drive ${revision.driveId})`,
+                );
+              }
+              // Check for revision status ^^
             } else {
               logger.warn(
                 `Received revision for untracked unit for listener ${listenerState.listener.listenerId}`,
@@ -324,23 +421,31 @@ export class ListenerManager implements IListenerManager {
           for (const revision of listenerRevisions) {
             const error = revision.status === "ERROR";
             if (revision.error?.includes("Missing operations")) {
-              const updates = await this._triggerUpdate(source, onError);
-              listenerUpdates.push(...updates);
-            } else {
-              listenerUpdates.push({
-                listenerId: listenerState.listener.listenerId,
-                listenerRevisions,
-              });
-              if (error) {
-                throw new OperationError(
-                  revision.status as ErrorStatus,
-                  undefined,
-                  revision.error,
-                  revision.error,
-                );
-              }
+              continuationNeeded = true;
+            } else if (error) {
+              throw new OperationError(
+                revision.status as ErrorStatus,
+                undefined,
+                revision.error,
+                revision.error,
+              );
             }
           }
+
+          if (!continuationNeeded) {
+            listenerUpdates.push({
+              listenerId: listenerState.listener.listenerId,
+              listenerRevisions,
+            });
+          } else {
+            const updates = await this._triggerUpdate(
+              source,
+              onError,
+              maxContinues - 1,
+            );
+            listenerUpdates.push(...updates);
+          }
+
           listenerState.listenerStatus = "SUCCESS";
         } catch (e) {
           // TODO: Handle error based on listener params (blocking, retry, etc)
@@ -350,6 +455,12 @@ export class ListenerManager implements IListenerManager {
         }
       }
     }
+
+    this.debugLog(
+      `Returning listener updates (maxContinues: ${maxContinues})`,
+      listenerUpdates,
+    );
+
     return listenerUpdates;
   }
 

package/src/server/listener/transmitter/pull-responder.ts

@@ -22,6 +22,8 @@ import {
   StrandUpdateSource,
 } from "./types";
 
+const ENABLE_SYNC_DEBUG = false;
+
 export type OperationUpdateGraphQL = Omit<OperationUpdate, "input"> & {
   input: string;
 };
@@ -44,16 +46,52 @@ export interface IPullResponderTransmitter extends ITransmitter {
   getStrands(options?: GetStrandsOptions): Promise<StrandUpdate[]>;
 }
 
+const STATIC_DEBUG_ID = `[PRT #static]`;
+
+function staticDebugLog(...data: any[]) {
+  if (!ENABLE_SYNC_DEBUG) {
+    return;
+  }
+
+  if (data.length > 0 && typeof data[0] === "string") {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    console.log(`${STATIC_DEBUG_ID} ${data[0]}`, ...data.slice(1));
+  } else {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    console.log(STATIC_DEBUG_ID, ...data);
+  }
+}
+
 export class PullResponderTransmitter implements IPullResponderTransmitter {
+  private debugID = `[PRT #${Math.floor(Math.random() * 999)}]`;
   private listener: Listener;
   private manager: IListenerManager;
 
   constructor(listener: Listener, manager: IListenerManager) {
     this.listener = listener;
     this.manager = manager;
+    this.debugLog(`constructor(listener: ${listener.listenerId})`);
+  }
+
+  private debugLog(...data: any[]) {
+    if (!ENABLE_SYNC_DEBUG) {
+      return;
+    }
+
+    if (data.length > 0 && typeof data[0] === "string") {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(`${this.debugID} ${data[0]}`, ...data.slice(1));
+    } else {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(this.debugID, ...data);
+    }
   }
 
   getStrands(options?: GetStrandsOptions): Promise<StrandUpdate[]> {
+    this.debugLog(
+      `getStrands(drive: ${this.listener.driveId}, listener: ${this.listener.listenerId})`,
+    );
+
     return this.manager.getStrands(
       this.listener.driveId,
       this.listener.listenerId,
@@ -71,6 +109,11 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     revisions: ListenerRevision[],
   ): Promise<boolean> {
+    this.debugLog(
+      `processAcknowledge(drive: ${driveId}, listener: ${listenerId})`,
+      revisions,
+    );
+
     const syncUnits = await this.manager.getListenerSyncUnitIds(
       driveId,
       listenerId,
@@ -106,6 +149,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     url: string,
     filter: ListenerFilter,
   ): Promise<Listener["listenerId"]> {
+    staticDebugLog(`registerPullResponder(url: ${url})`, filter);
     // graphql request to switchboard
     const result = await requestGraphql<{
       registerPullResponderListener: {
@@ -141,6 +185,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     options?: GetStrandsOptions, // TODO add support for since
   ): Promise<StrandUpdate[]> {
+    staticDebugLog(`pullStrands(url: ${url}, listener: ${listenerId})`);
     const result = await requestGraphql<PullStrandsGraphQL>(
       url,
       gql`
@@ -207,6 +252,11 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     listenerId: string,
     revisions: ListenerRevision[],
   ): Promise<boolean> {
+    staticDebugLog(
+      `acknowledgeStrands(url: ${url}, listener: ${listenerId})`,
+      revisions,
+    );
+
     const result = await requestGraphql<{ acknowledge: boolean }>(
       url,
       gql`
@@ -241,6 +291,8 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
     onAcknowledge?: (success: boolean) => void,
   ) {
+    staticDebugLog(`executePull(driveId: ${driveId}), trigger:`, trigger);
+
     try {
       const { url, listenerId } = trigger.data;
       const strands = await PullResponderTransmitter.pullStrands(
@@ -323,6 +375,8 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     onRevisions?: (revisions: ListenerRevisionWithError[]) => void,
     onAcknowledge?: (success: boolean) => void,
   ): CancelPullLoop {
+    staticDebugLog(`setupPull(drive: ${driveId}), trigger:`, trigger);
+
     const { interval } = trigger.data;
     let loopInterval = PULL_DRIVE_INTERVAL;
     if (interval) {
@@ -341,6 +395,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
 
     const executeLoop = async () => {
       while (!isCancelled) {
+        staticDebugLog("Execute loop...");
         await this.executePull(
           driveId,
           trigger,
@@ -350,6 +405,7 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
           onAcknowledge,
         );
         await new Promise((resolve) => {
+          staticDebugLog(`Scheduling next pull in ${loopInterval} ms`);
           timeout = setTimeout(resolve, loopInterval) as unknown as number;
         });
       }
@@ -370,6 +426,10 @@ export class PullResponderTransmitter implements IPullResponderTransmitter {
     url: string,
     options: Pick<RemoteDriveOptions, "pullInterval" | "pullFilter">,
   ): Promise<PullResponderTrigger> {
+    staticDebugLog(
+      `createPullResponderTrigger(drive: ${driveId}, url: ${url})`,
+    );
+
     const { pullFilter, pullInterval } = options;
     const listenerId = await PullResponderTransmitter.registerPullResponder(
       driveId,

package/src/server/listener/transmitter/switchboard-push.ts

@@ -4,13 +4,31 @@ import { logger } from "../../../utils/logger";
 import { ListenerRevision, StrandUpdate } from "../../types";
 import { ITransmitter, StrandUpdateSource } from "./types";
 
+const ENABLE_SYNC_DEBUG = false;
+const SYNC_OPS_BATCH_LIMIT = 10;
+
 export class SwitchboardPushTransmitter implements ITransmitter {
   private targetURL: string;
+  private debugID = `[SPT #${Math.floor(Math.random() * 999)}]`;
 
   constructor(targetURL: string) {
     this.targetURL = targetURL;
   }
 
+  private debugLog(...data: any[]) {
+    if (!ENABLE_SYNC_DEBUG) {
+      return false;
+    }
+
+    if (data.length > 0 && typeof data[0] === "string") {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(`${this.debugID} ${data[0]}`, ...data.slice(1));
+    } else {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      console.log(this.debugID, ...data);
+    }
+  }
+
   async transmit(
     strands: StrandUpdate[],
     source: StrandUpdateSource,
@@ -19,6 +37,7 @@ export class SwitchboardPushTransmitter implements ITransmitter {
       source.type === "trigger" &&
       source.trigger.data?.url === this.targetURL
     ) {
+      this.debugLog(`Cutting trigger loop from ${this.targetURL}.`);
       return strands.map((strand) => ({
         driveId: strand.driveId,
         documentId: strand.documentId,
@@ -29,6 +48,39 @@ export class SwitchboardPushTransmitter implements ITransmitter {
       }));
     }
 
+    const culledStrands: StrandUpdate[] = [];
+    let opsCounter = 0;
+
+    for (
+      let s = 0;
+      opsCounter <= SYNC_OPS_BATCH_LIMIT && s < strands.length;
+      s++
+    ) {
+      const currentStrand = strands.at(s);
+      if (!currentStrand) {
+        break;
+      }
+      const newOps = Math.min(
+        SYNC_OPS_BATCH_LIMIT - opsCounter,
+        currentStrand.operations.length,
+      );
+
+      culledStrands.push({
+        ...currentStrand,
+        operations: currentStrand.operations.slice(0, newOps),
+      });
+
+      opsCounter += newOps;
+    }
+
+    this.debugLog(
+      ` Total update: [${strands.map((s) => s.operations.length).join(", ")}] operations`,
+    );
+
+    this.debugLog(
+      `Culled update: [${culledStrands.map((s) => s.operations.length).join(", ")}] operations`,
+    );
+
     // Send Graphql mutation to switchboard
     try {
       const { pushUpdates } = await requestGraphql<{
@@ -49,7 +101,7 @@ export class SwitchboardPushTransmitter implements ITransmitter {
           }
         `,
         {
-          strands: strands.map((strand) => ({
+          strands: culledStrands.map((strand) => ({
             ...strand,
             operations: strand.operations.map((op) => ({
               ...op,