@powerhousedao/reactor-local 1.20.0 → 1.20.2

package/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 1.20.2 (2025-02-19)
+
+### 🚀 Features
+
+- **common:** improved drive story ([ec96a6b7](https://github.com/powerhouse-inc/powerhouse/commit/ec96a6b7))
+- **common:** document drive generic layout ([e7518094](https://github.com/powerhouse-inc/powerhouse/commit/e7518094))
+- **common:** initial commit ([f561b8c9](https://github.com/powerhouse-inc/powerhouse/commit/f561b8c9))
+
+### 🧱 Updated Dependencies
+
+- Updated document-model-libs to 1.133.0
+- Updated document-drive to 1.19.1
+- Updated @powerhousedao/reactor-api to 1.21.2
+- Updated @powerhousedao/scalars to 1.24.0
+
+### ❤️ Thank You
+
+- acaldas @acaldas
+
 ## 1.1.0 (2024-10-29)
 
 ### 🚀 Features

package/dist/{chunk-JKUEBNSV.js → chunk-PUCSQ2II.js}

@@ -1318,8 +1318,6 @@ var {
 var DefaultListenerManagerOptions = {
   sequentialUpdates: true
 };
-
-// ../document-drive/src/server/listener/manager.ts
 function debounce(func, delay = 250) {
   let timer;
   return (immediate, ...args) => {
@@ -1339,16 +1337,24 @@ function debounce(func, delay = 250) {
 }
 var ListenerManager = class _ListenerManager {
   static LISTENER_UPDATE_DELAY = 250;
+  debugID = `[LM  #${Math.floor(Math.random() * 999)}]`;
   driveServer;
+  options;
   // driveId -> listenerId -> listenerState
   listenerStateByDriveId = /* @__PURE__ */ new Map();
-  options;
   constructor(drive, listenerState = /* @__PURE__ */ new Map(), options = DefaultListenerManagerOptions) {
+    this.debugLog(`constructor(...)`);
     this.driveServer = drive;
     this.listenerStateByDriveId = listenerState;
     this.options = { ...DefaultListenerManagerOptions, ...options };
   }
+  debugLog(...data) {
+    {
+      return;
+    }
+  }
   async initialize(handler) {
+    this.debugLog("initialize(...)");
     if (typeof window !== "undefined") {
       window.addEventListener("online", () => {
         this.triggerUpdate(false, { type: "local" }, handler).catch((error) => {
@@ -1361,6 +1367,9 @@ var ListenerManager = class _ListenerManager {
     return this.listenerStateByDriveId.has(driveId);
   }
   async setListener(driveId, listener) {
+    this.debugLog(
+      `setListener(drive: ${driveId}, listener: ${listener.listenerId})`
+    );
     if (driveId !== listener.driveId) {
       throw new Error("Drive ID mismatch");
     }
@@ -1382,6 +1391,7 @@ var ListenerManager = class _ListenerManager {
     this.triggerUpdate(true, { type: "local" });
   }
   async removeListener(driveId, listenerId) {
+    this.debugLog("setListener()");
     const driveMap = this.listenerStateByDriveId.get(driveId);
     if (!driveMap) {
       return false;
@@ -1456,23 +1466,36 @@ var ListenerManager = class _ListenerManager {
     this._triggerUpdate.bind(this),
     _ListenerManager.LISTENER_UPDATE_DELAY
   );
-  async _triggerUpdate(source, onError) {
+  async _triggerUpdate(source, onError, maxContinues = 500) {
+    this.debugLog(
+      `_triggerUpdate(source: ${source.type}, maxContinues: ${maxContinues})`,
+      this.listenerStateByDriveId
+    );
+    if (maxContinues < 0) {
+      throw new Error("Maximum retries exhausted.");
+    }
     const listenerUpdates = [];
     for (const [driveId, drive] of this.listenerStateByDriveId) {
-      for (const [_, listenerState] of drive) {
+      for (const [listenerId, listenerState] of drive) {
         const transmitter = listenerState.listener.transmitter;
         if (!transmitter?.transmit) {
+          this.debugLog(`Transmitter not set on listener: ${listenerId}`);
           continue;
         }
-        const syncUnits = await this.getListenerSyncUnits(
-          driveId,
-          listenerState.listener.listenerId
-        );
+        const syncUnits = await this.getListenerSyncUnits(driveId, listenerId);
         const strandUpdates = [];
+        this.debugLog("syncUnits", syncUnits);
         const tasks = syncUnits.map((syncUnit) => async () => {
           const unitState = listenerState.syncUnits.get(syncUnit.syncId);
           if (unitState && unitState.listenerRev >= syncUnit.revision) {
+            this.debugLog(
+              `Abandoning push for sync unit ${syncUnit.syncId}: already up-to-date (${unitState.listenerRev} >= ${syncUnit.revision})`
+            );
             return;
+          } else {
+            this.debugLog(
+              `Listener out-of-date for sync unit ${syncUnit.syncId}: ${unitState?.listenerRev} < ${syncUnit.revision}`
+            );
           }
           const opData = [];
           try {
@@ -1489,6 +1512,9 @@ var ListenerManager = class _ListenerManager {
             logger.error(e);
           }
           if (!opData.length) {
+            this.debugLog(
+              `Abandoning push for ${syncUnit.syncId}: no operations found`
+            );
             return;
           }
           strandUpdates.push({
@@ -1500,13 +1526,20 @@ var ListenerManager = class _ListenerManager {
           });
         });
         if (this.options.sequentialUpdates) {
+          this.debugLog(
+            `Collecting ${tasks.length} syncUnit strandUpdates in sequence`
+          );
           for (const task of tasks) {
             await task();
           }
         } else {
+          this.debugLog(
+            `Collecting ${tasks.length} syncUnit strandUpdates in parallel`
+          );
           await Promise.all(tasks.map((task) => task()));
         }
         if (strandUpdates.length == 0) {
+          this.debugLog(`No strandUpdates needed for listener ${listenerId}`);
           continue;
         }
         listenerState.pendingTimeout = new Date(
@@ -1514,13 +1547,21 @@ var ListenerManager = class _ListenerManager {
         ).toISOString();
         listenerState.listenerStatus = "PENDING";
         try {
+          this.debugLog(
+            `_triggerUpdate(source: ${source.type}) > transmitter.transmit`
+          );
           const listenerRevisions = await transmitter.transmit(
             strandUpdates,
             source
           );
+          this.debugLog(
+            `_triggerUpdate(source: ${source.type}) > transmission succeeded`,
+            listenerRevisions
+          );
           listenerState.pendingTimeout = "0";
           listenerState.listenerStatus = "PENDING";
           const lastUpdated = (/* @__PURE__ */ new Date()).toISOString();
+          let continuationNeeded = false;
           for (const revision of listenerRevisions) {
             const syncUnit = syncUnits.find(
               (unit) => revision.documentId === unit.documentId && revision.scope === unit.scope && revision.branch === unit.branch
@@ -1530,6 +1571,28 @@ var ListenerManager = class _ListenerManager {
                 lastUpdated,
                 listenerRev: revision.revision
               });
+              const su = strandUpdates.find(
+                (su2) => su2.driveId === revision.driveId && su2.documentId === revision.documentId && su2.scope === revision.scope && su2.branch === revision.branch
+              );
+              if (su && su.operations.length > 0) {
+                const suIndex = su.operations.at(
+                  su.operations.length - 1
+                )?.index;
+                if (suIndex !== revision.revision) {
+                  this.debugLog(
+                    `Revision still out-of-date for ${su.documentId}:${su.scope}:${su.branch} ${suIndex} <> ${revision.revision}`
+                  );
+                  continuationNeeded = true;
+                } else {
+                  this.debugLog(
+                    `Revision match for ${su.documentId}:${su.scope}:${su.branch} ${suIndex}`
+                  );
+                }
+              } else {
+                this.debugLog(
+                  `Cannot find strand update for (${revision.documentId}:${revision.scope}:${revision.branch} in drive ${revision.driveId})`
+                );
+              }
             } else {
               logger.warn(
                 `Received revision for untracked unit for listener ${listenerState.listener.listenerId}`,
@@ -1540,23 +1603,29 @@ var ListenerManager = class _ListenerManager {
           for (const revision of listenerRevisions) {
             const error = revision.status === "ERROR";
             if (revision.error?.includes("Missing operations")) {
-              const updates = await this._triggerUpdate(source, onError);
-              listenerUpdates.push(...updates);
-            } else {
-              listenerUpdates.push({
-                listenerId: listenerState.listener.listenerId,
-                listenerRevisions
-              });
-              if (error) {
-                throw new OperationError(
-                  revision.status,
-                  void 0,
-                  revision.error,
-                  revision.error
-                );
-              }
+              continuationNeeded = true;
+            } else if (error) {
+              throw new OperationError(
+                revision.status,
+                void 0,
+                revision.error,
+                revision.error
+              );
             }
           }
+          if (!continuationNeeded) {
+            listenerUpdates.push({
+              listenerId: listenerState.listener.listenerId,
+              listenerRevisions
+            });
+          } else {
+            const updates = await this._triggerUpdate(
+              source,
+              onError,
+              maxContinues - 1
+            );
+            listenerUpdates.push(...updates);
+          }
           listenerState.listenerStatus = "SUCCESS";
         } catch (e) {
           onError?.(e, driveId, listenerState);
@@ -1564,6 +1633,10 @@ var ListenerManager = class _ListenerManager {
         }
       }
     }
+    this.debugLog(
+      `Returning listener updates (maxContinues: ${maxContinues})`,
+      listenerUpdates
+    );
     return listenerUpdates;
   }
   _checkFilter(filter, syncUnit) {
@@ -1782,16 +1855,24 @@ var InternalTransmitter = class {
     return this.listener;
   }
 };
-
-// ../document-drive/src/server/listener/transmitter/pull-responder.ts
 var PullResponderTransmitter = class _PullResponderTransmitter {
+  debugID = `[PRT #${Math.floor(Math.random() * 999)}]`;
   listener;
   manager;
   constructor(listener, manager) {
     this.listener = listener;
     this.manager = manager;
+    this.debugLog(`constructor(listener: ${listener.listenerId})`);
+  }
+  debugLog(...data) {
+    {
+      return;
+    }
   }
   getStrands(options) {
+    this.debugLog(
+      `getStrands(drive: ${this.listener.driveId}, listener: ${this.listener.listenerId})`
+    );
     return this.manager.getStrands(
       this.listener.driveId,
       this.listener.listenerId,
@@ -1802,6 +1883,10 @@ var PullResponderTransmitter = class _PullResponderTransmitter {
     return Promise.resolve();
   }
   async processAcknowledge(driveId, listenerId, revisions) {
+    this.debugLog(
+      `processAcknowledge(drive: ${driveId}, listener: ${listenerId})`,
+      revisions
+    );
     const syncUnits = await this.manager.getListenerSyncUnitIds(
       driveId,
       listenerId
@@ -2046,13 +2131,21 @@ var PullResponderTransmitter = class _PullResponderTransmitter {
     return trigger.type === "PullResponder";
   }
 };
+var SYNC_OPS_BATCH_LIMIT = 10;
 var SwitchboardPushTransmitter = class {
   targetURL;
+  debugID = `[SPT #${Math.floor(Math.random() * 999)}]`;
   constructor(targetURL) {
     this.targetURL = targetURL;
   }
+  debugLog(...data) {
+    {
+      return false;
+    }
+  }
   async transmit(strands, source) {
     if (source.type === "trigger" && source.trigger.data?.url === this.targetURL) {
+      this.debugLog(`Cutting trigger loop from ${this.targetURL}.`);
       return strands.map((strand) => ({
         driveId: strand.driveId,
         documentId: strand.documentId,
@@ -2062,6 +2155,29 @@ var SwitchboardPushTransmitter = class {
         revision: strand.operations.at(-1)?.index ?? -1
       }));
     }
+    const culledStrands = [];
+    let opsCounter = 0;
+    for (let s = 0; opsCounter <= SYNC_OPS_BATCH_LIMIT && s < strands.length; s++) {
+      const currentStrand = strands.at(s);
+      if (!currentStrand) {
+        break;
+      }
+      const newOps = Math.min(
+        SYNC_OPS_BATCH_LIMIT - opsCounter,
+        currentStrand.operations.length
+      );
+      culledStrands.push({
+        ...currentStrand,
+        operations: currentStrand.operations.slice(0, newOps)
+      });
+      opsCounter += newOps;
+    }
+    this.debugLog(
+      ` Total update: [${strands.map((s) => s.operations.length).join(", ")}] operations`
+    );
+    this.debugLog(
+      `Culled update: [${culledStrands.map((s) => s.operations.length).join(", ")}] operations`
+    );
     try {
       const { pushUpdates } = await requestGraphql(
         this.targetURL,
@@ -2079,7 +2195,7 @@ var SwitchboardPushTransmitter = class {
           }
         `,
         {
-          strands: strands.map((strand) => ({
+          strands: culledStrands.map((strand) => ({
             ...strand,
             operations: strand.operations.map((op) => ({
               ...op,
@@ -2525,7 +2641,15 @@ var BaseDocumentDriveServer = class {
     for (const zodListener of drive.state.local.listeners) {
       const transmitter = this.transmitterFactory.instance(
         zodListener.callInfo?.transmitterType ?? "",
-        zodListener,
+        {
+          driveId,
+          listenerId: zodListener.listenerId,
+          block: zodListener.block,
+          filter: zodListener.filter,
+          system: zodListener.system,
+          label: zodListener.label || undefined,
+          callInfo: zodListener.callInfo || undefined
+        },
         this
       );
       await this.listenerManager.setListener(driveId, {
@@ -3536,7 +3660,15 @@ var BaseDocumentDriveServer = class {
             const zodListener = operation.input.listener;
             const transmitter = this.transmitterFactory.instance(
               zodListener.callInfo?.transmitterType ?? "",
-              zodListener,
+              {
+                driveId,
+                listenerId: zodListener.listenerId,
+                block: zodListener.block,
+                filter: zodListener.filter,
+                system: zodListener.system,
+                label: zodListener.label || void 0,
+                callInfo: zodListener.callInfo || void 0
+              },
               this
             );
             const listener = {

package/dist/cli.js

@@ -1,5 +1,5 @@
 #! /usr/bin/env node
-import { startServer } from './chunk-JKUEBNSV.js';
+import { startServer } from './chunk-PUCSQ2II.js';
 import './chunk-HCTWYIZG.js';
 import { Command } from 'commander';
 

package/dist/index.js

@@ -1,2 +1,2 @@
-export { DefaultStartServerOptions, startServer } from './chunk-JKUEBNSV.js';
+export { DefaultStartServerOptions, startServer } from './chunk-PUCSQ2II.js';
 import './chunk-HCTWYIZG.js';

package/dist/server.js

@@ -1,2 +1,2 @@
-export { DefaultStartServerOptions, startServer } from './chunk-JKUEBNSV.js';
+export { DefaultStartServerOptions, startServer } from './chunk-PUCSQ2II.js';
 import './chunk-HCTWYIZG.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@powerhousedao/reactor-local",
   "type": "module",
-  "version": "1.20.0",
+  "version": "1.20.2",
   "main": "dist/server.js",
   "bin": {
     "reactor-local": "dist/cli.js"
@@ -27,10 +27,10 @@
     "nanoevents": "^9.0.0",
     "sanitize-filename": "^1.6.3",
     "uuid": "^11.0.2",
-    "@powerhousedao/reactor-api": "1.21.0",
-    "@powerhousedao/scalars": "1.23.0",
+    "@powerhousedao/reactor-api": "1.21.2",
+    "@powerhousedao/scalars": "1.24.0",
     "document-model": "2.20.0",
-    "document-model-libs": "1.132.0"
+    "document-model-libs": "1.133.0"
   },
   "devDependencies": {
     "@powerhousedao/analytics-engine-graphql": "^0.2.2",
@@ -41,7 +41,7 @@
     "@types/node": "^22.7.5",
     "tsup": "^8.3.5",
     "@powerhousedao/config": "1.17.0",
-    "document-drive": "1.18.0"
+    "document-drive": "1.19.1"
   },
   "scripts": {
     "start": "vite-node src/index.ts",