@liveblocks/server 1.4.0 → 1.4.2-pre1

package/dist/index.js

@@ -129,7 +129,9 @@ var feedMetadataRecordForCreate = record(
   (value) => Object.keys(value).length <= MAX_METADATA_COUNT,
   `Must be at most ${MAX_METADATA_COUNT} items`
 );
-var optionalFeedMetadataDecoder = optional(feedMetadataRecordForCreate);
+var optionalFeedMetadataDecoder = optional(
+  feedMetadataRecordForCreate
+);
 var feedMetadataUpdateDecoder = record(
   feedMetadataIdDecoder,
   feedMetadataNullableValueDecoder
@@ -476,6 +478,7 @@ function* iterJson(key, data, parent, state) {
       case "LiveMap":
         yield* iterMap(key, data.data, parent, state);
         return;
+      // istanbul ignore next
       default:
         assertNever(data, "Unknown `liveblocksType` field");
     }
@@ -560,7 +563,7 @@ function buildNode2(snapshot2, id) {
   } else if (node.type === CrdtType2.MAP) {
     return buildMap2(snapshot2, id);
   } else {
-    return node.data;
+    return node.data ?? null;
   }
 }
 function buildObject2(snapshot2, id, staticData) {
@@ -604,7 +607,7 @@ function* emit2(snapshot2, id) {
   } else if (node.type === CrdtType2.MAP) {
     yield* emitMap2(snapshot2, id);
   } else if (node.type === CrdtType2.REGISTER) {
-    yield JSON.stringify(node.data);
+    yield JSON.stringify(node.data ?? null);
   }
 }
 function* emitObject2(snapshot2, id, staticJson) {
@@ -819,7 +822,6 @@ function makeMetadataDB(driver) {
 import {
   assertNever as assertNever3,
   ClientMsgCode as ClientMsgCode2,
-  nodeStreamToCompactNodes,
   OpCode as OpCode3,
   raise as raise3,
   ServerMsgCode as CoreServerMsgCode,
@@ -828,7 +830,7 @@ import {
 } from "@liveblocks/core";
 import { Mutex, tryAcquire } from "async-mutex";
 import { array as array3, formatInline } from "decoders";
-import { chunked } from "itertools";
+import { chunkedByCost } from "itertools";
 import { nanoid as nanoid2 } from "nanoid";
 
 // src/lib/Logger.ts
@@ -940,12 +942,10 @@ var Logger = class _Logger {
       }
     });
     this.o = {
-      /* eslint-disable @typescript-eslint/no-unsafe-enum-comparison */
       debug: minLevel <= 0 /* DEBUG */ ? makeLogFn(0 /* DEBUG */) : void 0,
       info: minLevel <= 1 /* INFO */ ? makeLogFn(1 /* INFO */) : void 0,
       warn: minLevel <= 2 /* WARNING */ ? makeLogFn(2 /* WARNING */) : void 0,
       error: minLevel <= 3 /* ERROR */ ? makeLogFn(3 /* ERROR */) : void 0
-      /* eslint-enable @typescript-eslint/no-unsafe-enum-comparison */
     };
     this.debug = this.o.debug ?? noop;
     this.info = this.o.info ?? noop;
@@ -965,7 +965,13 @@ var Logger = class _Logger {
 };
 
 // src/plugins/InMemoryDriver.ts
-import { asPos, CrdtType as CrdtType4, isRootStorageNode as isRootStorageNode2, nn as nn2 } from "@liveblocks/core";
+import {
+  asPos,
+  CrdtType as CrdtType4,
+  isRootStorageNode as isRootStorageNode2,
+  nn as nn2,
+  nodeStreamToCompactNodes
+} from "@liveblocks/core";
 import { ifilter, imap } from "itertools";
 
 // src/lib/text.ts
@@ -1375,6 +1381,17 @@ var InMemoryDriver = class {
        * Yield all nodes as [id, node] pairs. Must always include the root node.
        */
       iter_nodes: () => nodes,
+      /**
+       * Yield each node as a pre-built CompactNode JSON tuple string.
+       *
+       * This implementation IS the canonical reference for the invariant:
+       * iter_nodes_optimized` ≡ `nodeStreamToCompactNodes(iter_nodes())
+       */
+      *iter_nodes_optimized() {
+        for (const compact of nodeStreamToCompactNodes(nodes)) {
+          yield JSON.stringify(compact);
+        }
+      },
       /**
        * Return true iff a node with the given id exists. Must return true for "root".
        */
@@ -1494,6 +1511,7 @@ function nodeFromCreateChildOp(op2) {
         parentKey: op2.parentKey,
         data: op2.data
       };
+    // istanbul ignore next
     default:
       return assertNever2(op2, "Unknown op code");
   }
@@ -1562,6 +1580,7 @@ var Storage = class {
         return await this.applyDeleteObjectKeyOp(op2);
       case OpCode2.DELETE_CRDT:
         return await this.applyDeleteCrdtOp(op2);
+      // istanbul ignore next
       default:
         if (process.env.NODE_ENV === "production") {
           return ignore(op2);
@@ -1584,6 +1603,7 @@ var Storage = class {
         if (op2.type === OpCode2.CREATE_REGISTER) {
           return ignore(op2);
         }
+      // fall through
       case CrdtType5.MAP:
         await this.loadedDriver.set_child(op2.id, node, true);
         return accept(op2);
@@ -1591,6 +1611,7 @@ var Storage = class {
         return this.createChildAsListItem(op2, node);
       case CrdtType5.REGISTER:
         return ignore(op2);
+      // istanbul ignore next
       default:
         return assertNever2(parent, "Unhandled CRDT type");
     }
@@ -1731,6 +1752,7 @@ import { Base64 } from "js-base64";
 import { nanoid } from "nanoid";
 import * as Y from "yjs";
 var UPDATE_COUNT_THRESHOLD = 1e3;
+var MERGE_SHRINK_THRESHOLD = 0.8;
 var YjsStorage = class {
   constructor(driver, updateCountThreshold = UPDATE_COUNT_THRESHOLD) {
     __publicField(this, "driver");
@@ -1748,15 +1770,16 @@ var YjsStorage = class {
       await this.driver.delete_y_updates(docId, storedKeys);
       this.storedKeysById.set(docId, [newKey]);
     });
-    __publicField(this, "_loadYDocFromDurableStorage", async (doc, docId) => {
+    __publicField(this, "_loadYDocFromDurableStorage", async (_logger, doc, docId) => {
       const docUpdates = Object.fromEntries(
         await this.driver.iter_y_updates(docId)
       );
       const updates = Object.values(docUpdates);
+      const beforeSize = updates.reduce((acc, update) => acc + update.length, 0);
       const newupdate = Y.mergeUpdates(updates);
       const storedKeys = Object.keys(docUpdates);
       Y.applyUpdate(doc, newupdate);
-      if (this.shouldCompact(storedKeys)) {
+      if (this.shouldCompactByKeyCount(storedKeys) || this.shouldCompactBySize(beforeSize, newupdate.length)) {
         await this._compactYJSUpdates(doc, docId, storedKeys);
       } else {
         this.storedKeysById.set(docId, storedKeys);
@@ -1775,8 +1798,8 @@ var YjsStorage = class {
   // ------------------------------------------------------------------------------------
   // Public API
   // ------------------------------------------------------------------------------------
-  async getYDoc(docId) {
-    const doc = await this.loadDocByIdIfNotAlreadyLoaded(docId);
+  async getYDoc(logger, docId) {
+    const doc = await this.loadDocByIdIfNotAlreadyLoaded(logger, docId);
     return doc;
   }
   /**
@@ -1796,7 +1819,7 @@ var YjsStorage = class {
     return Base64.fromUint8Array(update);
   }
   async getYDocUpdateBinary(logger, stateVector = "", guid, isV2 = false) {
-    const doc = guid !== void 0 ? await this.getYSubdoc(guid) : this.doc;
+    const doc = guid !== void 0 ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       return null;
     }
@@ -1813,15 +1836,15 @@ var YjsStorage = class {
     }
     return Y.encodeStateAsUpdate(doc, encodedTargetVector);
   }
-  async getYStateVector(guid) {
-    const doc = guid !== void 0 ? await this.getYSubdoc(guid) : this.doc;
+  async getYStateVector(logger, guid) {
+    const doc = guid !== void 0 ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       return null;
     }
     return Base64.fromUint8Array(Y.encodeStateVector(doc));
   }
-  async getSnapshotHash(options) {
-    const doc = options.guid !== void 0 ? await this.getYSubdoc(options.guid) : this.doc;
+  async getSnapshotHash(logger, options) {
+    const doc = options.guid !== void 0 ? await this.getYSubdoc(logger, options.guid) : this.doc;
     if (!doc) {
       return null;
     }
@@ -1835,7 +1858,7 @@ var YjsStorage = class {
    *   snapshotHash: the hash of the new snapshot
    */
   async addYDocUpdate(logger, update, guid, isV2) {
-    const doc = guid !== void 0 ? await this.getYSubdoc(guid) : this.doc;
+    const doc = guid !== void 0 ? await this.getYSubdoc(logger, guid) : this.doc;
     if (!doc) {
       throw new Error(`YDoc with guid ${guid} not found`);
     }
@@ -1860,20 +1883,20 @@ var YjsStorage = class {
       );
     }
   }
-  loadDocByIdIfNotAlreadyLoaded(docId) {
+  loadDocByIdIfNotAlreadyLoaded(logger, docId) {
     let loaded$ = this.initPromisesById.get(docId);
     let doc = docId === ROOT_YDOC_ID ? this.doc : this.findYSubdocByGuid(docId);
     if (!doc) {
       doc = new Y.Doc();
     }
     if (loaded$ === void 0) {
-      loaded$ = this._loadYDocFromDurableStorage(doc, docId);
+      loaded$ = this._loadYDocFromDurableStorage(logger, doc, docId);
       this.initPromisesById.set(docId, loaded$);
     }
     return loaded$;
   }
-  async load(_logger) {
-    await this.loadDocByIdIfNotAlreadyLoaded(ROOT_YDOC_ID);
+  async load(logger) {
+    await this.loadDocByIdIfNotAlreadyLoaded(logger, ROOT_YDOC_ID);
   }
   /**
    * Unloads the Yjs documents from memory.
@@ -1916,12 +1939,12 @@ var YjsStorage = class {
     );
   }
   // gets a subdoc, it will be loaded if not already loaded
-  async getYSubdoc(guid) {
+  async getYSubdoc(logger, guid) {
     const subdoc = this.findYSubdocByGuid(guid);
     if (!subdoc) {
       return null;
     }
-    await this.loadDocByIdIfNotAlreadyLoaded(guid);
+    await this.loadDocByIdIfNotAlreadyLoaded(logger, guid);
     return subdoc;
   }
   // When the YJS doc changes, update it in durable storage
@@ -1929,7 +1952,7 @@ var YjsStorage = class {
     const v1update = isV2 ? Y.convertUpdateFormatV2ToV1(update) : update;
     const docId = doc.guid === this.doc.guid ? ROOT_YDOC_ID : doc.guid;
     const storedKeys = this.storedKeysById.get(docId);
-    if (this.shouldCompact(storedKeys)) {
+    if (this.shouldCompactByKeyCount(storedKeys)) {
       await this._compactYJSUpdates(doc, docId, storedKeys || []);
       return;
     }
@@ -1941,7 +1964,10 @@ var YjsStorage = class {
       storedKeys.push(newKey);
     }
   }
-  shouldCompact(storedKeys) {
+  shouldCompactBySize(beforeSize, afterSize) {
+    return beforeSize * MERGE_SHRINK_THRESHOLD > afterSize;
+  }
+  shouldCompactByKeyCount(storedKeys) {
     if (!storedKeys) {
       return false;
     }
@@ -2027,6 +2053,7 @@ function isLeasedSessionExpired(leasedSession) {
 }
 
 // src/Room.ts
+var MB = 1024 * 1024;
 var messagesDecoder = array3(clientMsgDecoder);
 var ServerMsgCode2 = { ...CoreServerMsgCode, ...FeedMsgCode };
 var HIGHEST_PROTOCOL_VERSION = Math.max(
@@ -2040,6 +2067,11 @@ var SERVER_MSG_CODE_NAMES = Object.fromEntries(
 var BLACK_HOLE = new Logger([
   /* No targets, i.e. black hole logger */
 ]);
+function groupNodesForWebSocketMessages(input) {
+  const MAX_SIZE = 16 * MB;
+  const idealSize = (chunkIndex) => chunkIndex < 3 ? 1 * MB : Math.min(8, chunkIndex - 1) * MB;
+  return chunkedByCost(input, (tup) => tup.length, MAX_SIZE, idealSize);
+}
 function collectSideEffects() {
   const deferred = [];
   return {
@@ -2173,7 +2205,7 @@ var BackendSession = class extends BrowserSession {
     super(ticket, socket, debug);
   }
 };
-var __debug2, __allowStreaming;
+var __debug2;
 var Room = class {
   constructor(meta, options) {
     //              ^^^^^^^^^^ User-defined Room Metadata, Session Metadata, and Client Metadata
@@ -2195,12 +2227,10 @@ var Room = class {
     __publicField(this, "sessions", new UniqueMap((s) => s.actor));
     __publicField(this, "hooks");
     __privateAdd(this, __debug2);
-    __privateAdd(this, __allowStreaming);
     const driver = options?.storage ?? makeNewInMemoryDriver();
     this.meta = meta;
     this.driver = driver;
     this.logger = options?.logger ?? BLACK_HOLE;
-    __privateSet(this, __allowStreaming, options?.allowStreaming ?? true);
     this.hooks = {
       isClientMsgAllowed: options?.hooks?.isClientMsgAllowed ?? (() => {
         return {
@@ -2905,7 +2935,6 @@ var Room = class {
    * @internal
    * Handles an incoming ping, by sending a pong back.
    */
-  // eslint-disable-next-line @typescript-eslint/require-await
   async handlePing(sessionKey, ctx) {
     const session = this.sessions.get(sessionKey);
     if (session === void 0) {
@@ -2966,7 +2995,13 @@ var Room = class {
     const toReplyImmediately = [];
     const toReplyAfter = [];
     const replyImmediately = (msg) => {
-      if (Array.isArray(msg)) {
+      if (typeof msg === "string") {
+        if (toReplyImmediately.length > 0) {
+          session.send(toReplyImmediately);
+          toReplyImmediately.length = 0;
+        }
+        session.send(msg);
+      } else if (Array.isArray(msg)) {
         for (const m of msg) {
           toReplyImmediately.push(m);
         }
@@ -3024,24 +3059,10 @@ var Room = class {
       }
       case ClientMsgCode2.FETCH_STORAGE: {
         if (session.version >= 8 /* V8 */) {
-          if (__privateGet(this, __allowStreaming)) {
-            const NODES_PER_CHUNK = 250;
-            for (const chunk of chunked(
-              nodeStreamToCompactNodes(this.storage.loadedDriver.iter_nodes()),
-              NODES_PER_CHUNK
-            )) {
-              replyImmediately({
-                type: ServerMsgCode2.STORAGE_CHUNK,
-                nodes: chunk
-              });
-            }
-          } else {
-            replyImmediately({
-              type: ServerMsgCode2.STORAGE_CHUNK,
-              nodes: Array.from(
-                nodeStreamToCompactNodes(this.storage.loadedDriver.iter_nodes())
-              )
-            });
+          const rawStream = this.storage.loadedDriver.iter_nodes_optimized();
+          for (const chunk of groupNodesForWebSocketMessages(rawStream)) {
+            const frame = `{"type":${ServerMsgCode2.STORAGE_CHUNK},"nodes":[${chunk.join(",")}]}`;
+            replyImmediately(frame);
           }
           replyImmediately({ type: ServerMsgCode2.STORAGE_STREAM_END });
         } else {
@@ -3064,6 +3085,7 @@ var Room = class {
               return r.ignoredOpId !== void 0 ? [ackIgnoredOp(r.ignoredOpId)] : [];
             case "accepted":
               return r.fix !== void 0 ? [r.fix] : [];
+            // istanbul ignore next
             default:
               return assertNever3(r, "Unhandled case");
           }
@@ -3096,8 +3118,8 @@ var Room = class {
         const isV2 = msg.v2;
         const [update, stateVector, snapshotHash] = await Promise.all([
           this.yjsStorage.getYDocUpdate(this.logger, vector, guid, isV2),
-          this.yjsStorage.getYStateVector(guid),
-          this.yjsStorage.getSnapshotHash({ guid, isV2 })
+          this.yjsStorage.getYStateVector(this.logger, guid),
+          this.yjsStorage.getSnapshotHash(this.logger, { guid, isV2 })
         ]);
         if (update !== null && snapshotHash !== null) {
           replyImmediately({
@@ -3141,6 +3163,7 @@ var Room = class {
         }
         break;
       }
+      // Feed messages
       case FeedMsgCode.FETCH_FEEDS: {
         const fetchMsg = msg;
         const [result, err] = await tryCatch(
@@ -3190,8 +3213,7 @@ var Room = class {
         const [feed, err] = await tryCatch(
           this.createFeed({
             feedId: addMsg.feedId,
-            metadata: addMsg.metadata ?? {},
-            timestamp: addMsg.timestamp
+            metadata: addMsg.metadata ?? {}
           })
         );
         if (err) {
@@ -3253,8 +3275,7 @@ var Room = class {
         const [message, err] = await tryCatch(
           this.addFeedMessage(addMsg.feedId, {
             data: addMsg.data,
-            id: addMsg.id,
-            timestamp: addMsg.timestamp
+            id: addMsg.id
           })
         );
         if (err) {
@@ -3276,8 +3297,7 @@ var Room = class {
           this.updateFeedMessage(
             updateMsg.feedId,
             updateMsg.messageId,
-            updateMsg.data,
-            updateMsg.timestamp
+            updateMsg.data
           )
         );
         if (err) {
@@ -3321,7 +3341,6 @@ var Room = class {
   }
 };
 __debug2 = new WeakMap();
-__allowStreaming = new WeakMap();
 export {
   BackendSession,
   BrowserSession,