@liveblocks/server 1.0.14 → 1.1.0

package/README.md

@@ -11,10 +11,14 @@
   <a href="https://github.com/liveblocks/liveblocks/blob/main/licenses/LICENSE-AGPL-3.0"><img src="https://img.shields.io/badge/license-AGPL--3.0-green" alt="License" /></a>
 </p>
 
-`@liveblocks/server` provides the core Liveblocks server functionality. It
-powers both the Liveblocks production environment, and
-[our dev server](https://liveblocks.io/docs/tools/dev-server), so the behavior
-is identical. You typically don't need to install or use this package directly.
+`@liveblocks/server` provides the core Liveblocks server functionality, powering
+both the Liveblocks production environment, and [dev server][devserver], so the
+behavior is identical. **You typically don't need to install or use this package
+directly.**
+
+If you are interested in running Liveblocks locally or within your CI
+environment, check out the [dev server][devserver] instead (which has an actual
+web server wrapped around it, contains documentation, and a Dockerfile).
 
 While `@liveblocks/server` contains the same core technology that powers
 Liveblocks, we do not yet offer or recommend self-hosting or on-premises
@@ -29,3 +33,5 @@ Licensed under the GNU Affero General Public License v3.0 or later, Copyright ©
 2021-present [Liveblocks](https://liveblocks.io).
 
 See [LICENSE-AGPL-3.0](./LICENSE) for more information.
+
+[devserver]: https://liveblocks.io/docs/tools/dev-server

package/dist/index.cjs

@@ -1369,63 +1369,45 @@ var Storage = class {
 };
 
 // src/YjsStorage.ts
-
 var _jsbase64 = require('js-base64');
 
 var _yjs = require('yjs'); var Y = _interopRequireWildcard(_yjs);
-var MAX_Y_UPDATE_SIZE = 1e5;
+var UPDATE_COUNT_THRESHOLD = 1e3;
 var YjsStorage = class {
-  constructor(driver) {
+  constructor(driver, updateCountThreshold = UPDATE_COUNT_THRESHOLD) {
     __publicField(this, "driver");
+    __publicField(this, "updateCountThreshold");
     __publicField(this, "doc", new Y.Doc());
     // the root document
-    __publicField(this, "lastUpdatesById", /* @__PURE__ */ new Map());
     __publicField(this, "lastSnapshotById", /* @__PURE__ */ new Map());
-    // Keeps track of which keys are loaded, so we can clean them up without calling `.list()`
-    __publicField(this, "keysById", new (0, _core.DefaultMap)(
-      () => /* @__PURE__ */ new Set()
-    ));
     __publicField(this, "initPromisesById", /* @__PURE__ */ new Map());
-    /**
-     * Given a record of updates, merge them and compress if savings are significant
-     */
-    __publicField(this, "_loadAndCompressYJSUpdates", async (docUpdates, doc, docId) => {
-      const SAVINGS_THRESHOLD = 0.2;
-      const updates = Object.values(docUpdates);
-      const sizeOnDisk = updates.reduce((acc, update) => {
-        return acc + update.length;
-      }, 0);
-      if (updates.length > 0) {
-        const docKeys = Object.keys(docUpdates);
-        this.keysById.set(docId, new Set(docKeys));
-        const mergedUpdate = Y.mergeUpdates(updates);
-        Y.applyUpdate(doc, mergedUpdate);
-        const garbageCollectedUpdate = Y.encodeStateAsUpdate(doc);
-        if (garbageCollectedUpdate.length < sizeOnDisk * (1 - SAVINGS_THRESHOLD)) {
-          const newKey = _nanoid.nanoid.call(void 0, );
-          await this.driver.write_y_updates(
-            docId,
-            newKey,
-            garbageCollectedUpdate
-          );
-          await this.driver.delete_y_updates(docId, docKeys);
-          this.keysById.set(docId, /* @__PURE__ */ new Set([newKey]));
-        }
-      }
+    __publicField(this, "storedKeysById", /* @__PURE__ */ new Map());
+    // compact the updates into a single update and write it to the durable storage
+    __publicField(this, "_compactYJSUpdates", async (doc, docId, storedKeys) => {
+      const compactedUpdate = Y.encodeStateAsUpdate(doc);
+      const newKey = _nanoid.nanoid.call(void 0, );
+      await this.driver.write_y_updates(docId, newKey, compactedUpdate);
+      await this.driver.delete_y_updates(docId, storedKeys);
+      this.storedKeysById.set(docId, [newKey]);
     });
     __publicField(this, "_loadYDocFromDurableStorage", async (doc, docId) => {
       const docUpdates = Object.fromEntries(
         await this.driver.iter_y_updates(docId)
       );
-      await this._loadAndCompressYJSUpdates(docUpdates, doc, docId);
-      this.lastUpdatesById.set(docId, {
-        currentKey: _nanoid.nanoid.call(void 0, ),
-        lastVector: Y.encodeStateVector(doc)
-      });
+      const updates = Object.values(docUpdates);
+      const newupdate = Y.mergeUpdates(updates);
+      const storedKeys = Object.keys(docUpdates);
+      Y.applyUpdate(doc, newupdate);
+      if (this.shouldCompact(storedKeys)) {
+        await this._compactYJSUpdates(doc, docId, storedKeys);
+      } else {
+        this.storedKeysById.set(docId, storedKeys);
+      }
       doc.emit("load", [doc]);
       return doc;
     });
     this.driver = driver;
+    this.updateCountThreshold = updateCountThreshold;
     this.doc.on("subdocs", ({ removed }) => {
       removed.forEach((subdoc) => {
         subdoc.destroy();
@@ -1490,7 +1472,9 @@ var YjsStorage = class {
   }
   /**
    * @param update base64 encoded uint8array
-   * @returns
+   * @returns { isUpdated: boolean; snapshotHash: string }
+   *   isUpdated: true if the update had an effect on the YDoc
+   *   snapshotHash: the hash of the new snapshot
    */
   async addYDocUpdate(logger, update, guid, isV2) {
     const doc = guid !== void 0 ? await this.getYSubdoc(guid) : this.doc;
@@ -1505,7 +1489,7 @@ var YjsStorage = class {
       const afterSnapshot = this._putLastSnapshot(doc);
       const updated = !Y.equalSnapshots(beforeSnapshot, afterSnapshot);
       if (updated) {
-        await this.handleYDocUpdate(doc);
+        await this.handleYDocUpdate(doc, updateAsU8, isV2);
       }
       return {
         isUpdated: updated,
@@ -1583,43 +1567,27 @@ var YjsStorage = class {
     return subdoc;
   }
   // When the YJS doc changes, update it in durable storage
-  async handleYDocUpdate(doc) {
+  async handleYDocUpdate(doc, update, isV2) {
+    const v1update = isV2 ? Y.convertUpdateFormatV2ToV1(update) : update;
     const docId = doc.guid === this.doc.guid ? ROOT_YDOC_ID : doc.guid;
-    const docUpdateInfo = this.lastUpdatesById.get(docId);
-    const updateSinceLastVector = Y.encodeStateAsUpdate(
-      doc,
-      _optionalChain([docUpdateInfo, 'optionalAccess', _49 => _49.lastVector])
-    );
-    const storageKey = _nullishCoalesce(_optionalChain([docUpdateInfo, 'optionalAccess', _50 => _50.currentKey]), () => ( _nanoid.nanoid.call(void 0, )));
-    if (updateSinceLastVector.length > MAX_Y_UPDATE_SIZE) {
-      const newKey = _nanoid.nanoid.call(void 0, );
-      await this.driver.write_y_updates(
-        docId,
-        newKey,
-        Y.encodeStateAsUpdate(doc)
-      );
-      await this.driver.delete_y_updates(
-        docId,
-        Array.from(this.keysById.getOrCreate(docId))
-      );
-      this.keysById.set(docId, /* @__PURE__ */ new Set([newKey]));
-      this.lastUpdatesById.set(docId, {
-        currentKey: _nanoid.nanoid.call(void 0, ),
-        // start writing to a new key
-        lastVector: Y.encodeStateVector(doc)
-      });
+    const storedKeys = this.storedKeysById.get(docId);
+    if (this.shouldCompact(storedKeys)) {
+      await this._compactYJSUpdates(doc, docId, storedKeys || []);
+      return;
+    }
+    const newKey = _nanoid.nanoid.call(void 0, );
+    await this.driver.write_y_updates(docId, newKey, v1update);
+    if (!storedKeys) {
+      this.storedKeysById.set(docId, [newKey]);
     } else {
-      await this.driver.write_y_updates(
-        docId,
-        storageKey,
-        updateSinceLastVector
-      );
-      const keys = [storageKey];
-      const currentKeys = this.keysById.getOrCreate(docId);
-      for (const key of keys) {
-        currentKeys.add(key);
-      }
+      storedKeys.push(newKey);
+    }
+  }
+  shouldCompact(storedKeys) {
+    if (!storedKeys) {
+      return false;
     }
+    return storedKeys.length >= this.updateCountThreshold;
   }
 };
 
@@ -1768,7 +1736,7 @@ var BrowserSession = class {
     __privateSet(this, __hasNotifiedClientStorageUpdateError, false);
   }
   get lastActiveAt() {
-    const lastPing = _optionalChain([__privateGet, 'call', _51 => _51(this, __socket), 'access', _52 => _52.getLastPongTimestamp, 'optionalCall', _53 => _53()]);
+    const lastPing = _optionalChain([__privateGet, 'call', _49 => _49(this, __socket), 'access', _50 => _50.getLastPongTimestamp, 'optionalCall', _51 => _51()]);
     if (lastPing && lastPing > __privateGet(this, __lastActiveAt)) {
       return lastPing;
     } else {
@@ -1860,28 +1828,28 @@ var Room = class {
     __publicField(this, "hooks");
     __privateAdd(this, __debug2);
     __privateAdd(this, __allowStreaming);
-    const driver = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _54 => _54.storage]), () => ( makeNewInMemoryDriver()));
+    const driver = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _52 => _52.storage]), () => ( makeNewInMemoryDriver()));
     this.meta = meta;
     this.driver = driver;
-    this.logger = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _55 => _55.logger]), () => ( BLACK_HOLE));
-    __privateSet(this, __allowStreaming, _nullishCoalesce(_optionalChain([options, 'optionalAccess', _56 => _56.allowStreaming]), () => ( true)));
+    this.logger = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _53 => _53.logger]), () => ( BLACK_HOLE));
+    __privateSet(this, __allowStreaming, _nullishCoalesce(_optionalChain([options, 'optionalAccess', _54 => _54.allowStreaming]), () => ( true)));
     this.hooks = {
-      isClientMsgAllowed: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _57 => _57.hooks, 'optionalAccess', _58 => _58.isClientMsgAllowed]), () => ( (() => {
+      isClientMsgAllowed: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _55 => _55.hooks, 'optionalAccess', _56 => _56.isClientMsgAllowed]), () => ( (() => {
         return {
           allowed: true
         };
       }))),
       // YYY .load() isn't called on the RoomServer yet! As soon as it does, these hooks will get called
-      onRoomWillLoad: _optionalChain([options, 'optionalAccess', _59 => _59.hooks, 'optionalAccess', _60 => _60.onRoomWillLoad]),
-      onRoomDidLoad: _optionalChain([options, 'optionalAccess', _61 => _61.hooks, 'optionalAccess', _62 => _62.onRoomDidLoad]),
-      onRoomWillUnload: _optionalChain([options, 'optionalAccess', _63 => _63.hooks, 'optionalAccess', _64 => _64.onRoomWillUnload]),
-      onRoomDidUnload: _optionalChain([options, 'optionalAccess', _65 => _65.hooks, 'optionalAccess', _66 => _66.onRoomDidUnload]),
-      onSessionDidStart: _optionalChain([options, 'optionalAccess', _67 => _67.hooks, 'optionalAccess', _68 => _68.onSessionDidStart]),
-      onSessionDidEnd: _optionalChain([options, 'optionalAccess', _69 => _69.hooks, 'optionalAccess', _70 => _70.onSessionDidEnd]),
-      postClientMsgStorageDidUpdate: _optionalChain([options, 'optionalAccess', _71 => _71.hooks, 'optionalAccess', _72 => _72.postClientMsgStorageDidUpdate]),
-      postClientMsgYdocDidUpdate: _optionalChain([options, 'optionalAccess', _73 => _73.hooks, 'optionalAccess', _74 => _74.postClientMsgYdocDidUpdate])
+      onRoomWillLoad: _optionalChain([options, 'optionalAccess', _57 => _57.hooks, 'optionalAccess', _58 => _58.onRoomWillLoad]),
+      onRoomDidLoad: _optionalChain([options, 'optionalAccess', _59 => _59.hooks, 'optionalAccess', _60 => _60.onRoomDidLoad]),
+      onRoomWillUnload: _optionalChain([options, 'optionalAccess', _61 => _61.hooks, 'optionalAccess', _62 => _62.onRoomWillUnload]),
+      onRoomDidUnload: _optionalChain([options, 'optionalAccess', _63 => _63.hooks, 'optionalAccess', _64 => _64.onRoomDidUnload]),
+      onSessionDidStart: _optionalChain([options, 'optionalAccess', _65 => _65.hooks, 'optionalAccess', _66 => _66.onSessionDidStart]),
+      onSessionDidEnd: _optionalChain([options, 'optionalAccess', _67 => _67.hooks, 'optionalAccess', _68 => _68.onSessionDidEnd]),
+      postClientMsgStorageDidUpdate: _optionalChain([options, 'optionalAccess', _69 => _69.hooks, 'optionalAccess', _70 => _70.postClientMsgStorageDidUpdate]),
+      postClientMsgYdocDidUpdate: _optionalChain([options, 'optionalAccess', _71 => _71.hooks, 'optionalAccess', _72 => _72.postClientMsgYdocDidUpdate])
     };
-    __privateSet(this, __debug2, _nullishCoalesce(_optionalChain([options, 'optionalAccess', _75 => _75.enableDebugLogging]), () => ( false)));
+    __privateSet(this, __debug2, _nullishCoalesce(_optionalChain([options, 'optionalAccess', _73 => _73.enableDebugLogging]), () => ( false)));
   }
   get loadingState() {
     if (this._loadData$ === null) {
@@ -1937,13 +1905,13 @@ var Room = class {
    * room will be reloaded from storage.
    */
   unload(ctx) {
-    _optionalChain([this, 'access', _76 => _76.hooks, 'access', _77 => _77.onRoomWillUnload, 'optionalCall', _78 => _78(ctx)]);
+    _optionalChain([this, 'access', _74 => _74.hooks, 'access', _75 => _75.onRoomWillUnload, 'optionalCall', _76 => _76(ctx)]);
     if (this._data) {
       this.storage.unload();
       this.yjsStorage.unload();
     }
     this._loadData$ = null;
-    _optionalChain([this, 'access', _79 => _79.hooks, 'access', _80 => _80.onRoomDidUnload, 'optionalCall', _81 => _81(ctx)]);
+    _optionalChain([this, 'access', _77 => _77.hooks, 'access', _78 => _78.onRoomDidUnload, 'optionalCall', _79 => _79(ctx)]);
   }
   /**
    * Issues a Ticket with a new/unique actor ID
@@ -1957,17 +1925,17 @@ var Room = class {
    * unused Ticket will simply get garbage collected.
    */
   async createTicket(options) {
-    const actor$ = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _82 => _82.actor]), () => ( this.getNextActor()));
+    const actor$ = _nullishCoalesce(_optionalChain([options, 'optionalAccess', _80 => _80.actor]), () => ( this.getNextActor()));
     const sessionKey = _nanoid.nanoid.call(void 0, );
-    const info = _optionalChain([options, 'optionalAccess', _83 => _83.info]);
+    const info = _optionalChain([options, 'optionalAccess', _81 => _81.info]);
     const ticket = {
-      version: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _84 => _84.version]), () => ( HIGHEST_PROTOCOL_VERSION)),
+      version: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _82 => _82.version]), () => ( HIGHEST_PROTOCOL_VERSION)),
       actor: await actor$,
       sessionKey,
-      meta: _optionalChain([options, 'optionalAccess', _85 => _85.meta]),
-      publicMeta: _optionalChain([options, 'optionalAccess', _86 => _86.publicMeta]),
-      user: _optionalChain([options, 'optionalAccess', _87 => _87.id]) ? { id: options.id, info } : { anonymousId: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _88 => _88.anonymousId]), () => ( _nanoid.nanoid.call(void 0, ))), info },
-      scopes: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _89 => _89.scopes]), () => ( ["room:write"]))
+      meta: _optionalChain([options, 'optionalAccess', _83 => _83.meta]),
+      publicMeta: _optionalChain([options, 'optionalAccess', _84 => _84.publicMeta]),
+      user: _optionalChain([options, 'optionalAccess', _85 => _85.id]) ? { id: options.id, info } : { anonymousId: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _86 => _86.anonymousId]), () => ( _nanoid.nanoid.call(void 0, ))), info },
+      scopes: _nullishCoalesce(_optionalChain([options, 'optionalAccess', _87 => _87.scopes]), () => ( ["room:write"]))
     };
     if (__privateGet(this, __debug2)) {
       console.log(`new ticket created: ${JSON.stringify(ticket)}`);
@@ -2096,7 +2064,7 @@ var Room = class {
       ctx,
       defer
     );
-    const p$ = _optionalChain([this, 'access', _90 => _90.hooks, 'access', _91 => _91.onSessionDidStart, 'optionalCall', _92 => _92(newSession, ctx)]);
+    const p$ = _optionalChain([this, 'access', _88 => _88.hooks, 'access', _89 => _89.onSessionDidStart, 'optionalCall', _90 => _90(newSession, ctx)]);
     if (p$) defer(p$);
   }
   /**
@@ -2120,7 +2088,7 @@ var Room = class {
       for (const other of this.otherSessions(key)) {
         other.send({ type: _core.ServerMsgCode.USER_LEFT, actor: session.actor });
       }
-      const p$ = _optionalChain([this, 'access', _93 => _93.hooks, 'access', _94 => _94.onSessionDidEnd, 'optionalCall', _95 => _95(session, ctx)]);
+      const p$ = _optionalChain([this, 'access', _91 => _91.hooks, 'access', _92 => _92.onSessionDidEnd, 'optionalCall', _93 => _93(session, ctx)]);
       if (p$) defer(p$);
     }
   }
@@ -2424,7 +2392,7 @@ var Room = class {
   }
   // Don't ever manually call this!
   async _load(ctx) {
-    await _optionalChain([this, 'access', _96 => _96.hooks, 'access', _97 => _97.onRoomWillLoad, 'optionalCall', _98 => _98(ctx)]);
+    await _optionalChain([this, 'access', _94 => _94.hooks, 'access', _95 => _95.onRoomWillLoad, 'optionalCall', _96 => _96(ctx)]);
     const storage = await this._loadStorage();
     const yjsStorage = await this._loadYjsStorage();
     this._data = {
@@ -2432,7 +2400,7 @@ var Room = class {
       storage,
       yjsStorage
     };
-    await _optionalChain([this, 'access', _99 => _99.hooks, 'access', _100 => _100.onRoomDidLoad, 'optionalCall', _101 => _101(ctx)]);
+    await _optionalChain([this, 'access', _97 => _97.hooks, 'access', _98 => _98.onRoomDidLoad, 'optionalCall', _99 => _99(ctx)]);
   }
   /**
    * Returns a new, unique, actor ID.
@@ -2473,7 +2441,7 @@ var Room = class {
     }
     const sent = session.sendPong();
     if (sent !== 0) {
-      await _optionalChain([this, 'access', _102 => _102.hooks, 'access', _103 => _103.onDidPong, 'optionalCall', _104 => _104(ctx)]);
+      await _optionalChain([this, 'access', _100 => _100.hooks, 'access', _101 => _101.onDidPong, 'optionalCall', _102 => _102(ctx)]);
     }
   }
   async _processClientMsg_withExclusiveAccess(sessionKey, messages, ctx, defer) {
@@ -2612,7 +2580,7 @@ var Room = class {
         break;
       }
       case _core.ClientMsgCode.UPDATE_STORAGE: {
-        _optionalChain([this, 'access', _105 => _105.driver, 'access', _106 => _106.bump_storage_version, 'optionalCall', _107 => _107()]);
+        _optionalChain([this, 'access', _103 => _103.driver, 'access', _104 => _104.bump_storage_version, 'optionalCall', _105 => _105()]);
         const result = await this.storage.applyOps(msg.ops);
         const opsToForward = result.flatMap(
           (r) => r.action === "accepted" ? [r.op] : []
@@ -2644,7 +2612,7 @@ var Room = class {
           });
         }
         if (opsToForward.length > 0) {
-          const p$ = _optionalChain([this, 'access', _108 => _108.hooks, 'access', _109 => _109.postClientMsgStorageDidUpdate, 'optionalCall', _110 => _110(ctx)]);
+          const p$ = _optionalChain([this, 'access', _106 => _106.hooks, 'access', _107 => _107.postClientMsgStorageDidUpdate, 'optionalCall', _108 => _108(ctx)]);
           if (p$) defer(p$);
         }
         break;
@@ -2695,7 +2663,7 @@ var Room = class {
           defer
         );
         if (result.isUpdated) {
-          const p$ = _optionalChain([this, 'access', _111 => _111.hooks, 'access', _112 => _112.postClientMsgYdocDidUpdate, 'optionalCall', _113 => _113(ctx, session)]);
+          const p$ = _optionalChain([this, 'access', _109 => _109.hooks, 'access', _110 => _110.postClientMsgYdocDidUpdate, 'optionalCall', _111 => _111(ctx, session)]);
           if (p$) defer(p$);
         }
         break;