@liveblocks/server 1.0.14 → 1.1.0

package/dist/index.d.cts

@@ -824,12 +824,12 @@ declare class Storage {
 
 declare class YjsStorage {
     private readonly driver;
+    private readonly updateCountThreshold;
     private readonly doc;
-    private readonly lastUpdatesById;
     private readonly lastSnapshotById;
-    private readonly keysById;
     private readonly initPromisesById;
-    constructor(driver: IStorageDriver);
+    private readonly storedKeysById;
+    constructor(driver: IStorageDriver, updateCountThreshold?: number);
     getYDoc(docId: YDocId): Promise<Y.Doc>;
     /**
      * If passed a state vector, an update with diff will be returned, if not the entire doc is returned.
@@ -846,7 +846,9 @@ declare class YjsStorage {
     }): Promise<string | null>;
     /**
      * @param update base64 encoded uint8array
-     * @returns
+     * @returns { isUpdated: boolean; snapshotHash: string }
+     *   isUpdated: true if the update had an effect on the YDoc
+     *   snapshotHash: the hash of the new snapshot
      */
     addYDocUpdate(logger: Logger, update: string | Uint8Array, guid?: Guid, isV2?: boolean): Promise<{
         isUpdated: boolean;
@@ -860,15 +862,13 @@ declare class YjsStorage {
     unload(): void;
     private _getOrPutLastSnapshot;
     private _putLastSnapshot;
-    /**
-     * Given a record of updates, merge them and compress if savings are significant
-     */
-    private _loadAndCompressYJSUpdates;
+    private _compactYJSUpdates;
     private _loadYDocFromDurableStorage;
     private findYSubdocByGuid;
     private calculateSnapshotHash;
     private getYSubdoc;
     private handleYDocUpdate;
+    private shouldCompact;
 }
 
 /**

package/dist/index.d.ts

@@ -824,12 +824,12 @@ declare class Storage {
 
 declare class YjsStorage {
     private readonly driver;
+    private readonly updateCountThreshold;
     private readonly doc;
-    private readonly lastUpdatesById;
     private readonly lastSnapshotById;
-    private readonly keysById;
     private readonly initPromisesById;
-    constructor(driver: IStorageDriver);
+    private readonly storedKeysById;
+    constructor(driver: IStorageDriver, updateCountThreshold?: number);
     getYDoc(docId: YDocId): Promise<Y.Doc>;
     /**
      * If passed a state vector, an update with diff will be returned, if not the entire doc is returned.
@@ -846,7 +846,9 @@ declare class YjsStorage {
     }): Promise<string | null>;
     /**
      * @param update base64 encoded uint8array
-     * @returns
+     * @returns { isUpdated: boolean; snapshotHash: string }
+     *   isUpdated: true if the update had an effect on the YDoc
+     *   snapshotHash: the hash of the new snapshot
      */
     addYDocUpdate(logger: Logger, update: string | Uint8Array, guid?: Guid, isV2?: boolean): Promise<{
         isUpdated: boolean;
@@ -860,15 +862,13 @@ declare class YjsStorage {
     unload(): void;
     private _getOrPutLastSnapshot;
     private _putLastSnapshot;
-    /**
-     * Given a record of updates, merge them and compress if savings are significant
-     */
-    private _loadAndCompressYJSUpdates;
+    private _compactYJSUpdates;
     private _loadYDocFromDurableStorage;
     private findYSubdocByGuid;
     private calculateSnapshotHash;
     private getYSubdoc;
     private handleYDocUpdate;
+    private shouldCompact;
 }
 
 /**

package/dist/index.js

@@ -1369,63 +1369,45 @@ var Storage = class {
 };
 
 // src/YjsStorage.ts
-import { DefaultMap as DefaultMap2 } from "@liveblocks/core";
 import { Base64 } from "js-base64";
 import { nanoid } from "nanoid";
 import * as Y from "yjs";
-var MAX_Y_UPDATE_SIZE = 1e5;
+var UPDATE_COUNT_THRESHOLD = 1e3;
 var YjsStorage = class {
-  constructor(driver) {
+  constructor(driver, updateCountThreshold = UPDATE_COUNT_THRESHOLD) {
     __publicField(this, "driver");
+    __publicField(this, "updateCountThreshold");
     __publicField(this, "doc", new Y.Doc());
     // the root document
-    __publicField(this, "lastUpdatesById", /* @__PURE__ */ new Map());
     __publicField(this, "lastSnapshotById", /* @__PURE__ */ new Map());
-    // Keeps track of which keys are loaded, so we can clean them up without calling `.list()`
-    __publicField(this, "keysById", new DefaultMap2(
-      () => /* @__PURE__ */ new Set()
-    ));
     __publicField(this, "initPromisesById", /* @__PURE__ */ new Map());
-    /**
-     * Given a record of updates, merge them and compress if savings are significant
-     */
-    __publicField(this, "_loadAndCompressYJSUpdates", async (docUpdates, doc, docId) => {
-      const SAVINGS_THRESHOLD = 0.2;
-      const updates = Object.values(docUpdates);
-      const sizeOnDisk = updates.reduce((acc, update) => {
-        return acc + update.length;
-      }, 0);
-      if (updates.length > 0) {
-        const docKeys = Object.keys(docUpdates);
-        this.keysById.set(docId, new Set(docKeys));
-        const mergedUpdate = Y.mergeUpdates(updates);
-        Y.applyUpdate(doc, mergedUpdate);
-        const garbageCollectedUpdate = Y.encodeStateAsUpdate(doc);
-        if (garbageCollectedUpdate.length < sizeOnDisk * (1 - SAVINGS_THRESHOLD)) {
-          const newKey = nanoid();
-          await this.driver.write_y_updates(
-            docId,
-            newKey,
-            garbageCollectedUpdate
-          );
-          await this.driver.delete_y_updates(docId, docKeys);
-          this.keysById.set(docId, /* @__PURE__ */ new Set([newKey]));
-        }
-      }
+    __publicField(this, "storedKeysById", /* @__PURE__ */ new Map());
+    // compact the updates into a single update and write it to the durable storage
+    __publicField(this, "_compactYJSUpdates", async (doc, docId, storedKeys) => {
+      const compactedUpdate = Y.encodeStateAsUpdate(doc);
+      const newKey = nanoid();
+      await this.driver.write_y_updates(docId, newKey, compactedUpdate);
+      await this.driver.delete_y_updates(docId, storedKeys);
+      this.storedKeysById.set(docId, [newKey]);
     });
     __publicField(this, "_loadYDocFromDurableStorage", async (doc, docId) => {
       const docUpdates = Object.fromEntries(
         await this.driver.iter_y_updates(docId)
       );
-      await this._loadAndCompressYJSUpdates(docUpdates, doc, docId);
-      this.lastUpdatesById.set(docId, {
-        currentKey: nanoid(),
-        lastVector: Y.encodeStateVector(doc)
-      });
+      const updates = Object.values(docUpdates);
+      const newupdate = Y.mergeUpdates(updates);
+      const storedKeys = Object.keys(docUpdates);
+      Y.applyUpdate(doc, newupdate);
+      if (this.shouldCompact(storedKeys)) {
+        await this._compactYJSUpdates(doc, docId, storedKeys);
+      } else {
+        this.storedKeysById.set(docId, storedKeys);
+      }
       doc.emit("load", [doc]);
       return doc;
     });
     this.driver = driver;
+    this.updateCountThreshold = updateCountThreshold;
     this.doc.on("subdocs", ({ removed }) => {
       removed.forEach((subdoc) => {
         subdoc.destroy();
@@ -1490,7 +1472,9 @@ var YjsStorage = class {
   }
   /**
    * @param update base64 encoded uint8array
-   * @returns
+   * @returns { isUpdated: boolean; snapshotHash: string }
+   *   isUpdated: true if the update had an effect on the YDoc
+   *   snapshotHash: the hash of the new snapshot
    */
   async addYDocUpdate(logger, update, guid, isV2) {
     const doc = guid !== void 0 ? await this.getYSubdoc(guid) : this.doc;
@@ -1505,7 +1489,7 @@ var YjsStorage = class {
       const afterSnapshot = this._putLastSnapshot(doc);
       const updated = !Y.equalSnapshots(beforeSnapshot, afterSnapshot);
       if (updated) {
-        await this.handleYDocUpdate(doc);
+        await this.handleYDocUpdate(doc, updateAsU8, isV2);
       }
       return {
         isUpdated: updated,
@@ -1583,43 +1567,27 @@ var YjsStorage = class {
     return subdoc;
   }
   // When the YJS doc changes, update it in durable storage
-  async handleYDocUpdate(doc) {
+  async handleYDocUpdate(doc, update, isV2) {
+    const v1update = isV2 ? Y.convertUpdateFormatV2ToV1(update) : update;
     const docId = doc.guid === this.doc.guid ? ROOT_YDOC_ID : doc.guid;
-    const docUpdateInfo = this.lastUpdatesById.get(docId);
-    const updateSinceLastVector = Y.encodeStateAsUpdate(
-      doc,
-      docUpdateInfo?.lastVector
-    );
-    const storageKey = docUpdateInfo?.currentKey ?? nanoid();
-    if (updateSinceLastVector.length > MAX_Y_UPDATE_SIZE) {
-      const newKey = nanoid();
-      await this.driver.write_y_updates(
-        docId,
-        newKey,
-        Y.encodeStateAsUpdate(doc)
-      );
-      await this.driver.delete_y_updates(
-        docId,
-        Array.from(this.keysById.getOrCreate(docId))
-      );
-      this.keysById.set(docId, /* @__PURE__ */ new Set([newKey]));
-      this.lastUpdatesById.set(docId, {
-        currentKey: nanoid(),
-        // start writing to a new key
-        lastVector: Y.encodeStateVector(doc)
-      });
+    const storedKeys = this.storedKeysById.get(docId);
+    if (this.shouldCompact(storedKeys)) {
+      await this._compactYJSUpdates(doc, docId, storedKeys || []);
+      return;
+    }
+    const newKey = nanoid();
+    await this.driver.write_y_updates(docId, newKey, v1update);
+    if (!storedKeys) {
+      this.storedKeysById.set(docId, [newKey]);
     } else {
-      await this.driver.write_y_updates(
-        docId,
-        storageKey,
-        updateSinceLastVector
-      );
-      const keys = [storageKey];
-      const currentKeys = this.keysById.getOrCreate(docId);
-      for (const key of keys) {
-        currentKeys.add(key);
-      }
+      storedKeys.push(newKey);
+    }
+  }
+  shouldCompact(storedKeys) {
+    if (!storedKeys) {
+      return false;
     }
+    return storedKeys.length >= this.updateCountThreshold;
   }
 };