@dxos/echo-pipeline 0.4.10-main.3aa8c1b → 0.4.10-main.3c620fb

package/dist/lib/browser/{chunk-RA6MLCZM.mjs → chunk-REQJ6WI6.mjs}

@@ -1903,6 +1903,7 @@ SpaceManager = _ts_decorate7([
 ], SpaceManager);
 
 export {
+  Buffer,
   codec,
   valueEncoding,
   createMappedFeedWriter,
@@ -1924,4 +1925,4 @@ export {
   SpaceProtocolSession,
   SpaceManager
 };
-//# sourceMappingURL=chunk-RA6MLCZM.mjs.map
+//# sourceMappingURL=chunk-REQJ6WI6.mjs.map

package/dist/lib/browser/index.mjs

@@ -2,6 +2,7 @@ import "@dxos/node-std/globals";
 import {
   AuthExtension,
   AuthStatus,
+  Buffer,
   DataServiceImpl,
   MOCK_AUTH_PROVIDER,
   MOCK_AUTH_VERIFIER,
@@ -20,124 +21,100 @@ import {
   mapTimeframeToFeedIndexes,
   startAfter,
   valueEncoding
-} from "./chunk-RA6MLCZM.mjs";
+} from "./chunk-REQJ6WI6.mjs";
 
 // packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
 import { next as automerge, getHeads } from "@dxos/automerge/automerge";
 import { Repo } from "@dxos/automerge/automerge-repo";
-import { IndexedDBStorageAdapter } from "@dxos/automerge/automerge-repo-storage-indexeddb";
 import { Context } from "@dxos/context";
 import { PublicKey } from "@dxos/keys";
-import { log as log3 } from "@dxos/log";
+import { log as log4 } from "@dxos/log";
 import { idCodec } from "@dxos/protocols";
-import { StorageType } from "@dxos/random-access-storage";
 import { trace } from "@dxos/tracing";
 import { ComplexMap, ComplexSet, defaultMap, mapValues } from "@dxos/util";
 
-// packages/core/echo/echo-pipeline/src/automerge/automerge-storage-adapter.ts
-import { arrayToBuffer, bufferToArray } from "@dxos/util";
-var AutomergeStorageAdapter = class {
-  constructor(_directory) {
-    this._directory = _directory;
+// packages/core/echo/echo-pipeline/src/automerge/leveldb-storage-adapter.ts
+var LevelDBStorageAdapter = class {
+  constructor(_params) {
+    this._params = _params;
     this._state = "opened";
   }
-  async load(key) {
+  async load(keyArray) {
     if (this._state !== "opened") {
       return void 0;
     }
-    const filename = this._getFilename(key);
-    const file = this._directory.getOrCreateFile(filename);
-    const { size } = await file.stat();
-    if (!size || size === 0) {
-      return void 0;
-    }
-    const buffer = await file.read(0, size);
-    return bufferToArray(buffer);
+    return this._params.db.get(keyArray, {
+      ...encodingOptions
+    }).catch((err) => err.code === "LEVEL_NOT_FOUND" ? void 0 : Promise.reject(err));
   }
-  async save(key, data) {
+  async save(keyArray, binary) {
     if (this._state !== "opened") {
       return void 0;
     }
-    const filename = this._getFilename(key);
-    const file = this._directory.getOrCreateFile(filename);
-    await file.write(0, arrayToBuffer(data));
-    await file.truncate?.(data.length);
-    await file.flush?.();
+    await this._params.callbacks?.beforeSave?.(keyArray);
+    await this._params.db.put(keyArray, Buffer.from(binary), {
+      ...encodingOptions
+    });
+    await this._params.callbacks?.afterSave?.(keyArray);
   }
-  async remove(key) {
+  async remove(keyArray) {
     if (this._state !== "opened") {
       return void 0;
     }
-    const filename = this._getFilename(key);
-    const file = this._directory.getOrCreateFile(filename);
-    await file.destroy();
+    await this._params.db.del(keyArray, {
+      ...encodingOptions
+    });
   }
   async loadRange(keyPrefix) {
     if (this._state !== "opened") {
       return [];
     }
-    const filename = this._getFilename(keyPrefix);
-    const entries = await this._directory.list();
-    return Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
-      const file = this._directory.getOrCreateFile(entry);
-      const { size } = await file.stat();
-      const buffer = await file.read(0, size);
-      return {
-        key: this._getKeyFromFilename(entry),
-        data: bufferToArray(buffer)
-      };
-    }));
+    const result = [];
+    for await (const [key, value] of this._params.db.iterator({
+      gte: keyPrefix,
+      lte: [
+        ...keyPrefix,
+        "\uFFFF"
+      ],
+      ...encodingOptions
+    })) {
+      result.push({
+        key,
+        data: value
+      });
+    }
+    return result;
   }
   async removeRange(keyPrefix) {
     if (this._state !== "opened") {
       return void 0;
     }
-    const filename = this._getFilename(keyPrefix);
-    const entries = await this._directory.list();
-    await Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
-      const file = this._directory.getOrCreateFile(entry);
-      await file.destroy();
-    }));
+    const batch = this._params.db.batch();
+    for await (const [key] of this._params.db.iterator({
+      gte: keyPrefix,
+      lte: [
+        ...keyPrefix,
+        "\uFFFF"
+      ],
+      ...encodingOptions
+    })) {
+      batch.del(key, {
+        ...encodingOptions
+      });
+    }
+    await batch.write();
   }
-  async close() {
+  close() {
     this._state = "closed";
   }
-  _getFilename(key) {
-    return key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-");
-  }
-  _getKeyFromFilename(filename) {
-    return filename.split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%"));
-  }
 };
-
-// packages/core/echo/echo-pipeline/src/automerge/automerge-storage–wrapper.ts
-var AutomergeStorageWrapper = class {
-  constructor({ storage, callbacks }) {
-    this._storage = storage;
-    this._callbacks = callbacks;
-  }
-  async load(key) {
-    return this._storage.load(key);
-  }
-  async save(key, value) {
-    await this._callbacks.beforeSave?.(key);
-    await this._storage.save(key, value);
-    await this._callbacks.afterSave?.(key);
-  }
-  async remove(key) {
-    return this._storage.remove(key);
-  }
-  async loadRange(keyPrefix) {
-    return this._storage.loadRange(keyPrefix);
-  }
-  async removeRange(keyPrefix) {
-    return this._storage.removeRange(keyPrefix);
-  }
-  async close() {
-    if (this._storage instanceof AutomergeStorageAdapter) {
-      return this._storage.close();
-    }
-  }
+var keyEncoder = {
+  encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
+  decode: (key) => Buffer.from(key).toString().split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%"))
+};
+var encodingOptions = {
+  keyEncoding: keyEncoder,
+  valueEncoding: "buffer"
 };
 
 // packages/core/echo/echo-pipeline/src/automerge/local-host-network-adapter.ts
@@ -372,6 +349,118 @@ var MeshNetworkAdapter = class extends NetworkAdapter2 {
   }
 };
 
+// packages/core/echo/echo-pipeline/src/automerge/migrations.ts
+import { IndexedDBStorageAdapter } from "@dxos/automerge/automerge-repo-storage-indexeddb";
+import { log as log3 } from "@dxos/log";
+import { StorageType } from "@dxos/random-access-storage";
+
+// packages/core/echo/echo-pipeline/src/automerge/automerge-storage-adapter.ts
+import { arrayToBuffer, bufferToArray } from "@dxos/util";
+var AutomergeStorageAdapter = class {
+  constructor(_directory) {
+    this._directory = _directory;
+    this._state = "opened";
+  }
+  async load(key) {
+    if (this._state !== "opened") {
+      return void 0;
+    }
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    const { size } = await file.stat();
+    if (!size || size === 0) {
+      return void 0;
+    }
+    const buffer = await file.read(0, size);
+    return bufferToArray(buffer);
+  }
+  async save(key, data) {
+    if (this._state !== "opened") {
+      return void 0;
+    }
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    await file.write(0, arrayToBuffer(data));
+    await file.truncate?.(data.length);
+    await file.flush?.();
+  }
+  async remove(key) {
+    if (this._state !== "opened") {
+      return void 0;
+    }
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    await file.destroy();
+  }
+  async loadRange(keyPrefix) {
+    if (this._state !== "opened") {
+      return [];
+    }
+    const filename = this._getFilename(keyPrefix);
+    const entries = await this._directory.list();
+    return Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
+      const file = this._directory.getOrCreateFile(entry);
+      const { size } = await file.stat();
+      const buffer = await file.read(0, size);
+      return {
+        key: this._getKeyFromFilename(entry),
+        data: bufferToArray(buffer)
+      };
+    }));
+  }
+  async removeRange(keyPrefix) {
+    if (this._state !== "opened") {
+      return void 0;
+    }
+    const filename = this._getFilename(keyPrefix);
+    const entries = await this._directory.list();
+    await Promise.all(entries.filter((entry) => entry.startsWith(filename)).map(async (entry) => {
+      const file = this._directory.getOrCreateFile(entry);
+      await file.destroy();
+    }));
+  }
+  async close() {
+    this._state = "closed";
+  }
+  _getFilename(key) {
+    return key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-");
+  }
+  _getKeyFromFilename(filename) {
+    return filename.split("-").map((k) => k.replaceAll("%2D", "-").replaceAll("%25", "%"));
+  }
+};
+
+// packages/core/echo/echo-pipeline/src/automerge/migrations.ts
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/migrations.ts";
+var levelMigration = async ({ db, directory }) => {
+  const isNewLevel = !await db.iterator({
+    ...encodingOptions
+  }).next();
+  if (!isNewLevel) {
+    return;
+  }
+  const oldStorageAdapter = directory.type === StorageType.IDB ? new IndexedDBStorageAdapter(directory.path, "data") : new AutomergeStorageAdapter(directory);
+  const chunks = await oldStorageAdapter.loadRange([]);
+  if (chunks.length === 0) {
+    return;
+  }
+  const batch = db.batch();
+  log3.info("found chunks on old storage adapter", {
+    chunks: chunks.length
+  }, {
+    F: __dxlog_file3,
+    L: 36,
+    S: void 0,
+    C: (f, a) => f(...a)
+  });
+  for (const { key, data } of await oldStorageAdapter.loadRange([])) {
+    data && batch.put(key, data, {
+      ...encodingOptions
+    });
+  }
+  await batch.write();
+};
+
 // packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -383,9 +472,9 @@ function _ts_decorate(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
 var AutomergeHost = class {
-  constructor({ directory, metadata }) {
+  constructor({ directory, db, metadata }) {
     this._ctx = new Context();
     /**
     * spaceKey -> deviceKey[]
@@ -393,19 +482,24 @@ var AutomergeHost = class {
     this._authorizedDevices = new ComplexMap(PublicKey.hash);
     this._updatingMetadata = /* @__PURE__ */ new Map();
     this._requestedDocs = /* @__PURE__ */ new Set();
+    this._directory = directory;
+    this._db = db;
     this._metadata = metadata;
-    this._meshNetwork = new MeshNetworkAdapter();
-    this._clientNetwork = new LocalHostNetworkAdapter();
-    this._storage = new AutomergeStorageWrapper({
-      storage: (
-        // TODO(mykola): Delete specific handling of IDB storage.
-        directory.type === StorageType.IDB ? new IndexedDBStorageAdapter(directory.path, "data") : new AutomergeStorageAdapter(directory)
-      ),
+  }
+  async open() {
+    this._directory && await levelMigration({
+      db: await this._db,
+      directory: this._directory
+    });
+    this._storage = new LevelDBStorageAdapter({
+      db: await this._db,
       callbacks: {
         beforeSave: (params) => this._beforeSave(params)
       }
     });
     this._peerId = `host-${PublicKey.random().toHex()}`;
+    this._meshNetwork = new MeshNetworkAdapter();
+    this._clientNetwork = new LocalHostNetworkAdapter();
     this._repo = new Repo({
       peerId: this._peerId,
       network: [
@@ -425,13 +519,13 @@ var AutomergeHost = class {
         const doc = this._repo.handles[documentId]?.docSync();
         if (!doc) {
           const isRequested = this._requestedDocs.has(`automerge:${documentId}`);
-          log3("doc share policy check", {
+          log4("doc share policy check", {
             peerId,
             documentId,
             isRequested
           }, {
-            F: __dxlog_file3,
-            L: 101,
+            F: __dxlog_file4,
+            L: 111,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -440,12 +534,12 @@ var AutomergeHost = class {
         try {
           const spaceKey = getSpaceKeyFromDoc(doc);
           if (!spaceKey) {
-            log3("space key not found for share policy check", {
+            log4("space key not found for share policy check", {
               peerId,
               documentId
             }, {
-              F: __dxlog_file3,
-              L: 108,
+              F: __dxlog_file4,
+              L: 118,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -454,12 +548,12 @@ var AutomergeHost = class {
           const authorizedDevices = this._authorizedDevices.get(PublicKey.from(spaceKey));
           const deviceKeyHex = this.repo.peerMetadataByPeerId[peerId]?.dxos_deviceKey;
           if (!deviceKeyHex) {
-            log3("device key not found for share policy check", {
+            log4("device key not found for share policy check", {
               peerId,
               documentId
             }, {
-              F: __dxlog_file3,
-              L: 117,
+              F: __dxlog_file4,
+              L: 127,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -467,7 +561,7 @@ var AutomergeHost = class {
           }
           const deviceKey = PublicKey.from(deviceKeyHex);
           const isAuthorized = authorizedDevices?.has(deviceKey) ?? false;
-          log3("share policy check", {
+          log4("share policy check", {
             localPeer: this._peerId,
             remotePeer: peerId,
             documentId,
@@ -475,16 +569,16 @@ var AutomergeHost = class {
             spaceKey,
             isAuthorized
           }, {
-            F: __dxlog_file3,
-            L: 123,
+            F: __dxlog_file4,
+            L: 133,
             S: this,
             C: (f, a) => f(...a)
           });
           return isAuthorized;
         } catch (err) {
-          log3.catch(err, void 0, {
-            F: __dxlog_file3,
-            L: 133,
+          log4.catch(err, void 0, {
+            F: __dxlog_file4,
+            L: 143,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -503,6 +597,11 @@ var AutomergeHost = class {
       });
     }
   }
+  async close() {
+    this._storage.close?.();
+    await this._clientNetwork.close();
+    await this._ctx.dispose();
+  }
   get repo() {
     return this._repo;
   }
@@ -540,9 +639,9 @@ var AutomergeHost = class {
     const markingDirtyPromise = this._metadata.markDirty(idToLastHash).then(() => {
       this._updatingMetadata.delete(event.handle.documentId);
     }).catch((err) => {
-      this._ctx.disposed && log3.catch(err, void 0, {
-        F: __dxlog_file3,
-        L: 191,
+      this._ctx.disposed && log4.catch(err, void 0, {
+        F: __dxlog_file4,
+        L: 207,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -574,11 +673,6 @@ var AutomergeHost = class {
   _automergePeers() {
     return this._repo.peers;
   }
-  async close() {
-    await this._storage.close();
-    await this._clientNetwork.close();
-    await this._ctx.dispose();
-  }
   //
   // Methods for client-services.
   //
@@ -602,12 +696,12 @@ var AutomergeHost = class {
     return this._meshNetwork.createExtension();
   }
   authorizeDevice(spaceKey, deviceKey) {
-    log3("authorizeDevice", {
+    log4("authorizeDevice", {
       spaceKey,
       deviceKey
     }, {
-      F: __dxlog_file3,
-      L: 264,
+      F: __dxlog_file4,
+      L: 274,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -661,8 +755,8 @@ import { Event } from "@dxos/async";
 import { cancelWithContext } from "@dxos/context";
 import { warnAfterTimeout } from "@dxos/debug";
 import { invariant as invariant3 } from "@dxos/invariant";
-import { log as log4 } from "@dxos/log";
-var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-doc-loader.ts";
+import { log as log5 } from "@dxos/log";
+var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-doc-loader.ts";
 var AutomergeDocumentLoaderImpl = class {
   constructor(_spaceKey, _repo) {
     this._spaceKey = _spaceKey;
@@ -682,10 +776,10 @@ var AutomergeDocumentLoaderImpl = class {
       return;
     }
     if (!spaceState.rootUrl) {
-      log4.error("Database opened with no rootUrl", {
+      log5.error("Database opened with no rootUrl", {
         spaceKey: this._spaceKey
       }, {
-        F: __dxlog_file4,
+        F: __dxlog_file5,
         L: 66,
         S: this,
         C: (f, a) => f(...a)
@@ -695,7 +789,7 @@ var AutomergeDocumentLoaderImpl = class {
       const existingDocHandle = await this._initDocHandle(ctx, spaceState.rootUrl);
       const doc = existingDocHandle.docSync();
       invariant3(doc, void 0, {
-        F: __dxlog_file4,
+        F: __dxlog_file5,
         L: 71,
         S: this,
         A: [
@@ -711,7 +805,7 @@ var AutomergeDocumentLoaderImpl = class {
   }
   loadObjectDocument(objectId) {
     invariant3(this._spaceRootDocHandle, void 0, {
-      F: __dxlog_file4,
+      F: __dxlog_file5,
       L: 80,
       S: this,
       A: [
@@ -724,7 +818,7 @@ var AutomergeDocumentLoaderImpl = class {
     }
     const spaceRootDoc = this._spaceRootDocHandle.docSync();
     invariant3(spaceRootDoc, void 0, {
-      F: __dxlog_file4,
+      F: __dxlog_file5,
       L: 85,
       S: this,
       A: [
@@ -735,10 +829,10 @@ var AutomergeDocumentLoaderImpl = class {
     const documentUrl = (spaceRootDoc.links ?? {})[objectId];
     if (documentUrl == null) {
       this._objectsPendingDocumentLoad.add(objectId);
-      log4.info("loading delayed until object links are initialized", {
+      log5.info("loading delayed until object links are initialized", {
         objectId
       }, {
-        F: __dxlog_file4,
+        F: __dxlog_file5,
         L: 89,
         S: this,
         C: (f, a) => f(...a)
@@ -759,7 +853,7 @@ var AutomergeDocumentLoaderImpl = class {
   }
   getSpaceRootDocHandle() {
     invariant3(this._spaceRootDocHandle, void 0, {
-      F: __dxlog_file4,
+      F: __dxlog_file5,
       L: 107,
       S: this,
       A: [
@@ -771,7 +865,7 @@ var AutomergeDocumentLoaderImpl = class {
   }
   createDocumentForObject(objectId) {
     invariant3(this._spaceRootDocHandle, void 0, {
-      F: __dxlog_file4,
+      F: __dxlog_file5,
       L: 112,
       S: this,
       A: [
@@ -810,11 +904,11 @@ var AutomergeDocumentLoaderImpl = class {
       };
       const objectDocumentHandle = this._objectDocumentHandles.get(objectId);
       if (objectDocumentHandle != null && objectDocumentHandle.url !== automergeUrl) {
-        log4.warn("object already inlined in a different document, ignoring the link", {
+        log5.warn("object already inlined in a different document, ignoring the link", {
           ...logMeta,
           actualDocumentUrl: objectDocumentHandle.url
         }, {
-          F: __dxlog_file4,
+          F: __dxlog_file5,
           L: 142,
           S: this,
           C: (f, a) => f(...a)
@@ -822,8 +916,8 @@ var AutomergeDocumentLoaderImpl = class {
         continue;
       }
       if (objectDocumentHandle?.url === automergeUrl) {
-        log4.warn("object document was already loaded", logMeta, {
-          F: __dxlog_file4,
+        log5.warn("object document was already loaded", logMeta, {
+          F: __dxlog_file5,
           L: 149,
           S: this,
           C: (f, a) => f(...a)
@@ -831,8 +925,8 @@ var AutomergeDocumentLoaderImpl = class {
         continue;
       }
       const handle = this._repo.find(automergeUrl);
-      log4.debug("document loading triggered", logMeta, {
-        F: __dxlog_file4,
+      log5.debug("document loading triggered", logMeta, {
+        F: __dxlog_file5,
         L: 153,
         S: this,
         C: (f, a) => f(...a)
@@ -851,11 +945,11 @@ var AutomergeDocumentLoaderImpl = class {
         break;
       } catch (err) {
         if (`${err}`.includes("Timeout")) {
-          log4.info("wraparound", {
+          log5.info("wraparound", {
             id: docHandle.documentId,
             state: docHandle.state
           }, {
-            F: __dxlog_file4,
+            F: __dxlog_file5,
             L: 169,
             S: this,
             C: (f, a) => f(...a)
@@ -896,8 +990,8 @@ var AutomergeDocumentLoaderImpl = class {
         docUrl: handle.url
       };
       if (this.onObjectDocumentLoaded.listenerCount() === 0) {
-        log4.info("document loaded after all listeners were removed", logMeta, {
-          F: __dxlog_file4,
+        log5.info("document loaded after all listeners were removed", logMeta, {
+          F: __dxlog_file5,
           L: 205,
           S: this,
           C: (f, a) => f(...a)
@@ -906,8 +1000,8 @@ var AutomergeDocumentLoaderImpl = class {
       }
       const objectDocHandle = this._objectDocumentHandles.get(objectId);
       if (objectDocHandle?.url !== handle.url) {
-        log4.warn("object was rebound while a document was loading, discarding handle", logMeta, {
-          F: __dxlog_file4,
+        log5.warn("object was rebound while a document was loading, discarding handle", logMeta, {
+          F: __dxlog_file5,
           L: 210,
           S: this,
           C: (f, a) => f(...a)
@@ -920,13 +1014,13 @@ var AutomergeDocumentLoaderImpl = class {
       });
     } catch (err) {
       const shouldRetryLoading = this.onObjectDocumentLoaded.listenerCount() > 0;
-      log4.warn("failed to load a document", {
+      log5.warn("failed to load a document", {
         objectId,
         automergeUrl: handle.url,
         retryLoading: shouldRetryLoading,
         err
       }, {
-        F: __dxlog_file4,
+        F: __dxlog_file5,
         L: 216,
         S: this,
         C: (f, a) => f(...a)