@dxos/echo-pipeline 0.4.10-main.bf8d896 → 0.4.10-main.c16d37b

package/dist/lib/browser/index.mjs

@@ -25,41 +25,54 @@ import {
 } from "./chunk-SYE4EK33.mjs";
 
 // packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts
-import { next as automerge, getHeads } from "@dxos/automerge/automerge";
+import { asyncTimeout } from "@dxos/async";
+import { next as automerge } from "@dxos/automerge/automerge";
 import { Repo } from "@dxos/automerge/automerge-repo";
 import { Context } from "@dxos/context";
 import { PublicKey } from "@dxos/keys";
-import { log as log4 } from "@dxos/log";
-import { idCodec } from "@dxos/protocols";
+import { log as log3 } from "@dxos/log";
 import { trace } from "@dxos/tracing";
 import { ComplexMap, ComplexSet, defaultMap, mapValues } from "@dxos/util";
 
 // packages/core/echo/echo-pipeline/src/automerge/leveldb-storage-adapter.ts
-var LevelDBStorageAdapter = class {
+import { LifecycleState, Resource } from "@dxos/context";
+var LevelDBStorageAdapter = class extends Resource {
   constructor(_params) {
+    super();
     this._params = _params;
-    this._state = "opened";
   }
   async load(keyArray) {
-    if (this._state !== "opened") {
-      return void 0;
+    try {
+      if (this._lifecycleState !== LifecycleState.OPEN) {
+        return void 0;
+      }
+      return await this._params.db.get(keyArray, {
+        ...encodingOptions
+      });
+    } catch (err) {
+      if (isLevelDbNotFoundError(err)) {
+        return void 0;
+      }
+      throw err;
     }
-    return this._params.db.get(keyArray, {
-      ...encodingOptions
-    }).catch((err) => err.code === "LEVEL_NOT_FOUND" ? void 0 : Promise.reject(err));
   }
   async save(keyArray, binary) {
-    if (this._state !== "opened") {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
       return void 0;
     }
-    await this._params.callbacks?.beforeSave?.(keyArray);
-    await this._params.db.put(keyArray, Buffer.from(binary), {
+    const batch = this._params.db.batch();
+    await this._params.callbacks?.beforeSave?.({
+      path: keyArray,
+      batch
+    });
+    batch.put(keyArray, Buffer.from(binary), {
       ...encodingOptions
     });
+    await batch.write();
     await this._params.callbacks?.afterSave?.(keyArray);
   }
   async remove(keyArray) {
-    if (this._state !== "opened") {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
       return void 0;
     }
     await this._params.db.del(keyArray, {
@@ -67,7 +80,7 @@ var LevelDBStorageAdapter = class {
     });
   }
   async loadRange(keyPrefix) {
-    if (this._state !== "opened") {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
       return [];
     }
     const result = [];
@@ -87,7 +100,7 @@ var LevelDBStorageAdapter = class {
     return result;
   }
   async removeRange(keyPrefix) {
-    if (this._state !== "opened") {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
       return void 0;
     }
     const batch = this._params.db.batch();
@@ -105,9 +118,6 @@ var LevelDBStorageAdapter = class {
     }
     await batch.write();
   }
-  close() {
-    this._state = "closed";
-  }
 };
 var keyEncoder = {
   encode: (key) => Buffer.from(key.map((k) => k.replaceAll("%", "%25").replaceAll("-", "%2D")).join("-")),
@@ -117,19 +127,20 @@ var encodingOptions = {
   keyEncoding: keyEncoder,
   valueEncoding: "buffer"
 };
+var isLevelDbNotFoundError = (err) => err.code === "LEVEL_NOT_FOUND";
 
 // packages/core/echo/echo-pipeline/src/automerge/local-host-network-adapter.ts
 import { Trigger } from "@dxos/async";
 import { NetworkAdapter, cbor } from "@dxos/automerge/automerge-repo";
 import { Stream } from "@dxos/codec-protobuf";
 import { invariant } from "@dxos/invariant";
-import { log } from "@dxos/log";
 var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/local-host-network-adapter.ts";
 var LocalHostNetworkAdapter = class extends NetworkAdapter {
   constructor() {
     super(...arguments);
     this._peers = /* @__PURE__ */ new Map();
     this._connected = new Trigger();
+    this._isConnected = false;
   }
   /**
   * Emits `ready` event. That signals to `Repo` that it can start using the adapter.
@@ -139,15 +150,21 @@ var LocalHostNetworkAdapter = class extends NetworkAdapter {
       network: this
     });
   }
+  /**
+  * Called by `Repo` to connect to the network.
+  *
+  * @param peerId Our peer Id.
+  */
   connect(peerId) {
     this.peerId = peerId;
+    this._isConnected = true;
     this._connected.wake();
   }
   send(message) {
     const peer = this._peers.get(message.targetId);
     invariant(peer, "Peer not found.", {
       F: __dxlog_file,
-      L: 45,
+      L: 51,
       S: this,
       A: [
         "peer",
@@ -162,12 +179,17 @@ var LocalHostNetworkAdapter = class extends NetworkAdapter {
   }
   disconnect() {
   }
+  async whenConnected() {
+    await this._connected.wait({
+      timeout: 1e4
+    });
+  }
   syncRepo({ id, syncMessage }) {
     const peerId = this._getPeerId(id);
     return new Stream(({ next, close }) => {
       invariant(!this._peers.has(peerId), "Peer already connected.", {
         F: __dxlog_file,
-        L: 63,
+        L: 73,
         S: this,
         A: [
           "!this._peers.has(peerId)",
@@ -189,35 +211,47 @@ var LocalHostNetworkAdapter = class extends NetworkAdapter {
           });
         }
       });
-      this._connected.wait({
-        timeout: 1e3
-      }).then(() => {
-        this.emit("peer-candidate", {
-          peerMetadata: {},
-          peerId
-        });
-      }).catch((err) => log.catch(err, void 0, {
+      invariant(this._isConnected, void 0, {
         F: __dxlog_file,
-        L: 88,
+        L: 90,
         S: this,
-        C: (f, a) => f(...a)
-      }));
+        A: [
+          "this._isConnected",
+          ""
+        ]
+      });
+      this.emit("peer-candidate", {
+        peerMetadata: {},
+        peerId
+      });
     });
   }
   async sendSyncMessage({ id, syncMessage }) {
-    await this._connected.wait({
-      timeout: 1e3
+    invariant(this._isConnected, void 0, {
+      F: __dxlog_file,
+      L: 99,
+      S: this,
+      A: [
+        "this._isConnected",
+        ""
+      ]
     });
     const message = cbor.decode(syncMessage);
     this.emit("message", message);
   }
   async getHostInfo() {
-    await this._connected.wait({
-      timeout: 1e3
+    invariant(this._isConnected, void 0, {
+      F: __dxlog_file,
+      L: 105,
+      S: this,
+      A: [
+        "this._isConnected",
+        ""
+      ]
     });
     invariant(this.peerId, "Peer id not set.", {
       F: __dxlog_file,
-      L: 100,
+      L: 106,
       S: this,
       A: [
         "this.peerId",
@@ -237,7 +271,7 @@ var LocalHostNetworkAdapter = class extends NetworkAdapter {
 import { Trigger as Trigger2 } from "@dxos/async";
 import { NetworkAdapter as NetworkAdapter2, cbor as cbor2 } from "@dxos/automerge/automerge-repo";
 import { invariant as invariant2 } from "@dxos/invariant";
-import { log as log2 } from "@dxos/log";
+import { log } from "@dxos/log";
 import { AutomergeReplicator } from "@dxos/teleport-extension-automerge-replicator";
 var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/mesh-network-adapter.ts";
 var MeshNetworkAdapter = class extends NetworkAdapter2 {
@@ -272,7 +306,7 @@ var MeshNetworkAdapter = class extends NetworkAdapter2 {
     });
     extension.sendSyncMessage({
       payload: cbor2.encode(message)
-    }).catch((err) => log2.catch(err, void 0, {
+    }).catch((err) => log.catch(err, void 0, {
       F: __dxlog_file2,
       L: 39,
       S: this,
@@ -297,7 +331,7 @@ var MeshNetworkAdapter = class extends NetworkAdapter2 {
     }, {
       onStartReplication: async (info, remotePeerId) => {
         await this._connected.wait();
-        log2("onStartReplication", {
+        log("onStartReplication", {
           id: info.id,
           thisPeerId: this.peerId,
           remotePeerId: remotePeerId.toHex()
@@ -310,7 +344,7 @@ var MeshNetworkAdapter = class extends NetworkAdapter2 {
         if (!this._extensions.has(info.id)) {
           peerInfo = info;
           this._extensions.set(info.id, extension);
-          log2("peer-candidate", {
+          log("peer-candidate", {
             id: info.id,
             thisPeerId: this.peerId,
             remotePeerId: remotePeerId.toHex()
@@ -352,7 +386,7 @@ var MeshNetworkAdapter = class extends NetworkAdapter2 {
 
 // packages/core/echo/echo-pipeline/src/automerge/migrations.ts
 import { IndexedDBStorageAdapter } from "@dxos/automerge/automerge-repo-storage-indexeddb";
-import { log as log3 } from "@dxos/log";
+import { log as log2 } from "@dxos/log";
 import { StorageType } from "@dxos/random-access-storage";
 
 // packages/core/echo/echo-pipeline/src/automerge/automerge-storage-adapter.ts
@@ -446,7 +480,7 @@ var levelMigration = async ({ db, directory }) => {
     return;
   }
   const batch = db.batch();
-  log3.info("found chunks on old storage adapter", {
+  log2.info("found chunks on old storage adapter", {
     chunks: chunks.length
   }, {
     F: __dxlog_file3,
@@ -475,29 +509,27 @@ function _ts_decorate(decorators, target, key, desc) {
 }
 var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-host.ts";
 var AutomergeHost = class {
-  constructor({ directory, db, metadata }) {
+  constructor({ directory, db, storageCallbacks }) {
     this._ctx = new Context();
     /**
     * spaceKey -> deviceKey[]
     */
     this._authorizedDevices = new ComplexMap(PublicKey.hash);
-    this._updatingMetadata = /* @__PURE__ */ new Map();
     this._requestedDocs = /* @__PURE__ */ new Set();
     this._directory = directory;
     this._db = db;
-    this._metadata = metadata;
+    this._storageCallbacks = storageCallbacks;
   }
   async open() {
     this._directory && await levelMigration({
-      db: await this._db,
+      db: this._db,
       directory: this._directory
     });
     this._storage = new LevelDBStorageAdapter({
-      db: await this._db,
-      callbacks: {
-        beforeSave: (params) => this._beforeSave(params)
-      }
+      db: this._db,
+      callbacks: this._storageCallbacks
     });
+    await this._storage.open?.();
     this._peerId = `host-${PublicKey.random().toHex()}`;
     this._meshNetwork = new MeshNetworkAdapter();
     this._clientNetwork = new LocalHostNetworkAdapter();
@@ -520,13 +552,13 @@ var AutomergeHost = class {
         const doc = this._repo.handles[documentId]?.docSync();
         if (!doc) {
           const isRequested = this._requestedDocs.has(`automerge:${documentId}`);
-          log4("doc share policy check", {
+          log3("doc share policy check", {
             peerId,
             documentId,
             isRequested
           }, {
             F: __dxlog_file4,
-            L: 111,
+            L: 100,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -535,12 +567,12 @@ var AutomergeHost = class {
         try {
           const spaceKey = getSpaceKeyFromDoc(doc);
           if (!spaceKey) {
-            log4("space key not found for share policy check", {
+            log3("space key not found for share policy check", {
               peerId,
               documentId
             }, {
               F: __dxlog_file4,
-              L: 118,
+              L: 107,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -549,12 +581,12 @@ var AutomergeHost = class {
           const authorizedDevices = this._authorizedDevices.get(PublicKey.from(spaceKey));
           const deviceKeyHex = this.repo.peerMetadataByPeerId[peerId]?.dxos_deviceKey;
           if (!deviceKeyHex) {
-            log4("device key not found for share policy check", {
+            log3("device key not found for share policy check", {
               peerId,
               documentId
             }, {
               F: __dxlog_file4,
-              L: 127,
+              L: 116,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -562,7 +594,7 @@ var AutomergeHost = class {
           }
           const deviceKey = PublicKey.from(deviceKeyHex);
           const isAuthorized = authorizedDevices?.has(deviceKey) ?? false;
-          log4("share policy check", {
+          log3("share policy check", {
             localPeer: this._peerId,
             remotePeer: peerId,
             documentId,
@@ -571,15 +603,15 @@ var AutomergeHost = class {
             isAuthorized
           }, {
             F: __dxlog_file4,
-            L: 133,
+            L: 122,
             S: this,
             C: (f, a) => f(...a)
           });
           return isAuthorized;
         } catch (err) {
-          log4.catch(err, void 0, {
+          log3.catch(err, void 0, {
             F: __dxlog_file4,
-            L: 143,
+            L: 132,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -589,66 +621,16 @@ var AutomergeHost = class {
     });
     this._clientNetwork.ready();
     this._meshNetwork.ready();
-    {
-      const listener = ({ handle }) => this._onDocument(handle);
-      this._repo.on("document", listener);
-      this._ctx.onDispose(() => {
-        this._repo.off("document", listener);
-        Object.values(this._repo.handles).forEach((handle) => handle.off("change"));
-      });
-    }
+    await this._clientNetwork.whenConnected();
   }
   async close() {
-    this._storage.close?.();
+    await this._storage.close?.();
     await this._clientNetwork.close();
     await this._ctx.dispose();
   }
   get repo() {
     return this._repo;
   }
-  async _beforeSave(path) {
-    const id = path[0];
-    if (this._updatingMetadata.has(id)) {
-      return this._updatingMetadata.get(id);
-    }
-  }
-  _onDocument(handle) {
-    const listener = (event) => this._onUpdate(event);
-    handle.on("change", listener);
-  }
-  _onUpdate(event) {
-    if (this._metadata == null) {
-      return;
-    }
-    const objectIds = getInlineChanges(event);
-    if (objectIds.length === 0) {
-      return;
-    }
-    const heads = getHeads(event.doc);
-    const lastAvailableHash = heads.join("");
-    if (!lastAvailableHash) {
-      return;
-    }
-    const encodedIds = objectIds.map((objectId) => idCodec.encode({
-      documentId: event.handle.documentId,
-      objectId
-    }));
-    const idToLastHash = new Map(encodedIds.map((id) => [
-      id,
-      lastAvailableHash
-    ]));
-    const markingDirtyPromise = this._metadata.markDirty(idToLastHash).then(() => {
-      this._updatingMetadata.delete(event.handle.documentId);
-    }).catch((err) => {
-      this._ctx.disposed && log4.catch(err, void 0, {
-        F: __dxlog_file4,
-        L: 207,
-        S: this,
-        C: (f, a) => f(...a)
-      });
-    });
-    this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
-  }
   _automergeDocs() {
     return mapValues(this._repo.handles, (handle) => ({
       state: handle.state,
@@ -679,7 +661,19 @@ var AutomergeHost = class {
   //
   async flush({ documentIds }) {
     await Promise.all(documentIds?.map((id) => this._repo.find(id).whenReady()) ?? []);
-    await this._repo.flush(documentIds);
+    try {
+      await asyncTimeout(this._repo.flush(documentIds), 500);
+    } catch (err) {
+      log3.warn("flush error", {
+        documentIds,
+        err
+      }, {
+        F: __dxlog_file4,
+        L: 196,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
   }
   syncRepo(request) {
     return this._clientNetwork.syncRepo(request);
@@ -697,12 +691,12 @@ var AutomergeHost = class {
     return this._meshNetwork.createExtension();
   }
   authorizeDevice(spaceKey, deviceKey) {
-    log4("authorizeDevice", {
+    log3("authorizeDevice", {
       spaceKey,
       deviceKey
     }, {
       F: __dxlog_file4,
-      L: 274,
+      L: 221,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -725,24 +719,6 @@ _ts_decorate([
 AutomergeHost = _ts_decorate([
   trace.resource()
 ], AutomergeHost);
-var getInlineChanges = (event) => {
-  const inlineChangedObjectIds = /* @__PURE__ */ new Set();
-  for (const { path } of event.patches) {
-    if (path.length < 2) {
-      continue;
-    }
-    switch (path[0]) {
-      case "objects":
-        if (path.length >= 2) {
-          inlineChangedObjectIds.add(path[1]);
-        }
-        break;
-    }
-  }
-  return [
-    ...inlineChangedObjectIds
-  ];
-};
 var getSpaceKeyFromDoc = (doc) => {
   const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
   if (rawSpaceKey == null) {
@@ -756,7 +732,7 @@ import { Event } from "@dxos/async";
 import { cancelWithContext } from "@dxos/context";
 import { warnAfterTimeout } from "@dxos/debug";
 import { invariant as invariant3 } from "@dxos/invariant";
-import { log as log5 } from "@dxos/log";
+import { log as log4 } from "@dxos/log";
 var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-doc-loader.ts";
 var AutomergeDocumentLoaderImpl = class {
   constructor(_spaceKey, _repo) {
@@ -777,7 +753,7 @@ var AutomergeDocumentLoaderImpl = class {
       return;
     }
     if (!spaceState.rootUrl) {
-      log5.error("Database opened with no rootUrl", {
+      log4.error("Database opened with no rootUrl", {
         spaceKey: this._spaceKey
       }, {
         F: __dxlog_file5,
@@ -830,7 +806,7 @@ var AutomergeDocumentLoaderImpl = class {
     const documentUrl = (spaceRootDoc.links ?? {})[objectId];
     if (documentUrl == null) {
       this._objectsPendingDocumentLoad.add(objectId);
-      log5.info("loading delayed until object links are initialized", {
+      log4.info("loading delayed until object links are initialized", {
         objectId
       }, {
         F: __dxlog_file5,
@@ -905,7 +881,7 @@ var AutomergeDocumentLoaderImpl = class {
       };
       const objectDocumentHandle = this._objectDocumentHandles.get(objectId);
       if (objectDocumentHandle != null && objectDocumentHandle.url !== automergeUrl) {
-        log5.warn("object already inlined in a different document, ignoring the link", {
+        log4.warn("object already inlined in a different document, ignoring the link", {
           ...logMeta,
           actualDocumentUrl: objectDocumentHandle.url
         }, {
@@ -917,7 +893,7 @@ var AutomergeDocumentLoaderImpl = class {
         continue;
       }
       if (objectDocumentHandle?.url === automergeUrl) {
-        log5.warn("object document was already loaded", logMeta, {
+        log4.warn("object document was already loaded", logMeta, {
           F: __dxlog_file5,
           L: 149,
           S: this,
@@ -926,7 +902,7 @@ var AutomergeDocumentLoaderImpl = class {
         continue;
       }
       const handle = this._repo.find(automergeUrl);
-      log5.debug("document loading triggered", logMeta, {
+      log4.debug("document loading triggered", logMeta, {
         F: __dxlog_file5,
         L: 153,
         S: this,
@@ -946,7 +922,7 @@ var AutomergeDocumentLoaderImpl = class {
         break;
       } catch (err) {
         if (`${err}`.includes("Timeout")) {
-          log5.info("wraparound", {
+          log4.info("wraparound", {
             id: docHandle.documentId,
             state: docHandle.state
           }, {
@@ -991,7 +967,7 @@ var AutomergeDocumentLoaderImpl = class {
         docUrl: handle.url
       };
       if (this.onObjectDocumentLoaded.listenerCount() === 0) {
-        log5.info("document loaded after all listeners were removed", logMeta, {
+        log4.info("document loaded after all listeners were removed", logMeta, {
           F: __dxlog_file5,
           L: 205,
           S: this,
@@ -1001,7 +977,7 @@ var AutomergeDocumentLoaderImpl = class {
       }
       const objectDocHandle = this._objectDocumentHandles.get(objectId);
       if (objectDocHandle?.url !== handle.url) {
-        log5.warn("object was rebound while a document was loading, discarding handle", logMeta, {
+        log4.warn("object was rebound while a document was loading, discarding handle", logMeta, {
           F: __dxlog_file5,
           L: 210,
           S: this,
@@ -1015,7 +991,7 @@ var AutomergeDocumentLoaderImpl = class {
       });
     } catch (err) {
       const shouldRetryLoading = this.onObjectDocumentLoaded.listenerCount() > 0;
-      log5.warn("failed to load a document", {
+      log4.warn("failed to load a document", {
         objectId,
         automergeUrl: handle.url,
         retryLoading: shouldRetryLoading,
@@ -1034,7 +1010,7 @@ var AutomergeDocumentLoaderImpl = class {
 };
 
 // packages/core/echo/echo-pipeline/src/automerge/reference.ts
-import { Reference } from "@dxos/echo-db";
+import { Reference } from "@dxos/echo-schema";
 var REFERENCE_TYPE_TAG = "dxos.echo.model.document.Reference";
 var encodeReference = (reference) => ({
   "@type": REFERENCE_TYPE_TAG,
@@ -1052,6 +1028,7 @@ export {
   AutomergeHost,
   AutomergeStorageAdapter,
   DataServiceImpl,
+  LevelDBStorageAdapter,
   LocalHostNetworkAdapter,
   MOCK_AUTH_PROVIDER,
   MOCK_AUTH_VERIFIER,
@@ -1070,6 +1047,7 @@ export {
   createMappedFeedWriter,
   decodeReference,
   encodeReference,
+  encodingOptions,
   getSpaceKeyFromDoc,
   hasInvitationExpired,
   isEncodedReferenceObject,