@dxos/echo-pipeline 0.4.9 → 0.4.10-main.06ef97a

package/dist/lib/node/index.cjs

@@ -18,34 +18,39 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var node_exports = {};
 __export(node_exports, {
-  AuthExtension: () => import_chunk_7VZVCCNF.AuthExtension,
-  AuthStatus: () => import_chunk_7VZVCCNF.AuthStatus,
+  AuthExtension: () => import_chunk_KGIYLJBT.AuthExtension,
+  AuthStatus: () => import_chunk_KGIYLJBT.AuthStatus,
+  AutomergeDocumentLoaderImpl: () => AutomergeDocumentLoaderImpl,
   AutomergeHost: () => AutomergeHost,
   AutomergeStorageAdapter: () => AutomergeStorageAdapter,
-  DataServiceImpl: () => import_chunk_7VZVCCNF.DataServiceImpl,
+  DataServiceImpl: () => import_chunk_KGIYLJBT.DataServiceImpl,
   LocalHostNetworkAdapter: () => LocalHostNetworkAdapter,
-  MOCK_AUTH_PROVIDER: () => import_chunk_7VZVCCNF.MOCK_AUTH_PROVIDER,
-  MOCK_AUTH_VERIFIER: () => import_chunk_7VZVCCNF.MOCK_AUTH_VERIFIER,
+  MOCK_AUTH_PROVIDER: () => import_chunk_KGIYLJBT.MOCK_AUTH_PROVIDER,
+  MOCK_AUTH_VERIFIER: () => import_chunk_KGIYLJBT.MOCK_AUTH_VERIFIER,
   MeshNetworkAdapter: () => MeshNetworkAdapter,
-  MetadataStore: () => import_chunk_7VZVCCNF.MetadataStore,
-  Pipeline: () => import_chunk_7VZVCCNF.Pipeline,
-  SnapshotManager: () => import_chunk_7VZVCCNF.SnapshotManager,
-  SnapshotStore: () => import_chunk_7VZVCCNF.SnapshotStore,
-  Space: () => import_chunk_7VZVCCNF.Space,
-  SpaceManager: () => import_chunk_7VZVCCNF.SpaceManager,
-  SpaceProtocol: () => import_chunk_7VZVCCNF.SpaceProtocol,
-  SpaceProtocolSession: () => import_chunk_7VZVCCNF.SpaceProtocolSession,
-  TimeframeClock: () => import_chunk_7VZVCCNF.TimeframeClock,
-  codec: () => import_chunk_7VZVCCNF.codec,
-  createMappedFeedWriter: () => import_chunk_7VZVCCNF.createMappedFeedWriter,
+  MetadataStore: () => import_chunk_KGIYLJBT.MetadataStore,
+  Pipeline: () => import_chunk_KGIYLJBT.Pipeline,
+  REFERENCE_TYPE_TAG: () => REFERENCE_TYPE_TAG,
+  SnapshotManager: () => import_chunk_KGIYLJBT.SnapshotManager,
+  SnapshotStore: () => import_chunk_KGIYLJBT.SnapshotStore,
+  Space: () => import_chunk_KGIYLJBT.Space,
+  SpaceManager: () => import_chunk_KGIYLJBT.SpaceManager,
+  SpaceProtocol: () => import_chunk_KGIYLJBT.SpaceProtocol,
+  SpaceProtocolSession: () => import_chunk_KGIYLJBT.SpaceProtocolSession,
+  TimeframeClock: () => import_chunk_KGIYLJBT.TimeframeClock,
+  codec: () => import_chunk_KGIYLJBT.codec,
+  createMappedFeedWriter: () => import_chunk_KGIYLJBT.createMappedFeedWriter,
+  decodeReference: () => decodeReference,
+  encodeReference: () => encodeReference,
   getSpaceKeyFromDoc: () => getSpaceKeyFromDoc,
-  mapFeedIndexesToTimeframe: () => import_chunk_7VZVCCNF.mapFeedIndexesToTimeframe,
-  mapTimeframeToFeedIndexes: () => import_chunk_7VZVCCNF.mapTimeframeToFeedIndexes,
-  startAfter: () => import_chunk_7VZVCCNF.startAfter,
-  valueEncoding: () => import_chunk_7VZVCCNF.valueEncoding
+  isEncodedReferenceObject: () => isEncodedReferenceObject,
+  mapFeedIndexesToTimeframe: () => import_chunk_KGIYLJBT.mapFeedIndexesToTimeframe,
+  mapTimeframeToFeedIndexes: () => import_chunk_KGIYLJBT.mapTimeframeToFeedIndexes,
+  startAfter: () => import_chunk_KGIYLJBT.startAfter,
+  valueEncoding: () => import_chunk_KGIYLJBT.valueEncoding
 });
 module.exports = __toCommonJS(node_exports);
-var import_chunk_7VZVCCNF = require("./chunk-7VZVCCNF.cjs");
+var import_chunk_KGIYLJBT = require("./chunk-KGIYLJBT.cjs");
 var import_automerge = require("@dxos/automerge/automerge");
 var import_automerge_repo = require("@dxos/automerge/automerge-repo");
 var import_automerge_repo_storage_indexeddb = require("@dxos/automerge/automerge-repo-storage-indexeddb");
@@ -67,6 +72,12 @@ var import_automerge_repo3 = require("@dxos/automerge/automerge-repo");
 var import_invariant2 = require("@dxos/invariant");
 var import_log3 = require("@dxos/log");
 var import_teleport_extension_automerge_replicator = require("@dxos/teleport-extension-automerge-replicator");
+var import_async3 = require("@dxos/async");
+var import_context2 = require("@dxos/context");
+var import_debug = require("@dxos/debug");
+var import_invariant3 = require("@dxos/invariant");
+var import_log4 = require("@dxos/log");
+var import_echo_db = require("@dxos/echo-db");
 var AutomergeStorageAdapter = class {
   constructor(_directory) {
     this._directory = _directory;
@@ -441,7 +452,7 @@ var AutomergeHost = class {
             isRequested
           }, {
             F: __dxlog_file3,
-            L: 96,
+            L: 101,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -455,7 +466,7 @@ var AutomergeHost = class {
               documentId
             }, {
               F: __dxlog_file3,
-              L: 103,
+              L: 108,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -469,7 +480,7 @@ var AutomergeHost = class {
               documentId
             }, {
               F: __dxlog_file3,
-              L: 112,
+              L: 117,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -486,7 +497,7 @@ var AutomergeHost = class {
             isAuthorized
           }, {
             F: __dxlog_file3,
-            L: 118,
+            L: 123,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -494,7 +505,7 @@ var AutomergeHost = class {
         } catch (err) {
           import_log.log.catch(err, void 0, {
             F: __dxlog_file3,
-            L: 128,
+            L: 133,
             S: this,
             C: (f, a) => f(...a)
           });
@@ -509,6 +520,7 @@ var AutomergeHost = class {
       this._repo.on("document", listener);
       this._ctx.onDispose(() => {
         this._repo.off("document", listener);
+        Object.values(this._repo.handles).forEach((handle) => handle.off("change"));
       });
     }
   }
@@ -524,9 +536,6 @@ var AutomergeHost = class {
   _onDocument(handle) {
     const listener = (event) => this._onUpdate(event);
     handle.on("change", listener);
-    this._ctx.onDispose(() => {
-      handle.off("change", listener);
-    });
   }
   _onUpdate(event) {
     if (this._metadata == null) {
@@ -554,7 +563,7 @@ var AutomergeHost = class {
     }).catch((err) => {
       this._ctx.disposed && import_log.log.catch(err, void 0, {
         F: __dxlog_file3,
-        L: 188,
+        L: 191,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -566,7 +575,7 @@ var AutomergeHost = class {
       state: handle.state,
       hasDoc: !!handle.docSync(),
       heads: handle.docSync() ? import_automerge.next.getHeads(handle.docSync()) : null,
-      data: handle.docSync()?.doc && (0, import_util.mapValues)(handle.docSync()?.doc, (value, key) => {
+      data: handle.docSync() && (0, import_util.mapValues)(handle.docSync(), (value, key) => {
         try {
           switch (key) {
             case "access":
@@ -594,6 +603,10 @@ var AutomergeHost = class {
   //
   // Methods for client-services.
   //
+  async flush({ documentIds }) {
+    await Promise.all(documentIds?.map((id) => this._repo.find(id).whenReady()) ?? []);
+    await this._repo.flush(documentIds);
+  }
   syncRepo(request) {
     return this._clientNetwork.syncRepo(request);
   }
@@ -615,7 +628,7 @@ var AutomergeHost = class {
       deviceKey
     }, {
       F: __dxlog_file3,
-      L: 255,
+      L: 264,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -663,10 +676,296 @@ var getSpaceKeyFromDoc = (doc) => {
   }
   return String(rawSpaceKey);
 };
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/automerge/automerge-doc-loader.ts";
+var AutomergeDocumentLoaderImpl = class {
+  constructor(_spaceKey, _repo) {
+    this._spaceKey = _spaceKey;
+    this._repo = _repo;
+    this._spaceRootDocHandle = null;
+    this._objectDocumentHandles = /* @__PURE__ */ new Map();
+    this._objectsPendingDocumentLoad = /* @__PURE__ */ new Set();
+    this.onObjectDocumentLoaded = new import_async3.Event();
+  }
+  getAllHandles() {
+    return [
+      ...new Set(this._objectDocumentHandles.values())
+    ];
+  }
+  async loadSpaceRootDocHandle(ctx, spaceState) {
+    if (this._spaceRootDocHandle != null) {
+      return;
+    }
+    if (!spaceState.rootUrl) {
+      import_log4.log.error("Database opened with no rootUrl", {
+        spaceKey: this._spaceKey
+      }, {
+        F: __dxlog_file4,
+        L: 66,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      this._createContextBoundSpaceRootDocument(ctx);
+    } else {
+      const existingDocHandle = await this._initDocHandle(ctx, spaceState.rootUrl);
+      const doc = existingDocHandle.docSync();
+      (0, import_invariant3.invariant)(doc, void 0, {
+        F: __dxlog_file4,
+        L: 71,
+        S: this,
+        A: [
+          "doc",
+          ""
+        ]
+      });
+      if (doc.access == null) {
+        this._initDocAccess(existingDocHandle);
+      }
+      this._spaceRootDocHandle = existingDocHandle;
+    }
+  }
+  loadObjectDocument(objectId) {
+    (0, import_invariant3.invariant)(this._spaceRootDocHandle, void 0, {
+      F: __dxlog_file4,
+      L: 80,
+      S: this,
+      A: [
+        "this._spaceRootDocHandle",
+        ""
+      ]
+    });
+    if (this._objectDocumentHandles.has(objectId) || this._objectsPendingDocumentLoad.has(objectId)) {
+      return;
+    }
+    const spaceRootDoc = this._spaceRootDocHandle.docSync();
+    (0, import_invariant3.invariant)(spaceRootDoc, void 0, {
+      F: __dxlog_file4,
+      L: 85,
+      S: this,
+      A: [
+        "spaceRootDoc",
+        ""
+      ]
+    });
+    const documentUrl = (spaceRootDoc.links ?? {})[objectId];
+    if (documentUrl == null) {
+      this._objectsPendingDocumentLoad.add(objectId);
+      import_log4.log.info("loading delayed until object links are initialized", {
+        objectId
+      }, {
+        F: __dxlog_file4,
+        L: 89,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      return;
+    }
+    this._loadLinkedObjects({
+      [objectId]: documentUrl
+    });
+  }
+  onObjectLinksUpdated(links) {
+    if (!links) {
+      return;
+    }
+    const linksAwaitingLoad = Object.entries(links).filter(([objectId]) => this._objectsPendingDocumentLoad.has(objectId));
+    this._loadLinkedObjects(Object.fromEntries(linksAwaitingLoad));
+    linksAwaitingLoad.forEach(([objectId]) => this._objectsPendingDocumentLoad.delete(objectId));
+  }
+  getSpaceRootDocHandle() {
+    (0, import_invariant3.invariant)(this._spaceRootDocHandle, void 0, {
+      F: __dxlog_file4,
+      L: 107,
+      S: this,
+      A: [
+        "this._spaceRootDocHandle",
+        ""
+      ]
+    });
+    return this._spaceRootDocHandle;
+  }
+  createDocumentForObject(objectId) {
+    (0, import_invariant3.invariant)(this._spaceRootDocHandle, void 0, {
+      F: __dxlog_file4,
+      L: 112,
+      S: this,
+      A: [
+        "this._spaceRootDocHandle",
+        ""
+      ]
+    });
+    const spaceDocHandle = this._repo.create();
+    this._initDocAccess(spaceDocHandle);
+    this.onObjectBoundToDocument(spaceDocHandle, objectId);
+    this._spaceRootDocHandle.change((newDoc) => {
+      newDoc.links ??= {};
+      newDoc.links[objectId] = spaceDocHandle.url;
+    });
+    return spaceDocHandle;
+  }
+  onObjectBoundToDocument(handle, objectId) {
+    this._objectDocumentHandles.set(objectId, handle);
+  }
+  clearHandleReferences() {
+    const objectsWithHandles = [
+      ...this._objectDocumentHandles.keys()
+    ];
+    this._objectDocumentHandles.clear();
+    this._spaceRootDocHandle = null;
+    return objectsWithHandles;
+  }
+  _loadLinkedObjects(links) {
+    if (!links) {
+      return;
+    }
+    for (const [objectId, automergeUrl] of Object.entries(links)) {
+      const logMeta = {
+        objectId,
+        automergeUrl
+      };
+      const objectDocumentHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocumentHandle != null && objectDocumentHandle.url !== automergeUrl) {
+        import_log4.log.warn("object already inlined in a different document, ignoring the link", {
+          ...logMeta,
+          actualDocumentUrl: objectDocumentHandle.url
+        }, {
+          F: __dxlog_file4,
+          L: 142,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        continue;
+      }
+      if (objectDocumentHandle?.url === automergeUrl) {
+        import_log4.log.warn("object document was already loaded", logMeta, {
+          F: __dxlog_file4,
+          L: 149,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        continue;
+      }
+      const handle = this._repo.find(automergeUrl);
+      import_log4.log.debug("document loading triggered", logMeta, {
+        F: __dxlog_file4,
+        L: 153,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      this._objectDocumentHandles.set(objectId, handle);
+      void this._createObjectOnDocumentLoad(handle, objectId);
+    }
+  }
+  async _initDocHandle(ctx, url) {
+    const docHandle = this._repo.find(url);
+    while (true) {
+      try {
+        await (0, import_debug.warnAfterTimeout)(5e3, "Automerge root doc load timeout (AutomergeDb)", async () => {
+          await (0, import_context2.cancelWithContext)(ctx, docHandle.whenReady());
+        });
+        break;
+      } catch (err) {
+        if (`${err}`.includes("Timeout")) {
+          import_log4.log.info("wraparound", {
+            id: docHandle.documentId,
+            state: docHandle.state
+          }, {
+            F: __dxlog_file4,
+            L: 169,
+            S: this,
+            C: (f, a) => f(...a)
+          });
+          continue;
+        }
+        throw err;
+      }
+    }
+    if (docHandle.state === "unavailable") {
+      throw new Error("Automerge document is unavailable");
+    }
+    return docHandle;
+  }
+  _createContextBoundSpaceRootDocument(ctx) {
+    const docHandle = this._repo.create();
+    this._spaceRootDocHandle = docHandle;
+    ctx.onDispose(() => {
+      docHandle.delete();
+      this._spaceRootDocHandle = null;
+    });
+  }
+  _initDocAccess(handle) {
+    handle.change((newDoc) => {
+      newDoc.access ??= {
+        spaceKey: this._spaceKey.toHex()
+      };
+      newDoc.access.spaceKey = this._spaceKey.toHex();
+    });
+  }
+  async _createObjectOnDocumentLoad(handle, objectId) {
+    try {
+      await handle.doc([
+        "ready"
+      ]);
+      const logMeta = {
+        objectId,
+        docUrl: handle.url
+      };
+      if (this.onObjectDocumentLoaded.listenerCount() === 0) {
+        import_log4.log.info("document loaded after all listeners were removed", logMeta, {
+          F: __dxlog_file4,
+          L: 205,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        return;
+      }
+      const objectDocHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocHandle?.url !== handle.url) {
+        import_log4.log.warn("object was rebound while a document was loading, discarding handle", logMeta, {
+          F: __dxlog_file4,
+          L: 210,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        return;
+      }
+      this.onObjectDocumentLoaded.emit({
+        handle,
+        objectId
+      });
+    } catch (err) {
+      const shouldRetryLoading = this.onObjectDocumentLoaded.listenerCount() > 0;
+      import_log4.log.warn("failed to load a document", {
+        objectId,
+        automergeUrl: handle.url,
+        retryLoading: shouldRetryLoading,
+        err
+      }, {
+        F: __dxlog_file4,
+        L: 216,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      if (shouldRetryLoading) {
+        await this._createObjectOnDocumentLoad(handle, objectId);
+      }
+    }
+  }
+};
+var REFERENCE_TYPE_TAG = "dxos.echo.model.document.Reference";
+var encodeReference = (reference) => ({
+  "@type": REFERENCE_TYPE_TAG,
+  // NOTE: Automerge do not support undefined values, so we need to use null instead.
+  itemId: reference.itemId ?? null,
+  protocol: reference.protocol ?? null,
+  host: reference.host ?? null
+});
+var decodeReference = (value) => new import_echo_db.Reference(value.itemId, value.protocol ?? void 0, value.host ?? void 0);
+var isEncodedReferenceObject = (value) => typeof value === "object" && value !== null && value["@type"] === REFERENCE_TYPE_TAG;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AuthExtension,
   AuthStatus,
+  AutomergeDocumentLoaderImpl,
   AutomergeHost,
   AutomergeStorageAdapter,
   DataServiceImpl,
@@ -676,6 +975,7 @@ var getSpaceKeyFromDoc = (doc) => {
   MeshNetworkAdapter,
   MetadataStore,
   Pipeline,
+  REFERENCE_TYPE_TAG,
   SnapshotManager,
   SnapshotStore,
   Space,
@@ -685,7 +985,10 @@ var getSpaceKeyFromDoc = (doc) => {
   TimeframeClock,
   codec,
   createMappedFeedWriter,
+  decodeReference,
+  encodeReference,
   getSpaceKeyFromDoc,
+  isEncodedReferenceObject,
   mapFeedIndexesToTimeframe,
   mapTimeframeToFeedIndexes,
   startAfter,