@dxos/echo-pipeline 0.4.9 → 0.4.10-main.0be5154

package/dist/types/src/automerge/types.d.ts

@@ -0,0 +1,67 @@
+import { type Reference } from '@dxos/echo-db';
+export type SpaceState = {
+    rootUrl?: string;
+};
+export interface SpaceDoc {
+    access?: {
+        spaceKey: string;
+    };
+    /**
+     * Objects inlined in the current document.
+     */
+    objects?: {
+        [key: string]: ObjectStructure;
+    };
+    /**
+     * Object id points to an automerge doc url where the object is embedded.
+     */
+    links?: {
+        [echoId: string]: string;
+    };
+}
+/**
+ * Representation of an ECHO object in an AM document.
+ */
+export type ObjectStructure = {
+    data: Record<string, any>;
+    meta: ObjectMeta;
+    system: ObjectSystem;
+};
+/**
+ * Echo object metadata.
+ */
+export type ObjectMeta = {
+    /**
+     * Foreign keys.
+     */
+    keys: ForeignKey[];
+};
+/**
+ * Reference to an object in a foreign database.
+ */
+export type ForeignKey = {
+    /**
+     * Name of the foreign database/system.
+     * E.g. `github.com`.
+     */
+    source?: string;
+    /**
+     * Id within the foreign database.
+     */
+    id?: string;
+};
+/**
+ * Automerge object system properties.
+ * (Is automerge specific.)
+ */
+export type ObjectSystem = {
+    /**
+     * Deletion marker.
+     */
+    deleted?: boolean;
+    /**
+     * Object reference ('protobuf' protocol) type.
+     */
+    type?: Reference;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types/src/automerge/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/automerge/types.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,MAAM,UAAU,GAAG;IAEvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF;;OAEG;IACH,OAAO,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAAC;KAChC,CAAC;IACF;;OAEG;IACH,KAAK,CAAC,EAAE;QACN,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;KAC1B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,YAAY,CAAC;CACtB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB;;OAEG;IACH,IAAI,EAAE,UAAU,EAAE,CAAC;CACpB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,IAAI,CAAC,EAAE,SAAS,CAAC;CAClB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.4.9",
+  "version": "0.4.10-main.0be5154",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -36,35 +36,35 @@
   ],
   "dependencies": {
     "crc-32": "^1.2.2",
-    "@dxos/automerge": "0.4.9",
-    "@dxos/codec-protobuf": "0.4.9",
-    "@dxos/async": "0.4.9",
-    "@dxos/context": "0.4.9",
-    "@dxos/credentials": "0.4.9",
-    "@dxos/debug": "0.4.9",
-    "@dxos/crypto": "0.4.9",
-    "@dxos/echo-db": "0.4.9",
-    "@dxos/feed-store": "0.4.9",
-    "@dxos/hypercore": "0.4.9",
-    "@dxos/keyring": "0.4.9",
-    "@dxos/invariant": "0.4.9",
-    "@dxos/messaging": "0.4.9",
-    "@dxos/log": "0.4.9",
-    "@dxos/keys": "0.4.9",
-    "@dxos/network-manager": "0.4.9",
-    "@dxos/node-std": "0.4.9",
-    "@dxos/protocols": "0.4.9",
-    "@dxos/rpc": "0.4.9",
-    "@dxos/random-access-storage": "0.4.9",
-    "@dxos/teleport": "0.4.9",
-    "@dxos/teleport-extension-automerge-replicator": "0.4.9",
-    "@dxos/teleport-extension-gossip": "0.4.9",
-    "@dxos/teleport-extension-object-sync": "0.4.9",
-    "@dxos/timeframe": "0.4.9",
-    "@dxos/teleport-extension-replicator": "0.4.9",
-    "@dxos/tracing": "0.4.9",
-    "@dxos/typings": "0.4.9",
-    "@dxos/util": "0.4.9"
+    "@dxos/async": "0.4.10-main.0be5154",
+    "@dxos/codec-protobuf": "0.4.10-main.0be5154",
+    "@dxos/context": "0.4.10-main.0be5154",
+    "@dxos/credentials": "0.4.10-main.0be5154",
+    "@dxos/automerge": "0.4.10-main.0be5154",
+    "@dxos/echo-db": "0.4.10-main.0be5154",
+    "@dxos/crypto": "0.4.10-main.0be5154",
+    "@dxos/debug": "0.4.10-main.0be5154",
+    "@dxos/feed-store": "0.4.10-main.0be5154",
+    "@dxos/hypercore": "0.4.10-main.0be5154",
+    "@dxos/invariant": "0.4.10-main.0be5154",
+    "@dxos/keys": "0.4.10-main.0be5154",
+    "@dxos/keyring": "0.4.10-main.0be5154",
+    "@dxos/messaging": "0.4.10-main.0be5154",
+    "@dxos/log": "0.4.10-main.0be5154",
+    "@dxos/network-manager": "0.4.10-main.0be5154",
+    "@dxos/node-std": "0.4.10-main.0be5154",
+    "@dxos/protocols": "0.4.10-main.0be5154",
+    "@dxos/teleport": "0.4.10-main.0be5154",
+    "@dxos/rpc": "0.4.10-main.0be5154",
+    "@dxos/random-access-storage": "0.4.10-main.0be5154",
+    "@dxos/teleport-extension-automerge-replicator": "0.4.10-main.0be5154",
+    "@dxos/teleport-extension-gossip": "0.4.10-main.0be5154",
+    "@dxos/teleport-extension-replicator": "0.4.10-main.0be5154",
+    "@dxos/teleport-extension-object-sync": "0.4.10-main.0be5154",
+    "@dxos/timeframe": "0.4.10-main.0be5154",
+    "@dxos/tracing": "0.4.10-main.0be5154",
+    "@dxos/typings": "0.4.10-main.0be5154",
+    "@dxos/util": "0.4.10-main.0be5154"
   },
   "devDependencies": {
     "fast-check": "^3.15.1",

package/src/automerge/automerge-doc-loader.test.ts

@@ -0,0 +1,97 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { sleep } from '@dxos/async';
+import { Repo } from '@dxos/automerge/automerge-repo';
+import { Context } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
+import { describe, test } from '@dxos/test';
+
+import {
+  type AutomergeDocumentLoader,
+  AutomergeDocumentLoaderImpl,
+  type ObjectDocumentLoaded,
+} from './automerge-doc-loader';
+import { type SpaceDoc } from './types';
+
+const ctx = new Context();
+const SPACE_KEY = PublicKey.random();
+const randomId = () => PublicKey.random().toHex();
+describe('AutomergeDocumentLoader', () => {
+  test('space access is set on root doc handle and it is accessible', async () => {
+    const { loader, spaceRootDocHandle } = await setupTest();
+    expect(loader.getSpaceRootDocHandle()).not.to.throw;
+    expect(spaceRootDocHandle.docSync()?.access?.spaceKey).to.eq(SPACE_KEY.toHex());
+  });
+
+  test('new object document is linked with space and root document', async () => {
+    const objectId = randomId();
+    const { loader, spaceRootDocHandle } = await setupTest();
+    const objectDocHandle = loader.createDocumentForObject(objectId);
+    const handle = spaceRootDocHandle.docSync();
+    expect(objectDocHandle.docSync()?.access?.spaceKey).to.eq(SPACE_KEY.toHex());
+    expect(handle?.links[objectId]).to.eq(objectDocHandle.url);
+  });
+
+  test('listener is invoked after a document is loaded', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const handle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle });
+    loadLinkedObjects(loader, { [objectId]: handle.url });
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.true;
+  });
+
+  test('listener is not invoked if an object was rebound during document loading', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const oldDocHandle = repo.create<SpaceDoc>();
+    const newDocHandle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle: oldDocHandle });
+    loadLinkedObjects(loader, { [objectId]: oldDocHandle.url });
+    loader.onObjectBoundToDocument(newDocHandle, objectId);
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.false;
+  });
+
+  test('document link is not loaded if object exists as inline object', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const existingHandle = repo.create<SpaceDoc>();
+    loader.onObjectBoundToDocument(existingHandle, objectId);
+    const newDocHandle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle: newDocHandle });
+    loadLinkedObjects(loader, { [objectId]: existingHandle.url });
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.false;
+  });
+
+  const setupTest = async () => {
+    const repo = new Repo({ network: [] });
+    const loader = new AutomergeDocumentLoaderImpl(SPACE_KEY, repo);
+    const spaceRootDocHandle = repo.create<SpaceDoc>();
+    await loader.loadSpaceRootDocHandle(ctx, {
+      rootUrl: spaceRootDocHandle.url,
+    });
+    return { loader, spaceRootDocHandle, repo };
+  };
+
+  const loadLinkedObjects = (loader: AutomergeDocumentLoader, links: SpaceDoc['links']) => {
+    Object.keys(links ?? {}).forEach((objectId) => loader.loadObjectDocument(objectId));
+    loader.onObjectLinksUpdated(links);
+  };
+
+  const waitForDocumentLoad = (loader: AutomergeDocumentLoader, expected: ObjectDocumentLoaded) => {
+    const docLoadInfo = { loaded: false };
+    loader.onObjectDocumentLoaded.on((data) => {
+      expect(data.objectId).to.eq(expected.objectId);
+      expect(data.handle.url).to.eq(expected.handle.url);
+      docLoadInfo.loaded = true;
+    });
+    return docLoadInfo;
+  };
+});

package/src/automerge/automerge-doc-loader.ts

@@ -0,0 +1,235 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { type DocHandle, type AutomergeUrl, type DocumentId, type Repo } from '@dxos/automerge/automerge-repo';
+import { cancelWithContext, type Context } from '@dxos/context';
+import { warnAfterTimeout } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
+import { type PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { type SpaceState, type SpaceDoc } from './types';
+
+type SpaceDocumentLinks = SpaceDoc['links'];
+
+export interface AutomergeDocumentLoader {
+  onObjectDocumentLoaded: Event<ObjectDocumentLoaded>;
+
+  loadSpaceRootDocHandle(ctx: Context, spaceState: SpaceState): Promise<void>;
+  loadObjectDocument(objectId: string): void;
+  getSpaceRootDocHandle(): DocHandle<SpaceDoc>;
+  createDocumentForObject(objectId: string): DocHandle<SpaceDoc>;
+  onObjectLinksUpdated(links: SpaceDocumentLinks): void;
+  onObjectBoundToDocument(handle: DocHandle<SpaceDoc>, objectId: string): void;
+
+  /**
+   * @returns objectIds for which we had document handles or were loading one.
+   */
+  clearHandleReferences(): string[];
+}
+
+/**
+ * Manages object <-> docHandle binding and automerge document loading.
+ */
+export class AutomergeDocumentLoaderImpl implements AutomergeDocumentLoader {
+  private _spaceRootDocHandle: DocHandle<SpaceDoc> | null = null;
+  /**
+   * An object id pointer to a handle of the document where the object is stored inline.
+   */
+  private readonly _objectDocumentHandles = new Map<string, DocHandle<SpaceDoc>>();
+  /**
+   * If object was requested via loadObjectDocument but root document links weren't updated yet
+   * loading will be triggered in onObjectLinksUpdated callback.
+   */
+  private readonly _objectsPendingDocumentLoad = new Set<string>();
+
+  public readonly onObjectDocumentLoaded = new Event<ObjectDocumentLoaded>();
+
+  constructor(
+    private readonly _spaceKey: PublicKey,
+    private readonly _repo: Repo,
+  ) {}
+
+  public async loadSpaceRootDocHandle(ctx: Context, spaceState: SpaceState): Promise<void> {
+    if (this._spaceRootDocHandle != null) {
+      return;
+    }
+    if (!spaceState.rootUrl) {
+      log.error('Database opened with no rootUrl', { spaceKey: this._spaceKey });
+      this._createContextBoundSpaceRootDocument(ctx);
+    } else {
+      const existingDocHandle = await this._initDocHandle(ctx, spaceState.rootUrl);
+      const doc = existingDocHandle.docSync();
+      invariant(doc);
+      if (doc.access == null) {
+        this._initDocAccess(existingDocHandle);
+      }
+      this._spaceRootDocHandle = existingDocHandle;
+    }
+  }
+
+  public loadObjectDocument(objectId: string) {
+    invariant(this._spaceRootDocHandle);
+    if (this._objectDocumentHandles.has(objectId) || this._objectsPendingDocumentLoad.has(objectId)) {
+      return;
+    }
+    const spaceRootDoc = this._spaceRootDocHandle.docSync();
+    invariant(spaceRootDoc);
+    const documentUrl = (spaceRootDoc.links ?? {})[objectId];
+    if (documentUrl == null) {
+      this._objectsPendingDocumentLoad.add(objectId);
+      log.info('loading delayed until object links are initialized', { objectId });
+      return;
+    }
+    this._loadLinkedObjects({ [objectId]: documentUrl });
+  }
+
+  public onObjectLinksUpdated(links: SpaceDocumentLinks) {
+    if (!links) {
+      return;
+    }
+    const linksAwaitingLoad = Object.entries(links).filter(([objectId]) =>
+      this._objectsPendingDocumentLoad.has(objectId),
+    );
+    this._loadLinkedObjects(Object.fromEntries(linksAwaitingLoad));
+    linksAwaitingLoad.forEach(([objectId]) => this._objectsPendingDocumentLoad.delete(objectId));
+  }
+
+  public getSpaceRootDocHandle(): DocHandle<SpaceDoc> {
+    invariant(this._spaceRootDocHandle);
+    return this._spaceRootDocHandle;
+  }
+
+  public createDocumentForObject(objectId: string): DocHandle<SpaceDoc> {
+    invariant(this._spaceRootDocHandle);
+    const spaceDocHandle = this._repo.create<SpaceDoc>();
+    this._initDocAccess(spaceDocHandle);
+    this.onObjectBoundToDocument(spaceDocHandle, objectId);
+    this._spaceRootDocHandle.change((newDoc: SpaceDoc) => {
+      newDoc.links ??= {};
+      newDoc.links[objectId] = spaceDocHandle.url;
+    });
+    return spaceDocHandle;
+  }
+
+  public onObjectBoundToDocument(handle: DocHandle<SpaceDoc>, objectId: string) {
+    this._objectDocumentHandles.set(objectId, handle);
+  }
+
+  public clearHandleReferences(): string[] {
+    const objectsWithHandles = [...this._objectDocumentHandles.keys()];
+    this._objectDocumentHandles.clear();
+    this._spaceRootDocHandle = null;
+    return objectsWithHandles;
+  }
+
+  private _loadLinkedObjects(links: SpaceDocumentLinks) {
+    if (!links) {
+      return;
+    }
+    for (const [objectId, automergeUrl] of Object.entries(links)) {
+      const logMeta = { objectId, automergeUrl };
+      const objectDocumentHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocumentHandle != null && objectDocumentHandle.url !== automergeUrl) {
+        log.warn('object already inlined in a different document, ignoring the link', {
+          ...logMeta,
+          actualDocumentUrl: objectDocumentHandle.url,
+        });
+        continue;
+      }
+      if (objectDocumentHandle?.url === automergeUrl) {
+        log.warn('object document was already loaded', logMeta);
+        continue;
+      }
+      const handle = this._repo.find<SpaceDoc>(automergeUrl as DocumentId);
+      log.debug('document loading triggered', logMeta);
+      this._objectDocumentHandles.set(objectId, handle);
+      void this._createObjectOnDocumentLoad(handle, objectId);
+    }
+  }
+
+  private async _initDocHandle(ctx: Context, url: string) {
+    const docHandle = this._repo.find<SpaceDoc>(url as DocumentId);
+    while (true) {
+      try {
+        await warnAfterTimeout(5_000, 'Automerge root doc load timeout (AutomergeDb)', async () => {
+          await cancelWithContext(ctx, docHandle.whenReady()); // TODO(dmaretskyi): Temporary 5s timeout for debugging.
+        });
+        break;
+      } catch (err) {
+        if (`${err}`.includes('Timeout')) {
+          log.info('wraparound', { id: docHandle.documentId, state: docHandle.state });
+          continue;
+        }
+
+        throw err;
+      }
+    }
+
+    if (docHandle.state === 'unavailable') {
+      throw new Error('Automerge document is unavailable');
+    }
+
+    return docHandle;
+  }
+
+  private _createContextBoundSpaceRootDocument(ctx: Context) {
+    const docHandle = this._repo.create<SpaceDoc>();
+    this._spaceRootDocHandle = docHandle;
+    ctx.onDispose(() => {
+      docHandle.delete();
+      this._spaceRootDocHandle = null;
+    });
+  }
+
+  private _initDocAccess(handle: DocHandle<SpaceDoc>) {
+    handle.change((newDoc: SpaceDoc) => {
+      newDoc.access ??= { spaceKey: this._spaceKey.toHex() };
+      newDoc.access.spaceKey = this._spaceKey.toHex();
+    });
+  }
+
+  private async _createObjectOnDocumentLoad(handle: DocHandle<SpaceDoc>, objectId: string) {
+    try {
+      await handle.doc(['ready']);
+      const logMeta = { objectId, docUrl: handle.url };
+      if (this.onObjectDocumentLoaded.listenerCount() === 0) {
+        log.info('document loaded after all listeners were removed', logMeta);
+        return;
+      }
+      const objectDocHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocHandle?.url !== handle.url) {
+        log.warn('object was rebound while a document was loading, discarding handle', logMeta);
+        return;
+      }
+      this.onObjectDocumentLoaded.emit({ handle, objectId });
+    } catch (err) {
+      const shouldRetryLoading = this.onObjectDocumentLoaded.listenerCount() > 0;
+      log.warn('failed to load a document', {
+        objectId,
+        automergeUrl: handle.url,
+        retryLoading: shouldRetryLoading,
+        err,
+      });
+      if (shouldRetryLoading) {
+        await this._createObjectOnDocumentLoad(handle, objectId);
+      }
+    }
+  }
+}
+
+export interface ObjectDocumentLoaded {
+  handle: DocHandle<SpaceDoc>;
+  objectId: string;
+}
+
+export interface DocumentChanges {
+  createdObjectIds: string[];
+  updatedObjectIds: string[];
+  objectsToRebind: string[];
+  linkedDocuments: {
+    [echoId: string]: AutomergeUrl;
+  };
+}

package/src/automerge/index.ts

@@ -4,5 +4,7 @@
 
 export * from './automerge-host';
 export * from './automerge-storage-adapter';
+export * from './automerge-doc-loader';
 export * from './local-host-network-adapter';
 export * from './mesh-network-adapter';
+export * from './types';

package/src/automerge/types.ts

@@ -0,0 +1,83 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type Reference } from '@dxos/echo-db';
+
+export type SpaceState = {
+  // Url of the root automerge document.
+  rootUrl?: string;
+};
+
+export interface SpaceDoc {
+  access?: {
+    spaceKey: string;
+  };
+  /**
+   * Objects inlined in the current document.
+   */
+  objects?: {
+    [key: string]: ObjectStructure;
+  };
+  /**
+   * Object id points to an automerge doc url where the object is embedded.
+   */
+  links?: {
+    [echoId: string]: string;
+  };
+}
+
+/**
+ * Representation of an ECHO object in an AM document.
+ */
+export type ObjectStructure = {
+  data: Record<string, any>;
+  meta: ObjectMeta;
+  system: ObjectSystem;
+};
+
+/**
+ * Echo object metadata.
+ */
+export type ObjectMeta = {
+  /**
+   * Foreign keys.
+   */
+  keys: ForeignKey[];
+};
+
+/**
+ * Reference to an object in a foreign database.
+ */
+export type ForeignKey = {
+  /**
+   * Name of the foreign database/system.
+   * E.g. `github.com`.
+   */
+  source?: string;
+
+  /**
+   * Id within the foreign database.
+   */
+  id?: string;
+};
+
+/**
+ * Automerge object system properties.
+ * (Is automerge specific.)
+ */
+export type ObjectSystem = {
+  /**
+   * Deletion marker.
+   */
+  deleted?: boolean;
+
+  /**
+   * Object reference ('protobuf' protocol) type.
+   */
+  type?: Reference;
+};