@dxos/echo-pipeline 0.4.9 → 0.4.10-main.068c3d8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.4.9",
+  "version": "0.4.10-main.068c3d8",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -35,36 +35,39 @@
     "src"
   ],
   "dependencies": {
+    "abstract-level": "^1.0.2",
     "crc-32": "^1.2.2",
-    "@dxos/automerge": "0.4.9",
-    "@dxos/codec-protobuf": "0.4.9",
-    "@dxos/async": "0.4.9",
-    "@dxos/context": "0.4.9",
-    "@dxos/credentials": "0.4.9",
-    "@dxos/debug": "0.4.9",
-    "@dxos/crypto": "0.4.9",
-    "@dxos/echo-db": "0.4.9",
-    "@dxos/feed-store": "0.4.9",
-    "@dxos/hypercore": "0.4.9",
-    "@dxos/keyring": "0.4.9",
-    "@dxos/invariant": "0.4.9",
-    "@dxos/messaging": "0.4.9",
-    "@dxos/log": "0.4.9",
-    "@dxos/keys": "0.4.9",
-    "@dxos/network-manager": "0.4.9",
-    "@dxos/node-std": "0.4.9",
-    "@dxos/protocols": "0.4.9",
-    "@dxos/rpc": "0.4.9",
-    "@dxos/random-access-storage": "0.4.9",
-    "@dxos/teleport": "0.4.9",
-    "@dxos/teleport-extension-automerge-replicator": "0.4.9",
-    "@dxos/teleport-extension-gossip": "0.4.9",
-    "@dxos/teleport-extension-object-sync": "0.4.9",
-    "@dxos/timeframe": "0.4.9",
-    "@dxos/teleport-extension-replicator": "0.4.9",
-    "@dxos/tracing": "0.4.9",
-    "@dxos/typings": "0.4.9",
-    "@dxos/util": "0.4.9"
+    "level": "^8.0.1",
+    "level-transcoder": "^1.0.1",
+    "@dxos/async": "0.4.10-main.068c3d8",
+    "@dxos/automerge": "0.4.10-main.068c3d8",
+    "@dxos/codec-protobuf": "0.4.10-main.068c3d8",
+    "@dxos/context": "0.4.10-main.068c3d8",
+    "@dxos/credentials": "0.4.10-main.068c3d8",
+    "@dxos/crypto": "0.4.10-main.068c3d8",
+    "@dxos/echo-db": "0.4.10-main.068c3d8",
+    "@dxos/debug": "0.4.10-main.068c3d8",
+    "@dxos/feed-store": "0.4.10-main.068c3d8",
+    "@dxos/invariant": "0.4.10-main.068c3d8",
+    "@dxos/hypercore": "0.4.10-main.068c3d8",
+    "@dxos/keys": "0.4.10-main.068c3d8",
+    "@dxos/log": "0.4.10-main.068c3d8",
+    "@dxos/keyring": "0.4.10-main.068c3d8",
+    "@dxos/messaging": "0.4.10-main.068c3d8",
+    "@dxos/network-manager": "0.4.10-main.068c3d8",
+    "@dxos/node-std": "0.4.10-main.068c3d8",
+    "@dxos/protocols": "0.4.10-main.068c3d8",
+    "@dxos/random-access-storage": "0.4.10-main.068c3d8",
+    "@dxos/rpc": "0.4.10-main.068c3d8",
+    "@dxos/teleport-extension-automerge-replicator": "0.4.10-main.068c3d8",
+    "@dxos/teleport": "0.4.10-main.068c3d8",
+    "@dxos/teleport-extension-object-sync": "0.4.10-main.068c3d8",
+    "@dxos/teleport-extension-gossip": "0.4.10-main.068c3d8",
+    "@dxos/teleport-extension-replicator": "0.4.10-main.068c3d8",
+    "@dxos/timeframe": "0.4.10-main.068c3d8",
+    "@dxos/tracing": "0.4.10-main.068c3d8",
+    "@dxos/typings": "0.4.10-main.068c3d8",
+    "@dxos/util": "0.4.10-main.068c3d8"
   },
   "devDependencies": {
     "fast-check": "^3.15.1",

package/src/automerge/automerge-doc-loader.test.ts

@@ -0,0 +1,97 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { sleep } from '@dxos/async';
+import { Repo } from '@dxos/automerge/automerge-repo';
+import { Context } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
+import { describe, test } from '@dxos/test';
+
+import {
+  type AutomergeDocumentLoader,
+  AutomergeDocumentLoaderImpl,
+  type ObjectDocumentLoaded,
+} from './automerge-doc-loader';
+import { type SpaceDoc } from './types';
+
+const ctx = new Context();
+const SPACE_KEY = PublicKey.random();
+const randomId = () => PublicKey.random().toHex();
+describe('AutomergeDocumentLoader', () => {
+  test('space access is set on root doc handle and it is accessible', async () => {
+    const { loader, spaceRootDocHandle } = await setupTest();
+    expect(loader.getSpaceRootDocHandle()).not.to.throw;
+    expect(spaceRootDocHandle.docSync()?.access?.spaceKey).to.eq(SPACE_KEY.toHex());
+  });
+
+  test('new object document is linked with space and root document', async () => {
+    const objectId = randomId();
+    const { loader, spaceRootDocHandle } = await setupTest();
+    const objectDocHandle = loader.createDocumentForObject(objectId);
+    const handle = spaceRootDocHandle.docSync();
+    expect(objectDocHandle.docSync()?.access?.spaceKey).to.eq(SPACE_KEY.toHex());
+    expect(handle?.links[objectId]).to.eq(objectDocHandle.url);
+  });
+
+  test('listener is invoked after a document is loaded', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const handle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle });
+    loadLinkedObjects(loader, { [objectId]: handle.url });
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.true;
+  });
+
+  test('listener is not invoked if an object was rebound during document loading', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const oldDocHandle = repo.create<SpaceDoc>();
+    const newDocHandle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle: oldDocHandle });
+    loadLinkedObjects(loader, { [objectId]: oldDocHandle.url });
+    loader.onObjectBoundToDocument(newDocHandle, objectId);
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.false;
+  });
+
+  test('document link is not loaded if object exists as inline object', async () => {
+    const objectId = randomId();
+    const { loader, repo } = await setupTest();
+    const existingHandle = repo.create<SpaceDoc>();
+    loader.onObjectBoundToDocument(existingHandle, objectId);
+    const newDocHandle = repo.create<SpaceDoc>();
+    const docLoadInfo = waitForDocumentLoad(loader, { objectId, handle: newDocHandle });
+    loadLinkedObjects(loader, { [objectId]: existingHandle.url });
+    await sleep(10);
+    expect(docLoadInfo.loaded).to.be.false;
+  });
+
+  const setupTest = async () => {
+    const repo = new Repo({ network: [] });
+    const loader = new AutomergeDocumentLoaderImpl(SPACE_KEY, repo);
+    const spaceRootDocHandle = repo.create<SpaceDoc>();
+    await loader.loadSpaceRootDocHandle(ctx, {
+      rootUrl: spaceRootDocHandle.url,
+    });
+    return { loader, spaceRootDocHandle, repo };
+  };
+
+  const loadLinkedObjects = (loader: AutomergeDocumentLoader, links: SpaceDoc['links']) => {
+    Object.keys(links ?? {}).forEach((objectId) => loader.loadObjectDocument(objectId));
+    loader.onObjectLinksUpdated(links);
+  };
+
+  const waitForDocumentLoad = (loader: AutomergeDocumentLoader, expected: ObjectDocumentLoaded) => {
+    const docLoadInfo = { loaded: false };
+    loader.onObjectDocumentLoaded.on((data) => {
+      expect(data.objectId).to.eq(expected.objectId);
+      expect(data.handle.url).to.eq(expected.handle.url);
+      docLoadInfo.loaded = true;
+    });
+    return docLoadInfo;
+  };
+});

package/src/automerge/automerge-doc-loader.ts

@@ -0,0 +1,241 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { type DocHandle, type AutomergeUrl, type DocumentId, type Repo } from '@dxos/automerge/automerge-repo';
+import { cancelWithContext, type Context } from '@dxos/context';
+import { warnAfterTimeout } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
+import { type PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import { type SpaceState, type SpaceDoc } from './types';
+
+type SpaceDocumentLinks = SpaceDoc['links'];
+
+export interface AutomergeDocumentLoader {
+  onObjectDocumentLoaded: Event<ObjectDocumentLoaded>;
+
+  getAllHandles(): DocHandle<SpaceDoc>[];
+
+  loadSpaceRootDocHandle(ctx: Context, spaceState: SpaceState): Promise<void>;
+  loadObjectDocument(objectId: string): void;
+  getSpaceRootDocHandle(): DocHandle<SpaceDoc>;
+  createDocumentForObject(objectId: string): DocHandle<SpaceDoc>;
+  onObjectLinksUpdated(links: SpaceDocumentLinks): void;
+  onObjectBoundToDocument(handle: DocHandle<SpaceDoc>, objectId: string): void;
+
+  /**
+   * @returns objectIds for which we had document handles or were loading one.
+   */
+  clearHandleReferences(): string[];
+}
+
+/**
+ * Manages object <-> docHandle binding and automerge document loading.
+ */
+export class AutomergeDocumentLoaderImpl implements AutomergeDocumentLoader {
+  private _spaceRootDocHandle: DocHandle<SpaceDoc> | null = null;
+  /**
+   * An object id pointer to a handle of the document where the object is stored inline.
+   */
+  private readonly _objectDocumentHandles = new Map<string, DocHandle<SpaceDoc>>();
+  /**
+   * If object was requested via loadObjectDocument but root document links weren't updated yet
+   * loading will be triggered in onObjectLinksUpdated callback.
+   */
+  private readonly _objectsPendingDocumentLoad = new Set<string>();
+
+  public readonly onObjectDocumentLoaded = new Event<ObjectDocumentLoaded>();
+
+  constructor(
+    private readonly _spaceKey: PublicKey,
+    private readonly _repo: Repo,
+  ) {}
+
+  getAllHandles(): DocHandle<SpaceDoc>[] {
+    return [...new Set(this._objectDocumentHandles.values())];
+  }
+
+  public async loadSpaceRootDocHandle(ctx: Context, spaceState: SpaceState): Promise<void> {
+    if (this._spaceRootDocHandle != null) {
+      return;
+    }
+    if (!spaceState.rootUrl) {
+      log.error('Database opened with no rootUrl', { spaceKey: this._spaceKey });
+      this._createContextBoundSpaceRootDocument(ctx);
+    } else {
+      const existingDocHandle = await this._initDocHandle(ctx, spaceState.rootUrl);
+      const doc = existingDocHandle.docSync();
+      invariant(doc);
+      if (doc.access == null) {
+        this._initDocAccess(existingDocHandle);
+      }
+      this._spaceRootDocHandle = existingDocHandle;
+    }
+  }
+
+  public loadObjectDocument(objectId: string) {
+    invariant(this._spaceRootDocHandle);
+    if (this._objectDocumentHandles.has(objectId) || this._objectsPendingDocumentLoad.has(objectId)) {
+      return;
+    }
+    const spaceRootDoc = this._spaceRootDocHandle.docSync();
+    invariant(spaceRootDoc);
+    const documentUrl = (spaceRootDoc.links ?? {})[objectId];
+    if (documentUrl == null) {
+      this._objectsPendingDocumentLoad.add(objectId);
+      log.info('loading delayed until object links are initialized', { objectId });
+      return;
+    }
+    this._loadLinkedObjects({ [objectId]: documentUrl });
+  }
+
+  public onObjectLinksUpdated(links: SpaceDocumentLinks) {
+    if (!links) {
+      return;
+    }
+    const linksAwaitingLoad = Object.entries(links).filter(([objectId]) =>
+      this._objectsPendingDocumentLoad.has(objectId),
+    );
+    this._loadLinkedObjects(Object.fromEntries(linksAwaitingLoad));
+    linksAwaitingLoad.forEach(([objectId]) => this._objectsPendingDocumentLoad.delete(objectId));
+  }
+
+  public getSpaceRootDocHandle(): DocHandle<SpaceDoc> {
+    invariant(this._spaceRootDocHandle);
+    return this._spaceRootDocHandle;
+  }
+
+  public createDocumentForObject(objectId: string): DocHandle<SpaceDoc> {
+    invariant(this._spaceRootDocHandle);
+    const spaceDocHandle = this._repo.create<SpaceDoc>();
+    this._initDocAccess(spaceDocHandle);
+    this.onObjectBoundToDocument(spaceDocHandle, objectId);
+    this._spaceRootDocHandle.change((newDoc: SpaceDoc) => {
+      newDoc.links ??= {};
+      newDoc.links[objectId] = spaceDocHandle.url;
+    });
+    return spaceDocHandle;
+  }
+
+  public onObjectBoundToDocument(handle: DocHandle<SpaceDoc>, objectId: string) {
+    this._objectDocumentHandles.set(objectId, handle);
+  }
+
+  public clearHandleReferences(): string[] {
+    const objectsWithHandles = [...this._objectDocumentHandles.keys()];
+    this._objectDocumentHandles.clear();
+    this._spaceRootDocHandle = null;
+    return objectsWithHandles;
+  }
+
+  private _loadLinkedObjects(links: SpaceDocumentLinks) {
+    if (!links) {
+      return;
+    }
+    for (const [objectId, automergeUrl] of Object.entries(links)) {
+      const logMeta = { objectId, automergeUrl };
+      const objectDocumentHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocumentHandle != null && objectDocumentHandle.url !== automergeUrl) {
+        log.warn('object already inlined in a different document, ignoring the link', {
+          ...logMeta,
+          actualDocumentUrl: objectDocumentHandle.url,
+        });
+        continue;
+      }
+      if (objectDocumentHandle?.url === automergeUrl) {
+        log.warn('object document was already loaded', logMeta);
+        continue;
+      }
+      const handle = this._repo.find<SpaceDoc>(automergeUrl as DocumentId);
+      log.debug('document loading triggered', logMeta);
+      this._objectDocumentHandles.set(objectId, handle);
+      void this._createObjectOnDocumentLoad(handle, objectId);
+    }
+  }
+
+  private async _initDocHandle(ctx: Context, url: string) {
+    const docHandle = this._repo.find<SpaceDoc>(url as DocumentId);
+    while (true) {
+      try {
+        await warnAfterTimeout(5_000, 'Automerge root doc load timeout (AutomergeDb)', async () => {
+          await cancelWithContext(ctx, docHandle.whenReady()); // TODO(dmaretskyi): Temporary 5s timeout for debugging.
+        });
+        break;
+      } catch (err) {
+        if (`${err}`.includes('Timeout')) {
+          log.info('wraparound', { id: docHandle.documentId, state: docHandle.state });
+          continue;
+        }
+
+        throw err;
+      }
+    }
+
+    if (docHandle.state === 'unavailable') {
+      throw new Error('Automerge document is unavailable');
+    }
+
+    return docHandle;
+  }
+
+  private _createContextBoundSpaceRootDocument(ctx: Context) {
+    const docHandle = this._repo.create<SpaceDoc>();
+    this._spaceRootDocHandle = docHandle;
+    ctx.onDispose(() => {
+      docHandle.delete();
+      this._spaceRootDocHandle = null;
+    });
+  }
+
+  private _initDocAccess(handle: DocHandle<SpaceDoc>) {
+    handle.change((newDoc: SpaceDoc) => {
+      newDoc.access ??= { spaceKey: this._spaceKey.toHex() };
+      newDoc.access.spaceKey = this._spaceKey.toHex();
+    });
+  }
+
+  private async _createObjectOnDocumentLoad(handle: DocHandle<SpaceDoc>, objectId: string) {
+    try {
+      await handle.doc(['ready']);
+      const logMeta = { objectId, docUrl: handle.url };
+      if (this.onObjectDocumentLoaded.listenerCount() === 0) {
+        log.info('document loaded after all listeners were removed', logMeta);
+        return;
+      }
+      const objectDocHandle = this._objectDocumentHandles.get(objectId);
+      if (objectDocHandle?.url !== handle.url) {
+        log.warn('object was rebound while a document was loading, discarding handle', logMeta);
+        return;
+      }
+      this.onObjectDocumentLoaded.emit({ handle, objectId });
+    } catch (err) {
+      const shouldRetryLoading = this.onObjectDocumentLoaded.listenerCount() > 0;
+      log.warn('failed to load a document', {
+        objectId,
+        automergeUrl: handle.url,
+        retryLoading: shouldRetryLoading,
+        err,
+      });
+      if (shouldRetryLoading) {
+        await this._createObjectOnDocumentLoad(handle, objectId);
+      }
+    }
+  }
+}
+
+export interface ObjectDocumentLoaded {
+  handle: DocHandle<SpaceDoc>;
+  objectId: string;
+}
+
+export interface DocumentChanges {
+  createdObjectIds: string[];
+  updatedObjectIds: string[];
+  objectsToRebind: string[];
+  linkedDocuments: {
+    [echoId: string]: AutomergeUrl;
+  };
+}

package/src/automerge/automerge-host.test.ts

@@ -25,35 +25,50 @@ import { arrayToBuffer, bufferToArray } from '@dxos/util';
 import { AutomergeHost } from './automerge-host';
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { MeshNetworkAdapter } from './mesh-network-adapter';
+import { createTestLevel } from '../testing';
 
 describe('AutomergeHost', () => {
-  test('can create documents', () => {
-    const host = new AutomergeHost({ directory: createStorage({ type: StorageType.RAM }).createDirectory() });
+  test('can create documents', async () => {
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
+    afterTest(() => host.close());
 
     const handle = host.repo.create();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
+    await host.repo.flush();
     expect(handle.docSync().text).toEqual('Hello world');
   });
 
   test('changes are preserved in storage', async () => {
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({ db: level.sublevel('automerge') });
+    await host.open();
     const handle = host.repo.create();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
     const url = handle.url;
 
-    // TODO(dmaretskyi): Is there a way to know when automerge has finished saving?
-    await sleep(100);
+    await host.repo.flush();
+    await host.close();
 
-    const host2 = new AutomergeHost({ directory: storageDirectory });
+    const host2 = new AutomergeHost({ db: level.sublevel('automerge') });
+    await host2.open();
+    afterTest(() => host2.close());
     const handle2 = host2.repo.find(url);
     await handle2.whenReady();
     expect(handle2.docSync().text).toEqual('Hello world');
+    await host2.repo.flush();
   });
 
   test('basic networking', async () => {
@@ -168,7 +183,6 @@ describe('AutomergeHost', () => {
     const secondHandle = host.create();
     secondHandle.change((doc: any) => (doc.text = 'Hello world'));
     await host.find(secondHandle.url).whenReady();
-    // await sleep(100);
     allowedDocs.push(secondHandle.documentId);
 
     {