@dxos/echo-pipeline 0.6.12 → 0.6.13-main.548ca8d

package/src/automerge/mesh-echo-replicator.ts

@@ -2,6 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
+import type { CollectionId } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { PublicKey, type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -126,7 +127,7 @@ export class MeshEchoReplicator implements EchoReplicator {
         }
       },
       shouldSyncCollection: ({ collectionId }) => {
-        const spaceId = getSpaceIdFromCollectionId(collectionId);
+        const spaceId = getSpaceIdFromCollectionId(collectionId as CollectionId);
 
         const authorizedDevices = this._authorizedDevices.get(spaceId);
 

package/src/automerge/space-collection.ts

@@ -2,12 +2,13 @@
 // Copyright 2024 DXOS.org
 //
 
+import type { CollectionId } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { SpaceId } from '@dxos/keys';
 
-export const deriveCollectionIdFromSpaceId = (spaceId: SpaceId): string => `space:${spaceId}`;
+export const deriveCollectionIdFromSpaceId = (spaceId: SpaceId): CollectionId => `space:${spaceId}` as CollectionId;
 
-export const getSpaceIdFromCollectionId = (collectionId: string): SpaceId => {
+export const getSpaceIdFromCollectionId = (collectionId: CollectionId): SpaceId => {
   const spaceId = collectionId.replace(/^space:/, '');
   invariant(SpaceId.isValid(spaceId));
   return spaceId;

package/src/automerge/storage-adapter.test.ts

@@ -2,12 +2,11 @@
 // Copyright 2024 DXOS.org
 //
 
-import { expect } from 'chai';
+import { onTestFinished, describe, expect, test } from 'vitest';
 
 import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';
 import { createTestLevel } from '@dxos/kv-store/testing';
-import { afterTest, describe, test } from '@dxos/test';
 import { arrayToBuffer, bufferToArray } from '@dxos/util';
 
 import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
@@ -23,7 +22,7 @@ describe('LevelDBStorageAdapter', () => {
       await adapter.close();
       await level.close();
     };
-    afterTest(close);
+    onTestFinished(close);
     return {
       adapter,
       close,

package/src/db-host/automerge-metrics.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as A from '@dxos/automerge/automerge';
+import { log } from '@dxos/log';
+
+export type DocMetrics = {
+  compressedByteSize: number;
+  loadTime: number;
+  mutationCount: number;
+};
+
+/**
+ * WARN: Slow to run on large docs.
+ */
+export const measureDocMetrics = (doc: A.Doc<any>): DocMetrics => {
+  const snapshot = A.save(doc);
+
+  const start = Date.now();
+  const temp = A.load(snapshot);
+  const end = Date.now();
+  A.free(temp);
+
+  const getAllChangesStart = Date.now();
+  const mutationCount = A.getAllChanges(doc).length;
+  const getAllChangesEnd = Date.now();
+
+  if (getAllChangesEnd - getAllChangesStart > 300) {
+    log.warn('getAllChanges took too long', { elapsed: getAllChangesEnd - getAllChangesStart });
+  }
+
+  return {
+    compressedByteSize: snapshot.byteLength,
+    loadTime: end - start,
+    mutationCount,
+  };
+};

package/src/db-host/data-service.ts

@@ -2,8 +2,10 @@
 // Copyright 2021 DXOS.org
 //
 
+import { UpdateScheduler } from '@dxos/async';
 import { type DocumentId } from '@dxos/automerge/automerge-repo';
-import { type RequestOptions, Stream } from '@dxos/codec-protobuf';
+import { type RequestOptions } from '@dxos/codec-protobuf';
+import { Stream } from '@dxos/codec-protobuf/stream';
 import { invariant } from '@dxos/invariant';
 import { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -120,19 +122,32 @@ export class DataServiceImpl implements DataService {
     await this._updateIndexes();
   }
 
-  async getSpaceSyncState(
-    request: GetSpaceSyncStateRequest,
-    options?: RequestOptions | undefined,
-  ): Promise<SpaceSyncState> {
-    invariant(SpaceId.isValid(request.spaceId));
-    const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
-    const state = await this._automergeHost.getCollectionSyncState(collectionId);
+  subscribeSpaceSyncState(request: GetSpaceSyncStateRequest): Stream<SpaceSyncState> {
+    return new Stream<SpaceSyncState>(({ ctx, next, ready }) => {
+      invariant(SpaceId.isValid(request.spaceId));
+      const collectionId = deriveCollectionIdFromSpaceId(request.spaceId);
+
+      const scheduler = new UpdateScheduler(ctx, async () => {
+        const state = await this._automergeHost.getCollectionSyncState(collectionId);
+
+        next({
+          peers: state.peers.map((peer) => ({
+            peerId: peer.peerId,
+            missingOnRemote: peer.missingOnRemote,
+            missingOnLocal: peer.missingOnLocal,
+            differentDocuments: peer.differentDocuments,
+            localDocumentCount: peer.localDocumentCount,
+            remoteDocumentCount: peer.remoteDocumentCount,
+          })),
+        });
+      });
 
-    return {
-      peers: state.peers.map((peer) => ({
-        peerId: peer.peerId,
-        documentsToReconcile: peer.differentDocuments,
-      })),
-    };
+      this._automergeHost.collectionStateUpdated.on(ctx, (e) => {
+        if (e.collectionId === collectionId) {
+          scheduler.trigger();
+        }
+      });
+      scheduler.trigger();
+    });
   }
 }

package/src/db-host/database-root.ts

@@ -0,0 +1,86 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type * as A from '@dxos/automerge/automerge';
+import type { AutomergeUrl, DocHandle, DocumentId } from '@dxos/automerge/automerge-repo';
+import { type SpaceDoc, SpaceDocVersion } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+
+import { measureDocMetrics, type DocMetrics } from './automerge-metrics';
+import { getSpaceKeyFromDoc } from '../automerge';
+
+export class DatabaseRoot {
+  constructor(private readonly _rootHandle: DocHandle<SpaceDoc>) {}
+
+  get documentId(): DocumentId {
+    return this._rootHandle.documentId;
+  }
+
+  get url() {
+    return this._rootHandle.url;
+  }
+
+  get isLoaded(): boolean {
+    return !!this._rootHandle.docSync();
+  }
+
+  get handle(): DocHandle<SpaceDoc> {
+    return this._rootHandle;
+  }
+
+  docSync(): A.Doc<SpaceDoc> | null {
+    return this._rootHandle.docSync();
+  }
+
+  getVersion(): SpaceDocVersion | null {
+    const doc = this.docSync();
+    if (!doc) {
+      return null;
+    }
+
+    return doc.version ?? SpaceDocVersion.LEGACY;
+  }
+
+  getSpaceKey(): string | null {
+    const doc = this.docSync();
+    if (!doc) {
+      return null;
+    }
+
+    return getSpaceKeyFromDoc(doc);
+  }
+
+  getInlineObjectCount(): number | null {
+    const doc = this.docSync();
+    if (!doc) {
+      return null;
+    }
+
+    return Object.keys(doc.objects ?? {}).length;
+  }
+
+  getLinkedObjectCount(): number | null {
+    const doc = this.docSync();
+    if (!doc) {
+      return null;
+    }
+
+    return Object.keys(doc.links ?? {}).length;
+  }
+
+  getAllLinkedDocuments(): AutomergeUrl[] {
+    const doc = this.docSync();
+    invariant(doc);
+
+    return Object.values(doc.links ?? {}) as AutomergeUrl[];
+  }
+
+  measureMetrics(): DocMetrics | null {
+    const doc = this.docSync();
+    if (!doc) {
+      return null;
+    }
+    return measureDocMetrics(doc);
+  }
+}

package/src/db-host/documents-iterator.ts

@@ -0,0 +1,73 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as A from '@dxos/automerge/automerge';
+import { type DocumentId } from '@dxos/automerge/automerge-repo';
+import { Context } from '@dxos/context';
+import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
+import { type ObjectSnapshot, type IdToHeads } from '@dxos/indexing';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ObjectPointerVersion, objectPointerCodec } from '@dxos/protocols';
+
+import { type AutomergeHost, getSpaceKeyFromDoc } from '../automerge';
+
+const LOG_VIEW_OPERATION_THRESHOLD = 300;
+
+/**
+ * Factory for `loadDocuments` iterator.
+ */
+export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
+  /**
+   * Get object data blobs from Automerge Repo by ids.
+   */
+  // TODO(mykola): Unload automerge handles after usage.
+  async function* loadDocuments(objects: IdToHeads): AsyncGenerator<ObjectSnapshot[], void, void> {
+    for (const [id, heads] of objects.entries()) {
+      try {
+        const { documentId, objectId } = objectPointerCodec.decode(id);
+        const handle = await automergeHost.loadDoc(Context.default(), documentId as DocumentId);
+
+        let doc: A.Doc<SpaceDoc> = handle.docSync();
+        invariant(doc);
+
+        const currentHeads = A.getHeads(doc);
+
+        // Checkout the requested version of the document.
+        if (!A.equals(currentHeads, heads)) {
+          const begin = Date.now();
+          // `view` can take a long time even if the document is already at the right version.
+          doc = A.view(doc, heads);
+          const end = Date.now();
+          if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
+            log.info('Checking out document version is taking too long', {
+              duration: end - begin,
+              requestedHeads: heads,
+              originalHeads: currentHeads,
+            });
+          }
+        }
+
+        // Skip outdated docs.
+        if (doc.version !== SpaceDocVersion.CURRENT) {
+          continue;
+        }
+
+        if (!doc.objects?.[objectId]) {
+          continue;
+        }
+
+        // Upgrade V0 object pointers to V1.
+        let newId = id;
+        if (objectPointerCodec.getVersion(id) === ObjectPointerVersion.V0) {
+          const spaceKey = getSpaceKeyFromDoc(doc) ?? undefined;
+          newId = objectPointerCodec.encode({ documentId, objectId, spaceKey });
+        }
+
+        yield [{ id: newId, object: doc.objects![objectId], heads }];
+      } catch (error) {
+        log.error('Error loading document', { heads, id, error });
+      }
+    }
+  };

package/src/db-host/documents-synchronizer.test.ts

@@ -2,12 +2,12 @@
 // Copyright 2024 DXOS.org
 //
 
-import { expect } from 'chai';
+import { describe, expect, test } from 'vitest';
 
 import { sleep } from '@dxos/async';
 import { next as A } from '@dxos/automerge/automerge';
 import { generateAutomergeUrl, parseAutomergeUrl, Repo } from '@dxos/automerge/automerge-repo';
-import { describe, openAndClose, test } from '@dxos/test';
+import { openAndClose } from '@dxos/test-utils';
 
 import { DocumentsSynchronizer } from './documents-synchronizer';
 

package/src/db-host/echo-host.ts

@@ -0,0 +1,257 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import {
+  type AnyDocumentId,
+  type AutomergeUrl,
+  type DocHandle,
+  type DocumentId,
+  type Repo,
+} from '@dxos/automerge/automerge-repo';
+import { LifecycleState, Resource, type Context } from '@dxos/context';
+import { todo } from '@dxos/debug';
+import { createIdFromSpaceKey, SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
+import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
+import { invariant } from '@dxos/invariant';
+import { type PublicKey, type SpaceId } from '@dxos/keys';
+import { type LevelDB } from '@dxos/kv-store';
+import { IndexKind, type IndexConfig } from '@dxos/protocols/proto/dxos/echo/indexing';
+import { trace } from '@dxos/tracing';
+
+import { DataServiceImpl } from './data-service';
+import { type DatabaseRoot } from './database-root';
+import { createSelectedDocumentsIterator } from './documents-iterator';
+import { QueryServiceImpl } from './query-service';
+import { SpaceStateManager } from './space-state-manager';
+import {
+  AutomergeHost,
+  EchoDataMonitor,
+  deriveCollectionIdFromSpaceId,
+  type LoadDocOptions,
+  type CreateDocOptions,
+  type EchoReplicator,
+  type CollectionSyncState,
+  type EchoDataStats,
+} from '../automerge';
+
+const INDEXER_CONFIG: IndexConfig = {
+  enabled: true,
+  indexes: [{ kind: IndexKind.Kind.SCHEMA_MATCH }],
+};
+
+export type EchoHostParams = {
+  kv: LevelDB;
+};
+
+/**
+ * Host for the Echo database.
+ * Manages multiple spaces.
+ * Stores data to disk.
+ * Can sync with pluggable data replicators.
+ */
+export class EchoHost extends Resource {
+  private readonly _indexMetadataStore: IndexMetadataStore;
+  private readonly _indexer: Indexer;
+  private readonly _automergeHost: AutomergeHost;
+  private readonly _queryService: QueryServiceImpl;
+  private readonly _dataService: DataServiceImpl;
+  private readonly _spaceStateManager = new SpaceStateManager();
+  private readonly _echoDataMonitor: EchoDataMonitor;
+
+  constructor({ kv }: EchoHostParams) {
+    super();
+
+    this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
+
+    this._echoDataMonitor = new EchoDataMonitor();
+
+    this._automergeHost = new AutomergeHost({
+      db: kv,
+      dataMonitor: this._echoDataMonitor,
+      indexMetadataStore: this._indexMetadataStore,
+    });
+
+    this._indexer = new Indexer({
+      db: kv,
+      indexStore: new IndexStore({ db: kv.sublevel('index-storage') }),
+      metadataStore: this._indexMetadataStore,
+      loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
+      indexCooldownTime: process.env.NODE_ENV === 'test' ? 0 : undefined,
+    });
+    this._indexer.setConfig(INDEXER_CONFIG);
+
+    this._queryService = new QueryServiceImpl({
+      automergeHost: this._automergeHost,
+      indexer: this._indexer,
+    });
+
+    this._dataService = new DataServiceImpl({
+      automergeHost: this._automergeHost,
+      updateIndexes: async () => {
+        await this._indexer.updateIndexes();
+      },
+    });
+
+    trace.diagnostic<EchoStatsDiagnostic>({
+      id: 'echo-stats',
+      name: 'Echo Stats',
+      fetch: async () => {
+        return {
+          dataStats: this._echoDataMonitor.computeStats(),
+          loadedDocsCount: this._automergeHost.loadedDocsCount,
+        };
+      },
+    });
+
+    trace.diagnostic({
+      id: 'database-roots',
+      name: 'Database Roots',
+      fetch: async () => {
+        return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
+          url: root.url,
+          isLoaded: root.isLoaded,
+          spaceKey: root.getSpaceKey(),
+          inlineObjects: root.getInlineObjectCount(),
+          linkedObjects: root.getLinkedObjectCount(),
+        }));
+      },
+    });
+
+    trace.diagnostic({
+      id: 'database-root-metrics',
+      name: 'Database Roots (with metrics)',
+      fetch: async () => {
+        return Array.from(this._spaceStateManager.roots.values()).map((root) => ({
+          url: root.url,
+          isLoaded: root.isLoaded,
+          spaceKey: root.getSpaceKey(),
+          inlineObjects: root.getInlineObjectCount(),
+          linkedObjects: root.getLinkedObjectCount(),
+          ...(root.measureMetrics() ?? {}),
+        }));
+      },
+    });
+  }
+
+  get queryService(): QueryServiceImpl {
+    return this._queryService;
+  }
+
+  get dataService(): DataServiceImpl {
+    return this._dataService;
+  }
+
+  /**
+   * @deprecated To be abstracted away.
+   */
+  get automergeRepo(): Repo {
+    return this._automergeHost.repo;
+  }
+
+  get roots(): ReadonlyMap<DocumentId, DatabaseRoot> {
+    return this._spaceStateManager.roots;
+  }
+
+  protected override async _open(ctx: Context): Promise<void> {
+    await this._automergeHost.open();
+    await this._indexer.open(ctx);
+    await this._queryService.open(ctx);
+    await this._spaceStateManager.open(ctx);
+
+    this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
+      void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
+    });
+  }
+
+  protected override async _close(ctx: Context): Promise<void> {
+    await this._spaceStateManager.close();
+    await this._queryService.close(ctx);
+    await this._indexer.close(ctx);
+    await this._automergeHost.close();
+  }
+
+  /**
+   * Flush all pending writes to the underlying storage.
+   */
+  async flush() {
+    await this._automergeHost.repo.flush();
+  }
+
+  /**
+   * Perform any pending index updates.
+   */
+  async updateIndexes() {
+    await this._indexer.updateIndexes();
+  }
+
+  /**
+   * Loads the document handle from the repo and waits for it to be ready.
+   */
+  async loadDoc<T>(ctx: Context, documentId: AnyDocumentId, opts?: LoadDocOptions): Promise<DocHandle<T>> {
+    return await this._automergeHost.loadDoc(ctx, documentId, opts);
+  }
+
+  /**
+   * Create new persisted document.
+   */
+  createDoc<T>(initialValue?: T, opts?: CreateDocOptions): DocHandle<T> {
+    return this._automergeHost.createDoc(initialValue, opts);
+  }
+
+  /**
+   * Create new space root.
+   */
+  async createSpaceRoot(spaceKey: PublicKey): Promise<DatabaseRoot> {
+    invariant(this._lifecycleState === LifecycleState.OPEN);
+    const spaceId = await createIdFromSpaceKey(spaceKey);
+
+    const automergeRoot = this._automergeHost.createDoc<SpaceDoc>({
+      version: SpaceDocVersion.CURRENT,
+      access: { spaceKey: spaceKey.toHex() },
+    });
+
+    await this._automergeHost.flush({ documentIds: [automergeRoot.documentId] });
+
+    return await this.openSpaceRoot(spaceId, automergeRoot.url);
+  }
+
+  // TODO(dmaretskyi): Change to document id.
+  async openSpaceRoot(spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
+    invariant(this._lifecycleState === LifecycleState.OPEN);
+    const handle = this._automergeHost.repo.find(automergeUrl);
+
+    return this._spaceStateManager.assignRootToSpace(spaceId, handle);
+  }
+
+  // TODO(dmaretskyi): Change to document id.
+  async closeSpaceRoot(automergeUrl: AutomergeUrl): Promise<void> {
+    todo();
+  }
+
+  /**
+   * Install data replicator.
+   */
+  async addReplicator(replicator: EchoReplicator): Promise<void> {
+    await this._automergeHost.addReplicator(replicator);
+  }
+
+  /**
+   * Remove data replicator.
+   */
+  async removeReplicator(replicator: EchoReplicator): Promise<void> {
+    await this._automergeHost.removeReplicator(replicator);
+  }
+
+  async getSpaceSyncState(spaceId: SpaceId): Promise<CollectionSyncState> {
+    const collectionId = deriveCollectionIdFromSpaceId(spaceId);
+    return this._automergeHost.getCollectionSyncState(collectionId);
+  }
+}
+
+export type { EchoDataStats };
+
+export type EchoStatsDiagnostic = {
+  loadedDocsCount: number;
+  dataStats: EchoDataStats;
+};

package/src/db-host/index.ts

@@ -1,6 +1,11 @@
 //
-// Copyright 2023 DXOS.org
+// Copyright 2024 DXOS.org
 //
 
 export * from './data-service';
 export * from './documents-synchronizer';
+export * from './echo-host';
+export * from './migration';
+export * from './database-root';
+export * from './query-state';
+export * from './query-service';

package/src/db-host/migration.ts

@@ -0,0 +1,57 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { convertLegacyReference } from '@dxos/echo-protocol';
+import {
+  decodeReference,
+  encodeReference,
+  isLegacyReference,
+  LEGACY_TYPE_PROPERTIES,
+  type ObjectStructure,
+  Reference,
+  type SpaceDoc,
+  SpaceDocVersion,
+} from '@dxos/echo-protocol';
+import { TYPE_PROPERTIES } from '@dxos/echo-schema';
+import { deepMapValuesAsync } from '@dxos/util';
+
+export const convertLegacyReferences = async (doc: SpaceDoc): Promise<SpaceDoc> => {
+  const newDoc = await deepMapValuesAsync(doc, async (value, recurse) => {
+    if (isLegacyReference(value)) {
+      return convertLegacyReference(value);
+    }
+    return recurse(value);
+  });
+
+  newDoc.version = SpaceDocVersion.CURRENT;
+  return newDoc;
+};
+
+export const convertLegacySpaceRootDoc = async (root: SpaceDoc): Promise<SpaceDoc> => {
+  // Convert references.
+  const newDoc: SpaceDoc = await convertLegacyReferences(root);
+
+  // Update properties type.
+  const properties = findInlineObjectOfType(newDoc, LEGACY_TYPE_PROPERTIES);
+  if (properties) {
+    const [_, obj] = properties;
+    obj.system.type = encodeReference(Reference.fromLegacyTypename(TYPE_PROPERTIES));
+  }
+
+  return newDoc;
+};
+
+/**
+ * Assumes properties are at root.
+ */
+export const findInlineObjectOfType = (spaceDoc: SpaceDoc, typename: string): [string, ObjectStructure] | undefined => {
+  for (const id in spaceDoc.objects ?? {}) {
+    const obj = spaceDoc.objects![id];
+    if (obj.system.type && decodeReference(obj.system.type).objectId === typename) {
+      return [id, obj];
+    }
+  }
+
+  return undefined;
+};