@dxos/echo-pipeline 0.8.3 → 0.8.4-main.1068cf700f

package/src/db-host/documents-synchronizer.ts

@@ -3,21 +3,22 @@
 //
 
 import { next as A, type Heads } from '@automerge/automerge';
-import { type Repo, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
+import { type DocHandle, type DocumentId } from '@automerge/automerge-repo';
 
 import { UpdateScheduler } from '@dxos/async';
-import { Resource } from '@dxos/context';
+import { Context, LifecycleState, Resource } from '@dxos/context';
 import { type DatabaseDirectory } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { type BatchedDocumentUpdates, type DocumentUpdate } from '@dxos/protocols/proto/dxos/echo/service';
+import { retry } from '@dxos/util';
 
-import { FIND_PARAMS } from '../automerge';
+import type { AutomergeHost } from '../automerge';
 
 const MAX_UPDATE_FREQ = 10; // [updates/sec]
 
-export type DocumentsSynchronizerParams = {
-  repo: Repo;
+export type DocumentsSynchronizerProps = {
+  automergeHost: AutomergeHost;
   sendUpdates: (updates: BatchedDocumentUpdates) => void;
 };
 
@@ -27,6 +28,9 @@ interface DocSyncState {
   clearSubscriptions?: () => void;
 }
 
+const WRAP_AROUND_RETRY_LIMIT = 3;
+const WRAP_AROUND_RETRY_INITIAL_DELAY = 100; // [ms]
+
 /**
  * Manages a connection and replication between worker's Automerge Repo and the client's Repo.
  */
@@ -43,30 +47,38 @@ export class DocumentsSynchronizer extends Resource {
    */
   private _sendUpdatesJob?: UpdateScheduler = undefined;
 
-  constructor(private readonly _params: DocumentsSynchronizerParams) {
+  constructor(private readonly _params: DocumentsSynchronizerProps) {
     super();
   }
 
-  addDocuments(documentIds: DocumentId[], retryCounter = 0): void {
-    if (retryCounter > 3) {
-      log.warn('Failed to load document, retry limit reached', { documentIds });
-      return;
-    }
-
-    for (const documentId of documentIds) {
-      this._params.repo
-        .find<DatabaseDirectory>(documentId as DocumentId)
-        .then(async (doc) => {
-          await doc.whenReady();
-          this._startSync(doc);
-          this._pendingUpdates.add(doc.documentId);
-          this._sendUpdatesJob!.trigger();
-        })
-        .catch((error) => {
-          log.warn('Failed to load document, wraparound', { documentId, error });
-          this.addDocuments([documentId], retryCounter + 1);
-        });
-    }
+  async addDocuments(documentIds: DocumentId[]): Promise<void> {
+    await Promise.all(
+      documentIds.map(async (documentId) => {
+        try {
+          await retry(
+            { count: WRAP_AROUND_RETRY_LIMIT, delayMs: WRAP_AROUND_RETRY_INITIAL_DELAY, exponent: 2 },
+            async () => {
+              try {
+                log('loading document', { documentId });
+                const doc = await this._params.automergeHost.loadDoc<DatabaseDirectory>(
+                  Context.default(),
+                  documentId as DocumentId,
+                  { fetchFromNetwork: true },
+                );
+                this._startSync(doc);
+                this._pendingUpdates.add(doc.documentId);
+                this._sendUpdatesJob!.trigger();
+              } catch (err) {
+                log.warn('failed to load document', { err });
+                throw err;
+              }
+            },
+          );
+        } catch (err) {
+          log.catch(err);
+        }
+      }),
+    );
   }
 
   removeDocuments(documentIds: DocumentId[]): void {
@@ -89,18 +101,14 @@ export class DocumentsSynchronizer extends Resource {
   }
 
   async update(updates: DocumentUpdate[]): Promise<void> {
-    for (const { documentId, mutation, isNew } of updates) {
-      if (isNew) {
-        const doc = await this._params.repo.find<DatabaseDirectory>(documentId as DocumentId, FIND_PARAMS);
-        doc.update((doc) => A.loadIncremental(doc, mutation));
-        this._startSync(doc);
-      } else {
-        this._writeMutation(documentId as DocumentId, mutation);
-      }
+    for (const { documentId, mutation } of updates) {
+      await this._writeMutation(documentId as DocumentId, mutation);
     }
+    // TODO(mykola): This should not be required.
+    await this._params.automergeHost.flush({ documentIds: updates.map(({ documentId }) => documentId as DocumentId) });
   }
 
-  private _startSync(doc: DocHandle<DatabaseDirectory>): void {
+  private _startSync(doc: DocHandle<DatabaseDirectory>) {
     if (this._syncStates.has(doc.documentId)) {
       log('Document already being synced', { documentId: doc.documentId });
       return;
@@ -109,6 +117,7 @@ export class DocumentsSynchronizer extends Resource {
     const syncState: DocSyncState = { handle: doc };
     this._subscribeForChanges(syncState);
     this._syncStates.set(doc.documentId, syncState);
+    return syncState;
   }
 
   _subscribeForChanges(syncState: DocSyncState): void {
@@ -157,16 +166,21 @@ export class DocumentsSynchronizer extends Resource {
     return mutation;
   }
 
-  private _writeMutation(documentId: DocumentId, mutation: Uint8Array): void {
+  private async _writeMutation(documentId: DocumentId, mutation: Uint8Array): Promise<void> {
+    if (this._lifecycleState === LifecycleState.CLOSED) {
+      return;
+    }
+    log('write mutation', { documentId });
+
     const syncState = this._syncStates.get(documentId);
     invariant(syncState, 'Sync state for document not found');
-    syncState.handle.update((doc) => {
-      const headsBefore = A.getHeads(doc);
-      const newDoc = A.loadIncremental(doc, mutation);
-      if (A.equals(headsBefore, syncState.lastSentHead)) {
-        syncState.lastSentHead = A.getHeads(newDoc);
-      }
-      return newDoc;
-    });
+    const headsBefore = A.getHeads(syncState.handle.doc());
+    // This will update corresponding handle in the repo.
+    await this._params.automergeHost.createDoc(mutation, { documentId, preserveHistory: true });
+
+    if (A.equals(headsBefore, syncState.lastSentHead)) {
+      // No new mutations were discovered on network, so we do not need to send updates from worker to client.
+      syncState.lastSentHead = A.getHeads(syncState.handle.doc());
+    }
   }
 }

package/src/db-host/echo-host.ts

@@ -2,66 +2,63 @@
 // Copyright 2024 DXOS.org
 //
 
-import {
-  type AnyDocumentId,
-  type AutomergeUrl,
-  type DocHandle,
-  type DocumentId,
-  type Repo,
-} from '@automerge/automerge-repo';
-
-import { LifecycleState, Resource, type Context } from '@dxos/context';
+import { type AnyDocumentId, type AutomergeUrl, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
+import type * as SqlClient from '@effect/sql/SqlClient';
+
+import { DeferredTask, sleep } from '@dxos/async';
+import { Context, LifecycleState, Resource } from '@dxos/context';
 import { todo } from '@dxos/debug';
-import { createIdFromSpaceKey, SpaceDocVersion, type DatabaseDirectory } from '@dxos/echo-protocol';
-import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
+import { type DatabaseDirectory, SpaceDocVersion, createIdFromSpaceKey } from '@dxos/echo-protocol';
+import { RuntimeProvider } from '@dxos/effect';
+import { FeedStore } from '@dxos/feed';
+import { IndexEngine } from '@dxos/index-core';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey, type SpaceId } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
-import { IndexKind } from '@dxos/protocols/proto/dxos/echo/indexing';
+import { log } from '@dxos/log';
+import { type FeedProtocol } from '@dxos/protocols';
+import type * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 import { trace } from '@dxos/tracing';
 
-import { DataServiceImpl } from './data-service';
-import { type DatabaseRoot } from './database-root';
-import { createSelectedDocumentsIterator } from './documents-iterator';
-import { QueryServiceImpl } from './query-service';
-import { SpaceStateManager } from './space-state-manager';
 import {
   AutomergeHost,
-  FIND_PARAMS,
-  EchoDataMonitor,
-  deriveCollectionIdFromSpaceId,
-  type LoadDocOptions,
+  type AutomergeReplicator,
   type CreateDocOptions,
-  type EchoReplicator,
+  EchoDataMonitor,
   type EchoDataStats,
+  type LoadDocOptions,
   type PeerIdProvider,
   type RootDocumentSpaceKeyProvider,
+  deriveCollectionIdFromSpaceId,
 } from '../automerge';
 
-export interface EchoHostIndexingConfig {
+import { AutomergeDataSource } from './automerge-data-source';
+import { DataServiceImpl } from './data-service';
+import { type DatabaseRoot } from './database-root';
+import { LocalQueueServiceImpl } from './local-queue-service';
+import { QueryServiceImpl } from './query-service';
+import { QueueDataSource } from './queue-data-source';
+import { SpaceStateManager } from './space-state-manager';
+import { QueueServiceStub } from './stub';
+
+export type EchoHostProps = {
+  kv: LevelDB;
+  peerIdProvider?: PeerIdProvider;
+  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
+
+  runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
+
   /**
-   * @default true
+   * Use SQLite-backed local-first durable queues.
+   * Otherwise defaults to in-memory impl.
    */
-  fullText: boolean;
+  localQueues?: boolean;
 
   /**
+   * This peer is allowed to assign positions (global-order) to items appended to the queue.
    * @default false
    */
-  vector: boolean;
-}
-
-const DEFAULT_INDEXING_CONFIG: EchoHostIndexingConfig = {
-  // TODO(dmaretskyi): Disabled by default since embedding generation is expensive.
-  fullText: false,
-  vector: false,
-};
-
-export type EchoHostParams = {
-  kv: LevelDB;
-  peerIdProvider?: PeerIdProvider;
-  getSpaceKeyByRootDocumentId?: RootDocumentSpaceKeyProvider;
-
-  indexing?: Partial<EchoHostIndexingConfig>;
+  assignQueuePositions?: boolean;
 };
 
 /**
@@ -71,51 +68,64 @@ export type EchoHostParams = {
  * Can sync with pluggable data replicators.
  */
 export class EchoHost extends Resource {
-  private readonly _indexMetadataStore: IndexMetadataStore;
-  private readonly _indexer: Indexer;
   private readonly _automergeHost: AutomergeHost;
   private readonly _queryService: QueryServiceImpl;
   private readonly _dataService: DataServiceImpl;
   private readonly _spaceStateManager = new SpaceStateManager();
   private readonly _echoDataMonitor: EchoDataMonitor;
 
-  constructor({ kv, indexing = {}, peerIdProvider, getSpaceKeyByRootDocumentId }: EchoHostParams) {
-    super();
+  private readonly _automergeDataSource: AutomergeDataSource;
+  private readonly _indexEngine: IndexEngine;
+  private readonly _runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient | SqlTransaction.SqlTransaction>;
+  private readonly _feedStore?: FeedStore;
+  private readonly _queueDataSource?: QueueDataSource;
+
+  private _updateIndexes!: DeferredTask;
+
+  private _queuesService: FeedProtocol.QueueService;
+
+  private _indexesUpToDate = false;
 
-    const indexingConfig = { ...DEFAULT_INDEXING_CONFIG, ...indexing };
+  constructor({
+    kv,
+    localQueues,
+    peerIdProvider,
+    getSpaceKeyByRootDocumentId,
+    runtime,
+    assignQueuePositions = false,
+  }: EchoHostProps) {
+    super();
 
-    this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
     this._echoDataMonitor = new EchoDataMonitor();
     this._automergeHost = new AutomergeHost({
       db: kv,
       dataMonitor: this._echoDataMonitor,
-      indexMetadataStore: this._indexMetadataStore,
       peerIdProvider,
       getSpaceKeyByRootDocumentId,
     });
 
-    this._indexer = new Indexer({
-      db: kv,
-      indexStore: new IndexStore({ db: kv.sublevel('index-storage') }),
-      metadataStore: this._indexMetadataStore,
-      loadDocuments: createSelectedDocumentsIterator(this._automergeHost),
-      indexCooldownTime: process.env.NODE_ENV === 'test' ? 0 : undefined,
-    });
-    void this._indexer.setConfig({
-      enabled: true,
-      indexes: [
-        //
-        { kind: IndexKind.Kind.SCHEMA_MATCH },
-        { kind: IndexKind.Kind.GRAPH },
-
-        ...(indexingConfig.fullText ? [{ kind: IndexKind.Kind.FULL_TEXT }] : []),
-        ...(indexingConfig.vector ? [{ kind: IndexKind.Kind.VECTOR }] : []),
-      ],
-    });
+    this._runtime = runtime;
+    this._automergeDataSource = new AutomergeDataSource(this._automergeHost);
+
+    if (localQueues) {
+      this._feedStore = new FeedStore({ assignPositions: assignQueuePositions, localActorId: crypto.randomUUID() });
+      this._queueDataSource = new QueueDataSource({
+        feedStore: this._feedStore,
+        runtime: this._runtime,
+        getSpaceIds: () => this._spaceStateManager.spaceIds,
+      });
+      this._queuesService = new LocalQueueServiceImpl(runtime, this._feedStore);
+    } else {
+      this._queuesService = new QueueServiceStub();
+    }
+
+    // SQLite-based index engine for all queries.
+    this._indexEngine = new IndexEngine();
 
     this._queryService = new QueryServiceImpl({
       automergeHost: this._automergeHost,
-      indexer: this._indexer,
+      indexEngine: this._indexEngine,
+      runtime: this._runtime,
       spaceStateManager: this._spaceStateManager,
     });
 
@@ -123,7 +133,9 @@ export class EchoHost extends Resource {
       automergeHost: this._automergeHost,
       spaceStateManager: this._spaceStateManager,
       updateIndexes: async () => {
-        await this._indexer.updateIndexes();
+        do {
+          await this._updateIndexes.runBlocking();
+        } while (!this._indexesUpToDate);
       },
     });
 
@@ -168,6 +180,10 @@ export class EchoHost extends Resource {
     });
   }
 
+  get spaceIds(): SpaceId[] {
+    return this._spaceStateManager.spaceIds;
+  }
+
   get queryService(): QueryServiceImpl {
     return this._queryService;
   }
@@ -176,29 +192,45 @@ export class EchoHost extends Resource {
     return this._dataService;
   }
 
-  /**
-   * @deprecated To be abstracted away.
-   */
-  get automergeRepo(): Repo {
-    return this._automergeHost.repo;
+  get queuesService(): FeedProtocol.QueueService {
+    return this._queuesService;
   }
 
   get roots(): ReadonlyMap<DocumentId, DatabaseRoot> {
     return this._spaceStateManager.roots;
   }
 
+  get feedStore(): FeedStore | undefined {
+    return this._feedStore;
+  }
+
+  /**
+   * Index engine for queries.
+   */
+  get indexEngine(): IndexEngine {
+    return this._indexEngine;
+  }
+
   protected override async _open(ctx: Context): Promise<void> {
     await this._automergeHost.open();
-    await this._indexer.open(ctx);
     await this._queryService.open(ctx);
     await this._spaceStateManager.open(ctx);
 
+    if (this._feedStore) {
+      await RuntimeProvider.runPromise(this._runtime)(this._feedStore.migrate());
+      this._feedStore.onNewBlocks.on(this._ctx, () => {
+        this._queryService.invalidateQueries();
+        this._updateIndexes.schedule();
+      });
+    }
+
+    await RuntimeProvider.runPromise(this._runtime)(this._indexEngine.migrate());
+    this._updateIndexes = new DeferredTask(this._ctx, this._runUpdateIndexes);
+
     this._spaceStateManager.spaceDocumentListUpdated.on(this._ctx, (e) => {
       if (e.previousRootId) {
         void this._automergeHost.clearLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId, e.previousRootId));
       }
-      // TODO(yaroslav): remove collection without spaceRootId after release (production<->staging interop)
-      void this._automergeHost.updateLocalCollectionState(deriveCollectionIdFromSpaceId(e.spaceId), e.documentIds);
       void this._automergeHost.updateLocalCollectionState(
         deriveCollectionIdFromSpaceId(e.spaceId, e.spaceRootId),
         e.documentIds,
@@ -206,13 +238,14 @@ export class EchoHost extends Resource {
     });
     this._automergeHost.documentsSaved.on(this._ctx, () => {
       this._queryService.invalidateQueries();
+      this._updateIndexes.schedule();
     });
+    this._updateIndexes.schedule();
   }
 
   protected override async _close(ctx: Context): Promise<void> {
     await this._queryService.close(ctx);
     await this._spaceStateManager.close(ctx);
-    await this._indexer.close(ctx);
     await this._automergeHost.close();
   }
 
@@ -220,14 +253,16 @@ export class EchoHost extends Resource {
    * Flush all pending writes to the underlying storage.
    */
   async flush(): Promise<void> {
-    await this._automergeHost.repo.flush();
+    await this._automergeHost.flush();
   }
 
   /**
    * Perform any pending index updates.
    */
   async updateIndexes(): Promise<void> {
-    await this._indexer.updateIndexes();
+    do {
+      await this._updateIndexes.runBlocking();
+    } while (!this._indexesUpToDate);
   }
 
   /**
@@ -244,7 +279,7 @@ export class EchoHost extends Resource {
   /**
    * Create new persisted document.
    */
-  createDoc<T>(initialValue?: T, opts?: CreateDocOptions): DocHandle<T> {
+  async createDoc<T>(initialValue?: T, opts?: CreateDocOptions): Promise<DocHandle<T>> {
     return this._automergeHost.createDoc(initialValue, opts);
   }
 
@@ -255,7 +290,7 @@ export class EchoHost extends Resource {
     invariant(this._lifecycleState === LifecycleState.OPEN);
     const spaceId = await createIdFromSpaceKey(spaceKey);
 
-    const automergeRoot = this._automergeHost.createDoc<DatabaseDirectory>({
+    const automergeRoot = await this._automergeHost.createDoc<DatabaseDirectory>({
       version: SpaceDocVersion.CURRENT,
       access: { spaceKey: spaceKey.toHex() },
 
@@ -272,30 +307,105 @@ export class EchoHost extends Resource {
   // TODO(dmaretskyi): Change to document id.
   async openSpaceRoot(spaceId: SpaceId, automergeUrl: AutomergeUrl): Promise<DatabaseRoot> {
     invariant(this._lifecycleState === LifecycleState.OPEN);
-    const handle = await this._automergeHost.repo.find<DatabaseDirectory>(automergeUrl, FIND_PARAMS);
+    const handle = await this._automergeHost.loadDoc<DatabaseDirectory>(Context.default(), automergeUrl, {
+      fetchFromNetwork: true,
+    });
     await handle.whenReady();
 
     return this._spaceStateManager.assignRootToSpace(spaceId, handle);
   }
 
   // TODO(dmaretskyi): Change to document id.
-  async closeSpaceRoot(automergeUrl: AutomergeUrl): Promise<void> {
+  async closeSpaceRoot(_automergeUrl: AutomergeUrl): Promise<void> {
     todo();
   }
 
   /**
    * Install data replicator.
    */
-  async addReplicator(replicator: EchoReplicator): Promise<void> {
+  async addReplicator(replicator: AutomergeReplicator): Promise<void> {
     await this._automergeHost.addReplicator(replicator);
   }
 
   /**
    * Remove data replicator.
    */
-  async removeReplicator(replicator: EchoReplicator): Promise<void> {
+  async removeReplicator(replicator: AutomergeReplicator): Promise<void> {
     await this._automergeHost.removeReplicator(replicator);
   }
+
+  /**
+   * Run collection sync for the given space.
+   * Does not wait for the sync to complete.
+   */
+  async runCollectionSync(spaceId: SpaceId) {
+    const root = this._spaceStateManager.getRootBySpaceId(spaceId);
+    if (!root) {
+      throw new Error(`Space not found: ${spaceId}`);
+    }
+    this._automergeHost.refreshCollection(deriveCollectionIdFromSpaceId(spaceId, root.documentId));
+  }
+
+  private _runUpdateIndexes = async (): Promise<void> => {
+    let totalUpdated = 0;
+    let totalDone = true;
+
+    {
+      performance.mark('indexEngine.update.automerge:start');
+      const { updated, done } = await this._indexEngine
+        .update(this._automergeDataSource, { spaceId: null, limit: 50 })
+        .pipe(RuntimeProvider.runPromise(this._runtime));
+      totalUpdated += updated;
+      totalDone &&= done;
+      performance.measure('Index Automerge', {
+        start: 'indexEngine.update.automerge:start',
+        detail: {
+          devtools: {
+            dataType: 'track-entry',
+            track: 'Indexing',
+            trackGroup: 'ECHO', // Group related tracks together
+            color: 'tertiary-dark',
+            properties: [['count', updated]],
+          },
+        },
+      });
+    }
+
+    if (this._queueDataSource) {
+      performance.mark('indexEngine.update.queue:start');
+      const { updated, done } = await this._indexEngine
+        .update(this._queueDataSource, { spaceId: null, limit: 50 })
+        .pipe(RuntimeProvider.runPromise(this._runtime));
+      totalUpdated += updated;
+      totalDone &&= done;
+      performance.measure('Index Queues', {
+        start: 'indexEngine.update.queue:start',
+        detail: {
+          devtools: {
+            dataType: 'track-entry',
+            track: 'Indexing',
+            trackGroup: 'ECHO',
+            color: 'tertiary-dark',
+            properties: [['count', updated]],
+          },
+        },
+      });
+    }
+
+    log.verbose('indexEngine update completed', { updated: totalUpdated, done: totalDone });
+    await sleep(1);
+    if (!totalDone) {
+      this._indexesUpToDate = false;
+      this._updateIndexes!.schedule();
+    } else {
+      this._indexesUpToDate = true;
+    }
+
+    // Invalidate queries after index update completes so queries can see newly indexed data.
+    if (totalUpdated > 0) {
+      this._queryService.invalidateQueries();
+    }
+  };
 }
 
 export type { EchoDataStats };

package/src/db-host/index.ts

@@ -2,6 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
+export * from './automerge-data-source';
 export * from './data-service';
 export * from './documents-synchronizer';
 export * from './echo-host';