@dxos/echo-pipeline 0.6.1 → 0.6.2-main.8a232a5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.6.1",
+  "version": "0.6.2-main.8a232a5",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -39,38 +39,38 @@
     "crc-32": "^1.2.2",
     "level": "^8.0.1",
     "level-transcoder": "^1.0.1",
-    "@dxos/automerge": "0.6.1",
-    "@dxos/context": "0.6.1",
-    "@dxos/codec-protobuf": "0.6.1",
-    "@dxos/credentials": "0.6.1",
-    "@dxos/crypto": "0.6.1",
-    "@dxos/debug": "0.6.1",
-    "@dxos/echo-schema": "0.6.1",
-    "@dxos/feed-store": "0.6.1",
-    "@dxos/echo-protocol": "0.6.1",
-    "@dxos/hypercore": "0.6.1",
-    "@dxos/invariant": "0.6.1",
-    "@dxos/indexing": "0.6.1",
-    "@dxos/async": "0.6.1",
-    "@dxos/keyring": "0.6.1",
-    "@dxos/keys": "0.6.1",
-    "@dxos/kv-store": "0.6.1",
-    "@dxos/log": "0.6.1",
-    "@dxos/messaging": "0.6.1",
-    "@dxos/network-manager": "0.6.1",
-    "@dxos/node-std": "0.6.1",
-    "@dxos/protocols": "0.6.1",
-    "@dxos/random-access-storage": "0.6.1",
-    "@dxos/rpc": "0.6.1",
-    "@dxos/teleport": "0.6.1",
-    "@dxos/teleport-extension-automerge-replicator": "0.6.1",
-    "@dxos/teleport-extension-object-sync": "0.6.1",
-    "@dxos/teleport-extension-gossip": "0.6.1",
-    "@dxos/teleport-extension-replicator": "0.6.1",
-    "@dxos/timeframe": "0.6.1",
-    "@dxos/tracing": "0.6.1",
-    "@dxos/typings": "0.6.1",
-    "@dxos/util": "0.6.1"
+    "@dxos/async": "0.6.2-main.8a232a5",
+    "@dxos/codec-protobuf": "0.6.2-main.8a232a5",
+    "@dxos/automerge": "0.6.2-main.8a232a5",
+    "@dxos/context": "0.6.2-main.8a232a5",
+    "@dxos/credentials": "0.6.2-main.8a232a5",
+    "@dxos/crypto": "0.6.2-main.8a232a5",
+    "@dxos/echo-protocol": "0.6.2-main.8a232a5",
+    "@dxos/debug": "0.6.2-main.8a232a5",
+    "@dxos/echo-schema": "0.6.2-main.8a232a5",
+    "@dxos/feed-store": "0.6.2-main.8a232a5",
+    "@dxos/hypercore": "0.6.2-main.8a232a5",
+    "@dxos/indexing": "0.6.2-main.8a232a5",
+    "@dxos/invariant": "0.6.2-main.8a232a5",
+    "@dxos/keyring": "0.6.2-main.8a232a5",
+    "@dxos/keys": "0.6.2-main.8a232a5",
+    "@dxos/kv-store": "0.6.2-main.8a232a5",
+    "@dxos/messaging": "0.6.2-main.8a232a5",
+    "@dxos/log": "0.6.2-main.8a232a5",
+    "@dxos/network-manager": "0.6.2-main.8a232a5",
+    "@dxos/protocols": "0.6.2-main.8a232a5",
+    "@dxos/node-std": "0.6.2-main.8a232a5",
+    "@dxos/rpc": "0.6.2-main.8a232a5",
+    "@dxos/random-access-storage": "0.6.2-main.8a232a5",
+    "@dxos/teleport": "0.6.2-main.8a232a5",
+    "@dxos/teleport-extension-gossip": "0.6.2-main.8a232a5",
+    "@dxos/teleport-extension-automerge-replicator": "0.6.2-main.8a232a5",
+    "@dxos/teleport-extension-replicator": "0.6.2-main.8a232a5",
+    "@dxos/teleport-extension-object-sync": "0.6.2-main.8a232a5",
+    "@dxos/timeframe": "0.6.2-main.8a232a5",
+    "@dxos/tracing": "0.6.2-main.8a232a5",
+    "@dxos/typings": "0.6.2-main.8a232a5",
+    "@dxos/util": "0.6.2-main.8a232a5"
   },
   "devDependencies": {
     "fast-check": "^3.19.0",

package/src/automerge/automerge-host.ts

@@ -8,21 +8,20 @@ import {
   getBackend,
   getHeads,
   isAutomerge,
-  save,
   equals as headsEquals,
+  save,
   type Doc,
   type Heads,
 } from '@dxos/automerge/automerge';
 import {
+  type DocHandleChangePayload,
   Repo,
   type AnyDocumentId,
   type DocHandle,
-  type DocHandleChangePayload,
   type DocumentId,
   type PeerId,
   type StorageAdapterInterface,
 } from '@dxos/automerge/automerge-repo';
-import { type Stream } from '@dxos/codec-protobuf';
 import { Context, Resource, cancelWithContext, type Lifecycle } from '@dxos/context';
 import { type SpaceDoc } from '@dxos/echo-protocol';
 import { type IndexMetadataStore } from '@dxos/indexing';
@@ -31,13 +30,7 @@ import { PublicKey } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
 import { objectPointerCodec } from '@dxos/protocols';
-import {
-  type DocHeadsList,
-  type FlushRequest,
-  type HostInfo,
-  type SyncRepoRequest,
-  type SyncRepoResponse,
-} from '@dxos/protocols/proto/dxos/echo/service';
+import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
 import { trace } from '@dxos/tracing';
 import { mapValues } from '@dxos/util';
 
@@ -45,10 +38,6 @@ import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
 import { type EchoReplicator } from './echo-replicator';
 import { HeadsStore } from './heads-store';
 import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-adapter';
-import { LocalHostNetworkAdapter } from './local-host-network-adapter';
-
-// TODO: Remove
-export type { DocumentId };
 
 export type AutomergeHostParams = {
   db: LevelDB;
@@ -79,7 +68,6 @@ export class AutomergeHost extends Resource {
   });
 
   private _repo!: Repo;
-  private _clientNetwork!: LocalHostNetworkAdapter;
   private _storage!: StorageAdapterInterface & Lifecycle;
   private readonly _headsStore: HeadsStore;
 
@@ -105,7 +93,6 @@ export class AutomergeHost extends Resource {
     this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
 
     await this._storage.open?.();
-    this._clientNetwork = new LocalHostNetworkAdapter();
 
     // Construct the automerge repo.
     this._repo = new Repo({
@@ -113,22 +100,17 @@ export class AutomergeHost extends Resource {
       sharePolicy: this._sharePolicy.bind(this),
       storage: this._storage,
       network: [
-        // Downstream client.
-        this._clientNetwork,
         // Upstream swarm.
         this._echoNetworkAdapter,
       ],
     });
 
-    this._clientNetwork.ready();
     await this._echoNetworkAdapter.open();
-    await this._clientNetwork.whenConnected();
     await this._echoNetworkAdapter.whenConnected();
   }
 
   protected override async _close() {
     await this._storage.close?.();
-    await this._clientNetwork.close();
     await this._echoNetworkAdapter.close();
     await this._ctx.dispose();
   }
@@ -335,21 +317,10 @@ export class AutomergeHost extends Resource {
    * Flush documents to disk.
    */
   @trace.span({ showInBrowserTimeline: true })
-  async flush({ states }: FlushRequest = {}): Promise<void> {
-    // Note: Wait for all requested documents to be loaded/synced from thin-client.
-    if (states) {
-      await Promise.all(
-        states.map(async ({ heads, documentId }) => {
-          if (!heads) {
-            return;
-          }
-          const handle = this._repo.handles[documentId as DocumentId] ?? this._repo.find(documentId as DocumentId);
-          await waitForHeads(handle, heads);
-        }) ?? [],
-      );
-    }
+  async flush({ documentIds }: FlushRequest = {}): Promise<void> {
+    // Note: Sync protocol for client and services ensures that all handles should have all changes.
 
-    await this._repo.flush(states?.map(({ documentId }) => documentId as DocumentId));
+    await this._repo.flush(documentIds as DocumentId[] | undefined);
   }
 
   async getHeads(documentId: DocumentId): Promise<Heads | undefined> {
@@ -364,27 +335,6 @@ export class AutomergeHost extends Resource {
       return this._headsStore.getHeads(documentId);
     }
   }
-
-  /**
-   * Host <-> Client sync.
-   */
-  syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
-    return this._clientNetwork.syncRepo(request);
-  }
-
-  /**
-   * Host <-> Client sync.
-   */
-  sendSyncMessage(request: SyncRepoRequest): Promise<void> {
-    return this._clientNetwork.sendSyncMessage(request);
-  }
-
-  /**
-   * Host <-> Client sync.
-   */
-  async getHostInfo(): Promise<HostInfo> {
-    return this._clientNetwork.getHostInfo();
-  }
 }
 
 export const getSpaceKeyFromDoc = (doc: Doc<SpaceDoc>): string | null => {

package/src/automerge/automerge-repo.test.ts

@@ -6,7 +6,18 @@ import { expect } from 'chai';
 import waitForExpect from 'wait-for-expect';
 
 import { asyncTimeout, sleep } from '@dxos/async';
-import { type Heads, change, clone, equals, from, getBackend, getHeads } from '@dxos/automerge/automerge';
+import {
+  next as A,
+  type Heads,
+  change,
+  clone,
+  equals,
+  from,
+  getBackend,
+  getHeads,
+  save,
+  saveSince,
+} from '@dxos/automerge/automerge';
 import {
   type Message,
   Repo,
@@ -14,6 +25,8 @@ import {
   type DocumentId,
   type HandleState,
   type AutomergeUrl,
+  parseAutomergeUrl,
+  generateAutomergeUrl,
 } from '@dxos/automerge/automerge-repo';
 import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';
@@ -630,5 +643,115 @@ describe('AutomergeRepo', () => {
         expect(peer2.handles[hostHandle.documentId]).to.not.be.undefined;
       });
     });
+
+    test('client cold-starts and syncs doc from a Repo', async () => {
+      const repo = new Repo({ network: [] });
+      const serverHandle = repo.create<{ field?: string }>();
+
+      let clientDoc = A.from<{ field?: string }>({});
+      const receiveByClient = (blob: Uint8Array) => {
+        clientDoc = A.loadIncremental(clientDoc, blob);
+      };
+
+      // Sync handshake.
+      let syncedHeads = getHeads(serverHandle.docSync());
+      receiveByClient(save(serverHandle.docSync()));
+
+      serverHandle.on('change', ({ doc }) => {
+        // Note: This is mock of a sync protocol between client and server.
+        const blob = saveSince(doc, syncedHeads);
+        syncedHeads = getHeads(doc);
+        receiveByClient(blob);
+      });
+
+      {
+        const value = 'text to test if sync works';
+        serverHandle.change((doc: any) => {
+          doc.field = value;
+        });
+        expect(clientDoc.field).to.deep.equal(value);
+      }
+
+      {
+        const value = 'test if updates propagate';
+        serverHandle.change((doc: any) => {
+          doc.field = value;
+        });
+        expect(clientDoc.field).to.deep.equal(value);
+      }
+    });
+
+    test('client creates doc and syncs with a Repo', async () => {
+      const repo = new Repo({ network: [] });
+      const receiveByServer = async (blob: Uint8Array, docId: DocumentId) => {
+        const serverHandle = repo.find(docId);
+        serverHandle.update((doc) => {
+          return A.loadIncremental(doc, blob);
+        });
+      };
+
+      let clientDoc = A.from<{ field?: string }>({});
+      const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
+      // Sync handshake.
+      let sentHeads = getHeads(clientDoc);
+
+      // Sync protocol.
+      const sendDoc = async (doc: A.Doc<any>) => {
+        await receiveByServer(saveSince(doc, sentHeads), documentId);
+        sentHeads = getHeads(doc);
+      };
+
+      {
+        // Change doc and send changes to server.
+        const value = 'text to test if sync works';
+        clientDoc = A.change(clientDoc, (doc: any) => {
+          doc.field = value;
+        });
+        await sendDoc(clientDoc);
+
+        await repo.find(documentId).whenReady();
+        expect(repo.find(documentId).docSync().field).to.deep.equal(value);
+      }
+    });
+
+    test('two repo sync docs on `update` call', async () => {
+      const [adapter1, adapter2] = TestAdapter.createPair();
+      const repoA = new Repo({
+        peerId: 'A' as any,
+        network: [adapter1],
+        sharePolicy: async () => true,
+      });
+      const repoB = new Repo({
+        peerId: 'B' as any,
+        network: [adapter2],
+        sharePolicy: async () => true,
+      });
+
+      {
+        // Connect repos.
+        adapter1.ready();
+        adapter2.ready();
+        await adapter1.onConnect.wait();
+        await adapter2.onConnect.wait();
+        adapter1.peerCandidate(adapter2.peerId!);
+        adapter2.peerCandidate(adapter1.peerId!);
+      }
+
+      const handleA = repoA.create();
+      const handleB = repoB.find(handleA.url);
+
+      const text = 'Hello world';
+      handleA.update((doc: any) => {
+        const newDoc = A.change(doc, (doc: any) => {
+          doc.text = text;
+        });
+        return newDoc;
+      });
+
+      expect(handleA.docSync().text).to.equal(text);
+
+      await asyncTimeout(handleB.whenReady(), 1000);
+      expect(handleB.docSync().text).to.equal(text);
+    });
   });
 });

package/src/automerge/echo-network-adapter.ts

@@ -71,7 +71,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   @synchronized
   async close() {
     if (this._lifecycleState === LifecycleState.CLOSED) {
-      return;
+      return this;
     }
 
     for (const replicator of this._replicators) {

package/src/automerge/index.ts

@@ -3,8 +3,6 @@
 //
 
 export * from './automerge-host';
-export * from './automerge-doc-loader';
 export * from './leveldb-storage-adapter';
-export * from './local-host-network-adapter';
 export * from './mesh-echo-replicator';
 export * from './echo-replicator';

package/src/db-host/data-service.ts

@@ -2,25 +2,26 @@
 // Copyright 2021 DXOS.org
 //
 
-import { type RequestOptions, type Stream } from '@dxos/codec-protobuf';
+import { type DocumentId } from '@dxos/automerge/automerge-repo';
+import { type RequestOptions, Stream } from '@dxos/codec-protobuf';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
 import {
   type DataService,
   type DocHeadsList,
-  type EchoEvent,
   type FlushRequest,
+  type SubscribeRequest,
+  type BatchedDocumentUpdates,
+  type UpdateSubscriptionRequest,
   type GetDocumentHeadsRequest,
   type GetDocumentHeadsResponse,
-  type HostInfo,
-  type MutationReceipt,
   type ReIndexHeadsRequest,
-  type SubscribeRequest,
-  type SyncRepoRequest,
-  type SyncRepoResponse,
   type WaitUntilHeadsReplicatedRequest,
-  type WriteRequest,
+  type UpdateRequest,
 } from '@dxos/protocols/proto/dxos/echo/service';
 
-import { type AutomergeHost, type DocumentId } from '../automerge';
+import { DocumentsSynchronizer } from './documents-synchronizer';
+import { type AutomergeHost } from '../automerge';
 
 export type DataServiceParams = {
   automergeHost: AutomergeHost;
@@ -32,6 +33,12 @@ export type DataServiceParams = {
  */
 // TODO(burdon): Move to client-services.
 export class DataServiceImpl implements DataService {
+  /**
+   * Map of subscriptions.
+   * subscriptionId -> DocumentsSynchronizer
+   */
+  private readonly _subscriptions = new Map<string, DocumentsSynchronizer>();
+
   private readonly _automergeHost: AutomergeHost;
   private readonly _updateIndexes: () => Promise<void>;
 
@@ -40,30 +47,47 @@ export class DataServiceImpl implements DataService {
     this._updateIndexes = params.updateIndexes;
   }
 
-  subscribe(request: SubscribeRequest): Stream<EchoEvent> {
-    throw new Error('Deprecated.');
+  subscribe(request: SubscribeRequest): Stream<BatchedDocumentUpdates> {
+    return new Stream<BatchedDocumentUpdates>(({ next, ready }) => {
+      const synchronizer = new DocumentsSynchronizer({
+        repo: this._automergeHost.repo,
+        sendUpdates: (updates) => next(updates),
+      });
+      synchronizer
+        .open()
+        .then(() => {
+          this._subscriptions.set(request.subscriptionId, synchronizer);
+          ready();
+        })
+        .catch((err) => log.catch(err));
+      return () => synchronizer.close();
+    });
   }
 
-  write(request: WriteRequest): Promise<MutationReceipt> {
-    throw new Error('Deprecated.');
-  }
+  async updateSubscription(request: UpdateSubscriptionRequest) {
+    const synchronizer = this._subscriptions.get(request.subscriptionId);
+    invariant(synchronizer, 'Subscription not found');
 
-  async flush(request: FlushRequest): Promise<void> {
-    await this._automergeHost.flush(request);
+    if (request.addIds?.length) {
+      await synchronizer.addDocuments(request.addIds as DocumentId[]);
+    }
+    if (request.removeIds?.length) {
+      await synchronizer.removeDocuments(request.removeIds as DocumentId[]);
+    }
   }
 
-  // Automerge specific.
-
-  async getHostInfo(request: void): Promise<HostInfo> {
-    return this._automergeHost.getHostInfo();
-  }
+  async update(request: UpdateRequest): Promise<void> {
+    if (!request.updates) {
+      return;
+    }
+    const synchronizer = this._subscriptions.get(request.subscriptionId);
+    invariant(synchronizer, 'Subscription not found');
 
-  syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
-    return this._automergeHost.syncRepo(request);
+    synchronizer.update(request.updates);
   }
 
-  sendSyncMessage(request: SyncRepoRequest): Promise<void> {
-    return this._automergeHost.sendSyncMessage(request);
+  async flush(request: FlushRequest): Promise<void> {
+    await this._automergeHost.flush(request);
   }
 
   async getDocumentHeads(request: GetDocumentHeadsRequest): Promise<GetDocumentHeadsResponse> {

package/src/db-host/documents-synchronizer.test.ts

@@ -0,0 +1,40 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { sleep } from '@dxos/async';
+import { next as A } from '@dxos/automerge/automerge';
+import { generateAutomergeUrl, parseAutomergeUrl, Repo } from '@dxos/automerge/automerge-repo';
+import { describe, openAndClose, test } from '@dxos/test';
+
+import { DocumentsSynchronizer } from './documents-synchronizer';
+
+describe('DocumentsSynchronizer', () => {
+  test('do not get init changes for client created docs', async () => {
+    let counter = 0;
+    const serverRepo = new Repo({ network: [] });
+    const synchronizer = new DocumentsSynchronizer({
+      repo: serverRepo,
+      sendUpdates: () => {
+        counter++;
+      },
+    });
+    await openAndClose(synchronizer);
+
+    synchronizer.update([
+      {
+        documentId: parseAutomergeUrl(generateAutomergeUrl()).documentId,
+        mutation: A.save(A.from({ text: 'hello' })),
+        isNew: true,
+      },
+    ]);
+
+    // Wait for the changes to be processed.
+    await sleep(100);
+
+    // No updates should be sent.
+    expect(counter).to.eq(0);
+  });
+});