@dxos/echo-pipeline 0.8.2-staging.7ac8446 → 0.8.2

package/src/automerge/echo-network-adapter.ts

@@ -2,8 +2,9 @@
 // Copyright 2024 DXOS.org
 //
 
+import { NetworkAdapter, type Message, type PeerId, type PeerMetadata } from '@automerge/automerge-repo';
+
 import { synchronized, Trigger } from '@dxos/async';
-import { NetworkAdapter, type Message, type PeerId, type PeerMetadata } from '@dxos/automerge/automerge-repo';
 import { LifecycleState } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
@@ -53,11 +54,20 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   private readonly _connections = new Map<PeerId, ConnectionEntry>();
   private _lifecycleState: LifecycleState = LifecycleState.CLOSED;
   private readonly _connected = new Trigger();
+  private readonly _ready = new Trigger();
 
   constructor(private readonly _params: EchoNetworkAdapterParams) {
     super();
   }
 
+  override isReady(): boolean {
+    return this._lifecycleState === LifecycleState.OPEN;
+  }
+
+  override whenReady(): Promise<void> {
+    return this._ready.wait();
+  }
+
   override connect(peerId: PeerId, peerMetadata?: PeerMetadata | undefined): void {
     this.peerId = peerId;
     this.peerMetadata = peerMetadata;
@@ -73,20 +83,16 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   }
 
   @synchronized
-  async open() {
+  async open(): Promise<void> {
     if (this._lifecycleState === LifecycleState.OPEN) {
       return;
     }
     this._lifecycleState = LifecycleState.OPEN;
-
-    log('emit ready');
-    this.emit('ready', {
-      network: this,
-    });
+    this._ready.wake();
   }
 
   @synchronized
-  async close() {
+  async close(): Promise<this | undefined> {
     if (this._lifecycleState === LifecycleState.CLOSED) {
       return this;
     }
@@ -96,15 +102,16 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     }
     this._replicators.clear();
 
+    this._ready.reset();
     this._lifecycleState = LifecycleState.CLOSED;
   }
 
-  async whenConnected() {
+  async whenConnected(): Promise<void> {
     await this._connected.wait({ timeout: 10_000 });
   }
 
   @synchronized
-  async addReplicator(replicator: EchoReplicator) {
+  async addReplicator(replicator: EchoReplicator): Promise<void> {
     invariant(this._lifecycleState === LifecycleState.OPEN);
     invariant(this.peerId);
     invariant(!this._replicators.has(replicator));
@@ -125,7 +132,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   }
 
   @synchronized
-  async removeReplicator(replicator: EchoReplicator) {
+  async removeReplicator(replicator: EchoReplicator): Promise<void> {
     invariant(this._lifecycleState === LifecycleState.OPEN);
     invariant(this._replicators.has(replicator));
     await replicator.disconnect();
@@ -171,7 +178,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._send(message);
   }
 
-  private _send(message: Message) {
+  private _send(message: Message): void {
     const connectionEntry = this._connections.get(message.targetId);
     if (!connectionEntry) {
       throw new Error('Connection not found.');
@@ -204,7 +211,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
       .filter(isNonNullable);
   }
 
-  private _onConnectionOpen(connection: ReplicatorConnection) {
+  private _onConnectionOpen(connection: ReplicatorConnection): void {
     log('Connection opened', { peerId: connection.peerId });
     invariant(!this._connections.has(connection.peerId as PeerId));
     const reader = connection.readable.getReader();
@@ -235,7 +242,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._params.monitor?.recordPeerConnected(connection.peerId);
   }
 
-  private _onMessage(message: Message) {
+  private _onMessage(message: Message): void {
     if (isCollectionQueryMessage(message)) {
       this._params.onCollectionStateQueried(message.collectionId, message.senderId);
     } else if (isCollectionStateMessage(message)) {
@@ -246,7 +253,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._params.monitor?.recordMessageReceived(message);
   }
 
-  public onConnectionAuthScopeChanged(peer: PeerId) {
+  public onConnectionAuthScopeChanged(peer: PeerId): void {
     const entry = this._connections.get(peer);
     if (entry) {
       this._onConnectionAuthScopeChanged(entry.connection);
@@ -257,7 +264,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
    * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
    * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
    */
-  private _onConnectionAuthScopeChanged(connection: ReplicatorConnection) {
+  private _onConnectionAuthScopeChanged(connection: ReplicatorConnection): void {
     log('Connection auth scope changed', { peerId: connection.peerId });
     const entry = this._connections.get(connection.peerId as PeerId);
     invariant(entry);
@@ -265,7 +272,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._emitPeerCandidate(connection);
   }
 
-  private _onConnectionClosed(connection: ReplicatorConnection) {
+  private _onConnectionClosed(connection: ReplicatorConnection): void {
     log('Connection closed', { peerId: connection.peerId });
     const entry = this._connections.get(connection.peerId as PeerId);
     invariant(entry);
@@ -280,7 +287,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._connections.delete(connection.peerId as PeerId);
   }
 
-  private _emitPeerCandidate(connection: ReplicatorConnection) {
+  private _emitPeerCandidate(connection: ReplicatorConnection): void {
     this.emit('peer-candidate', {
       peerId: connection.peerId as PeerId,
       peerMetadata: createEchoPeerMetadata(),

package/src/automerge/heads-store.ts

@@ -2,8 +2,9 @@
 // Copyright 2024 DXOS.org
 //
 
-import type { Heads } from '@dxos/automerge/automerge';
-import type { DocumentId } from '@dxos/automerge/automerge-repo';
+import type { Heads } from '@automerge/automerge';
+import type { DocumentId } from '@automerge/automerge-repo';
+
 import { headsEncoding } from '@dxos/indexing';
 import type { BatchLevel, SublevelDB } from '@dxos/kv-store';
 
@@ -18,7 +19,7 @@ export class HeadsStore {
     this._db = db;
   }
 
-  setHeads(documentId: DocumentId, heads: Heads, batch: BatchLevel) {
+  setHeads(documentId: DocumentId, heads: Heads, batch: BatchLevel): void {
     batch.put<DocumentId, Heads>(documentId, heads, {
       sublevel: this._db,
       keyEncoding: 'utf8',

package/src/automerge/leveldb-storage-adapter.ts

@@ -2,9 +2,9 @@
 // Copyright 2024 DXOS.org
 // s
 
+import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@automerge/automerge-repo';
 import { type MixedEncoding } from 'level-transcoder';
 
-import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type BatchLevel, type SublevelDB } from '@dxos/kv-store';
 import { type MaybePromise } from '@dxos/util';

package/src/automerge/mesh-echo-replicator-connection.ts

@@ -2,8 +2,9 @@
 // Copyright 2024 DXOS.org
 //
 
-import * as A from '@dxos/automerge/automerge';
-import { cbor } from '@dxos/automerge/automerge-repo';
+import * as A from '@automerge/automerge';
+import { cbor } from '@automerge/automerge-repo';
+
 import { Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
@@ -95,7 +96,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
     ]);
   }
 
-  private _disconnectIfEnabled() {
+  private _disconnectIfEnabled(): void {
     if (this._isEnabled) {
       this._params.onRemoteDisconnected();
     }
@@ -122,7 +123,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
    * Start exchanging messages with the remote peer.
    * Call after the remote peer has connected.
    */
-  enable() {
+  enable(): void {
     invariant(this._remotePeerId != null, 'Remote peer has not connected yet.');
     this._isEnabled = true;
   }
@@ -130,7 +131,7 @@ export class MeshReplicatorConnection extends Resource implements ReplicatorConn
   /**
    * Stop exchanging messages with the remote peer.
    */
-  disable() {
+  disable(): void {
     this._isEnabled = false;
   }
 }

package/src/automerge/mesh-echo-replicator.ts

@@ -46,7 +46,7 @@ export class MeshEchoReplicator implements EchoReplicator {
     this._context = context;
   }
 
-  async disconnect() {
+  async disconnect(): Promise<void> {
     for (const connection of this._connections) {
       if (connection.isEnabled) {
         this._context?.onConnectionClosed(connection);
@@ -180,7 +180,7 @@ export class MeshEchoReplicator implements EchoReplicator {
     return connection.replicatorExtension;
   }
 
-  async authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey) {
+  async authorizeDevice(spaceKey: PublicKey, deviceKey: PublicKey): Promise<void> {
     log('authorizeDevice', { spaceKey, deviceKey });
     const spaceId = await createIdFromSpaceKey(spaceKey);
     defaultMap(this._authorizedDevices, spaceId, () => new ComplexSet(PublicKey.hash)).add(deviceKey);

package/src/automerge/network-protocol.ts

@@ -2,7 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-import type { Message } from '@dxos/automerge/automerge-repo';
+import type { Message } from '@automerge/automerge-repo';
+
 import {
   type CollectionQueryMessage,
   type CollectionStateMessage,

package/src/automerge/space-collection.ts

@@ -2,7 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-import { type DocumentId } from '@dxos/automerge/automerge-repo';
+import { type DocumentId } from '@automerge/automerge-repo';
+
 import type { CollectionId } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { SpaceId } from '@dxos/keys';

package/src/db-host/automerge-metrics.ts

@@ -2,7 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-import * as A from '@dxos/automerge/automerge';
+import * as A from '@automerge/automerge';
+
 import { log } from '@dxos/log';
 
 export type DocMetrics = {

package/src/db-host/data-service.ts

@@ -2,8 +2,9 @@
 // Copyright 2021 DXOS.org
 //
 
+import { type DocumentId } from '@automerge/automerge-repo';
+
 import { UpdateScheduler } from '@dxos/async';
-import { type DocumentId } from '@dxos/automerge/automerge-repo';
 import { type RequestOptions } from '@dxos/codec-protobuf';
 import { Stream } from '@dxos/codec-protobuf/stream';
 import { invariant } from '@dxos/invariant';
@@ -72,7 +73,7 @@ export class DataServiceImpl implements DataService {
     });
   }
 
-  async updateSubscription(request: UpdateSubscriptionRequest) {
+  async updateSubscription(request: UpdateSubscriptionRequest): Promise<void> {
     const synchronizer = this._subscriptions.get(request.subscriptionId);
     invariant(synchronizer, 'Subscription not found');
 
@@ -122,7 +123,7 @@ export class DataServiceImpl implements DataService {
     await this._automergeHost.reIndexHeads((request.documentIds ?? []) as DocumentId[]);
   }
 
-  async updateIndexes() {
+  async updateIndexes(): Promise<void> {
     await this._updateIndexes();
   }
 

package/src/db-host/database-root.ts

@@ -2,21 +2,16 @@
 // Copyright 2024 DXOS.org
 //
 
-import type * as A from '@dxos/automerge/automerge';
-import {
-  interpretAsDocumentId,
-  type AutomergeUrl,
-  type DocHandle,
-  type DocumentId,
-} from '@dxos/automerge/automerge-repo';
-import { type SpaceDoc, SpaceDocVersion } from '@dxos/echo-protocol';
+import type * as A from '@automerge/automerge';
+import { interpretAsDocumentId, type AutomergeUrl, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
+
+import { DatabaseDirectory, SpaceDocVersion } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 
 import { measureDocMetrics, type DocMetrics } from './automerge-metrics';
-import { getSpaceKeyFromDoc } from '../automerge';
 
 export class DatabaseRoot {
-  static mapLinks(doc: DocHandle<SpaceDoc>, mapping: Record<DocumentId, DocumentId>): void {
+  static mapLinks(doc: DocHandle<DatabaseDirectory>, mapping: Record<DocumentId, DocumentId>): void {
     doc.change((d) => {
       if (!d.links) {
         return;
@@ -30,7 +25,7 @@ export class DatabaseRoot {
     });
   }
 
-  constructor(private readonly _rootHandle: DocHandle<SpaceDoc>) {}
+  constructor(private readonly _rootHandle: DocHandle<DatabaseDirectory>) {}
 
   get documentId(): DocumentId {
     return this._rootHandle.documentId;
@@ -41,19 +36,19 @@ export class DatabaseRoot {
   }
 
   get isLoaded(): boolean {
-    return !!this._rootHandle.docSync();
+    return this._rootHandle.isReady();
   }
 
-  get handle(): DocHandle<SpaceDoc> {
+  get handle(): DocHandle<DatabaseDirectory> {
     return this._rootHandle;
   }
 
-  docSync(): A.Doc<SpaceDoc> | null {
-    return this._rootHandle.docSync() ?? null;
+  doc(): A.Doc<DatabaseDirectory> | null {
+    return this._rootHandle.isReady() ? this._rootHandle.doc() : null;
   }
 
   getVersion(): SpaceDocVersion | null {
-    const doc = this.docSync();
+    const doc = this.doc();
     if (!doc) {
       return null;
     }
@@ -62,16 +57,16 @@ export class DatabaseRoot {
   }
 
   getSpaceKey(): string | null {
-    const doc = this.docSync();
+    const doc = this.doc();
     if (!doc) {
       return null;
     }
 
-    return getSpaceKeyFromDoc(doc);
+    return DatabaseDirectory.getSpaceKey(doc);
   }
 
   getInlineObjectCount(): number | null {
-    const doc = this.docSync();
+    const doc = this.doc();
     if (!doc) {
       return null;
     }
@@ -80,7 +75,7 @@ export class DatabaseRoot {
   }
 
   getLinkedObjectCount(): number | null {
-    const doc = this.docSync();
+    const doc = this.doc();
     if (!doc) {
       return null;
     }
@@ -89,7 +84,7 @@ export class DatabaseRoot {
   }
 
   getAllLinkedDocuments(): AutomergeUrl[] {
-    const doc = this.docSync();
+    const doc = this.doc();
     invariant(doc);
 
     // .toString() to handle RawString.
@@ -97,7 +92,7 @@ export class DatabaseRoot {
   }
 
   measureMetrics(): DocMetrics | null {
-    const doc = this.docSync();
+    const doc = this.doc();
     if (!doc) {
       return null;
     }

package/src/db-host/documents-iterator.ts

@@ -2,16 +2,17 @@
 // Copyright 2024 DXOS.org
 //
 
-import * as A from '@dxos/automerge/automerge';
-import { type DocumentId } from '@dxos/automerge/automerge-repo';
+import * as A from '@automerge/automerge';
+import { type DocumentId } from '@automerge/automerge-repo';
+
 import { Context } from '@dxos/context';
-import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
-import { type ObjectSnapshot, type IdToHeads } from '@dxos/indexing';
+import { DatabaseDirectory, SpaceDocVersion } from '@dxos/echo-protocol';
+import { type IdToHeads, type ObjectSnapshot } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ObjectPointerVersion, objectPointerCodec } from '@dxos/protocols';
 
-import { type AutomergeHost, getSpaceKeyFromDoc } from '../automerge';
+import { type AutomergeHost } from '../automerge';
 
 const LOG_VIEW_OPERATION_THRESHOLD = 300;
 
@@ -27,9 +28,9 @@ export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
     for (const [id, heads] of objects.entries()) {
       try {
         const { documentId, objectId } = objectPointerCodec.decode(id);
-        const handle = await automergeHost.loadDoc<SpaceDoc>(Context.default(), documentId as DocumentId);
+        const handle = await automergeHost.loadDoc<DatabaseDirectory>(Context.default(), documentId as DocumentId);
 
-        let doc = handle.docSync();
+        let doc = handle.doc();
         invariant(doc);
 
         const currentHeads = A.getHeads(doc);
@@ -61,7 +62,7 @@ export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
         // Upgrade V0 object pointers to V1.
         let newId = id;
         if (objectPointerCodec.getVersion(id) === ObjectPointerVersion.V0) {
-          const spaceKey = getSpaceKeyFromDoc(doc) ?? undefined;
+          const spaceKey = DatabaseDirectory.getSpaceKey(doc) ?? undefined;
           newId = objectPointerCodec.encode({ documentId, objectId, spaceKey });
         }
 

package/src/db-host/documents-synchronizer.test.ts

@@ -2,11 +2,11 @@
 // Copyright 2024 DXOS.org
 //
 
+import { next as A } from '@automerge/automerge';
+import { generateAutomergeUrl, parseAutomergeUrl, Repo } from '@automerge/automerge-repo';
 import { describe, expect, test } from 'vitest';
 
 import { sleep } from '@dxos/async';
-import { next as A } from '@dxos/automerge/automerge';
-import { generateAutomergeUrl, parseAutomergeUrl, Repo } from '@dxos/automerge/automerge-repo';
 import { openAndClose } from '@dxos/test-utils';
 
 import { DocumentsSynchronizer } from './documents-synchronizer';

package/src/db-host/documents-synchronizer.ts

@@ -2,11 +2,12 @@
 // Copyright 2024 DXOS.org
 //
 
+import { next as A, type Heads } from '@automerge/automerge';
+import { type Repo, type DocHandle, type DocumentId } from '@automerge/automerge-repo';
+
 import { UpdateScheduler } from '@dxos/async';
-import { next as A, type Heads } from '@dxos/automerge/automerge';
-import { type Repo, type DocHandle, type DocumentId } from '@dxos/automerge/automerge-repo';
 import { Resource } from '@dxos/context';
-import { type SpaceDoc } from '@dxos/echo-protocol';
+import { type DatabaseDirectory } from '@dxos/echo-protocol';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { type BatchedDocumentUpdates, type DocumentUpdate } from '@dxos/protocols/proto/dxos/echo/service';
@@ -19,7 +20,7 @@ export type DocumentsSynchronizerParams = {
 };
 
 interface DocSyncState {
-  handle: DocHandle<SpaceDoc>;
+  handle: DocHandle<DatabaseDirectory>;
   lastSentHead?: Heads;
   clearSubscriptions?: () => void;
 }
@@ -44,17 +45,17 @@ export class DocumentsSynchronizer extends Resource {
     super();
   }
 
-  addDocuments(documentIds: DocumentId[], retryCounter = 0) {
+  addDocuments(documentIds: DocumentId[], retryCounter = 0): void {
     if (retryCounter > 3) {
       log.warn('Failed to load document, retry limit reached', { documentIds });
       return;
     }
 
     for (const documentId of documentIds) {
-      const doc = this._params.repo.find<SpaceDoc>(documentId as DocumentId);
-      doc
-        .whenReady()
-        .then(() => {
+      this._params.repo
+        .find<DatabaseDirectory>(documentId as DocumentId)
+        .then(async (doc) => {
+          await doc.whenReady();
           this._startSync(doc);
           this._pendingUpdates.add(doc.documentId);
           this._sendUpdatesJob!.trigger();
@@ -66,7 +67,7 @@ export class DocumentsSynchronizer extends Resource {
     }
   }
 
-  removeDocuments(documentIds: DocumentId[]) {
+  removeDocuments(documentIds: DocumentId[]): void {
     for (const documentId of documentIds) {
       this._syncStates.get(documentId)?.clearSubscriptions?.();
       this._syncStates.delete(documentId);
@@ -85,10 +86,10 @@ export class DocumentsSynchronizer extends Resource {
     this._syncStates.clear();
   }
 
-  update(updates: DocumentUpdate[]) {
+  update(updates: DocumentUpdate[]): void {
     for (const { documentId, mutation, isNew } of updates) {
       if (isNew) {
-        const doc = this._params.repo.find<SpaceDoc>(documentId as DocumentId);
+        const { handle: doc } = this._params.repo.findWithProgress<DatabaseDirectory>(documentId as DocumentId);
         doc.update((doc) => A.loadIncremental(doc, mutation));
         this._startSync(doc);
       } else {
@@ -97,7 +98,7 @@ export class DocumentsSynchronizer extends Resource {
     }
   }
 
-  private _startSync(doc: DocHandle<SpaceDoc>) {
+  private _startSync(doc: DocHandle<DatabaseDirectory>): void {
     if (this._syncStates.has(doc.documentId)) {
       log.info('Document already being synced', { documentId: doc.documentId });
       return;
@@ -108,7 +109,7 @@ export class DocumentsSynchronizer extends Resource {
     this._syncStates.set(doc.documentId, syncState);
   }
 
-  _subscribeForChanges(syncState: DocSyncState) {
+  _subscribeForChanges(syncState: DocSyncState): void {
     const handler = () => {
       this._pendingUpdates.add(syncState.handle.documentId);
       this._sendUpdatesJob!.trigger();
@@ -117,7 +118,7 @@ export class DocumentsSynchronizer extends Resource {
     syncState.clearSubscriptions = () => syncState.handle.off('heads-changed', handler);
   }
 
-  private async _checkAndSendUpdates() {
+  private async _checkAndSendUpdates(): Promise<void> {
     const updates: DocumentUpdate[] = [];
 
     const docsWithPendingUpdates = Array.from(this._pendingUpdates);
@@ -141,10 +142,11 @@ export class DocumentsSynchronizer extends Resource {
   private _getPendingChanges(documentId: DocumentId): Uint8Array | void {
     const syncState = this._syncStates.get(documentId);
     invariant(syncState, 'Sync state for document not found');
-    const doc = syncState.handle.docSync();
-    if (!doc) {
+    const handle = syncState.handle;
+    if (!handle || !handle.isReady() || !handle.doc()) {
       return;
     }
+    const doc = handle.doc();
     const mutation = syncState.lastSentHead ? A.saveSince(doc, syncState.lastSentHead) : A.save(doc);
     if (mutation.length === 0) {
       return;
@@ -153,7 +155,7 @@ export class DocumentsSynchronizer extends Resource {
     return mutation;
   }
 
-  private _writeMutation(documentId: DocumentId, mutation: Uint8Array) {
+  private _writeMutation(documentId: DocumentId, mutation: Uint8Array): void {
     const syncState = this._syncStates.get(documentId);
     invariant(syncState, 'Sync state for document not found');
     syncState.handle.update((doc) => {