@dxos/echo-pipeline 0.8.2 → 0.8.3-main.7f5a14c

package/src/automerge/echo-network-adapter.ts

@@ -43,6 +43,13 @@ export type EchoNetworkAdapterParams = {
   monitor?: NetworkDataMonitor;
 };
 
+type ConnectionEntry = {
+  isOpen: boolean;
+  connection: ReplicatorConnection;
+  reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
+  writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
+};
+
 /**
  * Manages a set of {@link EchoReplicator} instances.
  */
@@ -110,6 +117,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     await this._connected.wait({ timeout: 10_000 });
   }
 
+  public onConnectionAuthScopeChanged(peer: PeerId): void {
+    const entry = this._connections.get(peer);
+    if (entry) {
+      this._onConnectionAuthScopeChanged(entry.connection);
+    }
+  }
+
   @synchronized
   async addReplicator(replicator: EchoReplicator): Promise<void> {
     invariant(this._lifecycleState === LifecycleState.OPEN);
@@ -178,52 +192,57 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._send(message);
   }
 
+  // TODO(dmaretskyi): Remove.
+  getPeersInterestedInCollection(collectionId: string): PeerId[] {
+    return Array.from(this._connections.values())
+      .map((connection) => {
+        return connection.connection.shouldSyncCollection({ collectionId })
+          ? (connection.connection.peerId as PeerId)
+          : null;
+      })
+      .filter(isNonNullable);
+  }
+
   private _send(message: Message): void {
     const connectionEntry = this._connections.get(message.targetId);
     if (!connectionEntry) {
       throw new Error('Connection not found.');
     }
 
-    const writeStart = Date.now();
     // TODO(dmaretskyi): Find a way to enforce backpressure on AM-repo.
+    const start = Date.now();
     connectionEntry.writer
       .write(message as AutomergeProtocolMessage)
       .then(() => {
-        const durationMs = Date.now() - writeStart;
-        this._params.monitor?.recordMessageSent(message, durationMs);
+        this._params.monitor?.recordMessageSent(message, Date.now() - start);
       })
       .catch((err) => {
         if (connectionEntry.isOpen) {
           log.catch(err);
         }
+
         this._params.monitor?.recordMessageSendingFailed(message);
       });
   }
 
-  // TODO(dmaretskyi): Remove.
-  getPeersInterestedInCollection(collectionId: string): PeerId[] {
-    return Array.from(this._connections.values())
-      .map((connection) => {
-        return connection.connection.shouldSyncCollection({ collectionId })
-          ? (connection.connection.peerId as PeerId)
-          : null;
-      })
-      .filter(isNonNullable);
-  }
-
   private _onConnectionOpen(connection: ReplicatorConnection): void {
-    log('Connection opened', { peerId: connection.peerId });
+    log('connection opened', { peerId: connection.peerId });
     invariant(!this._connections.has(connection.peerId as PeerId));
-    const reader = connection.readable.getReader();
-    const writer = connection.writable.getWriter();
-    const connectionEntry: ConnectionEntry = { connection, reader, writer, isOpen: true };
+    const connectionEntry: ConnectionEntry = {
+      isOpen: true,
+      connection,
+      reader: connection.readable.getReader(),
+      writer: connection.writable.getWriter(),
+    };
+
     this._connections.set(connection.peerId as PeerId, connectionEntry);
 
+    // Read inbound messages.
     queueMicrotask(async () => {
       try {
         while (true) {
           // TODO(dmaretskyi): Find a way to enforce backpressure on AM-repo.
-          const { done, value } = await reader.read();
+          const { done, value } = await connectionEntry.reader.read();
           if (done) {
             break;
           }
@@ -253,11 +272,18 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._params.monitor?.recordMessageReceived(message);
   }
 
-  public onConnectionAuthScopeChanged(peer: PeerId): void {
-    const entry = this._connections.get(peer);
-    if (entry) {
-      this._onConnectionAuthScopeChanged(entry.connection);
-    }
+  private _onConnectionClosed(connection: ReplicatorConnection): void {
+    log('connection closed', { peerId: connection.peerId });
+    const entry = this._connections.get(connection.peerId as PeerId);
+    invariant(entry);
+
+    entry.isOpen = false;
+    this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
+    this._params.monitor?.recordPeerDisconnected(connection.peerId);
+
+    void entry.reader.cancel().catch((err) => log.catch(err));
+    void entry.writer.abort().catch((err) => log.catch(err));
+    this._connections.delete(connection.peerId as PeerId);
   }
 
   /**
@@ -272,21 +298,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     this._emitPeerCandidate(connection);
   }
 
-  private _onConnectionClosed(connection: ReplicatorConnection): void {
-    log('Connection closed', { peerId: connection.peerId });
-    const entry = this._connections.get(connection.peerId as PeerId);
-    invariant(entry);
-
-    entry.isOpen = false;
-    this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
-    this._params.monitor?.recordPeerDisconnected(connection.peerId);
-
-    void entry.reader.cancel().catch((err) => log.catch(err));
-    void entry.writer.abort().catch((err) => log.catch(err));
-
-    this._connections.delete(connection.peerId as PeerId);
-  }
-
   private _emitPeerCandidate(connection: ReplicatorConnection): void {
     this.emit('peer-candidate', {
       peerId: connection.peerId as PeerId,
@@ -295,13 +306,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   }
 }
 
-type ConnectionEntry = {
-  connection: ReplicatorConnection;
-  reader: ReadableStreamDefaultReader<AutomergeProtocolMessage>;
-  writer: WritableStreamDefaultWriter<AutomergeProtocolMessage>;
-  isOpen: boolean;
-};
-
 export const createEchoPeerMetadata = (): PeerMetadata =>
   ({
     // TODO(dmaretskyi): Refactor this.

package/src/automerge/mesh-echo-replicator.ts

@@ -125,6 +125,7 @@ export class MeshEchoReplicator implements EchoReplicator {
               documentId: params.documentId,
               acceptDocument: remoteDocumentExists,
             });
+
             // If a document is not present locally return true if another peer claims to have it.
             // Simply returning true will add the peer to "generous peers list" for this document which will
             // start replication of the document after we receive, even if the peer is not in the corresponding space.

package/src/db-host/data-service.ts

@@ -92,7 +92,7 @@ export class DataServiceImpl implements DataService {
     const synchronizer = this._subscriptions.get(request.subscriptionId);
     invariant(synchronizer, 'Subscription not found');
 
-    synchronizer.update(request.updates);
+    await synchronizer.update(request.updates);
   }
 
   async flush(request: FlushRequest): Promise<void> {

package/src/db-host/documents-iterator.ts

@@ -42,7 +42,7 @@ export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
           doc = A.view(doc, heads);
           const end = Date.now();
           if (end - begin > LOG_VIEW_OPERATION_THRESHOLD) {
-            log.info('Checking out document version is taking too long', {
+            log('Checking out document version is taking too long', {
               duration: end - begin,
               requestedHeads: heads,
               originalHeads: currentHeads,

package/src/db-host/documents-synchronizer.test.ts

@@ -23,7 +23,7 @@ describe('DocumentsSynchronizer', () => {
     });
     await openAndClose(synchronizer);
 
-    synchronizer.update([
+    await synchronizer.update([
       {
         documentId: parseAutomergeUrl(generateAutomergeUrl()).documentId,
         mutation: A.save(A.from({ text: 'hello' })),

package/src/db-host/documents-synchronizer.ts

@@ -12,6 +12,8 @@ import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { type BatchedDocumentUpdates, type DocumentUpdate } from '@dxos/protocols/proto/dxos/echo/service';
 
+import { FIND_PARAMS } from '../automerge';
+
 const MAX_UPDATE_FREQ = 10; // [updates/sec]
 
 export type DocumentsSynchronizerParams = {
@@ -86,10 +88,10 @@ export class DocumentsSynchronizer extends Resource {
     this._syncStates.clear();
   }
 
-  update(updates: DocumentUpdate[]): void {
+  async update(updates: DocumentUpdate[]): Promise<void> {
     for (const { documentId, mutation, isNew } of updates) {
       if (isNew) {
-        const { handle: doc } = this._params.repo.findWithProgress<DatabaseDirectory>(documentId as DocumentId);
+        const doc = await this._params.repo.find<DatabaseDirectory>(documentId as DocumentId, FIND_PARAMS);
         doc.update((doc) => A.loadIncremental(doc, mutation));
         this._startSync(doc);
       } else {
@@ -100,7 +102,7 @@ export class DocumentsSynchronizer extends Resource {
 
   private _startSync(doc: DocHandle<DatabaseDirectory>): void {
     if (this._syncStates.has(doc.documentId)) {
-      log.info('Document already being synced', { documentId: doc.documentId });
+      log('Document already being synced', { documentId: doc.documentId });
       return;
     }
 

package/src/db-host/query-service.ts

@@ -138,7 +138,7 @@ export class QueryServiceImpl extends Resource implements QueryService {
    * Re-index all loaded documents.
    */
   async reindex(): Promise<void> {
-    log.info('Reindexing all documents...');
+    log('Reindexing all documents...');
     const iterator = createDocumentsIterator(this._params.automergeHost);
     const ids: IdToHeads = new Map();
     for await (const documents of iterator()) {
@@ -146,11 +146,11 @@ export class QueryServiceImpl extends Resource implements QueryService {
         ids.set(id, heads);
       }
       if (ids.size % 100 === 0) {
-        log.info('Collected documents...', { count: ids.size });
+        log('Collected documents...', { count: ids.size });
       }
     }
 
-    log.info('Marking all documents as dirty...', { count: ids.size });
+    log('Marking all documents as dirty...', { count: ids.size });
     await this._params.indexer.reindex(ids);
   }
 }

package/src/edge/echo-edge-replicator.ts

@@ -58,9 +58,8 @@ export class EchoEdgeReplicator implements EchoReplicator {
   }
 
   async connect(context: EchoReplicatorContext): Promise<void> {
-    log.info('connecting...', { peerId: context.peerId, connectedSpaces: this._connectedSpaces.size });
+    log('connecting...', { peerId: context.peerId, connectedSpaces: this._connectedSpaces.size });
     this._context = context;
-
     this._ctx = Context.default();
     this._ctx.onDispose(
       this._edgeConnection.onReconnected(() => {
@@ -146,7 +145,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
         const restartDelay =
           Math.min(MAX_RESTART_DELAY, INITIAL_RESTART_DELAY * reconnects) + Math.random() * RESTART_DELAY_JITTER;
 
-        log.info('connection restart scheduled', { spaceId, reconnects, restartDelay });
+        log('connection restart scheduled', { spaceId, reconnects, restartDelay });
 
         restartScheduled = true;
         scheduleTask(

package/src/pipeline/pipeline.ts

@@ -409,7 +409,7 @@ export class Pipeline implements PipelineAccessor {
       if (err) {
         // log.warn(err); // TODO(burdon): Feed is closed/Download was cancelled.
       } else {
-        log.info('downloaded', { data }); // TODO(burdon): Never called.
+        log('downloaded', { data }); // TODO(burdon): Never called.
       }
     });
 

package/src/query/query-executor.ts

@@ -97,6 +97,8 @@ type StepExecutionResult = {
   trace: ExecutionTrace;
 };
 
+const TRACE_QUERY_EXECUTION = false;
+
 /**
  * Executes query plans against the Indexer and AutomergeHost.
  *
@@ -186,12 +188,10 @@ export class QueryExecutor extends Resource {
           workingSet[index].documentId !== item.documentId,
       );
 
-    // log.info('Query execution result', {
-    //   changed,
-    //   trace: ExecutionTrace.format(trace),
-    // });
-    // eslint-disable-next-line no-console
-    // console.log(ExecutionTrace.format(trace));
+    if (TRACE_QUERY_EXECUTION) {
+      // eslint-disable-next-line no-console
+      console.log(ExecutionTrace.format(trace));
+    }
 
     return {
       changed,

package/src/testing/change-metadata.ts

@@ -16,7 +16,7 @@ const EchoMetadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata');
  * Use this only for testing purposes.
  */
 export const changeStorageVersionInMetadata = async (storage: Storage, version: number) => {
-  log.info('Changing storage version in metadata. USE ONLY FOR TESTING.');
+  log('Changing storage version in metadata. USE ONLY FOR TESTING.');
   const metadata = new MetadataStore(storage.createDirectory('metadata'));
   await metadata.load();
   const echoMetadata = metadata.metadata;

package/src/testing/test-replicator.ts

@@ -83,7 +83,7 @@ export class TestReplicationNetwork extends Resource {
     const forward = new TransformStream({
       transform: async (message, controller) => {
         if (LOG) {
-          log.info('replicate', { from: peer1, to: peer2, message });
+          log('replicate', { from: peer1, to: peer2, message });
         }
 
         if (this._latency !== undefined) {
@@ -96,7 +96,7 @@ export class TestReplicationNetwork extends Resource {
     const backwards = new TransformStream({
       transform: async (message, controller) => {
         if (LOG) {
-          log.info('replicate', { from: peer2, to: peer1, message });
+          log('replicate', { from: peer2, to: peer1, message });
         }
 
         if (this._latency !== undefined) {