@dxos/echo-pipeline 0.7.2 → 0.7.3-staging.0905f03

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.7.2",
+  "version": "0.7.3-staging.0905f03",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -37,43 +37,44 @@
     "crc-32": "^1.2.2",
     "level-transcoder": "^1.0.1",
     "lodash.isequal": "^4.5.0",
-    "@dxos/async": "0.7.2",
-    "@dxos/automerge": "0.7.2",
-    "@dxos/codec-protobuf": "0.7.2",
-    "@dxos/crypto": "0.7.2",
-    "@dxos/credentials": "0.7.2",
-    "@dxos/debug": "0.7.2",
-    "@dxos/context": "0.7.2",
-    "@dxos/echo-protocol": "0.7.2",
-    "@dxos/echo-schema": "0.7.2",
-    "@dxos/edge-client": "0.7.2",
-    "@dxos/feed-store": "0.7.2",
-    "@dxos/hypercore": "0.7.2",
-    "@dxos/indexing": "0.7.2",
-    "@dxos/invariant": "0.7.2",
-    "@dxos/keyring": "0.7.2",
-    "@dxos/kv-store": "0.7.2",
-    "@dxos/log": "0.7.2",
-    "@dxos/keys": "0.7.2",
-    "@dxos/network-manager": "0.7.2",
-    "@dxos/messaging": "0.7.2",
-    "@dxos/node-std": "0.7.2",
-    "@dxos/protocols": "0.7.2",
-    "@dxos/random-access-storage": "0.7.2",
-    "@dxos/teleport-extension-automerge-replicator": "0.7.2",
-    "@dxos/teleport": "0.7.2",
-    "@dxos/teleport-extension-gossip": "0.7.2",
-    "@dxos/teleport-extension-object-sync": "0.7.2",
-    "@dxos/teleport-extension-replicator": "0.7.2",
-    "@dxos/timeframe": "0.7.2",
-    "@dxos/tracing": "0.7.2",
-    "@dxos/typings": "0.7.2",
-    "@dxos/util": "0.7.2"
+    "@dxos/async": "0.7.3-staging.0905f03",
+    "@dxos/codec-protobuf": "0.7.3-staging.0905f03",
+    "@dxos/automerge": "0.7.3-staging.0905f03",
+    "@dxos/context": "0.7.3-staging.0905f03",
+    "@dxos/credentials": "0.7.3-staging.0905f03",
+    "@dxos/crypto": "0.7.3-staging.0905f03",
+    "@dxos/debug": "0.7.3-staging.0905f03",
+    "@dxos/echo-protocol": "0.7.3-staging.0905f03",
+    "@dxos/echo-schema": "0.7.3-staging.0905f03",
+    "@dxos/edge-client": "0.7.3-staging.0905f03",
+    "@dxos/feed-store": "0.7.3-staging.0905f03",
+    "@dxos/indexing": "0.7.3-staging.0905f03",
+    "@dxos/hypercore": "0.7.3-staging.0905f03",
+    "@dxos/keyring": "0.7.3-staging.0905f03",
+    "@dxos/invariant": "0.7.3-staging.0905f03",
+    "@dxos/kv-store": "0.7.3-staging.0905f03",
+    "@dxos/log": "0.7.3-staging.0905f03",
+    "@dxos/messaging": "0.7.3-staging.0905f03",
+    "@dxos/keys": "0.7.3-staging.0905f03",
+    "@dxos/network-manager": "0.7.3-staging.0905f03",
+    "@dxos/node-std": "0.7.3-staging.0905f03",
+    "@dxos/random-access-storage": "0.7.3-staging.0905f03",
+    "@dxos/protocols": "0.7.3-staging.0905f03",
+    "@dxos/teleport": "0.7.3-staging.0905f03",
+    "@dxos/teleport-extension-automerge-replicator": "0.7.3-staging.0905f03",
+    "@dxos/teleport-extension-gossip": "0.7.3-staging.0905f03",
+    "@dxos/teleport-extension-object-sync": "0.7.3-staging.0905f03",
+    "@dxos/tracing": "0.7.3-staging.0905f03",
+    "@dxos/timeframe": "0.7.3-staging.0905f03",
+    "@dxos/teleport-extension-replicator": "0.7.3-staging.0905f03",
+    "@dxos/typings": "0.7.3-staging.0905f03",
+    "@dxos/util": "0.7.3-staging.0905f03"
   },
   "devDependencies": {
     "@types/lodash.isequal": "^4.5.0",
     "fast-check": "^3.19.0",
-    "@dxos/test-utils": "0.7.2"
+    "get-port-please": "^3.1.1",
+    "@dxos/test-utils": "0.7.3-staging.0905f03"
   },
   "publishConfig": {
     "access": "public"

package/src/automerge/automerge-host.ts

@@ -42,11 +42,18 @@ import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './
 import { HeadsStore } from './heads-store';
 import { LevelDBStorageAdapter, type BeforeSaveParams } from './leveldb-storage-adapter';
 
+export type PeerIdProvider = () => string | undefined;
+
 export type AutomergeHostParams = {
   db: LevelDB;
 
   indexMetadataStore: IndexMetadataStore;
   dataMonitor?: EchoDataMonitor;
+
+  /**
+   * Used for creating stable ids. A random key is generated on open, if no value is provided.
+   */
+  peerIdProvider?: PeerIdProvider;
 };
 
 export type LoadDocOptions = {
@@ -82,9 +89,11 @@ export class AutomergeHost extends Resource {
   @trace.info()
   private _peerId!: PeerId;
 
+  private readonly _peerIdProvider?: PeerIdProvider;
+
   public readonly collectionStateUpdated = new Event<{ collectionId: CollectionId }>();
 
-  constructor({ db, indexMetadataStore, dataMonitor }: AutomergeHostParams) {
+  constructor({ db, indexMetadataStore, dataMonitor, peerIdProvider }: AutomergeHostParams) {
     super();
     this._db = db;
     this._storage = new LevelDBStorageAdapter({
@@ -104,11 +113,11 @@ export class AutomergeHost extends Resource {
     });
     this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
     this._indexMetadataStore = indexMetadataStore;
+    this._peerIdProvider = peerIdProvider;
   }
 
   protected override async _open() {
-    // TODO(burdon): Should this be stable?
-    this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
+    this._peerId = `host-${this._peerIdProvider?.() ?? PublicKey.random().toHex()}` as PeerId;
 
     await this._storage.open?.();
 

package/src/db-host/echo-host.ts

@@ -33,6 +33,7 @@ import {
   type EchoReplicator,
   type CollectionSyncState,
   type EchoDataStats,
+  type PeerIdProvider,
 } from '../automerge';
 
 const INDEXER_CONFIG: IndexConfig = {
@@ -42,6 +43,7 @@ const INDEXER_CONFIG: IndexConfig = {
 
 export type EchoHostParams = {
   kv: LevelDB;
+  peerIdProvider?: PeerIdProvider;
 };
 
 /**
@@ -59,7 +61,7 @@ export class EchoHost extends Resource {
   private readonly _spaceStateManager = new SpaceStateManager();
   private readonly _echoDataMonitor: EchoDataMonitor;
 
-  constructor({ kv }: EchoHostParams) {
+  constructor({ kv, peerIdProvider }: EchoHostParams) {
     super();
 
     this._indexMetadataStore = new IndexMetadataStore({ db: kv.sublevel('index-metadata') });
@@ -70,6 +72,7 @@ export class EchoHost extends Resource {
       db: kv,
       dataMonitor: this._echoDataMonitor,
       indexMetadataStore: this._indexMetadataStore,
+      peerIdProvider,
     });
 
     this._indexer = new Indexer({

package/src/db-host/index.ts

@@ -5,7 +5,6 @@
 export * from './data-service';
 export * from './documents-synchronizer';
 export * from './echo-host';
-export * from './migration';
 export * from './database-root';
 export * from './query-state';
 export * from './query-service';

package/src/edge/echo-edge-replicator.test.ts

@@ -2,7 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-import { describe, test } from 'vitest';
+import { getRandomPort } from 'get-port-please';
+import { describe, expect, onTestFinished, test } from 'vitest';
 
 import { Event } from '@dxos/async';
 import { cbor } from '@dxos/automerge/automerge-repo';
@@ -16,69 +17,111 @@ import type { Peer } from '@dxos/protocols/proto/dxos/edge/messenger';
 import { openAndClose } from '@dxos/test-utils';
 
 import { EchoEdgeReplicator } from './echo-edge-replicator';
-import type { EchoReplicatorContext } from '../automerge';
+import type { EchoReplicatorContext, ReplicatorConnection } from '../automerge';
 
 describe('EchoEdgeReplicator', () => {
-  test('reconnects', async ({ onTestFinished }) => {
-    const { endpoint, cleanup, sendMessage } = await createTestEdgeWsServer(8001);
-    onTestFinished(cleanup);
-    const client = new EdgeClient(await createEphemeralEdgeIdentity(), { socketEndpoint: endpoint });
-    await openAndClose(client);
+  test('reconnects', async () => {
+    const { client, server } = await createClientServer();
 
     const spaceId = SpaceId.random();
 
-    const replicator = new EchoEdgeReplicator({ edgeConnection: client });
     const { context, connectionOpen } = createMockContext();
-    await replicator.connect(context);
+    const replicator = await connectReplicator(client, context);
     await replicator.connectToSpace(spaceId);
 
     client.setIdentity(await createEphemeralEdgeIdentity());
     await connectionOpen.waitForCount(1);
 
-    sendMessage(
-      createForbiddenMessage(
-        {
-          identityKey: client.identityKey,
-          peerKey: client.peerKey,
-        },
-        spaceId,
-      ),
-    );
+    const forbidden = createForbiddenMessage({ identityKey: client.identityKey, peerKey: client.peerKey }, spaceId);
+    server.sendMessage(forbidden);
     await connectionOpen.waitForCount(1);
 
     // Double restart to check for race conditions.
     client.setIdentity(await createEphemeralEdgeIdentity());
-    sendMessage(
-      createForbiddenMessage(
-        {
-          identityKey: client.identityKey,
-          peerKey: client.peerKey,
-        },
-        spaceId,
-      ),
-    );
+    server.sendMessage(forbidden);
     await connectionOpen.waitForCount(1);
 
     await replicator.disconnect();
   });
+
+  describe('shouldAdvertise', () => {
+    test('true if space document belongs to connection space', async () => {
+      const { client } = await createClientServer();
+
+      const spaceId = SpaceId.random();
+      const documentId = PublicKey.random().toHex();
+      const { context, openConnections } = createMockContext({
+        documentSpaceId: { [documentId]: spaceId },
+      });
+      const replicator = await connectReplicator(client, context);
+      await replicator.connectToSpace(spaceId);
+
+      await expect.poll(() => openConnections.length === 1).toBeTruthy();
+      expect(openConnections[0].shouldAdvertise({ documentId })).toBeTruthy();
+    });
+
+    test('checks remote collection if space id can not be resolved', async () => {
+      const { client } = await createClientServer();
+
+      const spaceId = SpaceId.random();
+      const documentId = PublicKey.random().toHex();
+      const remoteCollections: { [peerId: string]: { [documentId: string]: boolean } } = {};
+      const { context, openConnections } = createMockContext({ remoteCollections });
+      const replicator = await connectReplicator(client, context);
+      await replicator.connectToSpace(spaceId);
+
+      await expect.poll(() => openConnections.length === 1).toBeTruthy();
+      const connection = openConnections[0];
+      expect(await connection.shouldAdvertise({ documentId })).toBeFalsy();
+      remoteCollections[connection.peerId] = { [documentId]: true };
+      expect(await connection.shouldAdvertise({ documentId })).toBeTruthy();
+    });
+  });
+
+  const connectReplicator = async (client: EdgeClient, context: EchoReplicatorContext) => {
+    const replicator = new EchoEdgeReplicator({ edgeConnection: client });
+    await replicator.connect(context);
+    onTestFinished(() => replicator.disconnect());
+    return replicator;
+  };
+
+  const createClientServer = async () => {
+    const server = await createTestEdgeWsServer(await getRandomPort());
+    onTestFinished(server.cleanup);
+    const client = new EdgeClient(await createEphemeralEdgeIdentity(), { socketEndpoint: server.endpoint });
+    await openAndClose(client);
+    return { client, server };
+  };
 });
 
-const createMockContext = () => {
+const createMockContext = (args?: {
+  remoteCollections?: { [peerId: string]: { [documentId: string]: boolean } };
+  documentSpaceId?: { [documentId: string]: SpaceId };
+}) => {
   const connectionOpen = new Event();
+  const openConnections: ReplicatorConnection[] = [];
   return {
     context: {
-      getContainingSpaceIdForDocument: async (documentId) => null,
+      getContainingSpaceIdForDocument: async (documentId) => args?.documentSpaceId?.[documentId] ?? null,
       getContainingSpaceForDocument: async (documentId) => null,
       onConnectionAuthScopeChanged: (connection) => {},
-      isDocumentInRemoteCollection: async (params) => false,
-      onConnectionClosed: (connection) => {},
+      isDocumentInRemoteCollection: async (params) =>
+        args?.remoteCollections?.[params.peerId]?.[params.documentId] ?? false,
+      onConnectionClosed: (connection) => {
+        const idx = openConnections.indexOf(connection);
+        if (idx >= 0) {
+          openConnections.splice(idx, 1);
+        }
+      },
       onConnectionOpen: (connection) => {
+        openConnections.push(connection);
         connectionOpen.emit();
       },
 
       peerId: PublicKey.random().toHex(),
     } satisfies EchoReplicatorContext,
 
+    openConnections,
     connectionOpen,
   };
 };

package/src/edge/echo-edge-replicator.ts

@@ -97,6 +97,9 @@ export class EchoEdgeReplicator implements EchoReplicator {
   async connectToSpace(spaceId: SpaceId) {
     using _guard = await this._mutex.acquire();
 
+    if (this._connectedSpaces.has(spaceId)) {
+      return;
+    }
     this._connectedSpaces.add(spaceId);
 
     // Check if AM-repo requested that we connect to remote peers.
@@ -264,9 +267,20 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
     }
     const spaceId = await this._context.getContainingSpaceIdForDocument(params.documentId);
     if (!spaceId) {
-      // There's no spaceId if the document is not present locally. This means the sharePolicy check is being
-      // performed on message reception, so spaceId check was already performed in _onMessage.
-      return true;
+      const remoteDocumentExists = await this._context.isDocumentInRemoteCollection({
+        documentId: params.documentId,
+        peerId: this._remotePeerId as PeerId,
+      });
+
+      log.info('document not found locally for share policy check, accepting the remote document', {
+        documentId: params.documentId,
+        remoteDocumentExists,
+      });
+
+      // If a document is not present locally return true only if it already exists on edge.
+      // Simply returning true will add edge to "generous peers list" for this document which will
+      // start replication of the document after we receive it potentially pushing it to replicator of the wrong space.
+      return remoteDocumentExists;
     }
     return spaceId === this._spaceId;
   }

package/src/index.ts

@@ -9,3 +9,4 @@ export * from './pipeline';
 export * from './space';
 export * from './automerge';
 export * from './edge';
+export * from './util';

package/src/space/space.ts

@@ -48,6 +48,7 @@ export type CreatePipelineParams = {
  * Spaces are globally addressable databases with access control.
  */
 // TODO(dmaretskyi): Extract database stuff.
+// TODO(dmaretskyi): Rename HaloGraph move to HALO.
 @trackLeaks('open', 'close')
 @trace.resource()
 export class Space extends Resource {

package/src/util.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { decodeReference, type ObjectStructure, type SpaceDoc } from '@dxos/echo-protocol';
+
+/**
+ * Assumes properties are at root.
+ */
+export const findInlineObjectOfType = (spaceDoc: SpaceDoc, typename: string): [string, ObjectStructure] | undefined => {
+  for (const id in spaceDoc.objects ?? {}) {
+    const obj = spaceDoc.objects![id];
+    if (obj.system.type && decodeReference(obj.system.type).objectId === typename) {
+      return [id, obj];
+    }
+  }
+
+  return undefined;
+};

package/dist/types/src/db-host/migration.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"migration.d.ts","sourceRoot":"","sources":["../../../../src/db-host/migration.ts"],"names":[],"mappings":"AAKA,OAAO,EAKL,KAAK,eAAe,EAEpB,KAAK,QAAQ,EAEd,MAAM,qBAAqB,CAAC;AAI7B,eAAO,MAAM,uBAAuB,QAAe,QAAQ,KAAG,OAAO,CAAC,QAAQ,CAU7E,CAAC;AAEF,eAAO,MAAM,yBAAyB,SAAgB,QAAQ,KAAG,OAAO,CAAC,QAAQ,CAYhF,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,sBAAsB,aAAc,QAAQ,YAAY,MAAM,KAAG,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,SASzG,CAAC"}

package/src/db-host/migration.ts

@@ -1,57 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { convertLegacyReference } from '@dxos/echo-protocol';
-import {
-  decodeReference,
-  encodeReference,
-  isLegacyReference,
-  LEGACY_TYPE_PROPERTIES,
-  type ObjectStructure,
-  Reference,
-  type SpaceDoc,
-  SpaceDocVersion,
-} from '@dxos/echo-protocol';
-import { TYPE_PROPERTIES } from '@dxos/echo-schema';
-import { deepMapValuesAsync } from '@dxos/util';
-
-export const convertLegacyReferences = async (doc: SpaceDoc): Promise<SpaceDoc> => {
-  const newDoc = await deepMapValuesAsync(doc, async (value, recurse) => {
-    if (isLegacyReference(value)) {
-      return convertLegacyReference(value);
-    }
-    return recurse(value);
-  });
-
-  newDoc.version = SpaceDocVersion.CURRENT;
-  return newDoc;
-};
-
-export const convertLegacySpaceRootDoc = async (root: SpaceDoc): Promise<SpaceDoc> => {
-  // Convert references.
-  const newDoc: SpaceDoc = await convertLegacyReferences(root);
-
-  // Update properties type.
-  const properties = findInlineObjectOfType(newDoc, LEGACY_TYPE_PROPERTIES);
-  if (properties) {
-    const [_, obj] = properties;
-    obj.system.type = encodeReference(Reference.fromLegacyTypename(TYPE_PROPERTIES));
-  }
-
-  return newDoc;
-};
-
-/**
- * Assumes properties are at root.
- */
-export const findInlineObjectOfType = (spaceDoc: SpaceDoc, typename: string): [string, ObjectStructure] | undefined => {
-  for (const id in spaceDoc.objects ?? {}) {
-    const obj = spaceDoc.objects![id];
-    if (obj.system.type && decodeReference(obj.system.type).objectId === typename) {
-      return [id, obj];
-    }
-  }
-
-  return undefined;
-};