@dxos/echo-pipeline 0.5.8 → 0.5.9-main.0a0e87d

package/src/automerge/storage-adapter.test.ts

@@ -5,18 +5,24 @@
 import { expect } from 'chai';
 
 import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
+import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';
 import { createTestLevel } from '@dxos/kv-store/testing';
 import { StorageType, createStorage } from '@dxos/random-access-storage';
-import { afterTest, describe, openAndClose, test } from '@dxos/test';
-import { type MaybePromise } from '@dxos/util';
+import { afterTest, describe, test } from '@dxos/test';
+import { arrayToBuffer, bufferToArray, type MaybePromise } from '@dxos/util';
 
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
 
 const runTests = (
   testNamespace: string,
-  /** Run per test. Expects automatic clean-up with `afterTest`. */ createAdapter: () => MaybePromise<StorageAdapterInterface>,
+  /** Run per test. Expects automatic clean-up with `afterTest`. */
+  createAdapter: (root?: string) => MaybePromise<{
+    adapter: StorageAdapterInterface;
+    /** Would be called automatically with `afterTest`. Exposed for mid-test clean-up. */
+    close: () => MaybePromise<void>;
+  }>,
 ) => {
   describe(testNamespace, () => {
     const chunks = [
@@ -27,14 +33,14 @@ const runTests = (
     ];
 
     test('should store and retrieve data', async () => {
-      const adapter = await createAdapter();
+      const { adapter } = await createAdapter();
 
       await adapter.save(chunks[0].key, chunks[0].data);
       expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
     });
 
     test('loadRange return inputs with correct prefixes', async () => {
-      const adapter = await createAdapter();
+      const { adapter } = await createAdapter();
 
       for (const chunk of chunks) {
         await adapter.save(chunk.key, chunk.data);
@@ -46,7 +52,7 @@ const runTests = (
     });
 
     test('deletion works', async () => {
-      const adapter = await createAdapter();
+      const { adapter } = await createAdapter();
 
       for (const chunk of chunks) {
         await adapter.save(chunk.key, chunk.data);
@@ -64,27 +70,87 @@ const runTests = (
       expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
       expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
     });
+
+    test('loadRange', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const { adapter, close } = await createAdapter(root);
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+        await close();
+      }
+
+      {
+        const { adapter } = await createAdapter(root);
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['one', 'two']);
+        expect(range.map((chunk) => chunk.key)).to.deep.eq([
+          ['test', '1'],
+          ['test', '2'],
+        ]);
+      }
+    });
+
+    test('removeRange', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const { adapter, close } = await createAdapter(root);
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+        await close();
+      }
+
+      {
+        const { adapter } = await createAdapter(root);
+        await adapter.removeRange(['test']);
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq([]);
+        const range2 = await adapter.loadRange(['bar']);
+        expect(range2.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['bar']);
+        expect(range2.map((chunk) => chunk.key)).to.deep.eq([['bar', '1']]);
+      }
+    });
   });
 };
 
 /**
  * Run tests for AutomergeStorageAdapter.
  */
-runTests('AutomergeStorageAdapter', () => {
-  const storage = createStorage({ type: StorageType.RAM });
-  afterTest(() => storage.close());
+runTests('AutomergeStorageAdapter', (root?: string) => {
+  const storage = createStorage({ type: root ? StorageType.NODE : StorageType.RAM, root });
   const dir = storage.createDirectory('automerge');
   const adapter = new AutomergeStorageAdapter(dir);
-  afterTest(() => adapter.close());
-  return adapter;
+
+  const close = async () => {
+    await adapter.close();
+    await storage.close();
+  };
+  afterTest(close);
+
+  return {
+    adapter,
+    close,
+  };
 });
 
 /**
  * Run tests for LevelDBStorageAdapter.
  */
-runTests('LevelDBStorageAdapter', async () => {
-  const level = createTestLevel();
+runTests('LevelDBStorageAdapter', async (root?: string) => {
+  const level = createTestLevel(root);
+  await level.open();
   const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
-  await openAndClose(level, adapter as any);
-  return adapter;
+  await adapter.open?.();
+
+  const close = async () => {
+    await adapter.close();
+    await level.close();
+  };
+  afterTest(close);
+  return {
+    adapter,
+    close,
+  };
 });

package/src/space/space-manager.ts

@@ -8,7 +8,7 @@ import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { type NetworkManager } from '@dxos/network-manager';
+import { type SwarmNetworkManager } from '@dxos/network-manager';
 import { trace } from '@dxos/protocols';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
@@ -16,14 +16,14 @@ import { type Teleport } from '@dxos/teleport';
 import { type BlobStore } from '@dxos/teleport-extension-object-sync';
 import { ComplexMap } from '@dxos/util';
 
-import { Space } from './space';
+import { Space, createIdFromSpaceKey } from './space';
 import { SpaceProtocol, type SwarmIdentity } from './space-protocol';
 import { SnapshotManager, type SnapshotStore } from '../db-host';
 import { type MetadataStore } from '../metadata';
 
 export type SpaceManagerParams = {
   feedStore: FeedStore<FeedMessage>;
-  networkManager: NetworkManager;
+  networkManager: SwarmNetworkManager;
   metadataStore: MetadataStore;
 
   /**
@@ -54,7 +54,7 @@ export type ConstructSpaceParams = {
 export class SpaceManager {
   private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
   private readonly _feedStore: FeedStore<FeedMessage>;
-  private readonly _networkManager: NetworkManager;
+  private readonly _networkManager: SwarmNetworkManager;
   private readonly _metadataStore: MetadataStore;
   private readonly _snapshotStore: SnapshotStore;
   private readonly _blobStore: BlobStore;
@@ -98,6 +98,7 @@ export class SpaceManager {
     const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? failUndefined());
 
     const spaceKey = metadata.key;
+    const spaceId = await createIdFromSpaceKey(spaceKey);
     const protocol = new SpaceProtocol({
       topic: spaceKey,
       swarmIdentity,
@@ -109,6 +110,7 @@ export class SpaceManager {
     const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
 
     const space = new Space({
+      id: spaceId,
       spaceKey,
       protocol,
       genesisFeed,

package/src/space/space-protocol.test.ts

@@ -7,7 +7,7 @@ import waitForExpect from 'wait-for-expect';
 
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
-import { MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
+import { MemoryTransportFactory, SwarmNetworkManager } from '@dxos/network-manager';
 import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
 import { describe, test, afterTest } from '@dxos/test';
@@ -25,11 +25,13 @@ describe('space/space-protocol', () => {
     const peer1 = await builder.createPeer();
     const gossip1 = peer1.createGossip();
     const presence1 = peer1.createPresence(gossip1);
+    await presence1.open();
     const protocol1 = peer1.createSpaceProtocol(topic, gossip1);
 
     const peer2 = await builder.createPeer();
     const gossip2 = peer2.createGossip();
     const presence2 = peer2.createPresence(gossip2);
+    await presence2.open();
     const protocol2 = peer2.createSpaceProtocol(topic, gossip2);
 
     await protocol1.start();
@@ -56,7 +58,7 @@ describe('space/space-protocol', () => {
         credentialAuthenticator: async () => false, // Reject everyone.
       },
       blobStore: new BlobStore(createStorage({ type: StorageType.RAM }).createDirectory()),
-      networkManager: new NetworkManager({
+      networkManager: new SwarmNetworkManager({
         signalManager: new MemorySignalManager(signalContext),
         transportFactory: MemoryTransportFactory,
       }),
@@ -70,7 +72,7 @@ describe('space/space-protocol', () => {
         credentialAuthenticator: MOCK_AUTH_VERIFIER,
       },
       blobStore: new BlobStore(createStorage({ type: StorageType.RAM }).createDirectory()),
-      networkManager: new NetworkManager({
+      networkManager: new SwarmNetworkManager({
         signalManager: new MemorySignalManager(signalContext),
         transportFactory: MemoryTransportFactory,
       }),

package/src/space/space-protocol.ts

@@ -9,7 +9,7 @@ import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import {
   MMSTTopology,
-  type NetworkManager,
+  type SwarmNetworkManager,
   type SwarmConnection,
   type WireProtocol,
   type WireProtocolParams,
@@ -37,7 +37,7 @@ export interface SwarmIdentity {
 export type SpaceProtocolOptions = {
   topic: PublicKey; // TODO(burdon): Rename?
   swarmIdentity: SwarmIdentity;
-  networkManager: NetworkManager;
+  networkManager: SwarmNetworkManager;
 
   blobStore: BlobStore;
 
@@ -54,7 +54,7 @@ export type SpaceProtocolOptions = {
  */
 @trace.resource()
 export class SpaceProtocol {
-  private readonly _networkManager: NetworkManager;
+  private readonly _networkManager: SwarmNetworkManager;
   private readonly _swarmIdentity: SwarmIdentity;
   private readonly _onSessionAuth?: (session: Teleport) => void;
   private readonly _onAuthFailure?: (session: Teleport) => void;

package/src/space/space.ts

@@ -5,15 +5,16 @@
 import { Event, Mutex, synchronized, trackLeaks } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { type DelegateInvitationCredential, type FeedInfo, type MemberInfo } from '@dxos/credentials';
+import { subtleCrypto } from '@dxos/crypto';
 import { type FeedOptions, type FeedWrapper } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
-import { type PublicKey } from '@dxos/keys';
+import { PublicKey, SpaceId } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type Timeframe } from '@dxos/timeframe';
 import { trace } from '@dxos/tracing';
-import { type AsyncCallback, Callback } from '@dxos/util';
+import { type AsyncCallback, Callback, ComplexMap } from '@dxos/util';
 
 import { ControlPipeline } from './control-pipeline';
 import { type SpaceProtocol } from './space-protocol';
@@ -25,6 +26,7 @@ import { type PipelineAccessor } from '../pipeline';
 type FeedProvider = (feedKey: PublicKey, opts?: FeedOptions) => Promise<FeedWrapper<FeedMessage>>;
 
 export type SpaceParams = {
+  id: SpaceId;
   spaceKey: PublicKey;
   protocol: SpaceProtocol;
   genesisFeed: FeedWrapper<FeedMessage>;
@@ -59,6 +61,7 @@ export class Space extends Resource {
   @trace.info()
   public readonly protocol: SpaceProtocol;
 
+  private readonly _id: SpaceId;
   private readonly _key: PublicKey;
   private readonly _genesisFeedKey: PublicKey;
   private readonly _feedProvider: FeedProvider;
@@ -73,6 +76,7 @@ export class Space extends Resource {
   constructor(params: SpaceParams) {
     super();
     invariant(params.spaceKey && params.feedProvider);
+    this._id = params.id;
     this._key = params.spaceKey;
     this._genesisFeedKey = params.genesisFeed.key;
     this._feedProvider = params.feedProvider;
@@ -120,6 +124,12 @@ export class Space extends Resource {
     this.protocol.addFeed(params.genesisFeed);
   }
 
+  @logInfo
+  @trace.info()
+  get id() {
+    return this._id;
+  }
+
   @logInfo
   @trace.info()
   get key() {
@@ -199,3 +209,23 @@ export class Space extends Resource {
     log('closed');
   }
 }
+
+const SPACE_IDS_CACHE = new ComplexMap<PublicKey, SpaceId>(PublicKey.hash);
+
+/**
+ * Space keys are generated by creating a keypair, and then taking the first 20 bytes of the SHA-256 hash of the public key and encoding them to multibase RFC4648 base-32 format (prefixed with B, see Multibase Table).
+ * Inspired by how ethereum addresses are derived.
+ */
+export const createIdFromSpaceKey = async (spaceKey: PublicKey): Promise<SpaceId> => {
+  const cachedValue = SPACE_IDS_CACHE.get(spaceKey);
+  if (cachedValue !== undefined) {
+    return cachedValue;
+  }
+
+  const digest = await subtleCrypto.digest('SHA-256', spaceKey.asUint8Array());
+
+  const bytes = new Uint8Array(digest).slice(0, SpaceId.byteLength);
+  const spaceId = SpaceId.encode(bytes);
+  SPACE_IDS_CACHE.set(spaceKey, spaceId);
+  return spaceId;
+};

package/src/testing/index.ts

@@ -5,3 +5,4 @@
 export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
+export * from './test-network-adapter';

package/src/testing/test-agent-builder.ts

@@ -8,7 +8,7 @@ import { type FeedStore } from '@dxos/feed-store';
 import { type Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
-import { MemoryTransportFactory, NetworkManager, createSimplePeerTransportFactory } from '@dxos/network-manager';
+import { MemoryTransportFactory, SwarmNetworkManager, createSimplePeerTransportFactory } from '@dxos/network-manager';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
@@ -22,12 +22,12 @@ import { SnapshotStore } from '../db-host';
 import { MetadataStore } from '../metadata';
 import { MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceManager, SpaceProtocol, type Space } from '../space';
 
-export type NetworkManagerProvider = () => NetworkManager;
+export type NetworkManagerProvider = () => SwarmNetworkManager;
 
 export const MemoryNetworkManagerProvider =
   (signalContext: MemorySignalManagerContext): NetworkManagerProvider =>
   () =>
-    new NetworkManager({
+    new SwarmNetworkManager({
       signalManager: new MemorySignalManager(signalContext),
       transportFactory: MemoryTransportFactory,
     });
@@ -35,7 +35,7 @@ export const MemoryNetworkManagerProvider =
 export const WebsocketNetworkManagerProvider =
   (signalUrl: string): NetworkManagerProvider =>
   () =>
-    new NetworkManager({
+    new SwarmNetworkManager({
       signalManager: new WebsocketSignalManager([{ server: signalUrl }]),
       transportFactory: createSimplePeerTransportFactory(),
     });

package/src/testing/test-network-adapter.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Trigger, sleep } from '@dxos/async';
+import { type Message, NetworkAdapter, type PeerId } from '@dxos/automerge/automerge-repo';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+export class TestAdapter extends NetworkAdapter {
+  static createPair() {
+    const adapter1: TestAdapter = new TestAdapter({
+      send: (message: Message) => sleep(10).then(() => adapter2.receive(message)),
+    });
+    const adapter2: TestAdapter = new TestAdapter({
+      send: (message: Message) => sleep(10).then(() => adapter1.receive(message)),
+    });
+
+    return [adapter1, adapter2];
+  }
+
+  public onConnect = new Trigger();
+
+  constructor(private readonly _params: { send: (message: Message) => void }) {
+    super();
+  }
+
+  // NOTE: Emitting `ready` event in NetworkAdapter`s constructor causes a race condition
+  //       because `Repo` waits for `ready` event (which it never receives) before it starts using the adapter.
+  ready() {
+    this.emit('ready', { network: this });
+  }
+
+  override connect(peerId: PeerId) {
+    this.peerId = peerId;
+    this.onConnect.wake();
+  }
+
+  peerCandidate(peerId: PeerId) {
+    invariant(peerId, 'PeerId is required');
+    this.emit('peer-candidate', { peerId, peerMetadata: {} });
+  }
+
+  peerDisconnected(peerId: PeerId) {
+    invariant(peerId, 'PeerId is required');
+    this.emit('peer-disconnected', { peerId });
+  }
+
+  override send(message: Message) {
+    log('send', { from: message.senderId, to: message.targetId, type: message.type });
+    this._params.send(message);
+  }
+
+  override disconnect() {
+    this.peerId = undefined;
+  }
+
+  receive(message: Message) {
+    invariant(this.peerId, 'Peer id is not set');
+    this.emit('message', message);
+  }
+}