@dxos/echo-pipeline 0.1.24-next.6b1f434

package/src/space/space-manager.ts

@@ -0,0 +1,92 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { synchronized, trackLeaks } from '@dxos/async';
+import { CredentialSigner } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
+import { FeedStore } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { NetworkManager } from '@dxos/network-manager';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { Credential, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { Teleport } from '@dxos/teleport';
+import { ComplexMap } from '@dxos/util';
+
+import { Space } from './space';
+import { SpaceProtocol, SwarmIdentity } from './space-protocol';
+
+// TODO(burdon): Factor out to DataSpaceManager
+export interface SigningContext {
+  identityKey: PublicKey;
+  deviceKey: PublicKey;
+  credentialSigner: CredentialSigner; // TODO(burdon): Already has keyring.
+  recordCredential: (credential: Credential) => Promise<void>;
+  profile?: ProfileDocument;
+}
+
+export type SpaceManagerParams = {
+  feedStore: FeedStore<FeedMessage>;
+  networkManager: NetworkManager;
+};
+
+export type ConstructSpaceParams = {
+  metadata: SpaceMetadata;
+  swarmIdentity: SwarmIdentity;
+  onNetworkConnection: (session: Teleport) => void;
+};
+
+/**
+ * Manages a collection of ECHO (Data) Spaces.
+ */
+@trackLeaks('open', 'close')
+export class SpaceManager {
+  private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
+  private readonly _feedStore: FeedStore<FeedMessage>;
+  private readonly _networkManager: NetworkManager;
+
+  constructor({ feedStore, networkManager }: SpaceManagerParams) {
+    // TODO(burdon): Assert.
+    this._feedStore = feedStore;
+    this._networkManager = networkManager;
+  }
+
+  // TODO(burdon): Remove.
+  get spaces() {
+    return this._spaces;
+  }
+
+  @synchronized
+  async open() {}
+
+  @synchronized
+  async close() {
+    await Promise.all([...this._spaces.values()].map((space) => space.close()));
+  }
+
+  async constructSpace({ metadata, swarmIdentity, onNetworkConnection }: ConstructSpaceParams) {
+    log('constructing space...', { spaceKey: metadata.genesisFeedKey });
+
+    // The genesis feed will be the same as the control feed if the space was created by the local agent.
+    const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? failUndefined());
+
+    const spaceKey = metadata.key;
+    const protocol = new SpaceProtocol({
+      topic: spaceKey,
+      swarmIdentity,
+      networkManager: this._networkManager,
+      onSessionAuth: onNetworkConnection
+    });
+
+    const space = new Space({
+      spaceKey,
+      protocol,
+      genesisFeed,
+      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey)
+    });
+    this._spaces.set(space.key, space);
+    return space;
+  }
+}

package/src/space/space-protocol.browser.test.ts

@@ -0,0 +1,135 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @dxos/test platform=browser
+
+import expect from 'expect';
+import waitForExpect from 'wait-for-expect';
+
+import { Keyring } from '@dxos/keyring';
+import { PublicKey } from '@dxos/keys';
+import { createStorage } from '@dxos/random-access-storage';
+import { describe, test, afterTest } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+
+import { TestFeedBuilder, TestAgentBuilder, WebsocketNetworkManagerProvider } from '../testing';
+
+// TODO(burdon): Config.
+// Signal server will be started by the setup script.
+const SIGNAL_URL = 'ws://localhost:4000/.well-known/dx/signal';
+
+describe('space/space-protocol', () => {
+  test('two peers discover each other', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const presence1 = peer1.createPresence();
+    const protocol1 = peer1.createSpaceProtocol(topic, presence1);
+
+    const peer2 = await builder.createPeer();
+    const presence2 = peer2.createPresence();
+    const protocol2 = peer2.createSpaceProtocol(topic, presence2);
+
+    await protocol1.start();
+    await protocol2.start();
+
+    afterTest(() => protocol1.stop());
+    afterTest(() => protocol2.stop());
+
+    await waitForExpect(() => {
+      expect(presence1.getPeersOnline().map(({ peerId }) => peerId)).toContainEqual(peer2.deviceKey);
+      expect(presence2.getPeersOnline().map(({ peerId }) => peerId)).toContainEqual(peer1.deviceKey);
+    });
+  });
+
+  test('replicates a feed', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start();
+    await protocol2.start();
+
+    afterTest(() => protocol1.stop());
+    afterTest(() => protocol2.stop());
+
+    const builder1 = new TestFeedBuilder();
+    const feedStore1 = builder1.createFeedStore();
+
+    const builder2 = new TestFeedBuilder();
+    const feedStore2 = builder2.createFeedStore();
+
+    const feedKey = await builder1.keyring.createKey();
+    const feed1 = await feedStore1.openFeed(feedKey, { writable: true });
+    const feed2 = await feedStore2.openFeed(feedKey);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      expect(feed2.properties.length).toEqual(1);
+    });
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      expect(feed2.properties.length).toEqual(2);
+    });
+
+    await builder.close();
+  });
+
+  test('replicates a feed through a webrtc connection', async () => {
+    const builder = new TestAgentBuilder({
+      storage: createStorage(),
+      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL)
+    });
+    afterTest(async () => await builder.close());
+
+    const keyring = new Keyring();
+    const topic = await keyring.createKey();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start();
+    await protocol2.start();
+
+    afterTest(() => protocol1.stop());
+    afterTest(() => protocol2.stop());
+
+    const feedKey = await peer1.keyring.createKey();
+
+    const feed1 = await peer1.feedStore.openFeed(feedKey, { writable: true });
+    const feed2 = await peer2.feedStore.openFeed(feedKey);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      expect(feed2.properties.length).toEqual(1);
+    });
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      expect(feed2.properties.length).toEqual(2);
+    });
+
+    await builder.close();
+  })
+    .skipEnvironments('webkit')
+    .tag('e2e'); // Some storage drivers may break when there are multiple storage instances.
+});

package/src/space/space-protocol.test.ts

@@ -0,0 +1,136 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import expect from 'expect';
+import waitForExpect from 'wait-for-expect';
+
+import { PublicKey } from '@dxos/keys';
+import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
+import { MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
+import { describe, test, afterTest } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+
+import { TestAgentBuilder, TestFeedBuilder } from '../testing';
+import { AuthStatus, MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceProtocol } from './space-protocol';
+
+describe('space/space-protocol', () => {
+  test('two peers discover each other via presence', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const presence1 = peer1.createPresence();
+    const protocol1 = peer1.createSpaceProtocol(topic, presence1);
+
+    const peer2 = await builder.createPeer();
+    const presence2 = peer2.createPresence();
+    const protocol2 = peer2.createSpaceProtocol(topic, presence2);
+
+    await protocol1.start();
+    afterTest(() => protocol1.stop());
+
+    await protocol2.start();
+    afterTest(() => protocol2.stop());
+
+    await waitForExpect(() => {
+      expect(presence1.getPeersOnline().map(({ peerId }) => peerId)).toContainEqual(peer2.deviceKey);
+      expect(presence2.getPeersOnline().map(({ peerId }) => peerId)).toContainEqual(peer1.deviceKey);
+    }, 1_000);
+  });
+
+  test('failing authentication', async () => {
+    const [topic, peerId1, peerId2] = PublicKey.randomSequence();
+    const signalContext = new MemorySignalManagerContext();
+
+    const protocol1 = new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        peerKey: peerId1,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: async () => false // Reject everyone.
+      },
+      networkManager: new NetworkManager({
+        signalManager: new MemorySignalManager(signalContext),
+        transportFactory: MemoryTransportFactory
+      })
+    });
+
+    const protocol2 = new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        peerKey: peerId2,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER
+      },
+      networkManager: new NetworkManager({
+        signalManager: new MemorySignalManager(signalContext),
+        transportFactory: MemoryTransportFactory
+      })
+    });
+
+    await protocol1.start();
+    afterTest(() => protocol1.stop());
+
+    await protocol2.start();
+    afterTest(() => protocol2.stop());
+
+    await waitForExpect(() => {
+      expect(protocol1.sessions.get(peerId2)?.authStatus).toEqual(AuthStatus.FAILURE);
+    });
+  });
+
+  test('replicates a feed', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start();
+    await protocol2.start();
+
+    afterTest(() => protocol1.stop());
+    afterTest(() => protocol2.stop());
+
+    //
+    // Create feeds.
+    //
+
+    const builder1 = new TestFeedBuilder();
+    const feedStore1 = builder1.createFeedStore();
+
+    const builder2 = new TestFeedBuilder();
+    const feedStore2 = builder2.createFeedStore();
+
+    const feed1 = await feedStore1.openFeed(await builder1.keyring.createKey(), { writable: true });
+    const feed2 = await feedStore2.openFeed(feed1.key);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    //
+    // Append message.
+    //
+
+    // TODO(burdon): Append batch of messages.
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      // Received message appended before replication.
+      expect(feed2.properties.length).toEqual(1);
+    });
+
+    // TODO(burdon): Append batch of messages.
+    await feed1.append({ timeframe: new Timeframe() });
+    await waitForExpect(() => {
+      // Received message appended after replication.
+      expect(feed2.properties.length).toEqual(2);
+    });
+  });
+});

package/src/space/space-protocol.ts

@@ -0,0 +1,222 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { discoveryKey, sha256 } from '@dxos/crypto';
+import { FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log, logInfo } from '@dxos/log';
+import {
+  MMSTTopology,
+  NetworkManager,
+  SwarmConnection,
+  WireProtocol,
+  WireProtocolParams,
+  WireProtocolProvider
+} from '@dxos/network-manager';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { Teleport } from '@dxos/teleport';
+import { ReplicatorExtension } from '@dxos/teleport-extension-replicator';
+import { ComplexMap } from '@dxos/util';
+
+import { AuthExtension, AuthProvider, AuthVerifier } from './auth';
+
+export const MOCK_AUTH_PROVIDER: AuthProvider = async (nonce: Uint8Array) => Buffer.from('mock');
+export const MOCK_AUTH_VERIFIER: AuthVerifier = async (nonce: Uint8Array, credential: Uint8Array) => true;
+
+// TODO(burdon): Reconcile with SigningContext (define types together).
+export interface SwarmIdentity {
+  peerKey: PublicKey;
+  credentialProvider: AuthProvider;
+  credentialAuthenticator: AuthVerifier;
+}
+
+export type SpaceProtocolOptions = {
+  topic: PublicKey; // TODO(burdon): Rename?
+  swarmIdentity: SwarmIdentity;
+  networkManager: NetworkManager;
+  /**
+   * Called when new session is authenticated.
+   * Additional extensions can be added here.
+   */
+  onSessionAuth?: (session: Teleport) => void;
+};
+
+/**
+ * Manages Teleport protocol stream creation and joining swarms with replication and presence extensions.
+ */
+export class SpaceProtocol {
+  private readonly _networkManager: NetworkManager;
+  private readonly _swarmIdentity: SwarmIdentity;
+  private readonly _onSessionAuth?: (session: Teleport) => void;
+
+  @logInfo
+  private readonly _topic: PublicKey;
+
+  private readonly _feeds = new Set<FeedWrapper<FeedMessage>>();
+  private readonly _sessions = new ComplexMap<PublicKey, SpaceProtocolSession>(PublicKey.hash);
+
+  private _connection?: SwarmConnection;
+
+  get sessions(): ReadonlyMap<PublicKey, SpaceProtocolSession> {
+    return this._sessions;
+  }
+
+  get feeds(): ReadonlySet<FeedWrapper<FeedMessage>> {
+    return this._feeds;
+  }
+
+  @logInfo
+  private get _ownPeerKey() {
+    return this._swarmIdentity.peerKey;
+  }
+
+  constructor({ topic, swarmIdentity, networkManager, onSessionAuth }: SpaceProtocolOptions) {
+    this._networkManager = networkManager;
+    this._swarmIdentity = swarmIdentity;
+    this._onSessionAuth = onSessionAuth;
+
+    this._topic = PublicKey.from(discoveryKey(sha256(topic.toHex())));
+  }
+
+  // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
+  addFeed(feed: FeedWrapper<FeedMessage>) {
+    log('addFeed', { key: feed.key });
+
+    this._feeds.add(feed);
+    for (const session of this._sessions.values()) {
+      session.replicator.addFeed(feed);
+    }
+  }
+
+  async start() {
+    if (this._connection) {
+      return;
+    }
+
+    // TODO(burdon): Document why empty buffer.
+    const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(''));
+
+    // TODO(burdon): Move to config (with sensible defaults).
+    const topologyConfig = {
+      originateConnections: 4,
+      maxPeers: 10,
+      sampleSize: 20
+    };
+
+    log('starting...');
+    this._connection = await this._networkManager.joinSwarm({
+      protocolProvider: this._createProtocolProvider(credentials),
+      peerId: this._swarmIdentity.peerKey,
+      topic: this._topic,
+      topology: new MMSTTopology(topologyConfig),
+      label: `Protocol swarm: ${this._topic}`
+    });
+
+    log('started');
+  }
+
+  async stop() {
+    if (this._connection) {
+      log('stopping...');
+      await this._connection.close();
+      log('stopped');
+    }
+  }
+
+  private _createProtocolProvider(credentials: Uint8Array | undefined): WireProtocolProvider {
+    return (wireParams) => {
+      const session = new SpaceProtocolSession({
+        wireParams,
+        swarmIdentity: this._swarmIdentity,
+        onSessionAuth: this._onSessionAuth
+      });
+      this._sessions.set(wireParams.remotePeerId, session);
+
+      for (const feed of this._feeds) {
+        session.replicator.addFeed(feed);
+      }
+
+      return session;
+    };
+  }
+}
+
+export type SpaceProtocolSessionParams = {
+  wireParams: WireProtocolParams;
+  swarmIdentity: SwarmIdentity;
+  /**
+   * Called when new session is authenticated.
+   * Additional extensions can be added here.
+   */
+  onSessionAuth?: (session: Teleport) => void;
+};
+
+export enum AuthStatus {
+  INITIAL = 'INITIAL',
+  SUCCESS = 'SUCCESS',
+  FAILURE = 'FAILURE'
+}
+
+// TODO(dmaretskyi): Move to a separate file.
+/**
+ * Represents a single connection to a remote peer
+ */
+export class SpaceProtocolSession implements WireProtocol {
+  @logInfo
+  private readonly _wireParams: WireProtocolParams;
+
+  private readonly _onSessionAuth?: (session: Teleport) => void;
+  private readonly _swarmIdentity: SwarmIdentity;
+
+  private readonly _teleport: Teleport;
+
+  // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
+  public readonly replicator = new ReplicatorExtension().setOptions({ upload: true });
+
+  private _authStatus = AuthStatus.INITIAL;
+
+  @logInfo
+  get authStatus() {
+    return this._authStatus;
+  }
+
+  // TODO(dmaretskyi): Allow to pass in extra extensions.
+  constructor({ wireParams, swarmIdentity, onSessionAuth }: SpaceProtocolSessionParams) {
+    this._wireParams = wireParams;
+    this._swarmIdentity = swarmIdentity;
+    this._onSessionAuth = onSessionAuth;
+
+    this._teleport = new Teleport(wireParams);
+  }
+
+  get stream() {
+    return this._teleport.stream;
+  }
+
+  async initialize(): Promise<void> {
+    await this._teleport.open();
+    this._teleport.addExtension(
+      'dxos.mesh.teleport.auth',
+      new AuthExtension({
+        provider: this._swarmIdentity.credentialProvider,
+        verifier: this._swarmIdentity.credentialAuthenticator,
+        onAuthSuccess: () => {
+          this._authStatus = AuthStatus.SUCCESS;
+          log('Peer authenticated');
+          this._onSessionAuth?.(this._teleport);
+          // TODO(dmaretskyi): Configure replicator to upload.
+        },
+        onAuthFailure: () => {
+          log.warn('Auth failed');
+          this._authStatus = AuthStatus.FAILURE;
+        }
+      })
+    );
+    this._teleport.addExtension('dxos.mesh.teleport.replicator', this.replicator);
+  }
+
+  async destroy(): Promise<void> {
+    await this._teleport.close();
+  }
+}