@dxos/client-services 0.10.0 → 0.11.1

package/src/packlets/space/space-manager.ts

@@ -0,0 +1,224 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { type AutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
+import * as EffectContext from 'effect/Context';
+import * as Effect from 'effect/Effect';
+import * as Layer from 'effect/Layer';
+
+import { Trigger, synchronized, trackLeaks } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { type DelegateInvitationCredential, type MemberInfo, getCredentialAssertion } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
+import { createIdFromSpaceKey } from '@dxos/echo-protocol';
+import { type FeedStore, FeedStoreService } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type SwarmNetworkManager, SwarmNetworkManagerService } from '@dxos/network-manager';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import type { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { type Teleport } from '@dxos/teleport';
+import { type BlobStoreApi, BlobStoreApiService } from '@dxos/teleport-extension-object-sync';
+import { ComplexMap } from '@dxos/util';
+
+import { type IMetadataStore, IMetadataStoreService } from '../metadata';
+import { CredentialRetrieverExtension } from './admission-discovery-extension';
+import { Space } from './space';
+import { SpaceProtocol, type SwarmIdentity } from './space-protocol';
+
+export type SpaceManagerProps = {
+  feedStore: FeedStore<FeedMessage>;
+  networkManager: SwarmNetworkManager;
+  metadataStore: IMetadataStore;
+
+  blobStore: BlobStoreApi;
+
+  disableP2pReplication?: boolean;
+};
+
+export type ConstructSpaceProps = {
+  metadata: SpaceMetadata;
+  swarmIdentity: SwarmIdentity;
+  memberKey: PublicKey;
+  /**
+   * Called when connection auth passed successful.
+   */
+  onAuthorizedConnection: (session: Teleport) => void;
+  onAuthFailure?: (session: Teleport) => void;
+  onDelegatedInvitationStatusChange: (invitation: DelegateInvitationCredential, isActive: boolean) => Promise<void>;
+  onMemberRolesChanged: (member: MemberInfo[]) => Promise<void>;
+};
+
+export type RequestSpaceAdmissionCredentialProps = {
+  spaceKey: PublicKey;
+  identityKey: PublicKey;
+  swarmIdentity: SwarmIdentity;
+  timeout: number;
+};
+
+/**
+ * Effect service tag for {@link SpaceManager}.
+ */
+export class SpaceManagerService extends EffectContext.Tag('@dxos/client-services/SpaceManager')<
+  SpaceManagerService,
+  SpaceManager
+>() {}
+
+/**
+ * Manages a collection of ECHO (Data) Spaces.
+ */
+@trackLeaks('open', 'close')
+export class SpaceManager {
+  private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
+  private readonly _feedStore: FeedStore<FeedMessage>;
+  private readonly _networkManager: SwarmNetworkManager;
+  private readonly _metadataStore: IMetadataStore;
+  private readonly _blobStore: BlobStoreApi;
+  private readonly _disableP2pReplication: boolean;
+
+  constructor({ feedStore, networkManager, metadataStore, blobStore, disableP2pReplication }: SpaceManagerProps) {
+    // TODO(burdon): Assert.
+    this._feedStore = feedStore;
+    this._networkManager = networkManager;
+    this._metadataStore = metadataStore;
+    this._blobStore = blobStore;
+    this._disableP2pReplication = disableP2pReplication ?? false;
+  }
+
+  // TODO(burdon): Remove.
+  get spaces() {
+    return this._spaces;
+  }
+
+  @synchronized
+  async open(): Promise<void> {}
+
+  @synchronized
+  async close(): Promise<void> {
+    await Promise.all([...this._spaces.values()].map((space) => space.close()));
+  }
+
+  async constructSpace({
+    metadata,
+    swarmIdentity,
+    onAuthorizedConnection,
+    onAuthFailure,
+    onDelegatedInvitationStatusChange,
+    onMemberRolesChanged,
+    memberKey,
+  }: ConstructSpaceProps): Promise<Space> {
+    log('constructing space...', { spaceKey: metadata.genesisFeedKey });
+
+    // The genesis feed will be the same as the control feed if the space was created by the local agent.
+    const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? failUndefined());
+
+    const spaceKey = metadata.key;
+    const spaceId = await createIdFromSpaceKey(spaceKey);
+    const protocol = new SpaceProtocol({
+      topic: spaceKey,
+      swarmIdentity,
+      networkManager: this._networkManager,
+      onSessionAuth: onAuthorizedConnection,
+      onAuthFailure,
+      blobStore: this._blobStore,
+      disableP2pReplication: this._disableP2pReplication,
+    });
+
+    const space = new Space({
+      id: spaceId,
+      spaceKey,
+      protocol,
+      genesisFeed,
+      feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
+      metadataStore: this._metadataStore,
+      memberKey,
+      onDelegatedInvitationStatusChange,
+      onMemberRolesChanged,
+    });
+    this._spaces.set(space.key, space);
+
+    return space;
+  }
+
+  public async requestSpaceAdmissionCredential(
+    ctx: Context,
+    params: RequestSpaceAdmissionCredentialProps,
+  ): Promise<Credential> {
+    log('requesting space admission credential...', { spaceKey: params.spaceKey });
+
+    const onCredentialResolved = new Trigger<Credential>();
+    const protocol = new SpaceProtocol({
+      topic: params.spaceKey,
+      swarmIdentity: params.swarmIdentity,
+      networkManager: this._networkManager,
+      onSessionAuth: (session: Teleport) => {
+        session.addExtension(
+          'dxos.mesh.teleport.admission-discovery',
+          new CredentialRetrieverExtension(
+            { spaceKey: params.spaceKey, memberKey: params.identityKey },
+            onCredentialResolved,
+          ),
+        );
+      },
+      onAuthFailure: (session: Teleport) => session.close(),
+      blobStore: this._blobStore,
+      disableP2pReplication: this._disableP2pReplication,
+    });
+
+    try {
+      await protocol.start(ctx);
+      const credential = await onCredentialResolved.wait({ timeout: params.timeout });
+      return credential;
+    } catch (err: any) {
+      throw err;
+    } finally {
+      await protocol.stop(ctx);
+    }
+  }
+
+  public findSpaceByRootDocumentId(documentId: string): Space | undefined {
+    return [...this._spaces.values()].find((space) => {
+      return space.spaceState.credentials.some((credential) => {
+        const assertion = getCredentialAssertion(credential);
+        if (assertion['@type'] !== 'dxos.halo.credentials.Epoch') {
+          return false;
+        }
+        if (!assertion?.automergeRoot) {
+          return false;
+        }
+        return parseAutomergeUrl(assertion.automergeRoot as AutomergeUrl).documentId === documentId;
+      });
+    });
+  }
+}
+
+export type SpaceManagerLayerOptions = Pick<SpaceManagerProps, 'disableP2pReplication'>;
+
+/**
+ * Effect Layer constructing a {@link SpaceManager} from ambient service dependencies.
+ */
+export const SpaceManagerLayer = (
+  options: SpaceManagerLayerOptions = {},
+): Layer.Layer<
+  SpaceManagerService,
+  never,
+  FeedStoreService | SwarmNetworkManagerService | IMetadataStoreService | BlobStoreApiService
+> =>
+  Layer.effect(
+    SpaceManagerService,
+    Effect.gen(function* () {
+      const feedStore = yield* FeedStoreService;
+      const networkManager = yield* SwarmNetworkManagerService;
+      const metadataStore = yield* IMetadataStoreService;
+      const blobStore = yield* BlobStoreApiService;
+      return new SpaceManager({
+        feedStore,
+        networkManager,
+        metadataStore,
+        blobStore,
+        disableP2pReplication: options.disableP2pReplication,
+      });
+    }),
+  );

package/src/packlets/space/space-protocol.browser.test.ts

@@ -0,0 +1,129 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, expect, onTestFinished, test } from 'vitest';
+
+import { Context } from '@dxos/context';
+import { Keyring } from '@dxos/keyring';
+import { PublicKey } from '@dxos/keys';
+import { createStorage } from '@dxos/random-access-storage';
+import { Timeframe } from '@dxos/timeframe';
+
+import { TestAgentBuilder, TestFeedBuilder } from './testing';
+
+describe('space/space-protocol', () => {
+  // TODO(dmaretskyi): Fails with the vscode test-runner for some reason.
+  test('two peers discover each other', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const gossip1 = peer1.createGossip();
+    const presence1 = peer1.createPresence(gossip1);
+    const protocol1 = peer1.createSpaceProtocol(topic, gossip1);
+
+    const peer2 = await builder.createPeer();
+    const gossip2 = peer2.createGossip();
+    const presence2 = peer2.createPresence(gossip2);
+    const protocol2 = peer2.createSpaceProtocol(topic, gossip2);
+
+    await protocol1.start(Context.default());
+    await protocol2.start(Context.default());
+
+    onTestFinished(() => protocol1.stop(Context.default()));
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    await expect
+      .poll(() => presence1.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer2.identityKey)))
+      .toBeTruthy();
+    await expect
+      .poll(() => presence2.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer1.identityKey)))
+      .toBeTruthy();
+  });
+
+  test('replicates a feed', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start(Context.default());
+    await protocol2.start(Context.default());
+
+    onTestFinished(() => protocol1.stop(Context.default()));
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    const builder1 = new TestFeedBuilder();
+    const feedStore1 = builder1.createFeedStore();
+
+    const builder2 = new TestFeedBuilder();
+    const feedStore2 = builder2.createFeedStore();
+
+    const feedKey = await builder1.keyring.createKey();
+    const feed1 = await feedStore1.openFeed(feedKey, { writable: true });
+    const feed2 = await feedStore2.openFeed(feedKey);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await expect.poll(() => feed2.properties.length).toEqual(1);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await expect.poll(() => feed2.properties.length).toEqual(2);
+
+    await builder.close();
+  });
+
+  // TODO: Some storage drivers may break when there are multiple storage instances.
+  test.skip('replicates a feed through a webrtc connection', async () => {
+    const builder = new TestAgentBuilder({
+      storage: createStorage(),
+    });
+    onTestFinished(async () => {
+      await builder.close();
+    });
+
+    const keyring = new Keyring();
+    const topic = await keyring.createKey();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start(Context.default());
+    await protocol2.start(Context.default());
+
+    onTestFinished(() => protocol1.stop(Context.default()));
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    const feedKey = await peer1.keyring.createKey();
+
+    const feed1 = await peer1.feedStore.openFeed(feedKey, { writable: true });
+    const feed2 = await peer2.feedStore.openFeed(feedKey);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await expect.poll(() => feed2.properties.length).toEqual(1);
+
+    await feed1.append({ timeframe: new Timeframe() });
+    await expect.poll(() => feed2.properties.length).toEqual(2);
+
+    await builder.close();
+  });
+});

package/src/packlets/space/space-protocol.test.ts

@@ -0,0 +1,158 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, expect, onTestFinished, test } from 'vitest';
+
+import { Context } from '@dxos/context';
+import { PublicKey } from '@dxos/keys';
+import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
+import { MemoryTransportFactory, SwarmNetworkManager } from '@dxos/network-manager';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
+import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { Timeframe } from '@dxos/timeframe';
+
+import { AuthStatus, MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, SpaceProtocol } from './space-protocol';
+import { TestAgentBuilder, TestFeedBuilder } from './testing';
+
+describe('space/space-protocol', () => {
+  // Flaky.
+  test('two peers discover each other via presence', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const gossip1 = peer1.createGossip();
+    const presence1 = peer1.createPresence(gossip1);
+    await presence1.open();
+    const protocol1 = peer1.createSpaceProtocol(topic, gossip1);
+
+    const peer2 = await builder.createPeer();
+    const gossip2 = peer2.createGossip();
+    const presence2 = peer2.createPresence(gossip2);
+    await presence2.open();
+    const protocol2 = peer2.createSpaceProtocol(topic, gossip2);
+
+    await protocol1.start(Context.default());
+    onTestFinished(() => protocol1.stop(Context.default()));
+
+    await protocol2.start(Context.default());
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    await expect
+      .poll(() => presence1.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer2.identityKey)), {
+        timeout: 1_000,
+      })
+      .toBeTruthy();
+    await expect
+      .poll(() => presence2.getPeersOnline().some(({ identityKey }) => identityKey.equals(peer1.identityKey)), {
+        timeout: 1_000,
+      })
+      .toBeTruthy();
+  });
+
+  test('failing authentication', async () => {
+    const [topic, peerId1, peerId2] = PublicKey.randomSequence();
+    const signalContext = new MemorySignalManagerContext();
+
+    const protocol1 = new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        peerKey: peerId1,
+        identityKey: peerId1,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: async () => false, // Reject everyone.
+      },
+      blobStore: new BlobStore(createStorage({ type: StorageType.RAM }).createDirectory()),
+      networkManager: new SwarmNetworkManager({
+        signalManager: new MemorySignalManager(signalContext),
+        transportFactory: MemoryTransportFactory,
+        peerInfo: {
+          peerKey: peerId1.toHex(),
+          identityKey: peerId1.toHex(),
+        },
+      }),
+    });
+
+    const protocol2 = new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        peerKey: peerId2,
+        identityKey: peerId2,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER,
+      },
+      blobStore: new BlobStore(createStorage({ type: StorageType.RAM }).createDirectory()),
+      networkManager: new SwarmNetworkManager({
+        signalManager: new MemorySignalManager(signalContext),
+        transportFactory: MemoryTransportFactory,
+        peerInfo: {
+          peerKey: peerId2.toHex(),
+          identityKey: peerId2.toHex(),
+        },
+      }),
+    });
+
+    await protocol1.start(Context.default());
+    onTestFinished(() => protocol1.stop(Context.default()));
+
+    await protocol2.start(Context.default());
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    await expect.poll(() => protocol1.sessions.get(peerId2)?.authStatus).toEqual(AuthStatus.FAILURE);
+  });
+
+  test('replicates a feed', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+
+    const topic = PublicKey.random();
+
+    const peer1 = await builder.createPeer();
+    const protocol1 = peer1.createSpaceProtocol(topic);
+
+    const peer2 = await builder.createPeer();
+    const protocol2 = peer2.createSpaceProtocol(topic);
+
+    await protocol1.start(Context.default());
+    await protocol2.start(Context.default());
+
+    onTestFinished(() => protocol1.stop(Context.default()));
+    onTestFinished(() => protocol2.stop(Context.default()));
+
+    //
+    // Create feeds.
+    //
+
+    const builder1 = new TestFeedBuilder();
+    const feedStore1 = builder1.createFeedStore();
+
+    const builder2 = new TestFeedBuilder();
+    const feedStore2 = builder2.createFeedStore();
+
+    const feed1 = await feedStore1.openFeed(await builder1.keyring.createKey(), { writable: true });
+    const feed2 = await feedStore2.openFeed(feed1.key);
+
+    await protocol1.addFeed(feed1);
+    await protocol2.addFeed(feed2);
+
+    //
+    // Append message.
+    //
+
+    // TODO(burdon): Append batch of messages.
+    await feed1.append({ timeframe: new Timeframe() });
+    // Received message appended before replication.
+    await expect.poll(() => feed2.properties.length).toEqual(1);
+
+    // TODO(burdon): Append batch of messages.
+    await feed1.append({ timeframe: new Timeframe() });
+    // Received message appended after replication.
+    await expect.poll(() => feed2.properties.length).toEqual(2);
+  });
+});