@dxos/client-services 0.10.0 → 0.11.1

package/src/packlets/space/space-protocol.ts

@@ -0,0 +1,308 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { type Event } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { createIdFromSpaceKey } from '@dxos/echo-protocol';
+import { type FeedWrapper } from '@dxos/feed-store';
+import { PublicKey, SpaceId } from '@dxos/keys';
+import { log, logInfo } from '@dxos/log';
+import {
+  MMSTTopology,
+  type SwarmConnection,
+  type SwarmNetworkManager,
+  type WireProtocol,
+  type WireProtocolProps,
+  type WireProtocolProvider,
+} from '@dxos/network-manager';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { type MuxerStats, Teleport } from '@dxos/teleport';
+import { type BlobStoreApi, BlobSync } from '@dxos/teleport-extension-object-sync';
+import { ReplicatorExtension } from '@dxos/teleport-extension-replicator';
+import { type AsyncCallback, CallbackCollection, ComplexMap } from '@dxos/util';
+
+import { AuthExtension, type AuthProvider, type AuthVerifier } from './auth';
+
+export const MOCK_AUTH_PROVIDER: AuthProvider = async (nonce: Uint8Array) => Buffer.from('mock');
+export const MOCK_AUTH_VERIFIER: AuthVerifier = async (nonce: Uint8Array, credential: Uint8Array) => true;
+
+// TODO(burdon): Reconcile with SigningContext (define types together).
+export interface SwarmIdentity {
+  peerKey: PublicKey;
+  identityKey: PublicKey;
+  credentialProvider: AuthProvider;
+  credentialAuthenticator: AuthVerifier;
+}
+
+export type SpaceProtocolOptions = {
+  topic: PublicKey; // TODO(burdon): Rename?
+  swarmIdentity: SwarmIdentity;
+  networkManager: SwarmNetworkManager;
+
+  blobStore: BlobStoreApi;
+
+  onFeed?: (feed: FeedWrapper<FeedMessage>) => Promise<void>;
+
+  /**
+   * Called when new session is authenticated.
+   * Additional extensions can be added here.
+   */
+  onSessionAuth?: (session: Teleport) => void;
+  onAuthFailure?: (session: Teleport) => void;
+
+  disableP2pReplication?: boolean;
+};
+
+/**
+ * Manages Teleport protocol stream creation and joining swarms with replication and presence extensions.
+ */
+export class SpaceProtocol {
+  private readonly _networkManager: SwarmNetworkManager;
+  private readonly _swarmIdentity: SwarmIdentity;
+  private readonly _onSessionAuth?: (session: Teleport) => void;
+  private readonly _onAuthFailure?: (session: Teleport) => void;
+
+  public readonly blobSync: BlobSync;
+
+  private readonly _disableP2pReplication: boolean;
+
+  @logInfo
+  private readonly _topic: Promise<PublicKey>;
+
+  private readonly _spaceKey: PublicKey;
+
+  private readonly _feeds = new Set<FeedWrapper<FeedMessage>>();
+  private readonly _sessions = new ComplexMap<PublicKey, SpaceProtocolSession>(PublicKey.hash);
+  // TODO(burdon): Move to config (with sensible defaults).
+  private readonly _topology = new MMSTTopology({
+    originateConnections: 4,
+    maxPeers: 10,
+    sampleSize: 20,
+  });
+
+  private _connection?: SwarmConnection;
+
+  public readonly feedAdded = new CallbackCollection<AsyncCallback<FeedWrapper<FeedMessage>>>();
+
+  get sessions(): ReadonlyMap<PublicKey, SpaceProtocolSession> {
+    return this._sessions;
+  }
+
+  get feeds(): ReadonlySet<FeedWrapper<FeedMessage>> {
+    return this._feeds;
+  }
+
+  @logInfo
+  private get _ownPeerKey() {
+    return this._swarmIdentity.peerKey;
+  }
+
+  constructor({
+    topic,
+    swarmIdentity,
+    networkManager,
+    onSessionAuth,
+    onAuthFailure,
+    blobStore,
+    disableP2pReplication,
+  }: SpaceProtocolOptions) {
+    this._spaceKey = topic;
+    this._networkManager = networkManager;
+    this._swarmIdentity = swarmIdentity;
+    this._onSessionAuth = onSessionAuth;
+    this._onAuthFailure = onAuthFailure;
+    this.blobSync = new BlobSync({ blobStore });
+
+    // Space swarm topic contract (DX-1125): the SpaceId bytes (first 20 bytes of SHA-256(spaceKey)).
+    // Both peers and the edge must derive the identical swarm key; SpaceId is the only derivation
+    // computable on workerd (pure subtle-crypto — the previous hypercore `discoveryKey` needs keyed
+    // BLAKE2b, unavailable there). Edge counterpart: `TraceFeedWriter.getSpaceSwarmKey`.
+    this._topic = createIdFromSpaceKey(topic).then((spaceId) => PublicKey.from(SpaceId.decode(spaceId)));
+
+    this._disableP2pReplication = disableP2pReplication ?? false;
+  }
+
+  // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
+  async addFeed(feed: FeedWrapper<FeedMessage>): Promise<void> {
+    log('addFeed', { key: feed.key });
+
+    this._feeds.add(feed);
+    for (const session of this._sessions.values()) {
+      session.replicator.addFeed(feed);
+    }
+
+    await this.feedAdded.callSerial(feed);
+  }
+
+  // TODO(burdon): Rename open? Common open/close interfaces for all services?
+  async start(ctx: Context): Promise<void> {
+    if (this._connection) {
+      return;
+    }
+
+    // TODO(burdon): Document why empty buffer.
+    const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(''));
+
+    await this.blobSync.open();
+
+    log('starting...');
+    const topic = await this._topic;
+    this._connection = await this._networkManager.joinSwarm(ctx, {
+      protocolProvider: this._createProtocolProvider(credentials),
+      topic,
+      topology: this._topology,
+      label: `swarm ${topic.truncate()} for space ${this._spaceKey.truncate()}`,
+    });
+
+    log('started');
+  }
+
+  public updateTopology(): void {
+    this._topology.forceUpdate();
+  }
+
+  async stop(ctx: Context): Promise<void> {
+    await this.blobSync.close();
+
+    if (this._connection) {
+      log('stopping...');
+      await this._connection.close(ctx);
+      log('stopped');
+    }
+  }
+
+  private _createProtocolProvider(credentials: Uint8Array | undefined): WireProtocolProvider {
+    return (wireProps) => {
+      const session = new SpaceProtocolSession({
+        wireProps,
+        swarmIdentity: this._swarmIdentity,
+        onSessionAuth: this._onSessionAuth,
+        onAuthFailure: this._onAuthFailure,
+        blobSync: this.blobSync,
+        disableP2pReplication: this._disableP2pReplication,
+      });
+      this._sessions.set(wireProps.remotePeerId, session);
+
+      for (const feed of this._feeds) {
+        session.replicator.addFeed(feed);
+      }
+
+      return session;
+    };
+  }
+}
+
+export type SpaceProtocolSessionProps = {
+  wireProps: WireProtocolProps;
+  swarmIdentity: SwarmIdentity;
+
+  blobSync: BlobSync;
+
+  /**
+   * Called when new session is authenticated.
+   * Additional extensions can be added here.
+   */
+  onSessionAuth?: (session: Teleport) => void;
+
+  onAuthFailure?: (session: Teleport) => void;
+
+  disableP2pReplication?: boolean;
+};
+
+export enum AuthStatus {
+  INITIAL = 'INITIAL',
+  SUCCESS = 'SUCCESS',
+  FAILURE = 'FAILURE',
+}
+
+// TODO(dmaretskyi): Move to a separate file.
+/**
+ * Represents a single connection to a remote peer
+ */
+export class SpaceProtocolSession implements WireProtocol {
+  @logInfo
+  private readonly _wireProps: WireProtocolProps;
+
+  private readonly _disableP2pReplication: boolean;
+
+  private readonly _onSessionAuth?: (session: Teleport) => void;
+  private readonly _onAuthFailure?: (session: Teleport) => void;
+  private readonly _swarmIdentity: SwarmIdentity;
+  private readonly _blobSync: BlobSync;
+
+  private readonly _teleport: Teleport;
+
+  // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
+  public readonly replicator = new ReplicatorExtension().setOptions({ upload: true });
+
+  private _authStatus = AuthStatus.INITIAL;
+
+  @logInfo
+  get authStatus() {
+    return this._authStatus;
+  }
+
+  get stats(): Event<MuxerStats> {
+    return this._teleport.stats;
+  }
+
+  // TODO(dmaretskyi): Allow to pass in extra extensions.
+  constructor({
+    wireProps,
+    swarmIdentity,
+    onSessionAuth,
+    onAuthFailure,
+    blobSync,
+    disableP2pReplication,
+  }: SpaceProtocolSessionProps) {
+    this._wireProps = wireProps;
+    this._swarmIdentity = swarmIdentity;
+    this._onSessionAuth = onSessionAuth;
+    this._onAuthFailure = onAuthFailure;
+    this._blobSync = blobSync;
+
+    this._teleport = new Teleport(wireProps);
+
+    this._disableP2pReplication = disableP2pReplication ?? false;
+  }
+
+  get stream() {
+    return this._teleport.stream;
+  }
+
+  async open(sessionId?: PublicKey): Promise<void> {
+    await this._teleport.open(sessionId);
+    this._teleport.addExtension(
+      'dxos.mesh.teleport.auth',
+      new AuthExtension({
+        provider: this._swarmIdentity.credentialProvider,
+        verifier: this._swarmIdentity.credentialAuthenticator,
+        onAuthSuccess: () => {
+          log('Peer authenticated');
+          this._authStatus = AuthStatus.SUCCESS;
+          this._onSessionAuth?.(this._teleport);
+          // TODO(dmaretskyi): Configure replicator to upload.
+        },
+        onAuthFailure: () => {
+          this._authStatus = AuthStatus.FAILURE;
+          this._onAuthFailure?.(this._teleport);
+        },
+      }),
+    );
+
+    if (!this._disableP2pReplication) {
+      this._teleport.addExtension('dxos.mesh.teleport.replicator', this.replicator);
+    }
+    this._teleport.addExtension('dxos.mesh.teleport.blobsync', this._blobSync.createExtension());
+  }
+
+  async close(): Promise<void> {
+    log('close');
+    await this._teleport.close();
+  }
+
+  async abort(): Promise<void> {
+    await this._teleport.abort();
+  }
+}

package/src/packlets/space/space.test.ts

@@ -0,0 +1,181 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { describe, expect, onTestFinished, test } from 'vitest';
+
+import { Context } from '@dxos/context';
+import { CredentialGenerator } from '@dxos/credentials';
+
+import { TestAgentBuilder } from './testing';
+
+// TODO(burdon): Factor out?
+const run = <T>(cb: () => Promise<T>): Promise<T> => cb();
+
+describe('space/space', () => {
+  test('creates a database with object model', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const agent = await builder.createPeer();
+    const space = await agent.createSpace();
+
+    await space.open(Context.default());
+    expect(space.isOpen).toBeTruthy();
+    onTestFinished(async () => {
+      await space.close();
+    });
+
+    await agent.spaceGenesis(space);
+
+    await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+
+    await builder.close();
+    expect(space.isOpen).toBeFalsy();
+  });
+
+  test('two spaces replicating', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+
+    //
+    // Agent 1
+    //
+    const [agent1, space1] = await run(async () => {
+      const agent = await builder.createPeer();
+      const space = await agent.createSpace(agent.identityKey);
+
+      await space.open(Context.default());
+      await space.startProtocol(Context.default());
+      expect(space.isOpen).toBeTruthy();
+      onTestFinished(async () => {
+        await space.close();
+      });
+
+      await agent.spaceGenesis(space);
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+
+      return [agent, space];
+    });
+
+    //
+    // Agent 2
+    //
+    const [agent2, space2] = await run(async () => {
+      // NOTE: The genesisKey would be passed as part of the invitation.
+      const agent = await builder.createPeer();
+      const space = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, undefined, true);
+
+      await space.open(Context.default());
+      await space.startProtocol(Context.default());
+      expect(space.isOpen).toBeTruthy();
+      onTestFinished(async () => {
+        await space.close();
+      });
+
+      return [agent, space];
+    });
+
+    expect(agent1).toBeDefined();
+    expect(agent2).toBeDefined();
+
+    {
+      // Write invitation from agent 1.
+      const generator = new CredentialGenerator(agent1.keyring, agent1.identityKey, agent1.deviceKey);
+      const credentials = await generator.createMemberInvitation(
+        space1.key,
+        agent2.identityKey,
+        agent2.deviceKey,
+        space2.controlFeedKey!,
+        space2.dataFeedKey!,
+        space1.genesisFeedKey,
+      );
+
+      for (const credential of credentials) {
+        await space1.controlPipeline.writer.write({
+          credential: { credential },
+        });
+      }
+    }
+
+    {
+      // Initial data exchange.
+
+      // Agent 1 reads all feed messages.
+      await space1.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+
+      // Agent 2 reads all feed messages.
+      await space2.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+    }
+
+    // TODO(burdon): Write multiple items (extract for all tests).
+
+    await builder.close();
+    expect(space1.isOpen).toBeFalsy();
+    expect(space2.isOpen).toBeFalsy();
+  });
+
+  test('open & close', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const agent = await builder.createPeer();
+    const space1 = await agent.createSpace();
+
+    await space1.open(Context.default());
+    expect(space1.isOpen).toBeTruthy();
+    onTestFinished(async () => {
+      await space1.close();
+    });
+
+    await agent.spaceGenesis(space1);
+
+    await space1.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+
+    await space1.close();
+    expect(space1.isOpen).toBeFalsy();
+
+    // Re-open.
+    const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
+
+    await space2.open(Context.default());
+    await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
+  });
+
+  test('re-open', async () => {
+    const builder = new TestAgentBuilder();
+    onTestFinished(async () => {
+      await builder.close();
+    });
+    const agent = await builder.createPeer();
+    const space = await agent.createSpace();
+
+    {
+      await space.open(Context.default());
+      onTestFinished(async () => {
+        await space.close();
+      });
+      expect(space.isOpen).toBeTruthy();
+
+      await agent.spaceGenesis(space);
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+
+      await space.close();
+      expect(space.isOpen).toBeFalsy();
+    }
+
+    // Re-open.
+    {
+      await space.open(Context.default());
+      expect(space.isOpen).toBeTruthy();
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+    }
+  });
+});

package/src/packlets/space/space.ts

@@ -0,0 +1,196 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { Event, scheduleMicroTask, synchronized, trackLeaks } from '@dxos/async';
+import { type Context, Resource } from '@dxos/context';
+import { type DelegateInvitationCredential, type FeedInfo, type MemberInfo } from '@dxos/credentials';
+import { type FeedOptions, type FeedWrapper } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
+import { type PublicKey, type SpaceId } from '@dxos/keys';
+import { log, logInfo } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { type Timeframe } from '@dxos/timeframe';
+import { trace } from '@dxos/tracing';
+import { type AsyncCallback, Callback } from '@dxos/util';
+
+import { type IMetadataStore } from '../metadata';
+import { type PipelineAccessor } from '../pipeline';
+import { ControlPipeline } from './control-pipeline';
+import { type SpaceProtocol } from './space-protocol';
+
+// TODO(burdon): Factor out?
+type FeedProvider = (feedKey: PublicKey, opts?: FeedOptions) => Promise<FeedWrapper<FeedMessage>>;
+
+export type SpaceProps = {
+  id: SpaceId;
+  spaceKey: PublicKey;
+  protocol: SpaceProtocol;
+  genesisFeed: FeedWrapper<FeedMessage>;
+  feedProvider: FeedProvider;
+  metadataStore: IMetadataStore;
+  memberKey: PublicKey;
+
+  // TODO(dmaretskyi): Superseded by epochs.
+  snapshotId?: string | undefined;
+
+  onDelegatedInvitationStatusChange: (invitation: DelegateInvitationCredential, isActive: boolean) => Promise<void>;
+  onMemberRolesChanged: (member: MemberInfo[]) => Promise<void>;
+};
+
+export type CreatePipelineProps = {
+  start: Timeframe;
+  // designation: AdmittedFeed.Designation;
+};
+
+/**
+ * Spaces are globally addressable databases with access control.
+ */
+// TODO(dmaretskyi): Extract database stuff.
+// TODO(dmaretskyi): Rename HaloGraph move to HALO.
+@trackLeaks('open', 'close')
+export class Space extends Resource {
+  public readonly onCredentialProcessed = new Callback<AsyncCallback<Credential>>();
+  public readonly stateUpdate = new Event();
+  public readonly protocol: SpaceProtocol;
+
+  private readonly _id: SpaceId;
+  private readonly _key: PublicKey;
+  private readonly _genesisFeedKey: PublicKey;
+  private readonly _feedProvider: FeedProvider;
+  private readonly _controlPipeline: ControlPipeline;
+
+  private _controlFeed?: FeedWrapper<FeedMessage>;
+  private _dataFeed?: FeedWrapper<FeedMessage>;
+
+  constructor(params: SpaceProps) {
+    super();
+    invariant(params.spaceKey && params.feedProvider);
+    this._id = params.id;
+    this._key = params.spaceKey;
+    this._genesisFeedKey = params.genesisFeed.key;
+    this._feedProvider = params.feedProvider;
+
+    this._controlPipeline = new ControlPipeline({
+      spaceKey: params.spaceKey,
+      genesisFeed: params.genesisFeed,
+      feedProvider: params.feedProvider,
+      metadataStore: params.metadataStore,
+    });
+
+    // TODO(dmaretskyi): Feed set abstraction.
+    this._controlPipeline.onFeedAdmitted.set(async (info) => {
+      // Enable sparse replication to not download mutations covered by prior epochs.
+      const sparse = info.assertion.designation === AdmittedFeed.Designation.DATA;
+
+      if (!info.key.equals(params.genesisFeed.key)) {
+        scheduleMicroTask(this._ctx, async () => {
+          await this.protocol.addFeed(await params.feedProvider(info.key, { sparse }));
+        });
+      }
+    });
+
+    this._controlPipeline.onCredentialProcessed.set(async (credential) => {
+      await this.onCredentialProcessed.callIfSet(credential);
+      log('onCredentialProcessed', { credential });
+      this.stateUpdate.emit();
+    });
+    this._controlPipeline.onDelegatedInvitation.set(async (invitation) => {
+      log('onDelegatedInvitation', { invitation });
+      await params.onDelegatedInvitationStatusChange(invitation, true);
+    });
+    this._controlPipeline.onDelegatedInvitationRemoved.set(async (invitation) => {
+      log('onDelegatedInvitationRemoved', { invitation });
+      await params.onDelegatedInvitationStatusChange(invitation, false);
+    });
+    this._controlPipeline.onMemberRoleChanged.set(async (changedMembers) => {
+      log('onMemberRoleChanged', () => ({ changedMembers: changedMembers.map((m) => [m.key, m.role]) }));
+      await params.onMemberRolesChanged(changedMembers);
+    });
+
+    // Start replicating the genesis feed.
+    this.protocol = params.protocol;
+  }
+
+  @logInfo
+  get id() {
+    return this._id;
+  }
+
+  @logInfo
+  get key() {
+    return this._key;
+  }
+
+  get genesisFeedKey(): PublicKey {
+    return this._genesisFeedKey;
+  }
+
+  get controlFeedKey() {
+    return this._controlFeed?.key;
+  }
+
+  get dataFeedKey() {
+    return this._dataFeed?.key;
+  }
+
+  get spaceState() {
+    return this._controlPipeline.spaceState;
+  }
+
+  /**
+   * @test-only
+   */
+  get controlPipeline(): PipelineAccessor {
+    return this._controlPipeline.pipeline;
+  }
+
+  async setControlFeed(feed: FeedWrapper<FeedMessage>): Promise<this> {
+    invariant(!this._controlFeed, 'Control feed already set.');
+    this._controlFeed = feed;
+    await this._controlPipeline.setWriteFeed(feed);
+    return this;
+  }
+
+  async setDataFeed(feed: FeedWrapper<FeedMessage>): Promise<this> {
+    invariant(!this._dataFeed, 'Data feed already set.');
+    this._dataFeed = feed;
+    return this;
+  }
+
+  /**
+   * Use for diagnostics.
+   */
+  getControlFeeds(): FeedInfo[] {
+    return Array.from(this._controlPipeline.spaceState.feeds.values());
+  }
+
+  @trace.span({ op: 'lifecycle' })
+  protected override async _open(ctx: Context): Promise<void> {
+    log('opening...');
+
+    // Order is important.
+    await this._controlPipeline.start(ctx);
+
+    log('opened');
+  }
+
+  @synchronized
+  public async startProtocol(ctx: Context): Promise<void> {
+    invariant(this.isOpen);
+    await this.protocol.start(ctx);
+    await this.protocol.addFeed(await this._feedProvider(this._genesisFeedKey));
+  }
+
+  @synchronized
+  protected override async _close(ctx: Context): Promise<void> {
+    log('closing...', { key: this._key });
+
+    // Closes in reverse order to open.
+    await this.protocol.stop(ctx);
+    await this._controlPipeline.stop();
+
+    log('closed');
+  }
+}

package/src/packlets/space/testing/change-metadata.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { log } from '@dxos/log';
+import { schema } from '@dxos/protocols/proto';
+import type { Storage } from '@dxos/random-access-storage';
+
+import { MetadataStore } from '../../metadata';
+
+const EchoMetadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata');
+
+/**
+ * This function will change the storage version in the metadata.
+ * This will break your storage and make it unusable.
+ * Use this only for testing purposes.
+ */
+export const changeStorageVersionInMetadata = async (storage: Storage, version: number) => {
+  log('Changing storage version in metadata. USE ONLY FOR TESTING.');
+  const metadata = new MetadataStore(storage.createDirectory('metadata'));
+  await metadata.load();
+  const echoMetadata = metadata.metadata;
+  echoMetadata.version = version;
+  const file = metadata._directory.getOrCreateFile('EchoMetadata');
+  await metadata._writeFile(file, EchoMetadata, echoMetadata);
+  await metadata._directory.flush();
+};