@dxos/client-services 0.6.13 → 0.6.14-main.2b6a0f3

package/src/packlets/spaces/data-space-manager.ts

@@ -14,8 +14,11 @@ import {
   type DelegateInvitationCredential,
   type MemberInfo,
 } from '@dxos/credentials';
-import { convertLegacyReferences, findInlineObjectOfType, type EchoEdgeReplicator, type EchoHost } from '@dxos/echo-db';
 import {
+  convertLegacyReferences,
+  findInlineObjectOfType,
+  type EchoEdgeReplicator,
+  type EchoHost,
   AuthStatus,
   CredentialServerExtension,
   type MeshEchoReplicator,
@@ -32,8 +35,8 @@ import {
   type ObjectStructure,
   type SpaceDoc,
 } from '@dxos/echo-protocol';
-import { TYPE_PROPERTIES, generateEchoId, getTypeReference } from '@dxos/echo-schema';
-import type { EdgeConnection } from '@dxos/edge-client';
+import { TYPE_PROPERTIES, createObjectId, getTypeReference } from '@dxos/echo-schema';
+import type { EdgeConnection, EdgeHttpClient } from '@dxos/edge-client';
 import { writeMessages, type FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -43,7 +46,7 @@ import { AlreadyJoinedError, trace as Trace } from '@dxos/protocols';
 import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import { type Runtime } from '@dxos/protocols/proto/dxos/config';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { type SpaceMetadata, EdgeReplicationSetting } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { SpaceMember, type Credential, type ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type DelegateSpaceInvitation } from '@dxos/protocols/proto/dxos/halo/invitations';
 import { type PeerState } from '@dxos/protocols/proto/dxos/mesh/presence';
@@ -107,6 +110,7 @@ export type DataSpaceManagerParams = {
   echoHost: EchoHost;
   invitationsManager: InvitationsManager;
   edgeConnection?: EdgeConnection;
+  edgeHttpClient?: EdgeHttpClient;
   meshReplicator?: MeshEchoReplicator;
   echoEdgeReplicator?: EchoEdgeReplicator;
   runtimeParams?: DataSpaceManagerRuntimeParams;
@@ -116,6 +120,7 @@ export type DataSpaceManagerParams = {
 export type DataSpaceManagerRuntimeParams = {
   spaceMemberPresenceAnnounceInterval?: number;
   spaceMemberPresenceOfflineTimeout?: number;
+  activeEdgeNotarizationPollingInterval?: number;
   disableP2pReplication?: boolean;
 };
 
@@ -135,6 +140,7 @@ export class DataSpaceManager extends Resource {
   private readonly _echoHost: EchoHost;
   private readonly _invitationsManager: InvitationsManager;
   private readonly _edgeConnection?: EdgeConnection = undefined;
+  private readonly _edgeHttpClient?: EdgeHttpClient = undefined;
   private readonly _edgeFeatures?: Runtime.Client.EdgeFeatures = undefined;
   private readonly _meshReplicator?: MeshEchoReplicator = undefined;
   private readonly _echoEdgeReplicator?: EchoEdgeReplicator = undefined;
@@ -154,6 +160,7 @@ export class DataSpaceManager extends Resource {
     this._edgeConnection = params.edgeConnection;
     this._edgeFeatures = params.edgeFeatures;
     this._echoEdgeReplicator = params.echoEdgeReplicator;
+    this._edgeHttpClient = params.edgeHttpClient;
     this._runtimeParams = params.runtimeParams;
 
     trace.diagnostic({
@@ -290,7 +297,7 @@ export class DataSpaceManager extends Resource {
       },
     };
 
-    const propertiesId = generateEchoId();
+    const propertiesId = createObjectId();
     document.change((doc: SpaceDoc) => {
       setDeep(doc, ['objects', propertiesId], properties);
     });
@@ -388,6 +395,26 @@ export class DataSpaceManager extends Resource {
     });
   }
 
+  async setSpaceEdgeReplicationSetting(spaceKey: PublicKey, setting: EdgeReplicationSetting) {
+    const space = this._spaces.get(spaceKey);
+    invariant(space, 'Space not found.');
+
+    await this._metadataStore.setSpaceEdgeReplicationSetting(spaceKey, setting);
+
+    if (space.isOpen) {
+      switch (setting) {
+        case EdgeReplicationSetting.DISABLED:
+          await this._echoEdgeReplicator?.disconnectFromSpace(space.id);
+          break;
+        case EdgeReplicationSetting.ENABLED:
+          await this._echoEdgeReplicator?.connectToSpace(space.id);
+          break;
+      }
+    }
+
+    space.stateUpdate.emit();
+  }
+
   private async _constructSpace(metadata: SpaceMetadata) {
     log('construct space', { metadata });
     const gossip = new Gossip({
@@ -479,13 +506,23 @@ export class DataSpaceManager extends Resource {
       },
       cache: metadata.cache,
       edgeConnection: this._edgeConnection,
+      edgeHttpClient: this._edgeHttpClient,
       edgeFeatures: this._edgeFeatures,
+      activeEdgeNotarizationPollingInterval: this._runtimeParams?.activeEdgeNotarizationPollingInterval,
     });
     dataSpace.postOpen.append(async () => {
-      await this._echoEdgeReplicator?.connectToSpace(dataSpace.id);
+      const setting = dataSpace.getEdgeReplicationSetting();
+      if (setting === EdgeReplicationSetting.ENABLED) {
+        await this._echoEdgeReplicator?.connectToSpace(dataSpace.id);
+      } else if (this._echoEdgeReplicator) {
+        log('not connecting EchoEdgeReplicator because of EdgeReplicationSetting', { spaceId: dataSpace.id });
+      }
     });
     dataSpace.preClose.append(async () => {
-      await this._echoEdgeReplicator?.disconnectFromSpace(dataSpace.id);
+      const setting = dataSpace.getEdgeReplicationSetting();
+      if (setting === EdgeReplicationSetting.ENABLED) {
+        await this._echoEdgeReplicator?.disconnectFromSpace(dataSpace.id);
+      }
     });
 
     presence.newPeer.on((peerState) => {

package/src/packlets/spaces/data-space.ts

@@ -7,10 +7,15 @@ import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { Context, ContextDisposedError, cancelWithContext } from '@dxos/context';
 import type { SpecificCredential } from '@dxos/credentials';
 import { timed, warnAfterTimeout } from '@dxos/debug';
-import { type EchoHost, type DatabaseRoot } from '@dxos/echo-db';
-import { createMappedFeedWriter, type MetadataStore, type Space } from '@dxos/echo-pipeline';
+import {
+  type EchoHost,
+  type DatabaseRoot,
+  createMappedFeedWriter,
+  type MetadataStore,
+  type Space,
+} from '@dxos/echo-pipeline';
 import { SpaceDocVersion } from '@dxos/echo-protocol';
-import type { EdgeConnection } from '@dxos/edge-client';
+import type { EdgeConnection, EdgeHttpClient } from '@dxos/edge-client';
 import { type FeedStore, type FeedWrapper } from '@dxos/feed-store';
 import { failedInvariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -75,7 +80,9 @@ export type DataSpaceParams = {
   callbacks?: DataSpaceCallbacks;
   cache?: SpaceCache;
   edgeConnection?: EdgeConnection;
+  edgeHttpClient?: EdgeHttpClient;
   edgeFeatures?: Runtime.Client.EdgeFeatures;
+  activeEdgeNotarizationPollingInterval?: number;
 };
 
 export type CreateEpochOptions = {
@@ -96,7 +103,7 @@ export class DataSpace {
   private readonly _feedStore: FeedStore<FeedMessage>;
   private readonly _metadataStore: MetadataStore;
   private readonly _signingContext: SigningContext;
-  private readonly _notarizationPlugin = new NotarizationPlugin();
+  private readonly _notarizationPlugin: NotarizationPlugin;
   private readonly _callbacks: DataSpaceCallbacks;
   private readonly _cache?: SpaceCache = undefined;
   private readonly _echoHost: EchoHost;
@@ -136,6 +143,12 @@ export class DataSpace {
     this._signingContext = params.signingContext;
     this._callbacks = params.callbacks ?? {};
     this._echoHost = params.echoHost;
+    this._notarizationPlugin = new NotarizationPlugin({
+      spaceId: this._inner.id,
+      edgeClient: params.edgeHttpClient,
+      edgeFeatures: params.edgeFeatures,
+      activeEdgePollingInterval: params.activeEdgeNotarizationPollingInterval,
+    });
 
     this.authVerifier = new TrustedKeySetAuthVerifier({
       trustedKeysProvider: () =>
@@ -231,6 +244,7 @@ export class DataSpace {
     }
 
     await this._inner.open(new Context());
+    await this._inner.startProtocol();
 
     await this._edgeFeedReplicator?.open();
 
@@ -318,6 +332,7 @@ export class DataSpace {
     this._state = SpaceState.SPACE_INITIALIZING;
     log('new state', { state: SpaceState[this._state] });
 
+    log('initializing control pipeline');
     await this._initializeAndReadControlPipeline();
 
     // Allow other tasks to run before loading the data pipeline.
@@ -325,10 +340,13 @@ export class DataSpace {
 
     const ready = this.stateUpdate.waitForCondition(() => this._state === SpaceState.SPACE_READY);
 
+    log('initializing automerge root');
     this._automergeSpaceState.startProcessingRootDocs();
 
     // TODO(dmaretskyi): Change so `initializeDataPipeline` doesn't wait for the space to be READY, but rather any state with a valid root.
+    log('waiting for space to be ready');
     await ready;
+    log('space is ready');
   }
 
   private async _enterReadyState() {
@@ -345,6 +363,7 @@ export class DataSpace {
   private async _initializeAndReadControlPipeline() {
     await this._inner.controlPipeline.state.waitUntilReachedTargetTimeframe({
       ctx: this._ctx,
+      timeout: 10_000,
       breakOnStall: false,
     });
 
@@ -408,8 +427,16 @@ export class DataSpace {
     }
 
     if (credentials.length > 0) {
-      // Never times out
-      await this.notarizationPlugin.notarize({ ctx: this._ctx, credentials, timeout: 0 });
+      try {
+        log('will notarize credentials for feed admission', { count: credentials.length });
+        // Never times out
+        await this.notarizationPlugin.notarize({ ctx: this._ctx, credentials, timeout: 0 });
+
+        log('credentials notarized');
+      } catch (err) {
+        log.error('error notarizing credentials for feed admission', err);
+        throw err;
+      }
 
       // Set this after credentials are notarized so that on failure we will retry.
       await this._metadataStore.setWritableFeedKeys(this.key, this.inner.controlFeedKey!, this.inner.dataFeedKey!);
@@ -546,6 +573,10 @@ export class DataSpace {
     this.stateUpdate.emit();
   }
 
+  getEdgeReplicationSetting() {
+    return this._metadataStore.getSpaceEdgeReplicationSetting(this.key);
+  }
+
   private _onFeedAdded = async (feed: FeedWrapper<any>) => {
     await this._edgeFeedReplicator!.addFeed(feed);
   };

package/src/packlets/spaces/edge-feed-replicator.test.ts

@@ -0,0 +1,252 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { decode as decodeCbor, encode as encodeCbor } from 'cbor-x';
+import { getPort } from 'get-port-please';
+import { describe, expect, onTestFinished, test, vi } from 'vitest';
+
+import { sleep, Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { valueEncoding } from '@dxos/echo-pipeline';
+import { createEphemeralEdgeIdentity, EdgeClient, EdgeIdentityChangedError } from '@dxos/edge-client';
+import { createTestEdgeWsServer } from '@dxos/edge-client/testing';
+import { FeedFactory, FeedStore, type FeedWrapper } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import { SpaceId } from '@dxos/keys';
+import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { createStorage } from '@dxos/random-access-storage';
+import { openAndClose } from '@dxos/test-utils';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { EdgeFeedReplicator } from './edge-feed-replicator';
+
+describe('EdgeFeedReplicator', () => {
+  test('requests metadata after connection is open', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    await sleep(50);
+
+    expect(sendSpy).not.toHaveBeenCalled();
+    expect(messageSink.length).toEqual(0);
+
+    admitConnection.wake();
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(1);
+    expect(messageSink.length).toEqual(1);
+    expect(messageSink[0].type).toEqual('get-metadata');
+  });
+
+  test('replicates if added to a connected client', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+    admitConnection.wake();
+    await expect.poll(() => messenger.isConnected).toBeTruthy();
+
+    await attachReplicator(messenger);
+    await expect.poll(() => messageSink.length).toEqual(1);
+  });
+
+  test('sends a block', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+
+    admitConnection.wake();
+    await appendMessage(feed);
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink[1].type).toEqual('data');
+  });
+
+  test('re-requests metadata on reconnect', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    admitConnection.wake();
+    await expect.poll(() => messageSink.length).toEqual(1);
+
+    await updateIdentity(messenger);
+    await messenger.reconnect.waitForCount(1);
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink[1].type).toEqual('get-metadata');
+  });
+
+  test('recovers after query sending failure during identity change', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    sendSpy.mockImplementationOnce(() => {
+      throw new EdgeIdentityChangedError(); // Hard to mock the exact race condition for when this error is thrown
+    });
+    admitConnection.wake();
+
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(1);
+    expect(messageSink.length).toEqual(0);
+    await updateIdentity(messenger);
+
+    await expect.poll(() => messageSink.length).toEqual(1);
+    expect(messageSink[0].type).toEqual('get-metadata');
+  });
+
+  test('recovers after response sending failure during identity change', async () => {
+    const { endpoint, admitConnection, messageSink, sendResponseMessage } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    await appendMessage(feed);
+
+    sendSpy.mockImplementationOnce(async (request: any) => {
+      sendResponseMessage(request, encodeCbor({ type: 'metadata', feedKey: feed.key.toHex(), length: 0 }));
+      return Promise.resolve();
+    });
+    sendSpy.mockImplementationOnce(async () => {
+      throw new EdgeIdentityChangedError();
+    });
+    admitConnection.wake();
+
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(2);
+    sendSpy.mockRestore();
+    expect(messageSink.length).toEqual(0);
+    await updateIdentity(messenger);
+
+    await messenger.reconnect.waitForCount(1);
+    await expect.poll(() => messageSink.find((msg) => msg.type === 'data')).toBeDefined();
+  });
+
+  test('propagates errors unrelated to reconnect', async () => {
+    const { endpoint, admitConnection } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    const { replicator } = await attachReplicator(messenger, { skipOpen: true });
+    const raised = new Trigger();
+    await replicator.open(new Context({ onError: () => raised.wake() }));
+    onTestFinished(async () => {
+      await replicator.close();
+    });
+
+    sendSpy.mockImplementationOnce(() => {
+      throw new Error();
+    });
+    admitConnection.wake();
+
+    await raised.wait();
+  });
+
+  test('identity update before connected', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+    await updateIdentity(messenger);
+    await sleep(100);
+    admitConnection.wake();
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink.map((m) => m.type)).toStrictEqual(range(2, () => 'get-metadata'));
+  });
+
+  test('block appended during reconnect', async () => {
+    const { endpoint, admitConnection, feedLength } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    admitConnection.wake();
+    await sleep(10);
+
+    admitConnection.reset();
+    await updateIdentity(messenger);
+    await appendMessage(feed);
+    await sleep(20);
+    admitConnection.wake();
+
+    await expect.poll(() => feedLength()).toEqual(1);
+  });
+
+  test('reconnect during block append', async () => {
+    const { endpoint, admitConnection, feedLength } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    admitConnection.wake();
+    await sleep(10);
+
+    void appendMessage(feed);
+    await updateIdentity(messenger);
+
+    await expect.poll(() => feedLength()).toEqual(1);
+  });
+
+  const createEdge = async () => {
+    const port = await getPort({ host: 'localhost', port: 7200, portRange: [7200, 7299] });
+    let lastBlockIndex = -1;
+    const admitConnection = new Trigger();
+    const { cleanup, endpoint, messageSink, sendResponseMessage } = await createTestEdgeWsServer(port, {
+      admitConnection,
+      payloadDecoder: decodeCbor,
+      messageHandler: async (message: any) => {
+        if (message.type === 'get-metadata') {
+          return encodeCbor({ type: 'metadata', feedKey: message.feedKey, length: lastBlockIndex + 1 });
+        } else {
+          lastBlockIndex = Math.max(lastBlockIndex, message.blocks[message.blocks.length - 1].index);
+        }
+      },
+    });
+    onTestFinished(cleanup);
+
+    return {
+      endpoint,
+      messageSink,
+      admitConnection,
+      sendResponseMessage,
+      feedLength: () => lastBlockIndex + 1,
+    };
+  };
+
+  const createClient = async (endpoint: string) => {
+    const messenger = new EdgeClient(await createEphemeralEdgeIdentity(), { socketEndpoint: endpoint });
+    const sendSpy = vi.spyOn(messenger, 'send');
+    await openAndClose(messenger);
+    return { messenger, sendSpy };
+  };
+
+  const attachReplicator = async (messenger: EdgeClient, options?: { skipOpen?: boolean }) => {
+    const spaceId = SpaceId.random();
+    const feed = await createNewFeed();
+    const replicator = new EdgeFeedReplicator({ messenger, spaceId });
+    await replicator.addFeed(feed);
+    if (!options?.skipOpen) {
+      await openAndClose(replicator);
+    }
+    return { feed, replicator };
+  };
+
+  const createNewFeed = async () => {
+    const storage = createStorage();
+    const keyring = new Keyring();
+    const feedStore = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory(),
+        signer: keyring,
+        hypercore: { valueEncoding },
+      }),
+    });
+    onTestFinished(() => feedStore.close());
+    return feedStore.openFeed(await keyring.createKey(), { writable: true });
+  };
+
+  const updateIdentity = async (messenger: EdgeClient) => {
+    messenger.setIdentity(await createEphemeralEdgeIdentity());
+  };
+
+  const appendMessage = (feed: FeedWrapper<FeedMessage>) => feed.append({ timeframe: new Timeframe() });
+});