@dxos/client-services 0.6.13 → 0.6.14-main.69511f5

package/src/packlets/services/service-host.test.ts

@@ -2,9 +2,8 @@
 // Copyright 2023 DXOS.org
 //
 
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
 import { rmSync } from 'node:fs';
+import { afterEach, onTestFinished, describe, expect, test } from 'vitest';
 
 import { asyncTimeout, latch, Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
@@ -14,13 +13,10 @@ import { type PublicKey } from '@dxos/keys';
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { type Identity } from '@dxos/protocols/proto/dxos/client/services';
 import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { afterTest, describe, test } from '@dxos/test';
 import { isNode } from '@dxos/util';
 
 import { createMockCredential, createServiceHost } from '../testing';
 
-chai.use(chaiAsPromised);
-
 describe('ClientServicesHost', () => {
   const dataRoot = '/tmp/dxos/client-services/service-host/storage';
 
@@ -38,7 +34,7 @@ describe('ClientServicesHost', () => {
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
     await host.open(new Context());
-    afterTest(() => host.close());
+    onTestFinished(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
     const { spaceKey } = await host.services.SpacesService!.createSpace();
@@ -49,7 +45,7 @@ describe('ClientServicesHost', () => {
       tick();
       // console.log(credential);
     });
-    afterTest(() => stream.close());
+    onTestFinished(() => stream.close());
 
     await done();
   });
@@ -57,7 +53,7 @@ describe('ClientServicesHost', () => {
   test('write and query credentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
     await host.open(new Context());
-    afterTest(() => host.close());
+    onTestFinished(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
@@ -86,7 +82,7 @@ describe('ClientServicesHost', () => {
         queriedCredential.wake(credential);
       }
     });
-    afterTest(() => credentials.close());
+    onTestFinished(() => credentials.close());
 
     await queriedCredential.wait();
   });
@@ -94,7 +90,7 @@ describe('ClientServicesHost', () => {
   test('sign presentation', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
     await host.open(new Context());
-    afterTest(() => host.close());
+    onTestFinished(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
@@ -147,9 +143,9 @@ describe('ClientServicesHost', () => {
           trigger.wake(identity.identity);
         }
       });
-      await expect(asyncTimeout(trigger.wait(), 200)).to.be.rejectedWith();
+      await expect(asyncTimeout(trigger.wait(), 200)).rejects.toBeInstanceOf(Error);
       await stream?.close();
       await host.close();
     }
-  }).onlyEnvironments('nodejs', 'chromium', 'firefox');
+  });
 });

package/src/packlets/services/service-host.ts

@@ -6,7 +6,7 @@ import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, type ClientServices } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { EdgeClient, type EdgeConnection } from '@dxos/edge-client';
+import { EdgeClient, EdgeHttpClient, createStubEdgeIdentity, type EdgeConnection } from '@dxos/edge-client';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { type LevelDB } from '@dxos/kv-store';
@@ -15,7 +15,7 @@ import { EdgeSignalManager, WebsocketSignalManager, type SignalManager } from '@
 import {
   SwarmNetworkManager,
   createIceProvider,
-  createSimplePeerTransportFactory,
+  createRtcTransportFactory,
   type TransportFactory,
 } from '@dxos/network-manager';
 import { trace } from '@dxos/protocols';
@@ -26,12 +26,13 @@ import { WebsocketRpcClient } from '@dxos/websocket-rpc';
 
 import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
 import { ServiceRegistry } from './service-registry';
+import { EdgeAgentServiceImpl } from '../agents';
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsHostEvents, DevtoolsServiceImpl } from '../devtools';
 import {
-  type CollectDiagnosticsBroadcastHandler,
   createCollectDiagnosticsBroadcastHandler,
   createDiagnostics,
+  type CollectDiagnosticsBroadcastHandler,
 } from '../diagnostics';
 import { IdentityServiceImpl, type CreateIdentityOptions } from '../identity';
 import { ContactsServiceImpl } from '../identity/contacts-service';
@@ -89,6 +90,7 @@ export class ClientServicesHost {
   private _callbacks?: ClientServicesHostCallbacks;
   private _devtoolsProxy?: WebsocketRpcClient<{}, ClientServices>;
   private _edgeConnection?: EdgeConnection = undefined;
+  private _edgeHttpClient?: EdgeHttpClient = undefined;
 
   private _serviceContext!: ServiceContext;
   private readonly _runtimeParams: ServiceContextRuntimeParams;
@@ -100,6 +102,9 @@ export class ClientServicesHost {
   @Trace.info()
   private _open = false;
 
+  @Trace.info()
+  private _resetting = false;
+
   constructor({
     config,
     transportFactory,
@@ -140,7 +145,7 @@ export class ClientServicesHost {
     this._systemService = new SystemServiceImpl({
       config: () => this._config,
       statusUpdate: this._statusUpdate,
-      getCurrentStatus: () => (this.isOpen ? SystemStatus.ACTIVE : SystemStatus.INACTIVE),
+      getCurrentStatus: () => (this.isOpen && !this._resetting ? SystemStatus.ACTIVE : SystemStatus.INACTIVE),
       getDiagnostics: () => {
         return createDiagnostics(this._serviceRegistry.services, this._serviceContext, this._config!);
       },
@@ -212,13 +217,13 @@ export class ClientServicesHost {
 
     const edgeEndpoint = config?.get('runtime.services.edge.url');
     if (edgeEndpoint) {
-      const randomKey = PublicKey.random().toHex();
-      this._edgeConnection = new EdgeClient(randomKey, randomKey, { socketEndpoint: edgeEndpoint });
+      this._edgeConnection = new EdgeClient(createStubEdgeIdentity(), { socketEndpoint: edgeEndpoint });
+      this._edgeHttpClient = new EdgeHttpClient(edgeEndpoint);
     }
 
     const {
       connectionLog = true,
-      transportFactory = createSimplePeerTransportFactory(
+      transportFactory = createRtcTransportFactory(
         { iceServers: this._config?.get('runtime.services.ice') },
         this._config?.get('runtime.services.iceProviders') &&
           createIceProvider(this._config!.get('runtime.services.iceProviders')!),
@@ -278,6 +283,7 @@ export class ClientServicesHost {
       this._networkManager,
       this._signalManager,
       this._edgeConnection,
+      this._edgeHttpClient,
       this._runtimeParams,
       this._config.get('runtime.client.edgeFeatures'),
     );
@@ -287,8 +293,14 @@ export class ClientServicesHost {
       return this._serviceContext.dataSpaceManager!;
     };
 
+    const agentManagerProvider = async () => {
+      await this._serviceContext.initialized.wait();
+      return this._serviceContext.edgeAgentManager!;
+    };
+
     const identityService = new IdentityServiceImpl(
       this._serviceContext.identityManager,
+      this._serviceContext.recoveryManager,
       this._serviceContext.keyring,
       () => this._serviceContext.dataSpaceManager!,
       (params) => this._createIdentity(params),
@@ -328,6 +340,8 @@ export class ClientServicesHost {
         config: this._config,
         context: this._serviceContext,
       }),
+
+      EdgeAgentService: new EdgeAgentServiceImpl(agentManagerProvider),
     });
 
     await this._serviceContext.open(ctx);
@@ -378,6 +392,10 @@ export class ClientServicesHost {
     log.trace('dxos.sdk.client-services-host.reset', trace.begin({ id: traceId }));
 
     log.info('resetting...');
+    // Emit this status update immediately so app returns to fallback.
+    // This state is never cleared because the app reloads.
+    this._resetting = true;
+    this._statusUpdate.emit();
     await this._serviceContext?.close();
     await this._storage!.reset();
     log.info('reset');

package/src/packlets/services/service-registry.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { expect } from 'chai';
+import { describe, expect, test } from 'vitest';
 
 import { Event } from '@dxos/async';
 import { type ClientServices } from '@dxos/client-protocol';
@@ -12,7 +12,6 @@ import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols/proto';
 import { type SystemService, SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { createLinkedPorts, createProtoRpcPeer, createServiceBundle } from '@dxos/rpc';
-import { describe, test } from '@dxos/test';
 
 import { ServiceRegistry } from './service-registry';
 import { SystemServiceImpl } from '../system';

package/src/packlets/spaces/data-space-manager.test.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { expect } from 'chai';
+import { describe, expect, test } from 'vitest';
 
 import { asyncTimeout, latch } from '@dxos/async';
 import { createAdmissionCredentials } from '@dxos/credentials';
@@ -10,7 +10,7 @@ import { AuthStatus } from '@dxos/echo-pipeline';
 import { writeMessages } from '@dxos/feed-store';
 import { log } from '@dxos/log';
 import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
-import { describe, openAndClose, test } from '@dxos/test';
+import { openAndClose } from '@dxos/test-utils';
 
 import { TestBuilder, type TestPeer } from '../testing';
 

package/src/packlets/spaces/data-space-manager.ts

@@ -14,8 +14,11 @@ import {
   type DelegateInvitationCredential,
   type MemberInfo,
 } from '@dxos/credentials';
-import { convertLegacyReferences, findInlineObjectOfType, type EchoEdgeReplicator, type EchoHost } from '@dxos/echo-db';
 import {
+  convertLegacyReferences,
+  findInlineObjectOfType,
+  type EchoEdgeReplicator,
+  type EchoHost,
   AuthStatus,
   CredentialServerExtension,
   type MeshEchoReplicator,
@@ -32,8 +35,8 @@ import {
   type ObjectStructure,
   type SpaceDoc,
 } from '@dxos/echo-protocol';
-import { TYPE_PROPERTIES, generateEchoId, getTypeReference } from '@dxos/echo-schema';
-import type { EdgeConnection } from '@dxos/edge-client';
+import { TYPE_PROPERTIES, createObjectId, getTypeReference } from '@dxos/echo-schema';
+import type { EdgeConnection, EdgeHttpClient } from '@dxos/edge-client';
 import { writeMessages, type FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -43,7 +46,7 @@ import { AlreadyJoinedError, trace as Trace } from '@dxos/protocols';
 import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import { type Runtime } from '@dxos/protocols/proto/dxos/config';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { type SpaceMetadata, EdgeReplicationSetting } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { SpaceMember, type Credential, type ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type DelegateSpaceInvitation } from '@dxos/protocols/proto/dxos/halo/invitations';
 import { type PeerState } from '@dxos/protocols/proto/dxos/mesh/presence';
@@ -107,6 +110,7 @@ export type DataSpaceManagerParams = {
   echoHost: EchoHost;
   invitationsManager: InvitationsManager;
   edgeConnection?: EdgeConnection;
+  edgeHttpClient?: EdgeHttpClient;
   meshReplicator?: MeshEchoReplicator;
   echoEdgeReplicator?: EchoEdgeReplicator;
   runtimeParams?: DataSpaceManagerRuntimeParams;
@@ -116,6 +120,7 @@ export type DataSpaceManagerParams = {
 export type DataSpaceManagerRuntimeParams = {
   spaceMemberPresenceAnnounceInterval?: number;
   spaceMemberPresenceOfflineTimeout?: number;
+  activeEdgeNotarizationPollingInterval?: number;
   disableP2pReplication?: boolean;
 };
 
@@ -135,6 +140,7 @@ export class DataSpaceManager extends Resource {
   private readonly _echoHost: EchoHost;
   private readonly _invitationsManager: InvitationsManager;
   private readonly _edgeConnection?: EdgeConnection = undefined;
+  private readonly _edgeHttpClient?: EdgeHttpClient = undefined;
   private readonly _edgeFeatures?: Runtime.Client.EdgeFeatures = undefined;
   private readonly _meshReplicator?: MeshEchoReplicator = undefined;
   private readonly _echoEdgeReplicator?: EchoEdgeReplicator = undefined;
@@ -154,6 +160,7 @@ export class DataSpaceManager extends Resource {
     this._edgeConnection = params.edgeConnection;
     this._edgeFeatures = params.edgeFeatures;
     this._echoEdgeReplicator = params.echoEdgeReplicator;
+    this._edgeHttpClient = params.edgeHttpClient;
     this._runtimeParams = params.runtimeParams;
 
     trace.diagnostic({
@@ -290,7 +297,7 @@ export class DataSpaceManager extends Resource {
       },
     };
 
-    const propertiesId = generateEchoId();
+    const propertiesId = createObjectId();
     document.change((doc: SpaceDoc) => {
       setDeep(doc, ['objects', propertiesId], properties);
     });
@@ -388,6 +395,26 @@ export class DataSpaceManager extends Resource {
     });
   }
 
+  async setSpaceEdgeReplicationSetting(spaceKey: PublicKey, setting: EdgeReplicationSetting) {
+    const space = this._spaces.get(spaceKey);
+    invariant(space, 'Space not found.');
+
+    await this._metadataStore.setSpaceEdgeReplicationSetting(spaceKey, setting);
+
+    if (space.isOpen) {
+      switch (setting) {
+        case EdgeReplicationSetting.DISABLED:
+          await this._echoEdgeReplicator?.disconnectFromSpace(space.id);
+          break;
+        case EdgeReplicationSetting.ENABLED:
+          await this._echoEdgeReplicator?.connectToSpace(space.id);
+          break;
+      }
+    }
+
+    space.stateUpdate.emit();
+  }
+
   private async _constructSpace(metadata: SpaceMetadata) {
     log('construct space', { metadata });
     const gossip = new Gossip({
@@ -479,13 +506,23 @@ export class DataSpaceManager extends Resource {
       },
       cache: metadata.cache,
       edgeConnection: this._edgeConnection,
+      edgeHttpClient: this._edgeHttpClient,
       edgeFeatures: this._edgeFeatures,
+      activeEdgeNotarizationPollingInterval: this._runtimeParams?.activeEdgeNotarizationPollingInterval,
     });
     dataSpace.postOpen.append(async () => {
-      await this._echoEdgeReplicator?.connectToSpace(dataSpace.id);
+      const setting = dataSpace.getEdgeReplicationSetting();
+      if (setting === EdgeReplicationSetting.ENABLED) {
+        await this._echoEdgeReplicator?.connectToSpace(dataSpace.id);
+      } else if (this._echoEdgeReplicator) {
+        log('not connecting EchoEdgeReplicator because of EdgeReplicationSetting', { spaceId: dataSpace.id });
+      }
     });
     dataSpace.preClose.append(async () => {
-      await this._echoEdgeReplicator?.disconnectFromSpace(dataSpace.id);
+      const setting = dataSpace.getEdgeReplicationSetting();
+      if (setting === EdgeReplicationSetting.ENABLED) {
+        await this._echoEdgeReplicator?.disconnectFromSpace(dataSpace.id);
+      }
     });
 
     presence.newPeer.on((peerState) => {

package/src/packlets/spaces/data-space.ts

@@ -7,10 +7,15 @@ import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { Context, ContextDisposedError, cancelWithContext } from '@dxos/context';
 import type { SpecificCredential } from '@dxos/credentials';
 import { timed, warnAfterTimeout } from '@dxos/debug';
-import { type EchoHost, type DatabaseRoot } from '@dxos/echo-db';
-import { createMappedFeedWriter, type MetadataStore, type Space } from '@dxos/echo-pipeline';
+import {
+  type EchoHost,
+  type DatabaseRoot,
+  createMappedFeedWriter,
+  type MetadataStore,
+  type Space,
+} from '@dxos/echo-pipeline';
 import { SpaceDocVersion } from '@dxos/echo-protocol';
-import type { EdgeConnection } from '@dxos/edge-client';
+import type { EdgeConnection, EdgeHttpClient } from '@dxos/edge-client';
 import { type FeedStore, type FeedWrapper } from '@dxos/feed-store';
 import { failedInvariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -75,7 +80,9 @@ export type DataSpaceParams = {
   callbacks?: DataSpaceCallbacks;
   cache?: SpaceCache;
   edgeConnection?: EdgeConnection;
+  edgeHttpClient?: EdgeHttpClient;
   edgeFeatures?: Runtime.Client.EdgeFeatures;
+  activeEdgeNotarizationPollingInterval?: number;
 };
 
 export type CreateEpochOptions = {
@@ -96,7 +103,7 @@ export class DataSpace {
   private readonly _feedStore: FeedStore<FeedMessage>;
   private readonly _metadataStore: MetadataStore;
   private readonly _signingContext: SigningContext;
-  private readonly _notarizationPlugin = new NotarizationPlugin();
+  private readonly _notarizationPlugin: NotarizationPlugin;
   private readonly _callbacks: DataSpaceCallbacks;
   private readonly _cache?: SpaceCache = undefined;
   private readonly _echoHost: EchoHost;
@@ -136,6 +143,12 @@ export class DataSpace {
     this._signingContext = params.signingContext;
     this._callbacks = params.callbacks ?? {};
     this._echoHost = params.echoHost;
+    this._notarizationPlugin = new NotarizationPlugin({
+      spaceId: this._inner.id,
+      edgeClient: params.edgeHttpClient,
+      edgeFeatures: params.edgeFeatures,
+      activeEdgePollingInterval: params.activeEdgeNotarizationPollingInterval,
+    });
 
     this.authVerifier = new TrustedKeySetAuthVerifier({
       trustedKeysProvider: () =>
@@ -231,6 +244,7 @@ export class DataSpace {
     }
 
     await this._inner.open(new Context());
+    await this._inner.startProtocol();
 
     await this._edgeFeedReplicator?.open();
 
@@ -318,6 +332,7 @@ export class DataSpace {
     this._state = SpaceState.SPACE_INITIALIZING;
     log('new state', { state: SpaceState[this._state] });
 
+    log('initializing control pipeline');
     await this._initializeAndReadControlPipeline();
 
     // Allow other tasks to run before loading the data pipeline.
@@ -325,10 +340,13 @@ export class DataSpace {
 
     const ready = this.stateUpdate.waitForCondition(() => this._state === SpaceState.SPACE_READY);
 
+    log('initializing automerge root');
     this._automergeSpaceState.startProcessingRootDocs();
 
     // TODO(dmaretskyi): Change so `initializeDataPipeline` doesn't wait for the space to be READY, but rather any state with a valid root.
+    log('waiting for space to be ready');
     await ready;
+    log('space is ready');
   }
 
   private async _enterReadyState() {
@@ -345,6 +363,7 @@ export class DataSpace {
   private async _initializeAndReadControlPipeline() {
     await this._inner.controlPipeline.state.waitUntilReachedTargetTimeframe({
       ctx: this._ctx,
+      timeout: 10_000,
       breakOnStall: false,
     });
 
@@ -408,8 +427,16 @@ export class DataSpace {
     }
 
     if (credentials.length > 0) {
-      // Never times out
-      await this.notarizationPlugin.notarize({ ctx: this._ctx, credentials, timeout: 0 });
+      try {
+        log('will notarize credentials for feed admission', { count: credentials.length });
+        // Never times out
+        await this.notarizationPlugin.notarize({ ctx: this._ctx, credentials, timeout: 0 });
+
+        log('credentials notarized');
+      } catch (err) {
+        log.error('error notarizing credentials for feed admission', err);
+        throw err;
+      }
 
       // Set this after credentials are notarized so that on failure we will retry.
       await this._metadataStore.setWritableFeedKeys(this.key, this.inner.controlFeedKey!, this.inner.dataFeedKey!);
@@ -546,6 +573,10 @@ export class DataSpace {
     this.stateUpdate.emit();
   }
 
+  getEdgeReplicationSetting() {
+    return this._metadataStore.getSpaceEdgeReplicationSetting(this.key);
+  }
+
   private _onFeedAdded = async (feed: FeedWrapper<any>) => {
     await this._edgeFeedReplicator!.addFeed(feed);
   };