@dxos/client-services 0.4.10-main.c75170d → 0.4.10-main.c8e5c39

package/src/packlets/invitations/invitations-manager.ts

@@ -2,20 +2,27 @@
 // Copyright 2024 DXOS.org
 //
 
-import { Event } from '@dxos/async';
-import type { AuthenticatingInvitation, CancellableInvitation } from '@dxos/client-protocol';
-import { type Context } from '@dxos/context';
+import { Event, PushStream } from '@dxos/async';
+import {
+  type AuthenticatingInvitation,
+  AUTHENTICATION_CODE_LENGTH,
+  CancellableInvitation,
+  INVITATION_TIMEOUT,
+} from '@dxos/client-protocol';
+import { Context } from '@dxos/context';
+import { generatePasscode } from '@dxos/credentials';
 import { hasInvitationExpired, type MetadataStore } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
   type AcceptInvitationRequest,
-  type Invitation,
   type AuthenticationRequest,
+  Invitation,
 } from '@dxos/protocols/proto/dxos/client/services';
 
 import type { InvitationProtocol } from './invitation-protocol';
-import type { InvitationsHandler } from './invitations-handler';
+import { createAdmissionKeypair, type InvitationsHandler } from './invitations-handler';
 
 /**
  * Entry point for creating and accepting invitations, keeps track of existing invitation set and
@@ -36,36 +43,44 @@ export class InvitationsManager {
 
   constructor(
     private readonly _invitationsHandler: InvitationsHandler,
-    private readonly _getHandler: (invitation: Invitation) => InvitationProtocol,
+    private readonly _getHandler: (invitation: Partial<Invitation> & Pick<Invitation, 'kind'>) => InvitationProtocol,
     private readonly _metadataStore: MetadataStore,
   ) {}
 
-  createInvitation(options: Invitation): CancellableInvitation {
-    const existingInvitation = this._createInvitations.get(options.invitationId);
-    if (existingInvitation) {
-      return existingInvitation;
+  async createInvitation(options: Partial<Invitation> & Pick<Invitation, 'kind'>): Promise<CancellableInvitation> {
+    if (options.invitationId) {
+      const existingInvitation = this._createInvitations.get(options.invitationId);
+      if (existingInvitation) {
+        return existingInvitation;
+      }
     }
 
     const handler = this._getHandler(options);
-    const invitation = this._invitationsHandler.createInvitation(handler, options);
-    this._createInvitations.set(invitation.get().invitationId, invitation);
-    this.invitationCreated.emit(invitation.get());
-
-    const saveInvitationTask = invitation.get().persistent
-      ? this._safePersistInBackground(invitation)
-      : Promise.resolve();
+    const invitation = this._createInvitation(handler, options);
+    const { ctx, stream, observableInvitation } = this._createObservableInvitation(handler, invitation);
 
+    this._createInvitations.set(invitation.invitationId, observableInvitation);
+    this.invitationCreated.emit(invitation);
     // onComplete is called on cancel, expiration, or redemption of a single-use invitation
-    this._onInvitationComplete(invitation, async () => {
-      this._createInvitations.delete(invitation.get().invitationId);
-      this.removedCreated.emit(invitation.get());
-      if (invitation.get().persistent) {
-        await saveInvitationTask;
-        await this._safeDeleteInvitation(invitation.get());
+    this._onInvitationComplete(observableInvitation, async () => {
+      this._createInvitations.delete(observableInvitation.get().invitationId);
+      this.removedCreated.emit(observableInvitation.get());
+      if (observableInvitation.get().persistent) {
+        await this._safeDeleteInvitation(observableInvitation.get());
       }
     });
 
-    return invitation;
+    try {
+      await this._persistIfRequired(handler, stream, invitation);
+    } catch (err) {
+      log.catch(err);
+      await observableInvitation.cancel();
+      return observableInvitation;
+    }
+
+    this._invitationsHandler.handleInvitationFlow(ctx, stream, handler, observableInvitation.get());
+
+    return observableInvitation;
   }
 
   async loadPersistentInvitations(): Promise<{ invitations: Invitation[] }> {
@@ -78,12 +93,13 @@ export class InvitationsManager {
       // get saved persistent invitations, filter and remove from storage those that have expired.
       const freshInvitations = persistentInvitations.filter((invitation) => !hasInvitationExpired(invitation));
 
-      const cInvitations = freshInvitations.map((persistentInvitation) => {
+      const loadTasks = freshInvitations.map((persistentInvitation) => {
         invariant(!this._createInvitations.get(persistentInvitation.invitationId), 'invitation already exists');
-        return this.createInvitation({ ...persistentInvitation, persistent: false }).get();
+        return this.createInvitation({ ...persistentInvitation, persistent: false });
       });
+      const cInvitations = await Promise.all(loadTasks);
 
-      return { invitations: cInvitations };
+      return { invitations: cInvitations.map((invitation) => invitation.get()) };
     } catch (err) {
       log.catch(err);
       return { invitations: [] };
@@ -163,20 +179,78 @@ export class InvitationsManager {
     }
   }
 
-  private _safePersistInBackground(invitation: CancellableInvitation): Promise<void> {
-    return new Promise((resolve) => {
-      setTimeout(async () => {
-        try {
-          await this._metadataStore.addInvitation(invitation.get());
-          this.saved.emit(invitation.get());
-        } catch (err: any) {
-          log.catch(err);
-          await invitation.cancel();
-        } finally {
-          resolve();
-        }
-      });
+  private _createInvitation(protocol: InvitationProtocol, options?: Partial<Invitation>): Invitation {
+    const {
+      invitationId = PublicKey.random().toHex(),
+      type = Invitation.Type.INTERACTIVE,
+      authMethod = Invitation.AuthMethod.SHARED_SECRET,
+      state = Invitation.State.INIT,
+      timeout = INVITATION_TIMEOUT,
+      swarmKey = PublicKey.random(),
+      persistent = options?.authMethod !== Invitation.AuthMethod.KNOWN_PUBLIC_KEY, // default no not storing keypairs
+      created = new Date(),
+      guestKeypair = undefined,
+      lifetime = 86400, // 1 day,
+      multiUse = false,
+    } = options ?? {};
+    const authCode =
+      options?.authCode ??
+      (authMethod === Invitation.AuthMethod.SHARED_SECRET ? generatePasscode(AUTHENTICATION_CODE_LENGTH) : undefined);
+
+    return {
+      invitationId,
+      type,
+      authMethod,
+      state,
+      swarmKey,
+      authCode,
+      timeout,
+      persistent: persistent && type !== Invitation.Type.DELEGATED, // delegated invitations are persisted in control feed
+      guestKeypair:
+        guestKeypair ?? (authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY ? createAdmissionKeypair() : undefined),
+      created,
+      lifetime,
+      multiUse,
+      delegationCredentialId: options?.delegationCredentialId,
+      ...protocol.getInvitationContext(),
+    } satisfies Invitation;
+  }
+
+  private _createObservableInvitation(handler: InvitationProtocol, invitation: Invitation) {
+    const stream = new PushStream<Invitation>();
+    const ctx = new Context({
+      onError: (err) => {
+        stream.error(err);
+        void ctx.dispose();
+      },
     });
+    ctx.onDispose(() => {
+      log('complete', { ...handler.toJSON() });
+      stream.complete();
+    });
+    const observableInvitation = new CancellableInvitation({
+      initialInvitation: invitation,
+      subscriber: stream.observable,
+      onCancel: async () => {
+        stream.next({ ...invitation, state: Invitation.State.CANCELLED });
+        await ctx.dispose();
+      },
+    });
+    return { ctx, stream, observableInvitation };
+  }
+
+  private async _persistIfRequired(
+    handler: InvitationProtocol,
+    changeStream: PushStream<Invitation>,
+    invitation: Invitation,
+  ): Promise<void> {
+    if (invitation.type === Invitation.Type.DELEGATED && invitation.delegationCredentialId == null) {
+      const delegationCredentialId = await handler.delegate(invitation);
+      changeStream.next({ ...invitation, delegationCredentialId });
+    } else if (invitation.persistent) {
+      await this._metadataStore.addInvitation(invitation);
+      this.saved.emit(invitation);
+    }
   }
 
   private async _safeDeleteInvitation(invitation: Invitation): Promise<void> {

package/src/packlets/invitations/invitations-service.ts

@@ -27,9 +27,11 @@ export class InvitationsServiceImpl implements InvitationsService {
   }
 
   createInvitation(options: Invitation): Stream<Invitation> {
-    const invitation = this._invitationsManager.createInvitation(options);
     return new Stream<Invitation>(({ next, close }) => {
-      invitation.subscribe(next, close, close);
+      void this._invitationsManager
+        .createInvitation(options)
+        .then((invitation) => invitation.subscribe(next, close, close))
+        .catch(close);
     });
   }
 

package/src/packlets/invitations/space-invitation-protocol.ts

@@ -2,7 +2,11 @@
 // Copyright 2023 DXOS.org
 //
 
-import { createAdmissionCredentials, getCredentialAssertion } from '@dxos/credentials';
+import {
+  createAdmissionCredentials,
+  createDelegatedSpaceInvitationCredential,
+  getCredentialAssertion,
+} from '@dxos/credentials';
 import { writeMessages } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -11,7 +15,7 @@ import { log } from '@dxos/log';
 import { AlreadyJoinedError } from '@dxos/protocols';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { SpaceMember, type ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import {
   type AdmissionRequest,
   type AdmissionResponse,
@@ -43,7 +47,11 @@ export class SpaceInvitationProtocol implements InvitationProtocol {
     };
   }
 
-  async admit(request: AdmissionRequest, guestProfile?: ProfileDocument | undefined): Promise<AdmissionResponse> {
+  async admit(
+    invitation: Invitation,
+    request: AdmissionRequest,
+    guestProfile?: ProfileDocument | undefined,
+  ): Promise<AdmissionResponse> {
     invariant(this._spaceKey);
     const space = await this._spaceManager.spaces.get(this._spaceKey);
     invariant(space);
@@ -59,6 +67,7 @@ export class SpaceInvitationProtocol implements InvitationProtocol {
       space.key,
       space.inner.genesisFeedKey,
       guestProfile,
+      invitation.delegationCredentialId,
     );
 
     // TODO(dmaretskyi): Refactor.
@@ -76,6 +85,39 @@ export class SpaceInvitationProtocol implements InvitationProtocol {
     };
   }
 
+  async delegate(invitation: Invitation): Promise<PublicKey> {
+    invariant(this._spaceKey);
+    const space = await this._spaceManager.spaces.get(this._spaceKey);
+    invariant(space);
+    if (invitation.authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY) {
+      invariant(invitation.guestKeypair?.publicKey);
+    }
+
+    log('writing delegate space invitation', { host: this._signingContext.deviceKey, id: invitation.invitationId });
+    const credential = await createDelegatedSpaceInvitationCredential(
+      this._signingContext.credentialSigner,
+      space.key,
+      {
+        invitationId: invitation.invitationId,
+        authMethod: invitation.authMethod,
+        swarmKey: invitation.swarmKey,
+        role: SpaceMember.Role.ADMIN,
+        expiresOn: invitation.lifetime
+          ? new Date((invitation.created?.getTime() ?? Date.now()) + invitation.lifetime)
+          : undefined,
+        multiUse: invitation.multiUse ?? false,
+        guestKey:
+          invitation.authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY
+            ? invitation.guestKeypair!.publicKey
+            : undefined,
+      },
+    );
+
+    invariant(credential.credential);
+    await writeMessages(space.inner.controlPipeline.writer, [credential]);
+    return credential.credential.credential.id!;
+  }
+
   checkInvitation(invitation: Partial<Invitation>) {
     if (invitation.spaceKey && this._spaceManager.spaces.has(invitation.spaceKey)) {
       return new AlreadyJoinedError('Already joined space.');

package/src/packlets/services/automerge-host.test.ts

@@ -4,10 +4,11 @@
 
 import { expect } from 'chai';
 
-import { asyncTimeout, sleep } from '@dxos/async';
+import { asyncTimeout } from '@dxos/async';
+import { getHeads } from '@dxos/automerge/automerge';
+import { AutomergeContext } from '@dxos/echo-db';
 import { AutomergeHost, DataServiceImpl } from '@dxos/echo-pipeline';
 import { createTestLevel } from '@dxos/echo-pipeline/testing';
-import { AutomergeContext } from '@dxos/echo-schema';
 import { afterTest, describe, test } from '@dxos/test';
 
 describe('AutomergeHost', () => {
@@ -50,9 +51,8 @@ describe('AutomergeHost', () => {
     doc.change((doc: any) => {
       doc.text = newText;
     });
+    await client.flush({ states: [{ documentId: doc.documentId, heads: getHeads(doc.docSync()) }] });
 
-    // TODO(mykola): Is there a way to know when automerge has started replication?
-    await sleep(100);
     await asyncTimeout(handle.whenReady(), 1_000);
     expect(handle.docSync().text).to.equal(newText);
   });

package/src/packlets/services/service-context.test.ts

@@ -27,15 +27,15 @@ describe('services/ServiceContext', () => {
   test('joined space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
     const device1 = await createServiceContext({ signalContext: networkContext });
-    await openAndClose(device1.automergeHost);
+    await openAndClose(device1.echoHost);
     await device1.createIdentity();
 
     const device2 = await createServiceContext({ signalContext: networkContext });
-    await openAndClose(device2.automergeHost);
+    await openAndClose(device2.echoHost);
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     const identity2 = await createServiceContext({ signalContext: networkContext });
-    await openAndClose(identity2.automergeHost);
+    await openAndClose(identity2.echoHost);
     await identity2.createIdentity();
     const space1 = await identity2.dataSpaceManager!.createSpace();
     await Promise.all(

package/src/packlets/services/service-context.ts

@@ -8,9 +8,9 @@ import { Trigger } from '@dxos/async';
 import { Context, Resource } from '@dxos/context';
 import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
+import { EchoHost } from '@dxos/echo-db';
+import { MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
-import { IndexMetadataStore, IndexStore, Indexer, createStorageCallbacks } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
@@ -32,7 +32,6 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
-import { createDocumentsIterator, createSelectedDocumentsIterator } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -64,9 +63,7 @@ export class ServiceContext extends Resource {
   public readonly identityManager: IdentityManager;
   public readonly invitations: InvitationsHandler;
   public readonly invitationsManager: InvitationsManager;
-  public readonly automergeHost: AutomergeHost;
-  public readonly indexMetadata: IndexMetadataStore;
-  public readonly indexer: Indexer;
+  public readonly echoHost: EchoHost;
 
   // Initialized after identity is initialized.
   public dataSpaceManager?: DataSpaceManager;
@@ -122,19 +119,9 @@ export class ServiceContext extends Resource {
       this._runtimeParams as IdentityManagerRuntimeParams,
     );
 
-    this.indexMetadata = new IndexMetadataStore({ db: level.sublevel('index-metadata') });
-
-    this.automergeHost = new AutomergeHost({
-      directory: storage.createDirectory('automerge'),
-      db: level.sublevel('automerge'),
-      storageCallbacks: createStorageCallbacks({ host: () => this.automergeHost, metadata: this.indexMetadata }),
-    });
-
-    this.indexer = new Indexer({
-      indexStore: new IndexStore({ db: level.sublevel('index-storage') }),
-      metadataStore: this.indexMetadata,
-      loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
-      getAllDocuments: createDocumentsIterator(this.automergeHost),
+    this.echoHost = new EchoHost({
+      kv: this.level,
+      storage: this.storage,
     });
 
     this.invitations = new InvitationsHandler(this.networkManager);
@@ -166,7 +153,7 @@ export class ServiceContext extends Resource {
     await this.signalManager.open();
     await this.networkManager.open();
 
-    await this.automergeHost.open();
+    await this.echoHost.open(ctx);
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);
@@ -181,20 +168,19 @@ export class ServiceContext extends Resource {
     log('opened');
   }
 
-  protected override async _close() {
+  protected override async _close(ctx: Context) {
     log('closing...');
     if (this._deviceSpaceSync && this.identityManager.identity) {
       await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);
     }
-    await this.automergeHost.close();
     await this.dataSpaceManager?.close();
     await this.identityManager.close();
     await this.spaceManager.close();
     await this.feedStore.close();
+    await this.metadataStore.close();
+    await this.echoHost.close(ctx);
     await this.networkManager.close();
     await this.signalManager.close();
-    await this.metadataStore.close();
-    await this.indexer.destroy();
     log('closed');
   }
 
@@ -255,7 +241,8 @@ export class ServiceContext extends Resource {
       this.keyring,
       signingContext,
       this.feedStore,
-      this.automergeHost,
+      this.echoHost,
+      this.invitationsManager,
       this._runtimeParams as DataSpaceManagerRuntimeParams,
     );
     await this.dataSpaceManager.open();

package/src/packlets/services/service-host.test.ts

@@ -29,6 +29,12 @@ describe('ClientServicesHost', () => {
     isNode() && rmSync(dataRoot, { recursive: true, force: true });
   });
 
+  test('open and close', async () => {
+    const host = createServiceHost(new Config(), new MemorySignalManagerContext());
+    await host.open(new Context());
+    await host.close();
+  });
+
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
     await host.open(new Context());

package/src/packlets/services/service-host.ts

@@ -8,15 +8,8 @@ import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, defaultKey, type ClientServices, Properties } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import {
-  DataServiceImpl,
-  type ObjectStructure,
-  encodeReference,
-  type SpaceDoc,
-  type LevelDB,
-} from '@dxos/echo-pipeline';
+import { type ObjectStructure, encodeReference, type SpaceDoc, type LevelDB } from '@dxos/echo-pipeline';
 import { getTypeReference } from '@dxos/echo-schema';
-import { IndexServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -285,12 +278,8 @@ export class ClientServicesHost {
         },
       ),
 
-      DataService: new DataServiceImpl(this._serviceContext.automergeHost),
-
-      IndexService: new IndexServiceImpl({
-        indexer: this._serviceContext.indexer,
-        automergeHost: this._serviceContext.automergeHost,
-      }),
+      DataService: this._serviceContext.echoHost.dataService,
+      QueryService: this._serviceContext.echoHost.queryService,
 
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
@@ -368,7 +357,7 @@ export class ClientServicesHost {
 
     const automergeIndex = space.automergeSpaceState.rootUrl;
     invariant(automergeIndex);
-    const document = await this._serviceContext.automergeHost.repo.find<SpaceDoc>(automergeIndex as any);
+    const document = await this._serviceContext.echoHost.automergeRepo.find<SpaceDoc>(automergeIndex as any);
     await document.whenReady();
 
     // TODO(dmaretskyi): Better API for low-level data access.
@@ -388,7 +377,7 @@ export class ClientServicesHost {
       assignDeep(doc, ['objects', propertiesId], properties);
     });
 
-    await this._serviceContext.automergeHost.repo.flush();
+    await this._serviceContext.echoHost.flush();
 
     return identity;
   }

package/src/packlets/spaces/data-space-manager.test.ts

@@ -20,7 +20,7 @@ describe('DataSpaceManager', () => {
 
     const peer = builder.createPeer();
     await peer.createIdentity();
-    await openAndClose(peer.automergeHost, peer.dataSpaceManager);
+    await openAndClose(peer.echoHost, peer.dataSpaceManager);
 
     const space = await peer.dataSpaceManager.createSpace();
 
@@ -42,7 +42,7 @@ describe('DataSpaceManager', () => {
     const peer2 = builder.createPeer();
     await peer2.createIdentity();
 
-    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
+    await openAndClose(peer1.echoHost, peer1.dataSpaceManager, peer2.echoHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -111,7 +111,7 @@ describe('DataSpaceManager', () => {
     await peer2.createIdentity();
     await peer2.dataSpaceManager.open();
 
-    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
+    await openAndClose(peer1.echoHost, peer1.dataSpaceManager, peer2.echoHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -153,7 +153,7 @@ describe('DataSpaceManager', () => {
 
       const peer = builder.createPeer();
       await peer.createIdentity();
-      await openAndClose(peer.automergeHost, peer.dataSpaceManager);
+      await openAndClose(peer.echoHost, peer.dataSpaceManager);
 
       const space = await peer.dataSpaceManager.createSpace();
       await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);