@dxos/client-services 0.4.10-main.c42bfdb → 0.4.10-main.c5e8686

package/src/packlets/invitations/invitations-handler.ts

@@ -11,6 +11,7 @@ import {
 } from '@dxos/client-protocol';
 import { Context } from '@dxos/context';
 import { generatePasscode } from '@dxos/credentials';
+import { createKeyPair, sign } from '@dxos/crypto';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -21,9 +22,9 @@ import {
   type SwarmConnection,
 } from '@dxos/network-manager';
 import { InvalidInvitationExtensionRoleError, trace } from '@dxos/protocols';
-import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
+import { type AdmissionKeypair, Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type DeviceProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { AuthenticationResponse } from '@dxos/protocols/proto/dxos/halo/invitations';
+import { AuthenticationResponse, type IntroductionResponse } from '@dxos/protocols/proto/dxos/halo/invitations';
 
 import {
   InvitationGuestExtension,
@@ -74,9 +75,11 @@ export class InvitationsHandler {
       state = Invitation.State.INIT,
       timeout = INVITATION_TIMEOUT,
       swarmKey = PublicKey.random(),
-      persistent = true,
+      persistent = options?.authMethod !== Invitation.AuthMethod.KNOWN_PUBLIC_KEY, // default no not storing keypairs
       created = new Date(),
-      lifetime = 86400, // 1 day
+      guestKeypair = undefined,
+      lifetime = 86400, // 1 day,
+      multiUse = false,
     } = options ?? {};
     const authCode =
       options?.authCode ??
@@ -91,9 +94,12 @@ export class InvitationsHandler {
       swarmKey,
       authCode,
       timeout,
-      persistent,
+      persistent: persistent && type !== Invitation.Type.DELEGATED, // delegated invitations are persisted in control feed
+      guestKeypair:
+        guestKeypair ?? (authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY ? createAdmissionKeypair() : undefined),
       created,
       lifetime,
+      multiUse,
       ...protocol.getInvitationContext(),
     };
 
@@ -162,7 +168,7 @@ export class InvitationsHandler {
               }
               log.trace('dxos.sdk.invitations-handler.host.onOpen', trace.error({ id: traceId, error: err }));
             } finally {
-              if (type !== Invitation.Type.MULTIUSE) {
+              if (!multiUse) {
                 // Wait for graceful close before disposing.
                 await swarmConnection.close();
                 await ctx.dispose();
@@ -317,26 +323,13 @@ export class InvitationsHandler {
 
               // 2. Get authentication code.
               if (isAuthenticationRequired(invitation)) {
-                for (let attempt = 1; attempt <= MAX_OTP_ATTEMPTS; attempt++) {
-                  log('guest waiting for authentication code...');
-                  setState({ state: Invitation.State.READY_FOR_AUTHENTICATION });
-                  const authCode = await authenticated.wait({ timeout });
-
-                  log('sending authentication request');
-                  setState({ state: Invitation.State.AUTHENTICATING });
-                  const response = await extension.rpc.InvitationHostService.authenticate({ authCode });
-                  if (response.status === undefined || response.status === AuthenticationResponse.Status.OK) {
+                switch (invitation.authMethod) {
+                  case Invitation.AuthMethod.SHARED_SECRET:
+                    await this._handleGuestOtpAuth(extension, setState, authenticated, { timeout });
+                    break;
+                  case Invitation.AuthMethod.KNOWN_PUBLIC_KEY:
+                    await this._handleGuestKpkAuth(extension, setState, invitation, introductionResponse);
                     break;
-                  }
-
-                  if (response.status === AuthenticationResponse.Status.INVALID_OTP) {
-                    if (attempt === MAX_OTP_ATTEMPTS) {
-                      throw new Error(`Maximum retry attempts: ${MAX_OTP_ATTEMPTS}`);
-                    } else {
-                      log('retrying invalid code', { attempt });
-                      authenticated.reset();
-                    }
-                  }
                 }
               }
 
@@ -423,13 +416,61 @@ export class InvitationsHandler {
 
     return observable;
   }
+
+  private async _handleGuestOtpAuth(
+    extension: InvitationGuestExtension,
+    setState: (newState: Partial<Invitation>) => void,
+    authenticated: Trigger<string>,
+    options: { timeout: number },
+  ) {
+    for (let attempt = 1; attempt <= MAX_OTP_ATTEMPTS; attempt++) {
+      log('guest waiting for authentication code...');
+      setState({ state: Invitation.State.READY_FOR_AUTHENTICATION });
+      const authCode = await authenticated.wait(options);
+
+      log('sending authentication request');
+      setState({ state: Invitation.State.AUTHENTICATING });
+      const response = await extension.rpc.InvitationHostService.authenticate({ authCode });
+      if (response.status === undefined || response.status === AuthenticationResponse.Status.OK) {
+        break;
+      }
+
+      if (response.status === AuthenticationResponse.Status.INVALID_OTP) {
+        if (attempt === MAX_OTP_ATTEMPTS) {
+          throw new Error(`Maximum retry attempts: ${MAX_OTP_ATTEMPTS}`);
+        } else {
+          log('retrying invalid code', { attempt });
+          authenticated.reset();
+        }
+      }
+    }
+  }
+
+  private async _handleGuestKpkAuth(
+    extension: InvitationGuestExtension,
+    setState: (newState: Partial<Invitation>) => void,
+    invitation: Invitation,
+    introductionResponse: IntroductionResponse,
+  ) {
+    if (invitation.guestKeypair?.privateKey == null) {
+      throw new Error('keypair missing in the invitation');
+    }
+    if (introductionResponse.challenge == null) {
+      throw new Error('challenge missing in the introduction');
+    }
+    log('sending authentication request');
+    setState({ state: Invitation.State.AUTHENTICATING });
+    const signature = sign(Buffer.from(introductionResponse.challenge), invitation.guestKeypair.privateKey);
+    const response = await extension.rpc.InvitationHostService.authenticate({
+      signedChallenge: signature,
+    });
+    if (response.status !== AuthenticationResponse.Status.OK) {
+      throw new Error(`Authentication failed with code: ${response.status}`);
+    }
+  }
 }
 
-export const invitationExpired = (invitation: Invitation) => {
-  return (
-    invitation.created &&
-    invitation.lifetime &&
-    invitation.lifetime !== 0 &&
-    invitation.created.getTime() + invitation.lifetime * 1000 < Date.now()
-  );
+export const createAdmissionKeypair = (): AdmissionKeypair => {
+  const keypair = createKeyPair();
+  return { publicKey: PublicKey.from(keypair.publicKey), privateKey: keypair.secretKey };
 };

package/src/packlets/invitations/invitations-service.ts

@@ -6,7 +6,7 @@ import { Event, scheduleTask } from '@dxos/async';
 import { type AuthenticatingInvitation, type CancellableInvitation } from '@dxos/client-protocol';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
-import { type MetadataStore } from '@dxos/echo-pipeline';
+import { hasInvitationExpired, type MetadataStore } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import {
@@ -18,7 +18,7 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 
 import { type InvitationProtocol } from './invitation-protocol';
-import { invitationExpired, type InvitationsHandler } from './invitations-handler';
+import { type InvitationsHandler } from './invitations-handler';
 
 /**
  * Adapts invitation service observable to client/service stream.
@@ -91,7 +91,7 @@ export class InvitationsServiceImpl implements InvitationsService {
           }
 
           this._createInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedCreated.emit(invitation.get());
           }
         },
@@ -103,7 +103,7 @@ export class InvitationsServiceImpl implements InvitationsService {
     const persistentInvitations = this._metadataStore.getInvitations();
 
     // get saved persistent invitations, filter and remove from storage those that have expired.
-    const freshInvitations = persistentInvitations.filter(async (invitation) => !invitationExpired(invitation));
+    const freshInvitations = persistentInvitations.filter(async (invitation) => !hasInvitationExpired(invitation));
 
     const cInvitations = freshInvitations.map((persistentInvitation) => {
       invariant(!this._createInvitations.get(persistentInvitation.invitationId), 'invitation already exists');
@@ -148,7 +148,7 @@ export class InvitationsServiceImpl implements InvitationsService {
         () => {
           close();
           this._acceptInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedAccepted.emit(invitation.get());
           }
         },

package/src/packlets/services/automerge-host.test.ts

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { AutomergeHost, DataServiceImpl } from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { AutomergeContext } from '@dxos/echo-schema';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, test } from '@dxos/test';
 
 describe('AutomergeHost', () => {
@@ -19,10 +19,16 @@ describe('AutomergeHost', () => {
     // creates repo and document | replicates repo | finds document in repo
     //
 
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
     afterTest(() => host.close());
+
     const dataService = new DataServiceImpl(host);
     const client = new AutomergeContext(dataService);
     afterTest(() => client.close());

package/src/packlets/services/service-context.test.ts

@@ -4,7 +4,7 @@
 
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
-import { describe, test } from '@dxos/test';
+import { describe, openAndClose, test } from '@dxos/test';
 
 import { createServiceContext } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -27,12 +27,15 @@ describe('services/ServiceContext', () => {
   test('joined space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
     const device1 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device1.automergeHost);
     await device1.createIdentity();
 
     const device2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device2.automergeHost);
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     const identity2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(identity2.automergeHost);
     await identity2.createIdentity();
     const space1 = await identity2.dataSpaceManager!.createSpace();
     await Promise.all(

package/src/packlets/services/service-context.ts

@@ -10,7 +10,7 @@ import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credenti
 import { failUndefined } from '@dxos/debug';
 import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
-import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
+import { IndexMetadataStore, IndexStore, Indexer, createStorageCallbacks } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
@@ -124,11 +124,12 @@ export class ServiceContext extends Resource {
 
     this.automergeHost = new AutomergeHost({
       directory: storage.createDirectory('automerge'),
-      metadata: this.indexMetadata,
+      db: level.sublevel('automerge'),
+      storageCallbacks: createStorageCallbacks({ host: () => this.automergeHost, metadata: this.indexMetadata }),
     });
 
     this.indexer = new Indexer({
-      indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
+      indexStore: new IndexStore({ db: level.sublevel('index-storage') }),
       metadataStore: this.indexMetadata,
       loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
       getAllDocuments: createDocumentsIterator(this.automergeHost),
@@ -158,6 +159,7 @@ export class ServiceContext extends Resource {
     await this.signalManager.open();
     await this.networkManager.open();
 
+    await this.automergeHost.open();
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);

package/src/packlets/services/service-host.ts

@@ -8,8 +8,14 @@ import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, defaultKey, type ClientServices, Properties } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { DataServiceImpl, type ObjectStructure, encodeReference, type SpaceDoc } from '@dxos/echo-pipeline';
-import * as E from '@dxos/echo-schema';
+import {
+  DataServiceImpl,
+  type ObjectStructure,
+  encodeReference,
+  type SpaceDoc,
+  type LevelDB,
+} from '@dxos/echo-pipeline';
+import { getTypeReference } from '@dxos/echo-schema';
 import { IndexServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
@@ -50,6 +56,7 @@ export type ClientServicesHostParams = {
   signalManager?: SignalManager;
   connectionLog?: boolean;
   storage?: Storage;
+  level?: LevelDB;
   lockKey?: string;
   callbacks?: ClientServicesHostCallbacks;
   runtimeParams?: ServiceContextRuntimeParams;
@@ -101,12 +108,14 @@ export class ClientServicesHost {
     transportFactory,
     signalManager,
     storage,
+    level,
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
     callbacks,
     runtimeParams,
   }: ClientServicesHostParams = {}) {
     this._storage = storage;
+    this._level = level;
     this._callbacks = callbacks;
     this._runtimeParams = runtimeParams;
 
@@ -239,6 +248,8 @@ export class ClientServicesHost {
     if (!this._level) {
       this._level = await createLevel(this._config.get('runtime.client.storage', {})!);
     }
+    await this._level.open();
+
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
@@ -372,7 +383,7 @@ export class ClientServicesHost {
     // TODO(dmaretskyi): Better API for low-level data access.
     const properties: ObjectStructure = {
       system: {
-        type: encodeReference(E.getTypeReference(Properties)!),
+        type: encodeReference(getTypeReference(Properties)!),
       },
       data: {
         [defaultKey]: identity.identityKey.toHex(),

package/src/packlets/spaces/data-space-manager.test.ts

@@ -20,7 +20,7 @@ describe('DataSpaceManager', () => {
 
     const peer = builder.createPeer();
     await peer.createIdentity();
-    await openAndClose(peer.dataSpaceManager);
+    await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
     const space = await peer.dataSpaceManager.createSpace();
 
@@ -42,7 +42,7 @@ describe('DataSpaceManager', () => {
     const peer2 = builder.createPeer();
     await peer2.createIdentity();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -111,7 +111,7 @@ describe('DataSpaceManager', () => {
     await peer2.createIdentity();
     await peer2.dataSpaceManager.open();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -153,7 +153,7 @@ describe('DataSpaceManager', () => {
 
       const peer = builder.createPeer();
       await peer.createIdentity();
-      await openAndClose(peer.dataSpaceManager);
+      await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
       const space = await peer.dataSpaceManager.createSpace();
       await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);

package/src/packlets/storage/level.ts

@@ -12,7 +12,7 @@ import { getRootPath, isPersistent } from './util';
 
 export const createLevel = async (config: Runtime.Client.Storage) => {
   const persistent = isPersistent(config);
-  const storagePath = persistent ? getRootPath(config) : path.join('tmp', 'level', PublicKey.random().toHex());
+  const storagePath = persistent ? path.join(getRootPath(config), 'level') : `/tmp/dxos-${PublicKey.random().toHex()}`;
   const level = new Level<string, string>(storagePath);
   await level.open();
   return level;

package/src/packlets/testing/test-builder.ts

@@ -6,9 +6,16 @@ import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
 import { createCredentialSignerWithChain, CredentialGenerator } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
+import {
+  AutomergeHost,
+  MetadataStore,
+  type LevelDB,
+  SnapshotStore,
+  SpaceManager,
+  valueEncoding,
+} from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
-import { createTestLevel } from '@dxos/indexing/testing';
 import { Keyring } from '@dxos/keyring';
 import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
 import { MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
@@ -42,7 +49,8 @@ export const createServiceContext = async ({
     signalManager,
     transportFactory: MemoryTransportFactory,
   });
-  const level = await createTestLevel();
+  const level = createTestLevel();
+  await level.open();
 
   return new ServiceContext(storage, level, networkManager, signalManager);
 };
@@ -85,6 +93,7 @@ export type TestPeerOpts = {
 
 export type TestPeerProps = {
   storage?: Storage;
+  level?: LevelDB;
   feedStore?: FeedStore<any>;
   metadataStore?: MetadataStore;
   keyring?: Keyring;
@@ -117,6 +126,10 @@ export class TestPeer {
     return (this._props.keyring ??= new Keyring(this.storage.createDirectory('keyring')));
   }
 
+  get level() {
+    return (this._props.level ??= createTestLevel());
+  }
+
   get feedStore() {
     return (this._props.feedStore ??= new FeedStore({
       factory: new FeedFactory({
@@ -163,7 +176,9 @@ export class TestPeer {
   }
 
   get automergeHost() {
-    return (this._props.automergeHost ??= new AutomergeHost({ directory: this.storage.createDirectory('automerge') }));
+    return (this._props.automergeHost ??= new AutomergeHost({
+      db: this.level.sublevel('automerge'),
+    }));
   }
 
   get dataSpaceManager() {
@@ -182,6 +197,7 @@ export class TestPeer {
   }
 
   async destroy() {
+    await this.level.close();
     await this.storage.reset();
   }
 }

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.4.10-main.c42bfdb";
+export const DXOS_VERSION = "0.4.10-main.c5e8686";