@dxos/client-services 0.4.10-main.ef6fbc2 → 0.4.10-main.f635fb0

package/src/packlets/diagnostics/browser-diagnostics-broadcast.ts

@@ -0,0 +1,94 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Trigger } from '@dxos/async';
+import { log } from '@dxos/log';
+import { type SystemService } from '@dxos/protocols/proto/dxos/client/services';
+
+import {
+  type CollectDiagnosticsBroadcastSender,
+  type CollectDiagnosticsBroadcastHandler,
+} from './diagnostics-collector';
+
+const CHANNEL_NAME = 'dxos.diagnostics.broadcast';
+
+enum MessageType {
+  PROBE = 'probe',
+  PROBE_ACK = 'probe-ack',
+  REQUEST_DIAGNOSTICS = 'request-diagnostics',
+  RECEIVE_DIAGNOSTICS = 'receive-diagnostics',
+}
+
+interface Message {
+  type: MessageType;
+  payload?: any;
+}
+
+export const createCollectDiagnosticsBroadcastSender = (): CollectDiagnosticsBroadcastSender => {
+  return {
+    broadcastDiagnosticsRequest: async () => {
+      let expectedResponse = MessageType.PROBE_ACK;
+      let channel: BroadcastChannel | undefined;
+      try {
+        const trigger = new Trigger<Message>();
+        channel = new BroadcastChannel(CHANNEL_NAME);
+        channel.onmessage = (msg) => {
+          if (expectedResponse === msg.data.type) {
+            trigger.wake(msg.data);
+          }
+        };
+        channel.postMessage({ type: MessageType.PROBE });
+        await trigger.wait({ timeout: 200 });
+        expectedResponse = MessageType.RECEIVE_DIAGNOSTICS;
+        trigger.reset();
+        channel.postMessage({ type: MessageType.REQUEST_DIAGNOSTICS });
+        const diagnostics = await trigger.wait({ timeout: 5000 });
+        return diagnostics.payload;
+      } catch (e) {
+        const errorDescription = e instanceof Error ? e.message : JSON.stringify(e);
+        return { expectedResponse, errorDescription };
+      } finally {
+        safeClose(channel);
+      }
+    },
+  };
+};
+
+export const createCollectDiagnosticsBroadcastHandler = (
+  systemService: SystemService,
+): CollectDiagnosticsBroadcastHandler => {
+  let channel: BroadcastChannel | undefined;
+  return {
+    start: () => {
+      channel = new BroadcastChannel(CHANNEL_NAME);
+      channel.onmessage = async (message) => {
+        try {
+          if (message.data.type === MessageType.PROBE) {
+            channel?.postMessage({ type: MessageType.PROBE_ACK });
+          } else if (message.data.type === MessageType.REQUEST_DIAGNOSTICS) {
+            const diagnostics = await systemService.getDiagnostics({});
+            channel?.postMessage({
+              type: MessageType.RECEIVE_DIAGNOSTICS,
+              payload: diagnostics,
+            });
+          }
+        } catch (error) {
+          log.catch(error);
+        }
+      };
+    },
+    stop: () => {
+      safeClose(channel);
+      channel = undefined;
+    },
+  };
+};
+
+const safeClose = (channel?: BroadcastChannel) => {
+  try {
+    channel?.close();
+  } catch (e) {
+    // ignored
+  }
+};

package/src/packlets/diagnostics/diagnostics-broadcast.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2024 DXOS.org
+//
+import { type SystemService } from '@dxos/protocols/proto/dxos/client/services';
+
+import {
+  type CollectDiagnosticsBroadcastSender,
+  type CollectDiagnosticsBroadcastHandler,
+} from './diagnostics-collector';
+
+export const createCollectDiagnosticsBroadcastSender = (): CollectDiagnosticsBroadcastSender => {
+  return { broadcastDiagnosticsRequest: async () => undefined };
+};
+
+export const createCollectDiagnosticsBroadcastHandler = (_: SystemService): CollectDiagnosticsBroadcastHandler => {
+  return {
+    start: () => {},
+    stop: () => {},
+  };
+};

package/src/packlets/diagnostics/diagnostics-collector.ts

@@ -0,0 +1,65 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ClientServicesProvider } from '@dxos/client-protocol';
+import { type Config, ConfigResource } from '@dxos/config';
+import { GetDiagnosticsRequest } from '@dxos/protocols/proto/dxos/client/services';
+import { TRACE_PROCESSOR } from '@dxos/tracing';
+import { type JsonKeyOptions, jsonKeyReplacer, nonNullable } from '@dxos/util';
+
+import { createCollectDiagnosticsBroadcastSender } from './diagnostics-broadcast';
+import { ClientServicesProviderResource } from '../services';
+
+export interface CollectDiagnosticsBroadcastSender {
+  broadcastDiagnosticsRequest(): any;
+}
+
+export interface CollectDiagnosticsBroadcastHandler {
+  start(): void;
+  stop(): void;
+}
+
+export class DiagnosticsCollector {
+  private static broadcastSender = createCollectDiagnosticsBroadcastSender();
+
+  public static async collect(
+    config: Config | Config[] = findConfigs(),
+    services: ClientServicesProvider | null = findSystemServiceProvider(),
+    options: JsonKeyOptions = {},
+  ): Promise<any> {
+    const serviceDiagnostics = await services?.services?.SystemService?.getDiagnostics({
+      keys: options.humanize
+        ? GetDiagnosticsRequest.KEY_OPTION.HUMANIZE
+        : options.truncate
+          ? GetDiagnosticsRequest.KEY_OPTION.TRUNCATE
+          : undefined,
+    });
+
+    const clientDiagnostics = {
+      config,
+      trace: TRACE_PROCESSOR.getDiagnostics(),
+    };
+
+    const diagnostics =
+      serviceDiagnostics != null
+        ? { client: clientDiagnostics, services: serviceDiagnostics }
+        : {
+            client: clientDiagnostics,
+            broadcast: await this.broadcastSender.broadcastDiagnosticsRequest(),
+          };
+
+    return JSON.parse(JSON.stringify(diagnostics, jsonKeyReplacer(options)));
+  }
+}
+
+const findSystemServiceProvider = (): ClientServicesProvider | null => {
+  const serviceProviders = TRACE_PROCESSOR.findByAnnotation(ClientServicesProviderResource);
+  const providerResource = serviceProviders.find((r) => r.instance.deref()?.services?.SystemService != null);
+  return providerResource?.instance?.deref() ?? null;
+};
+
+const findConfigs = (): Config[] => {
+  const configs = TRACE_PROCESSOR.findByAnnotation(ConfigResource);
+  return configs.map((r) => r.instance.deref()).filter(nonNullable);
+};

package/src/packlets/{services → diagnostics}/diagnostics.ts

@@ -25,9 +25,9 @@ import { type Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type Resource, type Span } from '@dxos/protocols/proto/dxos/tracing';
 import { TRACE_PROCESSOR } from '@dxos/tracing';
 
-import { getPlatform } from './platform';
-import { type ServiceContext } from './service-context';
 import { DXOS_VERSION } from '../../version';
+import { type ServiceContext } from '../services';
+import { getPlatform } from '../services/platform';
 import { type DataSpace } from '../spaces';
 
 const DEFAULT_TIMEOUT = 1_000;

package/src/packlets/diagnostics/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './diagnostics';
+export * from './diagnostics-collector';
+export * from './diagnostics-broadcast';

package/src/packlets/indexing/util.ts

@@ -7,7 +7,7 @@ import { type DocHandle } from '@dxos/automerge/automerge-repo';
 import { warnAfterTimeout } from '@dxos/debug';
 import { type AutomergeHost } from '@dxos/echo-pipeline';
 import { type ObjectSnapshot } from '@dxos/indexing';
-import { idCodec } from '@dxos/protocols';
+import { type ObjectPointerEncoded, idCodec } from '@dxos/protocols';
 
 /**
  * Factory for `loadDocuments` iterator.
@@ -18,7 +18,7 @@ export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
    * @param ids
    */
   // TODO(mykola): Unload automerge handles after usage.
-  async function* loadDocuments(ids: string[]) {
+  async function* loadDocuments(ids: ObjectPointerEncoded[]) {
     for (const id of ids) {
       const { documentId, objectId } = idCodec.decode(id);
       const handle = automergeHost.repo.find(documentId as any);

package/src/packlets/invitations/invitation-extension.ts

@@ -4,6 +4,7 @@
 
 import { Trigger } from '@dxos/async';
 import { cancelWithContext, Context } from '@dxos/context';
+import { randomBytes, verify } from '@dxos/crypto';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -51,6 +52,8 @@ export class InvitationHostExtension extends RpcExtension<
   private _remoteOptions?: Options;
   private _remoteOptionsTrigger = new Trigger();
 
+  private _challenge?: Buffer = undefined;
+
   public invitation?: Invitation = undefined;
 
   public guestProfile?: ProfileDocument = undefined;
@@ -113,13 +116,17 @@ export class InvitationHostExtension extends RpcExtension<
 
           this._callbacks.onStateUpdate({ ...this.invitation, state: Invitation.State.READY_FOR_AUTHENTICATION });
 
+          this._challenge =
+            this.invitation.authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY ? randomBytes(32) : undefined;
+
           log.trace('dxos.sdk.invitation-handler.host.introduce', trace.end({ id: traceId }));
           return {
             authMethod: this.invitation.authMethod,
+            challenge: this._challenge,
           };
         },
 
-        authenticate: async ({ authCode: code }) => {
+        authenticate: async ({ authCode: code, signedChallenge }) => {
           const traceId = PublicKey.random().toHex();
           log.trace('dxos.sdk.invitation-handler.host.authenticate', trace.begin({ id: traceId }));
           log('received authentication request', { authCode: code });
@@ -145,6 +152,26 @@ export class InvitationHostExtension extends RpcExtension<
               break;
             }
 
+            case Invitation.AuthMethod.KNOWN_PUBLIC_KEY: {
+              if (!this.invitation.guestKeypair) {
+                status = AuthenticationResponse.Status.INTERNAL_ERROR;
+                break;
+              }
+              const isSignatureValid =
+                this._challenge &&
+                verify(
+                  this._challenge,
+                  Buffer.from(signedChallenge ?? []),
+                  this.invitation.guestKeypair.publicKey.asBuffer(),
+                );
+              if (isSignatureValid) {
+                this.authenticationPassed = true;
+              } else {
+                status = AuthenticationResponse.Status.INVALID_SIGNATURE;
+              }
+              break;
+            }
+
             default: {
               log.error('invalid authentication method', { authMethod: this.invitation.authMethod });
               status = AuthenticationResponse.Status.INTERNAL_ERROR;

package/src/packlets/invitations/invitations-handler.ts

@@ -11,6 +11,7 @@ import {
 } from '@dxos/client-protocol';
 import { Context } from '@dxos/context';
 import { generatePasscode } from '@dxos/credentials';
+import { createKeyPair, sign } from '@dxos/crypto';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -21,9 +22,9 @@ import {
   type SwarmConnection,
 } from '@dxos/network-manager';
 import { InvalidInvitationExtensionRoleError, trace } from '@dxos/protocols';
-import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
+import { type AdmissionKeypair, Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type DeviceProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { AuthenticationResponse } from '@dxos/protocols/proto/dxos/halo/invitations';
+import { AuthenticationResponse, type IntroductionResponse } from '@dxos/protocols/proto/dxos/halo/invitations';
 
 import {
   InvitationGuestExtension,
@@ -74,9 +75,11 @@ export class InvitationsHandler {
       state = Invitation.State.INIT,
       timeout = INVITATION_TIMEOUT,
       swarmKey = PublicKey.random(),
-      persistent = true,
+      persistent = options?.authMethod !== Invitation.AuthMethod.KNOWN_PUBLIC_KEY, // default no not storing keypairs
       created = new Date(),
-      lifetime = 86400, // 1 day
+      guestKeypair = undefined,
+      lifetime = 86400, // 1 day,
+      multiUse = false,
     } = options ?? {};
     const authCode =
       options?.authCode ??
@@ -91,9 +94,12 @@ export class InvitationsHandler {
       swarmKey,
       authCode,
       timeout,
-      persistent,
+      persistent: persistent && type !== Invitation.Type.DELEGATED, // delegated invitations are persisted in control feed
+      guestKeypair:
+        guestKeypair ?? (authMethod === Invitation.AuthMethod.KNOWN_PUBLIC_KEY ? createAdmissionKeypair() : undefined),
       created,
       lifetime,
+      multiUse,
       ...protocol.getInvitationContext(),
     };
 
@@ -162,7 +168,7 @@ export class InvitationsHandler {
               }
               log.trace('dxos.sdk.invitations-handler.host.onOpen', trace.error({ id: traceId, error: err }));
             } finally {
-              if (type !== Invitation.Type.MULTIUSE) {
+              if (!multiUse) {
                 // Wait for graceful close before disposing.
                 await swarmConnection.close();
                 await ctx.dispose();
@@ -317,26 +323,13 @@ export class InvitationsHandler {
 
               // 2. Get authentication code.
               if (isAuthenticationRequired(invitation)) {
-                for (let attempt = 1; attempt <= MAX_OTP_ATTEMPTS; attempt++) {
-                  log('guest waiting for authentication code...');
-                  setState({ state: Invitation.State.READY_FOR_AUTHENTICATION });
-                  const authCode = await authenticated.wait({ timeout });
-
-                  log('sending authentication request');
-                  setState({ state: Invitation.State.AUTHENTICATING });
-                  const response = await extension.rpc.InvitationHostService.authenticate({ authCode });
-                  if (response.status === undefined || response.status === AuthenticationResponse.Status.OK) {
+                switch (invitation.authMethod) {
+                  case Invitation.AuthMethod.SHARED_SECRET:
+                    await this._handleGuestOtpAuth(extension, setState, authenticated, { timeout });
+                    break;
+                  case Invitation.AuthMethod.KNOWN_PUBLIC_KEY:
+                    await this._handleGuestKpkAuth(extension, setState, invitation, introductionResponse);
                     break;
-                  }
-
-                  if (response.status === AuthenticationResponse.Status.INVALID_OTP) {
-                    if (attempt === MAX_OTP_ATTEMPTS) {
-                      throw new Error(`Maximum retry attempts: ${MAX_OTP_ATTEMPTS}`);
-                    } else {
-                      log('retrying invalid code', { attempt });
-                      authenticated.reset();
-                    }
-                  }
                 }
               }
 
@@ -423,13 +416,61 @@ export class InvitationsHandler {
 
     return observable;
   }
+
+  private async _handleGuestOtpAuth(
+    extension: InvitationGuestExtension,
+    setState: (newState: Partial<Invitation>) => void,
+    authenticated: Trigger<string>,
+    options: { timeout: number },
+  ) {
+    for (let attempt = 1; attempt <= MAX_OTP_ATTEMPTS; attempt++) {
+      log('guest waiting for authentication code...');
+      setState({ state: Invitation.State.READY_FOR_AUTHENTICATION });
+      const authCode = await authenticated.wait(options);
+
+      log('sending authentication request');
+      setState({ state: Invitation.State.AUTHENTICATING });
+      const response = await extension.rpc.InvitationHostService.authenticate({ authCode });
+      if (response.status === undefined || response.status === AuthenticationResponse.Status.OK) {
+        break;
+      }
+
+      if (response.status === AuthenticationResponse.Status.INVALID_OTP) {
+        if (attempt === MAX_OTP_ATTEMPTS) {
+          throw new Error(`Maximum retry attempts: ${MAX_OTP_ATTEMPTS}`);
+        } else {
+          log('retrying invalid code', { attempt });
+          authenticated.reset();
+        }
+      }
+    }
+  }
+
+  private async _handleGuestKpkAuth(
+    extension: InvitationGuestExtension,
+    setState: (newState: Partial<Invitation>) => void,
+    invitation: Invitation,
+    introductionResponse: IntroductionResponse,
+  ) {
+    if (invitation.guestKeypair?.privateKey == null) {
+      throw new Error('keypair missing in the invitation');
+    }
+    if (introductionResponse.challenge == null) {
+      throw new Error('challenge missing in the introduction');
+    }
+    log('sending authentication request');
+    setState({ state: Invitation.State.AUTHENTICATING });
+    const signature = sign(Buffer.from(introductionResponse.challenge), invitation.guestKeypair.privateKey);
+    const response = await extension.rpc.InvitationHostService.authenticate({
+      signedChallenge: signature,
+    });
+    if (response.status !== AuthenticationResponse.Status.OK) {
+      throw new Error(`Authentication failed with code: ${response.status}`);
+    }
+  }
 }
 
-export const invitationExpired = (invitation: Invitation) => {
-  return (
-    invitation.created &&
-    invitation.lifetime &&
-    invitation.lifetime !== 0 &&
-    invitation.created.getTime() + invitation.lifetime * 1000 < Date.now()
-  );
+export const createAdmissionKeypair = (): AdmissionKeypair => {
+  const keypair = createKeyPair();
+  return { publicKey: PublicKey.from(keypair.publicKey), privateKey: keypair.secretKey };
 };

package/src/packlets/invitations/invitations-service.ts

@@ -6,7 +6,7 @@ import { Event, scheduleTask } from '@dxos/async';
 import { type AuthenticatingInvitation, type CancellableInvitation } from '@dxos/client-protocol';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
-import { type MetadataStore } from '@dxos/echo-pipeline';
+import { hasInvitationExpired, type MetadataStore } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import {
@@ -18,7 +18,7 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 
 import { type InvitationProtocol } from './invitation-protocol';
-import { invitationExpired, type InvitationsHandler } from './invitations-handler';
+import { type InvitationsHandler } from './invitations-handler';
 
 /**
  * Adapts invitation service observable to client/service stream.
@@ -91,7 +91,7 @@ export class InvitationsServiceImpl implements InvitationsService {
           }
 
           this._createInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedCreated.emit(invitation.get());
           }
         },
@@ -103,7 +103,7 @@ export class InvitationsServiceImpl implements InvitationsService {
     const persistentInvitations = this._metadataStore.getInvitations();
 
     // get saved persistent invitations, filter and remove from storage those that have expired.
-    const freshInvitations = persistentInvitations.filter(async (invitation) => !invitationExpired(invitation));
+    const freshInvitations = persistentInvitations.filter(async (invitation) => !hasInvitationExpired(invitation));
 
     const cInvitations = freshInvitations.map((persistentInvitation) => {
       invariant(!this._createInvitations.get(persistentInvitation.invitationId), 'invitation already exists');
@@ -148,7 +148,7 @@ export class InvitationsServiceImpl implements InvitationsService {
         () => {
           close();
           this._acceptInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedAccepted.emit(invitation.get());
           }
         },

package/src/packlets/services/automerge-host.test.ts

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { AutomergeHost, DataServiceImpl } from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { AutomergeContext } from '@dxos/echo-schema';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, test } from '@dxos/test';
 
 describe('AutomergeHost', () => {
@@ -19,10 +19,16 @@ describe('AutomergeHost', () => {
     // creates repo and document | replicates repo | finds document in repo
     //
 
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
     afterTest(() => host.close());
+
     const dataService = new DataServiceImpl(host);
     const client = new AutomergeContext(dataService);
     afterTest(() => client.close());

package/src/packlets/services/index.ts

@@ -3,7 +3,7 @@
 //
 
 export * from './client-rpc-server';
-export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';
+export { ClientServicesProviderResource } from './util';

package/src/packlets/services/service-context.test.ts

@@ -4,7 +4,7 @@
 
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
-import { describe, test } from '@dxos/test';
+import { describe, openAndClose, test } from '@dxos/test';
 
 import { createServiceContext } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -27,12 +27,15 @@ describe('services/ServiceContext', () => {
   test('joined space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
     const device1 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device1.automergeHost);
     await device1.createIdentity();
 
     const device2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device2.automergeHost);
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     const identity2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(identity2.automergeHost);
     await identity2.createIdentity();
     const space1 = await identity2.dataSpaceManager!.createSpace();
     await Promise.all(

package/src/packlets/services/service-context.ts

@@ -10,7 +10,7 @@ import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credenti
 import { failUndefined } from '@dxos/debug';
 import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
-import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
+import { IndexMetadataStore, IndexStore, Indexer, createStorageCallbacks } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
@@ -124,11 +124,12 @@ export class ServiceContext extends Resource {
 
     this.automergeHost = new AutomergeHost({
       directory: storage.createDirectory('automerge'),
-      metadata: this.indexMetadata,
+      db: level.sublevel('automerge'),
+      storageCallbacks: createStorageCallbacks({ host: () => this.automergeHost, metadata: this.indexMetadata }),
     });
 
     this.indexer = new Indexer({
-      indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
+      indexStore: new IndexStore({ db: level.sublevel('index-store') }),
       metadataStore: this.indexMetadata,
       loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
       getAllDocuments: createDocumentsIterator(this.automergeHost),
@@ -158,6 +159,7 @@ export class ServiceContext extends Resource {
     await this.signalManager.open();
     await this.networkManager.open();
 
+    await this.automergeHost.open();
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);