@dxos/client-services 0.4.10-main.826d4ee → 0.4.10-main.8583d5a

package/src/index.ts

@@ -10,3 +10,4 @@ export * from './packlets/spaces';
 export * from './packlets/storage';
 export * from './packlets/vault';
 export * from './packlets/locks';
+export * from './packlets/diagnostics';

package/src/packlets/devices/devices-service.test.ts

@@ -19,7 +19,7 @@ describe('DevicesService', () => {
   let devicesService: DevicesService;
 
   beforeEach(async () => {
-    serviceContext = createServiceContext();
+    serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
     devicesService = new DevicesServiceImpl(serviceContext.identityManager);
   });

package/src/packlets/diagnostics/browser-diagnostics-broadcast.ts

@@ -0,0 +1,94 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Trigger } from '@dxos/async';
+import { log } from '@dxos/log';
+import { type SystemService } from '@dxos/protocols/proto/dxos/client/services';
+
+import {
+  type CollectDiagnosticsBroadcastSender,
+  type CollectDiagnosticsBroadcastHandler,
+} from './diagnostics-collector';
+
+const CHANNEL_NAME = 'dxos.diagnostics.broadcast';
+
+enum MessageType {
+  PROBE = 'probe',
+  PROBE_ACK = 'probe-ack',
+  REQUEST_DIAGNOSTICS = 'request-diagnostics',
+  RECEIVE_DIAGNOSTICS = 'receive-diagnostics',
+}
+
+interface Message {
+  type: MessageType;
+  payload?: any;
+}
+
+export const createCollectDiagnosticsBroadcastSender = (): CollectDiagnosticsBroadcastSender => {
+  return {
+    broadcastDiagnosticsRequest: async () => {
+      let expectedResponse = MessageType.PROBE_ACK;
+      let channel: BroadcastChannel | undefined;
+      try {
+        const trigger = new Trigger<Message>();
+        channel = new BroadcastChannel(CHANNEL_NAME);
+        channel.onmessage = (msg) => {
+          if (expectedResponse === msg.data.type) {
+            trigger.wake(msg.data);
+          }
+        };
+        channel.postMessage({ type: MessageType.PROBE });
+        await trigger.wait({ timeout: 200 });
+        expectedResponse = MessageType.RECEIVE_DIAGNOSTICS;
+        trigger.reset();
+        channel.postMessage({ type: MessageType.REQUEST_DIAGNOSTICS });
+        const diagnostics = await trigger.wait({ timeout: 5000 });
+        return diagnostics.payload;
+      } catch (e) {
+        const errorDescription = e instanceof Error ? e.message : JSON.stringify(e);
+        return { expectedResponse, errorDescription };
+      } finally {
+        safeClose(channel);
+      }
+    },
+  };
+};
+
+export const createCollectDiagnosticsBroadcastHandler = (
+  systemService: SystemService,
+): CollectDiagnosticsBroadcastHandler => {
+  let channel: BroadcastChannel | undefined;
+  return {
+    start: () => {
+      channel = new BroadcastChannel(CHANNEL_NAME);
+      channel.onmessage = async (message) => {
+        try {
+          if (message.data.type === MessageType.PROBE) {
+            channel?.postMessage({ type: MessageType.PROBE_ACK });
+          } else if (message.data.type === MessageType.REQUEST_DIAGNOSTICS) {
+            const diagnostics = await systemService.getDiagnostics({});
+            channel?.postMessage({
+              type: MessageType.RECEIVE_DIAGNOSTICS,
+              payload: diagnostics,
+            });
+          }
+        } catch (error) {
+          log.catch(error);
+        }
+      };
+    },
+    stop: () => {
+      safeClose(channel);
+      channel = undefined;
+    },
+  };
+};
+
+const safeClose = (channel?: BroadcastChannel) => {
+  try {
+    channel?.close();
+  } catch (e) {
+    // ignored
+  }
+};

package/src/packlets/diagnostics/diagnostics-broadcast.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2024 DXOS.org
+//
+import { type SystemService } from '@dxos/protocols/proto/dxos/client/services';
+
+import {
+  type CollectDiagnosticsBroadcastSender,
+  type CollectDiagnosticsBroadcastHandler,
+} from './diagnostics-collector';
+
+export const createCollectDiagnosticsBroadcastSender = (): CollectDiagnosticsBroadcastSender => {
+  return { broadcastDiagnosticsRequest: async () => undefined };
+};
+
+export const createCollectDiagnosticsBroadcastHandler = (_: SystemService): CollectDiagnosticsBroadcastHandler => {
+  return {
+    start: () => {},
+    stop: () => {},
+  };
+};

package/src/packlets/diagnostics/diagnostics-collector.ts

@@ -0,0 +1,65 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ClientServicesProvider } from '@dxos/client-protocol';
+import { type Config, ConfigResource } from '@dxos/config';
+import { GetDiagnosticsRequest } from '@dxos/protocols/proto/dxos/client/services';
+import { TRACE_PROCESSOR } from '@dxos/tracing';
+import { type JsonKeyOptions, jsonKeyReplacer, nonNullable } from '@dxos/util';
+
+import { createCollectDiagnosticsBroadcastSender } from './diagnostics-broadcast';
+import { ClientServicesProviderResource } from '../services';
+
+export interface CollectDiagnosticsBroadcastSender {
+  broadcastDiagnosticsRequest(): any;
+}
+
+export interface CollectDiagnosticsBroadcastHandler {
+  start(): void;
+  stop(): void;
+}
+
+export class DiagnosticsCollector {
+  private static broadcastSender = createCollectDiagnosticsBroadcastSender();
+
+  public static async collect(
+    config: Config | Config[] = findConfigs(),
+    services: ClientServicesProvider | null = findSystemServiceProvider(),
+    options: JsonKeyOptions = {},
+  ): Promise<any> {
+    const serviceDiagnostics = await services?.services?.SystemService?.getDiagnostics({
+      keys: options.humanize
+        ? GetDiagnosticsRequest.KEY_OPTION.HUMANIZE
+        : options.truncate
+          ? GetDiagnosticsRequest.KEY_OPTION.TRUNCATE
+          : undefined,
+    });
+
+    const clientDiagnostics = {
+      config,
+      trace: TRACE_PROCESSOR.getDiagnostics(),
+    };
+
+    const diagnostics =
+      serviceDiagnostics != null
+        ? { client: clientDiagnostics, services: serviceDiagnostics }
+        : {
+            client: clientDiagnostics,
+            broadcast: await this.broadcastSender.broadcastDiagnosticsRequest(),
+          };
+
+    return JSON.parse(JSON.stringify(diagnostics, jsonKeyReplacer(options)));
+  }
+}
+
+const findSystemServiceProvider = (): ClientServicesProvider | null => {
+  const serviceProviders = TRACE_PROCESSOR.findByAnnotation(ClientServicesProviderResource);
+  const providerResource = serviceProviders.find((r) => r.instance.deref()?.services?.SystemService != null);
+  return providerResource?.instance?.deref() ?? null;
+};
+
+const findConfigs = (): Config[] => {
+  const configs = TRACE_PROCESSOR.findByAnnotation(ConfigResource);
+  return configs.map((r) => r.instance.deref()).filter(nonNullable);
+};

package/src/packlets/{services → diagnostics}/diagnostics.ts

@@ -25,9 +25,9 @@ import { type Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type Resource, type Span } from '@dxos/protocols/proto/dxos/tracing';
 import { TRACE_PROCESSOR } from '@dxos/tracing';
 
-import { getPlatform } from './platform';
-import { type ServiceContext } from './service-context';
 import { DXOS_VERSION } from '../../version';
+import { type ServiceContext } from '../services';
+import { getPlatform } from '../services/platform';
 import { type DataSpace } from '../spaces';
 
 const DEFAULT_TIMEOUT = 1_000;

package/src/packlets/diagnostics/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './diagnostics';
+export * from './diagnostics-collector';
+export * from './diagnostics-broadcast';

package/src/packlets/identity/identity-service.test.ts

@@ -22,7 +22,7 @@ describe('IdentityService', () => {
   let identityService: IdentityService;
 
   beforeEach(async () => {
-    serviceContext = createServiceContext();
+    serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
     identityService = new IdentityServiceImpl(
       (options) => serviceContext.createIdentity(options),

package/src/packlets/indexing/util.ts

@@ -12,7 +12,7 @@ import { idCodec } from '@dxos/protocols';
 /**
  * Factory for `loadDocuments` iterator.
  */
-export const createLoadDocuments = (automergeHost: AutomergeHost) =>
+export const createSelectedDocumentsIterator = (automergeHost: AutomergeHost) =>
   /**
    * Get object data blobs from Automerge Repo by ids.
    * @param ids
@@ -25,14 +25,14 @@ export const createLoadDocuments = (automergeHost: AutomergeHost) =>
       await warnAfterTimeout(5000, 'to long to load doc', () => handle.whenReady());
       const doc = handle.docSync();
       const hash = getHeads(doc).join('');
-      yield [{ id, object: doc.objects[objectId], currentHash: hash }];
+      yield doc.objects?.[objectId] ? [{ id, object: doc.objects[objectId], currentHash: hash }] : [];
     }
   };
 
 /**
  * Factory for `getAllDocuments` iterator.
  */
-export const createGetAllDocuments = (automergeHost: AutomergeHost) =>
+export const createDocumentsIterator = (automergeHost: AutomergeHost) =>
   /**
    * Recursively get all object data blobs from Automerge Repo.
    * @param ids
@@ -57,7 +57,7 @@ export const createGetAllDocuments = (automergeHost: AutomergeHost) =>
           return {
             id: idCodec.encode({ documentId: handle.documentId, objectId }),
             object,
-            currentHash: heads.at(-1)!,
+            currentHash: heads.join(''),
           };
         });
       }

package/src/packlets/invitations/device-invitation-protocol.test.ts

@@ -20,7 +20,7 @@ const closeAfterTest = async (peer: ServiceContext) => {
 
 describe('services/device', () => {
   test('creates identity', async () => {
-    const peer = createServiceContext();
+    const peer = await createServiceContext();
     await peer.open(new Context());
     afterTest(() => peer.close());
 

package/src/packlets/invitations/invitations-handler.ts

@@ -76,7 +76,8 @@ export class InvitationsHandler {
       swarmKey = PublicKey.random(),
       persistent = true,
       created = new Date(),
-      lifetime = 86400, // 1 day
+      lifetime = 86400, // 1 day,
+      multiUse = false,
     } = options ?? {};
     const authCode =
       options?.authCode ??
@@ -91,9 +92,10 @@ export class InvitationsHandler {
       swarmKey,
       authCode,
       timeout,
-      persistent,
+      persistent: persistent && type !== Invitation.Type.OFFLINE, // offline invitations are persisted in control feed
       created,
       lifetime,
+      multiUse,
       ...protocol.getInvitationContext(),
     };
 
@@ -162,7 +164,7 @@ export class InvitationsHandler {
               }
               log.trace('dxos.sdk.invitations-handler.host.onOpen', trace.error({ id: traceId, error: err }));
             } finally {
-              if (type !== Invitation.Type.MULTIUSE) {
+              if (!multiUse) {
                 // Wait for graceful close before disposing.
                 await swarmConnection.close();
                 await ctx.dispose();
@@ -424,12 +426,3 @@ export class InvitationsHandler {
     return observable;
   }
 }
-
-export const invitationExpired = (invitation: Invitation) => {
-  return (
-    invitation.created &&
-    invitation.lifetime &&
-    invitation.lifetime !== 0 &&
-    invitation.created.getTime() + invitation.lifetime * 1000 < Date.now()
-  );
-};

package/src/packlets/invitations/invitations-service.ts

@@ -6,7 +6,7 @@ import { Event, scheduleTask } from '@dxos/async';
 import { type AuthenticatingInvitation, type CancellableInvitation } from '@dxos/client-protocol';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
-import { type MetadataStore } from '@dxos/echo-pipeline';
+import { hasInvitationExpired, type MetadataStore } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import {
@@ -18,7 +18,7 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 
 import { type InvitationProtocol } from './invitation-protocol';
-import { invitationExpired, type InvitationsHandler } from './invitations-handler';
+import { type InvitationsHandler } from './invitations-handler';
 
 /**
  * Adapts invitation service observable to client/service stream.
@@ -91,7 +91,7 @@ export class InvitationsServiceImpl implements InvitationsService {
           }
 
           this._createInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedCreated.emit(invitation.get());
           }
         },
@@ -103,7 +103,7 @@ export class InvitationsServiceImpl implements InvitationsService {
     const persistentInvitations = this._metadataStore.getInvitations();
 
     // get saved persistent invitations, filter and remove from storage those that have expired.
-    const freshInvitations = persistentInvitations.filter(async (invitation) => !invitationExpired(invitation));
+    const freshInvitations = persistentInvitations.filter(async (invitation) => !hasInvitationExpired(invitation));
 
     const cInvitations = freshInvitations.map((persistentInvitation) => {
       invariant(!this._createInvitations.get(persistentInvitation.invitationId), 'invitation already exists');
@@ -148,7 +148,7 @@ export class InvitationsServiceImpl implements InvitationsService {
         () => {
           close();
           this._acceptInvitations.delete(invitation.get().invitationId);
-          if (invitation.get().type !== Invitation.Type.MULTIUSE) {
+          if (!invitation.get().multiUse) {
             this._removedAccepted.emit(invitation.get());
           }
         },

package/src/packlets/network/network-service.test.ts

@@ -18,7 +18,7 @@ describe('NetworkService', () => {
   let networkService: NetworkService;
 
   beforeEach(async () => {
-    serviceContext = createServiceContext();
+    serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
     networkService = new NetworkServiceImpl(serviceContext.networkManager, serviceContext.signalManager);
   });

package/src/packlets/services/automerge-host.test.ts

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { AutomergeHost, DataServiceImpl } from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { AutomergeContext } from '@dxos/echo-schema';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, test } from '@dxos/test';
 
 describe('AutomergeHost', () => {
@@ -19,10 +19,16 @@ describe('AutomergeHost', () => {
     // creates repo and document | replicates repo | finds document in repo
     //
 
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
     afterTest(() => host.close());
+
     const dataService = new DataServiceImpl(host);
     const client = new AutomergeContext(dataService);
     afterTest(() => client.close());

package/src/packlets/services/index.ts

@@ -3,7 +3,7 @@
 //
 
 export * from './client-rpc-server';
-export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';
+export { ClientServicesProviderResource } from './util';

package/src/packlets/services/service-context.test.ts

@@ -4,7 +4,7 @@
 
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
-import { describe, test } from '@dxos/test';
+import { describe, openAndClose, test } from '@dxos/test';
 
 import { createServiceContext } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -12,10 +12,10 @@ import { performInvitation } from '../testing/invitation-utils';
 describe('services/ServiceContext', () => {
   test('new space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
-    const device1 = createServiceContext({ signalContext: networkContext });
+    const device1 = await createServiceContext({ signalContext: networkContext });
     await device1.createIdentity();
 
-    const device2 = createServiceContext({ signalContext: networkContext });
+    const device2 = await createServiceContext({ signalContext: networkContext });
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     const space1 = await device1.dataSpaceManager!.createSpace();
@@ -26,13 +26,16 @@ describe('services/ServiceContext', () => {
 
   test('joined space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
-    const device1 = createServiceContext({ signalContext: networkContext });
+    const device1 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device1.automergeHost);
     await device1.createIdentity();
 
-    const device2 = createServiceContext({ signalContext: networkContext });
+    const device2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device2.automergeHost);
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
-    const identity2 = createServiceContext({ signalContext: networkContext });
+    const identity2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(identity2.automergeHost);
     await identity2.createIdentity();
     const space1 = await identity2.dataSpaceManager!.createSpace();
     await Promise.all(

package/src/packlets/services/service-context.ts

@@ -2,8 +2,10 @@
 // Copyright 2022 DXOS.org
 //
 
+import { type Level } from 'level';
+
 import { Trigger } from '@dxos/async';
-import { Context } from '@dxos/context';
+import { Context, Resource } from '@dxos/context';
 import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
@@ -30,7 +32,7 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
-import { createGetAllDocuments, createLoadDocuments } from '../indexing';
+import { createDocumentsIterator, createSelectedDocumentsIterator } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -47,7 +49,7 @@ export type ServiceContextRuntimeParams = IdentityManagerRuntimeParams & DataSpa
 // TODO(dmaretskyi): Gets duplicated in CJS build between normal and testing bundles.
 @safeInstanceof('dxos.client-services.ServiceContext')
 @Trace.resource()
-export class ServiceContext {
+export class ServiceContext extends Resource {
   public readonly initialized = new Trigger();
   public readonly metadataStore: MetadataStore;
   /**
@@ -78,10 +80,13 @@ export class ServiceContext {
 
   constructor(
     public readonly storage: Storage,
+    public readonly level: Level<string, string>,
     public readonly networkManager: NetworkManager,
     public readonly signalManager: SignalManager,
     public readonly _runtimeParams?: IdentityManagerRuntimeParams & DataSpaceManagerRuntimeParams,
   ) {
+    super();
+
     // TODO(burdon): Move strings to constants.
     this.metadataStore = new MetadataStore(storage.createDirectory('metadata'));
     this.snapshotStore = new SnapshotStore(storage.createDirectory('snapshots'));
@@ -115,18 +120,19 @@ export class ServiceContext {
       this._runtimeParams as IdentityManagerRuntimeParams,
     );
 
-    this.indexMetadata = new IndexMetadataStore({ directory: storage.createDirectory('index-metadata') });
+    this.indexMetadata = new IndexMetadataStore({ db: level.sublevel('index-metadata') });
 
     this.automergeHost = new AutomergeHost({
       directory: storage.createDirectory('automerge'),
+      db: level.sublevel('automerge'),
       metadata: this.indexMetadata,
     });
 
     this.indexer = new Indexer({
       indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
       metadataStore: this.indexMetadata,
-      loadDocuments: createLoadDocuments(this.automergeHost),
-      getAllDocuments: createGetAllDocuments(this.automergeHost),
+      loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
+      getAllDocuments: createDocumentsIterator(this.automergeHost),
     });
 
     this.invitations = new InvitationsHandler(this.networkManager);
@@ -145,7 +151,7 @@ export class ServiceContext {
   }
 
   @Trace.span()
-  async open(ctx: Context) {
+  protected override async _open(ctx: Context) {
     await this._checkStorageVersion();
 
     log('opening...');
@@ -153,6 +159,7 @@ export class ServiceContext {
     await this.signalManager.open();
     await this.networkManager.open();
 
+    await this.automergeHost.open();
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);
@@ -163,7 +170,7 @@ export class ServiceContext {
     log('opened');
   }
 
-  async close() {
+  protected override async _close() {
     log('closing...');
     if (this._deviceSpaceSync && this.identityManager.identity) {
       await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);